From 378b2f21e22b11e8b912b077f2032e63c1ca0c81 Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Tue, 26 Nov 2024 15:41:38 +0800
Subject: [PATCH 001/399] [opt](exec)lazy deserialize pblock in
 VDataStreamRecvr::SenderQueue  (#44378)

### What problem does this PR solve?

Previously, for a `pblock` (serialized block), the block would be
deserialized immediately
after receiving the RPC request and then placed into the `data_queue`.
This approach caused significant time consumption during RPC processing
due to the
deserialization process, impacting overall performance.
The new approach defers deserialization until `getBlock` is called. This
has the following advantages:
1. Reduces time spent during the RPC handling phase.
2. Memory allocation for deserialization happens within the execution
thread, improving cache locality
   and reducing contention on memory resources.

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [x] No need to test or manual test. Explain why:
- [x] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [x] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 be/src/vec/runtime/vdata_stream_mgr.cpp   | 11 +++-
 be/src/vec/runtime/vdata_stream_recvr.cpp | 68 ++++++++++-------------
 be/src/vec/runtime/vdata_stream_recvr.h   | 49 +++++++++++++---
 3 files changed, 76 insertions(+), 52 deletions(-)

diff --git a/be/src/vec/runtime/vdata_stream_mgr.cpp b/be/src/vec/runtime/vdata_stream_mgr.cpp
index 382a6d0e6e3ece..7dad3d2c86705e 100644
--- a/be/src/vec/runtime/vdata_stream_mgr.cpp
+++ b/be/src/vec/runtime/vdata_stream_mgr.cpp
@@ -18,10 +18,12 @@
 #include "vec/runtime/vdata_stream_mgr.h"
 
 #include <gen_cpp/Types_types.h>
+#include <gen_cpp/data.pb.h>
 #include <gen_cpp/internal_service.pb.h>
 #include <gen_cpp/types.pb.h>
 #include <stddef.h>
 
+#include <memory>
 #include <ostream>
 #include <string>
 #include <vector>
@@ -141,9 +143,12 @@ Status VDataStreamMgr::transmit_block(const PTransmitDataParams* request,
 
     bool eos = request->eos();
     if (request->has_block()) {
-        RETURN_IF_ERROR(recvr->add_block(
-                request->block(), request->sender_id(), request->be_number(), request->packet_seq(),
-                eos ? nullptr : done, wait_for_worker, cpu_time_stop_watch.elapsed_time()));
+        std::unique_ptr<PBlock> pblock_ptr {
+                const_cast<PTransmitDataParams*>(request)->release_block()};
+        RETURN_IF_ERROR(recvr->add_block(std::move(pblock_ptr), request->sender_id(),
+                                         request->be_number(), request->packet_seq(),
+                                         eos ? nullptr : done, wait_for_worker,
+                                         cpu_time_stop_watch.elapsed_time()));
     }
 
     if (eos) {
diff --git a/be/src/vec/runtime/vdata_stream_recvr.cpp b/be/src/vec/runtime/vdata_stream_recvr.cpp
index b48b9f780b8754..81e4e1cd5f037e 100644
--- a/be/src/vec/runtime/vdata_stream_recvr.cpp
+++ b/be/src/vec/runtime/vdata_stream_recvr.cpp
@@ -69,7 +69,6 @@ VDataStreamRecvr::SenderQueue::~SenderQueue() {
 }
 
 Status VDataStreamRecvr::SenderQueue::get_batch(Block* block, bool* eos) {
-    std::lock_guard<std::mutex> l(_lock); // protect _block_queue
 #ifndef NDEBUG
     if (!_is_cancelled && _block_queue.empty() && _num_remaining_senders > 0) {
         throw doris::Exception(ErrorCode::INTERNAL_ERROR,
@@ -79,25 +78,33 @@ Status VDataStreamRecvr::SenderQueue::get_batch(Block* block, bool* eos) {
                                _debug_string_info());
     }
 #endif
-    return _inner_get_batch_without_lock(block, eos);
-}
+    BlockItem block_item;
+    {
+        std::lock_guard<std::mutex> l(_lock);
+        //check and get block_item from data_queue
+        if (_is_cancelled) {
+            RETURN_IF_ERROR(_cancel_status);
+            return Status::Cancelled("Cancelled");
+        }
 
-Status VDataStreamRecvr::SenderQueue::_inner_get_batch_without_lock(Block* block, bool* eos) {
-    if (_is_cancelled) {
-        RETURN_IF_ERROR(_cancel_status);
-        return Status::Cancelled("Cancelled");
-    }
+        if (_block_queue.empty()) {
+            DCHECK_EQ(_num_remaining_senders, 0);
+            *eos = true;
+            return Status::OK();
+        }
 
-    if (_block_queue.empty()) {
-        DCHECK_EQ(_num_remaining_senders, 0);
-        *eos = true;
-        return Status::OK();
+        DCHECK(!_block_queue.empty());
+        block_item = std::move(_block_queue.front());
+        _block_queue.pop_front();
     }
-
-    DCHECK(!_block_queue.empty());
-    auto [next_block, block_byte_size] = std::move(_block_queue.front());
-    _block_queue.pop_front();
+    BlockUPtr next_block;
+    RETURN_IF_ERROR(block_item.get_block(next_block));
+    size_t block_byte_size = block_item.block_byte_size();
+    COUNTER_UPDATE(_recvr->_deserialize_row_batch_timer, block_item.deserialize_time());
+    COUNTER_UPDATE(_recvr->_decompress_timer, block->get_decompress_time());
+    COUNTER_UPDATE(_recvr->_decompress_bytes, block->get_decompressed_bytes());
     _recvr->_parent->memory_used_counter()->update(-(int64_t)block_byte_size);
+    std::lock_guard<std::mutex> l(_lock);
     sub_blocks_memory_usage(block_byte_size);
     _record_debug_info();
     if (_block_queue.empty() && _source_dependency) {
@@ -133,7 +140,7 @@ void VDataStreamRecvr::SenderQueue::try_set_dep_ready_without_lock() {
     }
 }
 
-Status VDataStreamRecvr::SenderQueue::add_block(const PBlock& pblock, int be_number,
+Status VDataStreamRecvr::SenderQueue::add_block(std::unique_ptr<PBlock> pblock, int be_number,
                                                 int64_t packet_seq,
                                                 ::google::protobuf::Closure** done,
                                                 const int64_t wait_for_worker,
@@ -163,30 +170,12 @@ Status VDataStreamRecvr::SenderQueue::add_block(const PBlock& pblock, int be_num
         }
     }
 
-    BlockUPtr block = nullptr;
-    int64_t deserialize_time = 0;
-    {
-        SCOPED_RAW_TIMER(&deserialize_time);
-        block = Block::create_unique();
-        RETURN_IF_ERROR_OR_CATCH_EXCEPTION(block->deserialize(pblock));
-    }
-
-    const auto rows = block->rows();
-    if (rows == 0) {
-        return Status::OK();
-    }
-    auto block_byte_size = block->allocated_bytes();
-    VLOG_ROW << "added #rows=" << rows << " batch_size=" << block_byte_size << "\n";
-
     std::lock_guard<std::mutex> l(_lock);
     if (_is_cancelled) {
         return Status::OK();
     }
 
-    COUNTER_UPDATE(_recvr->_deserialize_row_batch_timer, deserialize_time);
-    COUNTER_UPDATE(_recvr->_decompress_timer, block->get_decompress_time());
-    COUNTER_UPDATE(_recvr->_decompress_bytes, block->get_decompressed_bytes());
-    COUNTER_UPDATE(_recvr->_rows_produced_counter, rows);
+    const auto block_byte_size = pblock->ByteSizeLong();
     COUNTER_UPDATE(_recvr->_blocks_produced_counter, 1);
     if (_recvr->_max_wait_worker_time->value() < wait_for_worker) {
         _recvr->_max_wait_worker_time->set(wait_for_worker);
@@ -196,7 +185,7 @@ Status VDataStreamRecvr::SenderQueue::add_block(const PBlock& pblock, int be_num
         _recvr->_max_find_recvr_time->set((int64_t)time_to_find_recvr);
     }
 
-    _block_queue.emplace_back(std::move(block), block_byte_size);
+    _block_queue.emplace_back(std::move(pblock), block_byte_size);
     COUNTER_UPDATE(_recvr->_remote_bytes_received_counter, block_byte_size);
     _record_debug_info();
     try_set_dep_ready_without_lock();
@@ -370,7 +359,6 @@ VDataStreamRecvr::VDataStreamRecvr(VDataStreamMgr* stream_mgr, pipeline::Exchang
     _first_batch_wait_total_timer = ADD_TIMER(_profile, "FirstBatchArrivalWaitTime");
     _decompress_timer = ADD_TIMER(_profile, "DecompressTime");
     _decompress_bytes = ADD_COUNTER(_profile, "DecompressBytes", TUnit::BYTES);
-    _rows_produced_counter = ADD_COUNTER(_profile, "RowsProduced", TUnit::UNIT);
     _blocks_produced_counter = ADD_COUNTER(_profile, "BlocksProduced", TUnit::UNIT);
     _max_wait_worker_time = ADD_COUNTER(_profile, "MaxWaitForWorkerTime", TUnit::UNIT);
     _max_wait_to_process_time = ADD_COUNTER(_profile, "MaxWaitToProcessTime", TUnit::UNIT);
@@ -401,13 +389,13 @@ Status VDataStreamRecvr::create_merger(const VExprContextSPtrs& ordering_expr,
     return Status::OK();
 }
 
-Status VDataStreamRecvr::add_block(const PBlock& pblock, int sender_id, int be_number,
+Status VDataStreamRecvr::add_block(std::unique_ptr<PBlock> pblock, int sender_id, int be_number,
                                    int64_t packet_seq, ::google::protobuf::Closure** done,
                                    const int64_t wait_for_worker,
                                    const uint64_t time_to_find_recvr) {
     SCOPED_ATTACH_TASK(_query_thread_context);
     int use_sender_id = _is_merging ? sender_id : 0;
-    return _sender_queues[use_sender_id]->add_block(pblock, be_number, packet_seq, done,
+    return _sender_queues[use_sender_id]->add_block(std::move(pblock), be_number, packet_seq, done,
                                                     wait_for_worker, time_to_find_recvr);
 }
 
diff --git a/be/src/vec/runtime/vdata_stream_recvr.h b/be/src/vec/runtime/vdata_stream_recvr.h
index 08fb004f3b1a8e..1639366c8b83d6 100644
--- a/be/src/vec/runtime/vdata_stream_recvr.h
+++ b/be/src/vec/runtime/vdata_stream_recvr.h
@@ -18,6 +18,7 @@
 #pragma once
 
 #include <gen_cpp/Types_types.h>
+#include <gen_cpp/data.pb.h>
 #include <glog/logging.h>
 #include <google/protobuf/stubs/callback.h>
 
@@ -84,9 +85,9 @@ class VDataStreamRecvr : public HasTaskExecutionCtx {
 
     std::vector<SenderQueue*> sender_queues() const { return _sender_queues; }
 
-    Status add_block(const PBlock& pblock, int sender_id, int be_number, int64_t packet_seq,
-                     ::google::protobuf::Closure** done, const int64_t wait_for_worker,
-                     const uint64_t time_to_find_recvr);
+    Status add_block(std::unique_ptr<PBlock> pblock, int sender_id, int be_number,
+                     int64_t packet_seq, ::google::protobuf::Closure** done,
+                     const int64_t wait_for_worker, const uint64_t time_to_find_recvr);
 
     void add_block(Block* block, int sender_id, bool use_move);
 
@@ -157,8 +158,6 @@ class VDataStreamRecvr : public HasTaskExecutionCtx {
     RuntimeProfile::Counter* _decompress_timer = nullptr;
     RuntimeProfile::Counter* _decompress_bytes = nullptr;
 
-    // Number of rows received
-    RuntimeProfile::Counter* _rows_produced_counter = nullptr;
     // Number of blocks received
     RuntimeProfile::Counter* _blocks_produced_counter = nullptr;
     RuntimeProfile::Counter* _max_wait_worker_time = nullptr;
@@ -181,7 +180,7 @@ class VDataStreamRecvr::SenderQueue {
 
     Status get_batch(Block* next_block, bool* eos);
 
-    Status add_block(const PBlock& pblock, int be_number, int64_t packet_seq,
+    Status add_block(std::unique_ptr<PBlock> pblock, int be_number, int64_t packet_seq,
                      ::google::protobuf::Closure** done, const int64_t wait_for_worker,
                      const uint64_t time_to_find_recvr);
 
@@ -205,8 +204,6 @@ class VDataStreamRecvr::SenderQueue {
 
 protected:
     friend class pipeline::ExchangeLocalState;
-    Status _inner_get_batch_without_lock(Block* block, bool* eos);
-
     void try_set_dep_ready_without_lock();
 
     // To record information about several variables in the event of a DCHECK failure.
@@ -260,7 +257,41 @@ class VDataStreamRecvr::SenderQueue {
     Status _cancel_status;
     int _num_remaining_senders;
     std::unique_ptr<MemTracker> _queue_mem_tracker;
-    std::list<std::pair<BlockUPtr, size_t>> _block_queue;
+
+    // `BlockItem` is used in `_block_queue` to handle both local and remote exchange blocks.
+    // For local exchange blocks, `BlockUPtr` is used directly without any modification.
+    // For remote exchange blocks, the `pblock` is stored in `BlockItem`.
+    // When `getBlock` is called, the `pblock` is deserialized into a usable block.
+    struct BlockItem {
+        Status get_block(BlockUPtr& block) {
+            if (!_block) {
+                DCHECK(_pblock);
+                SCOPED_RAW_TIMER(&_deserialize_time);
+                _block = Block::create_unique();
+                RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_block->deserialize(*_pblock));
+            }
+            block.swap(_block);
+            _block.reset();
+            return Status::OK();
+        }
+
+        size_t block_byte_size() const { return _block_byte_size; }
+        int64_t deserialize_time() const { return _deserialize_time; }
+        BlockItem() = default;
+        BlockItem(BlockUPtr&& block, size_t block_byte_size)
+                : _block(std::move(block)), _block_byte_size(block_byte_size) {}
+
+        BlockItem(std::unique_ptr<PBlock>&& pblock, size_t block_byte_size)
+                : _block(nullptr), _pblock(std::move(pblock)), _block_byte_size(block_byte_size) {}
+
+    private:
+        BlockUPtr _block;
+        std::unique_ptr<PBlock> _pblock;
+        size_t _block_byte_size = 0;
+        int64_t _deserialize_time = 0;
+    };
+
+    std::list<BlockItem> _block_queue;
 
     // sender_id
     std::unordered_set<int> _sender_eos_set;

From 3358156cf14531e9a9573bbcc965e608d05cdae7 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Tue, 26 Nov 2024 13:29:42 +0530
Subject: [PATCH 002/399] [Enhancement] (nereids)implement
 showLoadProfileCommand in nereids (#44440)

Issue Number: close #42773
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  8 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/ShowLoadProfileCommand.java      | 57 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../show/test_show_commands_nereids.groovy    |  2 +-
 6 files changed, 73 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLoadProfileCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 47214c8c271af6..30b924ed878a5c 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -211,6 +211,7 @@ supportedShowStatement
     | SHOW DELETE ((FROM | IN) database=multipartIdentifier)?                       #showDelete
     | SHOW ALL? GRANTS                                                              #showGrants
     | SHOW GRANTS FOR userIdentify                                                  #showGrantsForUser
+    | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL                                   #showLoadProfile
     | SHOW VIEW
         (FROM |IN) tableName=multipartIdentifier
         ((FROM | IN) database=identifier)?                                          #showView
@@ -330,7 +331,6 @@ unsupportedShowStatement
         ((FROM | IN) database=multipartIdentifier)?                                 #showIndex
     | SHOW TRANSACTION ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showTransaction
     | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL                                 #showQueryProfile
-    | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL                                   #showLoadProfile
     | SHOW CACHE HOTSPOT tablePath=STRING_LITERAL                                   #showCacheHotSpot
     | SHOW ENCRYPTKEYS ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showEncryptKeys
     | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)?                     #showSyncJob
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 5d31b284e15562..b77c141e2fd695 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -216,6 +216,7 @@
 import org.apache.doris.nereids.DorisParser.ShowGrantsContext;
 import org.apache.doris.nereids.DorisParser.ShowGrantsForUserContext;
 import org.apache.doris.nereids.DorisParser.ShowLastInsertContext;
+import org.apache.doris.nereids.DorisParser.ShowLoadProfileContext;
 import org.apache.doris.nereids.DorisParser.ShowPartitionIdContext;
 import org.apache.doris.nereids.DorisParser.ShowPluginsContext;
 import org.apache.doris.nereids.DorisParser.ShowPrivilegesContext;
@@ -483,6 +484,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowFrontendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowGrantsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowLastInsertCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowLoadProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowPartitionIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowPluginsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand;
@@ -4136,10 +4138,16 @@ public Object visitRefreshCatalog(RefreshCatalogContext ctx) {
         throw new AnalysisException("catalog name can not be null");
     }
 
+    @Override
     public LogicalPlan visitShowLastInsert(ShowLastInsertContext ctx) {
         return new ShowLastInsertCommand();
     }
 
+    @Override
+    public LogicalPlan visitShowLoadProfile(ShowLoadProfileContext ctx) {
+        return new ShowLoadProfileCommand(ctx.loadIdPath.getText());
+    }
+
     @Override
     public LogicalPlan visitShowGrants(ShowGrantsContext ctx) {
         boolean all = (ctx.ALL() != null) ? true : false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index f36af26e89273e..e649a905b4a0c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -196,6 +196,7 @@ public enum PlanType {
     SHOW_FRONTENDS_COMMAND,
     SHOW_GRANTS_COMMAND,
     SHOW_LAST_INSERT_COMMAND,
+    SHOW_LOAD_PROFILE_COMMAND,
     SHOW_PARTITIONID_COMMAND,
     SHOW_PROC_COMMAND,
     SHOW_PLUGINS_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLoadProfileCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLoadProfileCommand.java
new file mode 100644
index 00000000000000..5320671ad7794d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLoadProfileCommand.java
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.StmtExecutor;
+
+/**
+ * show load profile command
+ */
+public class ShowLoadProfileCommand extends ShowCommand {
+    private String loadIdPath;
+
+    /**
+     * constructor
+     */
+    public ShowLoadProfileCommand(String path) {
+        super(PlanType.SHOW_LOAD_PROFILE_COMMAND);
+        this.loadIdPath = path;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        String selfHost = Env.getCurrentEnv().getSelfNode().getHost();
+        int httpPort = Config.http_port;
+        String terminalMsg = String.format(
+                "try visit http://%s:%d/QueryProfile/%s, show query/load profile syntax is a deprecated feature",
+                selfHost, httpPort, this.loadIdPath);
+        throw new UserException(terminalMsg);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowLoadProfileCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 4383ebc01a0b9e..1c2a49b1b4e757 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -75,6 +75,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowFrontendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowGrantsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowLastInsertCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowLoadProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowPartitionIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowPluginsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand;
@@ -402,6 +403,10 @@ default R visitShowBrokerCommand(ShowBrokerCommand showBrokerCommand, C context)
         return visitCommand(showBrokerCommand, context);
     }
 
+    default R visitShowLoadProfileCommand(ShowLoadProfileCommand showLoadProfileCommand, C context) {
+        return visitCommand(showLoadProfileCommand, context);
+    }
+
     default R visitDropRoleCommand(DropRoleCommand dropRoleCommand, C context) {
         return visitCommand(dropRoleCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
index e994e22ec99709..998033deff9863 100644
--- a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
@@ -26,5 +26,5 @@ suite("test_show_commands_nereids") {
     checkNereidsExecute("""show backends;""")
     checkNereidsExecute("""show whitelist;""")
     checkNereidsExecute("""show triggers;""")
-
+    checkNereidsExecute("""show load profile \"\\";""")
 }

From 8b9809e1c02c942f6fffa52e18bbb8b35ee21829 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Tue, 26 Nov 2024 13:31:39 +0530
Subject: [PATCH 003/399] [Enhancement] (nereids)implement
 showSmallFilesCommand in nereids (#44453)

Issue Number: close #42770
---
 .../org/apache/doris/nereids/DorisParser.g4   |  4 +-
 .../nereids/parser/LogicalPlanBuilder.java    | 12 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/ShowSmallFilesCommand.java | 91 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 +
 .../auth_call/test_ddl_file_auth.groovy       |  2 +
 6 files changed, 113 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSmallFilesCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 30b924ed878a5c..f4394fa02c60f3 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -221,7 +221,8 @@ supportedShowStatement
     | SHOW ROLES                                                                    #showRoles        
     | SHOW PARTITION partitionId=INTEGER_VALUE                                      #showPartitionId
     | SHOW PRIVILEGES                                                               #showPrivileges
-    | SHOW PROC path=STRING_LITERAL                                                 #showProc        
+    | SHOW PROC path=STRING_LITERAL                                                 #showProc       
+    | SHOW FILE ((FROM | IN) database=multipartIdentifier)?                         #showSmallFiles 
     | SHOW STORAGE? ENGINES                                                         #showStorageEngines
     | SHOW CREATE CATALOG name=identifier                                           #showCreateCatalog
     | SHOW SQL_BLOCK_RULE (FOR ruleName=identifier)?                                #showSqlBlockRule
@@ -325,7 +326,6 @@ unsupportedShowStatement
         ((FROM | IN) database=multipartIdentifier)? wildWhere?                      #showFunctions
     | SHOW GLOBAL FULL? FUNCTIONS wildWhere?                                        #showGlobalFunctions
     | SHOW TYPECAST ((FROM | IN) database=multipartIdentifier)?                     #showTypeCast
-    | SHOW FILE ((FROM | IN) database=multipartIdentifier)?                         #showSmallFiles
     | SHOW (KEY | KEYS | INDEX | INDEXES)
         (FROM |IN) tableName=multipartIdentifier
         ((FROM | IN) database=multipartIdentifier)?                                 #showIndex
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index b77c141e2fd695..ab4ecb44fd4425 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -224,6 +224,7 @@
 import org.apache.doris.nereids.DorisParser.ShowProcedureStatusContext;
 import org.apache.doris.nereids.DorisParser.ShowRepositoriesContext;
 import org.apache.doris.nereids.DorisParser.ShowRolesContext;
+import org.apache.doris.nereids.DorisParser.ShowSmallFilesContext;
 import org.apache.doris.nereids.DorisParser.ShowSqlBlockRuleContext;
 import org.apache.doris.nereids.DorisParser.ShowStorageEnginesContext;
 import org.apache.doris.nereids.DorisParser.ShowTableIdContext;
@@ -492,6 +493,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
@@ -4232,6 +4234,16 @@ public LogicalPlan visitShowPlugins(ShowPluginsContext ctx) {
         return new ShowPluginsCommand();
     }
 
+    @Override
+    public LogicalPlan visitShowSmallFiles(ShowSmallFilesContext ctx) {
+        String dbName = null;
+        if (ctx.database != null) {
+            List<String> nameParts = visitMultipartIdentifier(ctx.database);
+            dbName = nameParts.get(0); // only one entry possible
+        }
+        return new ShowSmallFilesCommand(dbName);
+    }
+
     @Override
     public LogicalPlan visitShowSqlBlockRule(ShowSqlBlockRuleContext ctx) {
         String ruleName = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index e649a905b4a0c7..8698fd9855a9de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -203,6 +203,7 @@ public enum PlanType {
     SHOW_PRIVILEGES_COMMAND,
     SHOW_REPOSITORIES_COMMAND,
     SHOW_ROLE_COMMAND,
+    SHOW_SMALL_FILES_COMMAND,
     SHOW_STORAGE_ENGINES_COMMAND,
     SHOW_TABLE_ID_COMMAND,
     SHOW_TRIGGERS_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSmallFilesCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSmallFilesCommand.java
new file mode 100644
index 00000000000000..608fc8d4eff59d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSmallFilesCommand.java
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+import java.util.List;
+
+/**
+ * show small file command
+ */
+public class ShowSmallFilesCommand extends ShowCommand {
+    private static final ShowResultSetMetaData META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("Id", ScalarType.createVarchar(32)))
+                    .addColumn(new Column("DbName", ScalarType.createVarchar(256)))
+                    .addColumn(new Column("Catalog", ScalarType.createVarchar(32)))
+                    .addColumn(new Column("FileName", ScalarType.createVarchar(16)))
+                    .addColumn(new Column("FileSize", ScalarType.createVarchar(16)))
+                    .addColumn(new Column("IsContent", ScalarType.createVarchar(16)))
+                    .addColumn(new Column("MD5", ScalarType.createVarchar(16)))
+                    .build();
+    private String dbName; // if empty we will use current db;
+
+    /**
+     * constructor
+     */
+    public ShowSmallFilesCommand(String dbName) {
+        super(PlanType.SHOW_SMALL_FILES_COMMAND);
+        this.dbName = dbName;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        if (Strings.isNullOrEmpty(dbName)) {
+            dbName = ctx.getDatabase();
+            if (Strings.isNullOrEmpty(dbName)) {
+                ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
+            }
+        }
+        if (!Env.getCurrentEnv().getAccessManager()
+                .checkDbPriv(ConnectContext.get(), InternalCatalog.INTERNAL_CATALOG_NAME, dbName, PrivPredicate.SHOW)) {
+            ErrorReport.reportAnalysisException(
+                    ErrorCode.ERR_DBACCESS_DENIED_ERROR, ConnectContext.get().getQualifiedUser(), dbName);
+        }
+
+        List<List<String>> results;
+        try {
+            results = Env.getCurrentEnv().getSmallFileMgr().getInfo(dbName);
+        } catch (DdlException e) {
+            throw new AnalysisException(e.getMessage());
+        }
+        return new ShowResultSet(META_DATA, results);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowSmallFilesCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 1c2a49b1b4e757..2e7108613c2297 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -83,6 +83,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
@@ -330,6 +331,10 @@ default R visitShowCreateTableCommand(ShowCreateTableCommand showCreateTableComm
         return visitCommand(showCreateTableCommand, context);
     }
 
+    default R visitShowSmallFilesCommand(ShowSmallFilesCommand showSmallFilesCommand, C context) {
+        return visitCommand(showSmallFilesCommand, context);
+    }
+
     default R visitShowSqlBlockRuleCommand(ShowSqlBlockRuleCommand showblockruleCommand, C context) {
         return visitCommand(showblockruleCommand, context);
     }
diff --git a/regression-test/suites/auth_call/test_ddl_file_auth.groovy b/regression-test/suites/auth_call/test_ddl_file_auth.groovy
index 77ca5e6703f977..99b71eabddf140 100644
--- a/regression-test/suites/auth_call/test_ddl_file_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_file_auth.groovy
@@ -76,6 +76,8 @@ suite("test_ddl_file_auth","p0,auth_call") {
                     "catalog" = "internal"
                 );"""
         sql """use ${dbName}"""
+        checkNereidsExecute("SHOW FILE;")
+        checkNereidsExecute("SHOW FILE FROM ${dbName};")
         def res = sql """SHOW FILE FROM ${dbName};"""
         assertTrue(res.size() == 1)
 

From 78717e32bca0b9220f8c8e8dd86416ebd916daeb Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Tue, 26 Nov 2024 13:32:44 +0530
Subject: [PATCH 004/399] [Enhancement] (nereids)implement
 DropWorkloadGroupCommand in nereids (#44482)

Issue Number: close #42621
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  7 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/DropWorkloadGroupCommand.java    | 58 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../workloadgroup/WorkloadGroupMgr.java       |  7 ++-
 .../test_nereids_workload_test.out            |  6 ++
 .../test_nereids_workload_test.groovy         | 27 +++++++++
 8 files changed, 110 insertions(+), 3 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropWorkloadGroupCommand.java
 create mode 100644 regression-test/data/workload_manager_p0/test_nereids_workload_test.out
 create mode 100644 regression-test/suites/workload_manager_p0/test_nereids_workload_test.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index f4394fa02c60f3..c5a3b7d6efb0e8 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -200,6 +200,7 @@ supportedDropStatement
     | DROP ROLE (IF EXISTS)? name=identifier                                    #dropRole
     | DROP SQL_BLOCK_RULE (IF EXISTS)? identifierSeq                            #dropSqlBlockRule
     | DROP USER (IF EXISTS)? userIdentify                                       #dropUser
+    | DROP WORKLOAD GROUP (IF EXISTS)? name=identifierOrText                    #dropWorkloadGroup
     ;
 
 supportedShowStatement
@@ -671,7 +672,6 @@ unsupportedDropStatement
         ((FROM | IN) database=identifier)? properties=propertyClause            #dropFile
     | DROP INDEX (IF EXISTS)? name=identifier ON tableName=multipartIdentifier  #dropIndex
     | DROP RESOURCE (IF EXISTS)? name=identifierOrText                          #dropResource
-    | DROP WORKLOAD GROUP (IF EXISTS)? name=identifierOrText                    #dropWorkloadGroup
     | DROP WORKLOAD POLICY (IF EXISTS)? name=identifierOrText                   #dropWorkloadPolicy
     | DROP ENCRYPTKEY (IF EXISTS)? name=multipartIdentifier                     #dropEncryptkey
     | DROP ROW POLICY (IF EXISTS)? policyName=identifier
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index ab4ecb44fd4425..537ff71320b1e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -104,6 +104,7 @@
 import org.apache.doris.nereids.DorisParser.DropRoleContext;
 import org.apache.doris.nereids.DorisParser.DropSqlBlockRuleContext;
 import org.apache.doris.nereids.DorisParser.DropUserContext;
+import org.apache.doris.nereids.DorisParser.DropWorkloadGroupContext;
 import org.apache.doris.nereids.DorisParser.ElementAtContext;
 import org.apache.doris.nereids.DorisParser.ExceptContext;
 import org.apache.doris.nereids.DorisParser.ExceptOrReplaceContext;
@@ -453,6 +454,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DropRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropUserCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropWorkloadGroupCommand;
 import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
 import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
 import org.apache.doris.nereids.trees.plans.commands.ExportCommand;
@@ -4374,6 +4376,11 @@ public LogicalPlan visitDropUser(DropUserContext ctx) {
         return new DropUserCommand(userIdent, ctx.EXISTS() != null);
     }
 
+    @Override
+    public LogicalPlan visitDropWorkloadGroup(DropWorkloadGroupContext ctx) {
+        return new DropWorkloadGroupCommand(ctx.name.getText(), ctx.EXISTS() != null);
+    }
+
     @Override
     public LogicalPlan visitShowTableId(ShowTableIdContext ctx) {
         long tableId = -1;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 8698fd9855a9de..8ce5caac661c92 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -184,6 +184,7 @@ public enum PlanType {
     EXECUTE_COMMAND,
     DROP_SQL_BLOCK_RULE_COMMAND,
     DROP_USER_COMMAND,
+    DROP_WORKLOAD_GROUP_NAME,
     SHOW_BACKENDS_COMMAND,
     SHOW_BLOCK_RULE_COMMAND,
     SHOW_BROKER_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropWorkloadGroupCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropWorkloadGroupCommand.java
new file mode 100644
index 00000000000000..694fc1411db127
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropWorkloadGroupCommand.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+/**
+ * drop workload group command
+ */
+public class DropWorkloadGroupCommand extends DropCommand {
+    private final boolean ifExists;
+    private final String workloadGroupName;
+
+    /**
+     * constructor
+     */
+    public DropWorkloadGroupCommand(String workloadGroupName, boolean ifExists) {
+        super(PlanType.DROP_WORKLOAD_GROUP_NAME);
+        this.workloadGroupName = workloadGroupName;
+        this.ifExists = ifExists;
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+        Env.getCurrentEnv().getWorkloadGroupMgr().dropWorkloadGroup(workloadGroupName, ifExists);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitDropWorkloadGroupCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 2e7108613c2297..9e451fac4f1371 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -44,6 +44,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DropRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropUserCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropWorkloadGroupCommand;
 import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
 import org.apache.doris.nereids.trees.plans.commands.ExportCommand;
 import org.apache.doris.nereids.trees.plans.commands.LoadCommand;
@@ -424,6 +425,10 @@ default R visitDropUserCommand(DropUserCommand dropUserCommand, C context) {
         return visitCommand(dropUserCommand, context);
     }
 
+    default R visitDropWorkloadGroupCommand(DropWorkloadGroupCommand dropWorkloadGroupCommand, C context) {
+        return visitCommand(dropWorkloadGroupCommand, context);
+    }
+
     default R visitShowTableIdCommand(ShowTableIdCommand showTableIdCommand, C context) {
         return visitCommand(showTableIdCommand, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
index 5e61f48bec6f2e..b4c25524c3a8fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
@@ -492,7 +492,10 @@ public void alterWorkloadGroup(AlterWorkloadGroupStmt stmt) throws DdlException
     }
 
     public void dropWorkloadGroup(DropWorkloadGroupStmt stmt) throws DdlException {
-        String workloadGroupName = stmt.getWorkloadGroupName();
+        dropWorkloadGroup(stmt.getWorkloadGroupName(), stmt.isIfExists());
+    }
+
+    public void dropWorkloadGroup(String workloadGroupName, boolean ifExists) throws DdlException {
         if (DEFAULT_GROUP_NAME.equals(workloadGroupName) || INTERNAL_GROUP_NAME.equals(workloadGroupName)) {
             throw new DdlException("Dropping workload group " + workloadGroupName + " is not allowed");
         }
@@ -521,7 +524,7 @@ public void dropWorkloadGroup(DropWorkloadGroupStmt stmt) throws DdlException {
         writeLock();
         try {
             if (!nameToWorkloadGroup.containsKey(workloadGroupName)) {
-                if (stmt.isIfExists()) {
+                if (ifExists) {
                     return;
                 }
                 throw new DdlException("workload group " + workloadGroupName + " does not exist");
diff --git a/regression-test/data/workload_manager_p0/test_nereids_workload_test.out b/regression-test/data/workload_manager_p0/test_nereids_workload_test.out
new file mode 100644
index 00000000000000..c188c28c35c3d4
--- /dev/null
+++ b/regression-test/data/workload_manager_p0/test_nereids_workload_test.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !check_workload_check1 --
+test_nereids_wg1
+
+-- !check_workload_check2 --
+
diff --git a/regression-test/suites/workload_manager_p0/test_nereids_workload_test.groovy b/regression-test/suites/workload_manager_p0/test_nereids_workload_test.groovy
new file mode 100644
index 00000000000000..3f7c55f5c99cf9
--- /dev/null
+++ b/regression-test/suites/workload_manager_p0/test_nereids_workload_test.groovy
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_nereids_workload_test") {
+    sql "drop workload group if exists test_nereids_wg1;"
+    sql "drop workload group if exists test_nereids_wg2;"
+    sql "create workload group test_nereids_wg1 properties('cpu_share'='1024');"
+    sql "create workload group test_nereids_wg2 properties('cpu_share'='1024');"
+    qt_check_workload_check1("select NAME from information_schema.workload_groups where NAME='test_nereids_wg1';")
+    checkNereidsExecute("drop workload group  test_nereids_wg1;")
+    qt_check_workload_check2("select NAME from information_schema.workload_groups where NAME='test_nereids_wg1';")
+    checkNereidsExecute("drop workload group if exists test_nereids_wg2;")
+}
\ No newline at end of file

From 1b8f7624e7a6a9c12c549157880ed81f698aa55d Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Tue, 26 Nov 2024 13:34:15 +0530
Subject: [PATCH 005/399] [Enhancement] (nereids)implement
 alterWorkloadGroupCommand in nereids (#44500)

Issue Number: close #42793
---
 .../org/apache/doris/nereids/DorisParser.g4   |  4 +-
 .../nereids/parser/LogicalPlanBuilder.java    | 10 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/AlterWorkloadGroupCommand.java   | 81 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../workloadgroup/WorkloadGroupMgr.java       |  6 +-
 .../test_nereids_workload_alter_test.out      |  7 ++
 .../test_nereids_workload_alter_test.groovy   | 25 ++++++
 8 files changed, 135 insertions(+), 4 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterWorkloadGroupCommand.java
 create mode 100644 regression-test/data/workload_manager_p0/test_nereids_workload_alter_test.out
 create mode 100644 regression-test/suites/workload_manager_p0/test_nereids_workload_alter_test.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index c5a3b7d6efb0e8..c662ee295b6f49 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -193,6 +193,8 @@ supportedAlterStatement
         AS query                                                          #alterView
     | ALTER STORAGE VAULT name=multipartIdentifier properties=propertyClause   #alterStorageVault
     | ALTER ROLE role=identifier commentSpec                                        #alterRole
+    | ALTER WORKLOAD GROUP name=identifierOrText
+        properties=propertyClause?                                                  #alterWorkloadGroup
     ;
 
 supportedDropStatement
@@ -558,8 +560,6 @@ unsupportedAlterStatement
     | ALTER RESOURCE name=identifierOrText properties=propertyClause?               #alterResource
     | ALTER COLOCATE GROUP name=multipartIdentifier
         SET LEFT_PAREN propertyItemList RIGHT_PAREN                                 #alterColocateGroup
-    | ALTER WORKLOAD GROUP name=identifierOrText
-        properties=propertyClause?                                                  #alterWorkloadGroup
     | ALTER WORKLOAD POLICY name=identifierOrText
         properties=propertyClause?                                                  #alterWorkloadPolicy
     | ALTER ROUTINE LOAD FOR name=multipartIdentifier properties=propertyClause?
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 537ff71320b1e1..dc0fe2aa4a10d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -56,6 +56,7 @@
 import org.apache.doris.nereids.DorisParser.AlterRoleContext;
 import org.apache.doris.nereids.DorisParser.AlterStorageVaultContext;
 import org.apache.doris.nereids.DorisParser.AlterViewContext;
+import org.apache.doris.nereids.DorisParser.AlterWorkloadGroupContext;
 import org.apache.doris.nereids.DorisParser.ArithmeticBinaryContext;
 import org.apache.doris.nereids.DorisParser.ArithmeticUnaryContext;
 import org.apache.doris.nereids.DorisParser.ArrayLiteralContext;
@@ -430,6 +431,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AlterRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterStorageVaultCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
 import org.apache.doris.nereids.trees.plans.commands.CallCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelJobTaskCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
@@ -4298,12 +4300,20 @@ public LogicalPlan visitShowCreateMaterializedView(ShowCreateMaterializedViewCon
         return new ShowCreateMaterializedViewCommand(stripQuotes(ctx.mvName.getText()), new TableNameInfo(nameParts));
     }
 
+    @Override
+    public LogicalPlan visitAlterWorkloadGroup(AlterWorkloadGroupContext ctx) {
+        Map<String, String> properties = ctx.propertyClause() != null
+                        ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+        return new AlterWorkloadGroupCommand(ctx.name.getText(), properties);
+    }
+
     @Override
     public LogicalPlan visitAlterRole(AlterRoleContext ctx) {
         String comment = visitCommentSpec(ctx.commentSpec());
         return new AlterRoleCommand(ctx.role.getText(), comment);
     }
 
+    @Override
     public LogicalPlan visitShowFrontends(ShowFrontendsContext ctx) {
         String detail = (ctx.name != null) ? ctx.name.getText() : null;
         return new ShowFrontendsCommand(detail);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 8ce5caac661c92..04551da904bfa9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -170,6 +170,7 @@ public enum PlanType {
     ALTER_ROLE_COMMAND,
     ALTER_VIEW_COMMAND,
     ALTER_STORAGE_VAULT,
+    ALTER_WORKLOAD_GROUP_COMMAND,
     DROP_CATALOG_RECYCLE_BIN_COMMAND,
     UNSET_VARIABLE_COMMAND,
     UNSET_DEFAULT_STORAGE_VAULT_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterWorkloadGroupCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterWorkloadGroupCommand.java
new file mode 100644
index 00000000000000..8cceaa20fdce9a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterWorkloadGroupCommand.java
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.resource.workloadgroup.WorkloadGroup;
+import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Map;
+
+/**
+ * alter workload group command
+ */
+public class AlterWorkloadGroupCommand extends AlterCommand {
+    private final String workloadGroupName;
+    private final Map<String, String> properties;
+
+    /**
+     * constructor
+     */
+    public AlterWorkloadGroupCommand(String workloadGroupName, Map<String, String> properties) {
+        super(PlanType.ALTER_WORKLOAD_GROUP_COMMAND);
+        this.workloadGroupName = workloadGroupName;
+        this.properties = properties;
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        if (properties == null || properties.isEmpty()) {
+            throw new AnalysisException("Workload Group properties can't be empty");
+        }
+
+        if (properties.containsKey(WorkloadGroup.INTERNAL_TYPE)) {
+            throw new AnalysisException(WorkloadGroup.INTERNAL_TYPE + " can not be create or modified ");
+        }
+        String tagStr = properties.get(WorkloadGroup.TAG);
+        if (!StringUtils.isEmpty(tagStr) && (WorkloadGroupMgr.DEFAULT_GROUP_NAME.equals(workloadGroupName)
+                || WorkloadGroupMgr.INTERNAL_GROUP_NAME.equals(workloadGroupName))) {
+            throw new AnalysisException(
+                    WorkloadGroupMgr.INTERNAL_GROUP_NAME + " and " + WorkloadGroupMgr.DEFAULT_GROUP_NAME
+                            + " group can not set tag");
+        }
+
+        Env.getCurrentEnv().getWorkloadGroupMgr().alterWorkloadGroup(workloadGroupName, properties);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAlterWorkloadGroupCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 9e451fac4f1371..13eeef182641fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -22,6 +22,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
 import org.apache.doris.nereids.trees.plans.commands.CallCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelJobTaskCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
@@ -377,6 +378,10 @@ default R visitAlterRoleCommand(AlterRoleCommand alterRoleCommand, C context) {
         return visitCommand(alterRoleCommand, context);
     }
 
+    default R visitAlterWorkloadGroupCommand(AlterWorkloadGroupCommand alterWorkloadGroupCommand, C context) {
+        return visitCommand(alterWorkloadGroupCommand, context);
+    }
+
     default R visitCleanAllProfileCommand(CleanAllProfileCommand cleanAllProfileCommand, C context) {
         return visitCommand(cleanAllProfileCommand, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
index b4c25524c3a8fa..22cbcf4a0a1502 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
@@ -464,8 +464,10 @@ private void checkGlobalUnlock(WorkloadGroup newWg, WorkloadGroup oldWg) throws
     }
 
     public void alterWorkloadGroup(AlterWorkloadGroupStmt stmt) throws DdlException {
-        String workloadGroupName = stmt.getWorkloadGroupName();
-        Map<String, String> properties = stmt.getProperties();
+        alterWorkloadGroup(stmt.getWorkloadGroupName(), stmt.getProperties());
+    }
+
+    public void alterWorkloadGroup(String workloadGroupName, Map<String, String> properties) throws DdlException {
         if (properties.size() == 0) {
             throw new DdlException("alter workload group should contain at least one property");
         }
diff --git a/regression-test/data/workload_manager_p0/test_nereids_workload_alter_test.out b/regression-test/data/workload_manager_p0/test_nereids_workload_alter_test.out
new file mode 100644
index 00000000000000..897cef4c230a51
--- /dev/null
+++ b/regression-test/data/workload_manager_p0/test_nereids_workload_alter_test.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !check_workload_check1 --
+1024	10
+
+-- !check_workload_check2 --
+20	8
+
diff --git a/regression-test/suites/workload_manager_p0/test_nereids_workload_alter_test.groovy b/regression-test/suites/workload_manager_p0/test_nereids_workload_alter_test.groovy
new file mode 100644
index 00000000000000..b3f600b3f186b4
--- /dev/null
+++ b/regression-test/suites/workload_manager_p0/test_nereids_workload_alter_test.groovy
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_nereids_workload_test") {
+    sql "drop workload group if exists test_nereids_alter_wg1;"
+    sql "create workload group test_nereids_alter_wg1 properties('cpu_share'='1024', 'scan_thread_num'='10');"
+    qt_check_workload_check1("select CPU_SHARE,scan_thread_num from information_schema.workload_groups where NAME='test_nereids_alter_wg1';")
+    checkNereidsExecute("Alter workload group  test_nereids_alter_wg1 properties('cpu_share'='20', 'scan_thread_num'='8');")
+    qt_check_workload_check2("select CPU_SHARE,scan_thread_num from information_schema.workload_groups where NAME='test_nereids_alter_wg1';")
+    checkNereidsExecute("drop workload group if exists test_nereids_alter_wg1;")
+}
\ No newline at end of file

From 95fdd45600b4357a0ea8b6a0d1c0ac6182dd0500 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Tue, 26 Nov 2024 13:35:46 +0530
Subject: [PATCH 006/399] [Enhancement] (nereids)implement
 DropEncryptKeyCommand in nereids (#44506)

Issue Number: close #42623
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../apache/doris/analysis/EncryptKeyName.java | 25 +++++++
 .../nereids/parser/LogicalPlanBuilder.java    |  9 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/DropEncryptkeyCommand.java | 65 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../nereids_p0/test_nereids_encrypt_test.out  |  8 +++
 .../test_nereids_encrypt_test.groovy          | 29 +++++++++
 8 files changed, 143 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropEncryptkeyCommand.java
 create mode 100644 regression-test/data/nereids_p0/test_nereids_encrypt_test.out
 create mode 100644 regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index c662ee295b6f49..932f644fca4e7a 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -199,6 +199,7 @@ supportedAlterStatement
 
 supportedDropStatement
     : DROP CATALOG RECYCLE BIN WHERE idType=STRING_LITERAL EQ id=INTEGER_VALUE  #dropCatalogRecycleBin
+    | DROP ENCRYPTKEY (IF EXISTS)? name=multipartIdentifier                     #dropEncryptkey
     | DROP ROLE (IF EXISTS)? name=identifier                                    #dropRole
     | DROP SQL_BLOCK_RULE (IF EXISTS)? identifierSeq                            #dropSqlBlockRule
     | DROP USER (IF EXISTS)? userIdentify                                       #dropUser
@@ -673,7 +674,6 @@ unsupportedDropStatement
     | DROP INDEX (IF EXISTS)? name=identifier ON tableName=multipartIdentifier  #dropIndex
     | DROP RESOURCE (IF EXISTS)? name=identifierOrText                          #dropResource
     | DROP WORKLOAD POLICY (IF EXISTS)? name=identifierOrText                   #dropWorkloadPolicy
-    | DROP ENCRYPTKEY (IF EXISTS)? name=multipartIdentifier                     #dropEncryptkey
     | DROP ROW POLICY (IF EXISTS)? policyName=identifier
         ON tableName=multipartIdentifier
         (FOR (userIdentify | ROLE roleName=identifier))?                        #dropRowPolicy
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/EncryptKeyName.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/EncryptKeyName.java
index 155f1753bf4a34..e76bfb7e5b20a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/EncryptKeyName.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/EncryptKeyName.java
@@ -25,6 +25,7 @@
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.base.Strings;
 import com.google.gson.annotations.SerializedName;
@@ -34,6 +35,7 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.List;
 import java.util.Objects;
 
 public class EncryptKeyName implements Writable {
@@ -52,6 +54,19 @@ public EncryptKeyName(String db, String keyName) {
         }
     }
 
+    /**
+     * EncryptKeyName
+     * @param parts like [db1,keyName] or [keyName]
+     */
+    public EncryptKeyName(List<String> parts) {
+        int size = parts.size();
+        keyName = parts.get(size - 1);
+        keyName = keyName.toLowerCase();
+        if (size >= 2) {
+            db = parts.get(size - 2);
+        }
+    }
+
     public EncryptKeyName(String keyName) {
         this.db = null;
         this.keyName = keyName.toLowerCase();
@@ -67,6 +82,16 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
         }
     }
 
+    public void analyze(ConnectContext ctx) throws AnalysisException {
+        FeNameFormat.checkCommonName("EncryptKey", keyName);
+        if (db == null) {
+            db = ctx.getDatabase();
+            if (Strings.isNullOrEmpty(db)) {
+                ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
+            }
+        }
+    }
+
     public String getDb() {
         return db;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index dc0fe2aa4a10d6..320043e12b8b53 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -20,6 +20,7 @@
 import org.apache.doris.analysis.ArithmeticExpr.Operator;
 import org.apache.doris.analysis.BrokerDesc;
 import org.apache.doris.analysis.ColumnNullableType;
+import org.apache.doris.analysis.EncryptKeyName;
 import org.apache.doris.analysis.PassVar;
 import org.apache.doris.analysis.SetType;
 import org.apache.doris.analysis.StorageBackend;
@@ -100,6 +101,7 @@
 import org.apache.doris.nereids.DorisParser.DereferenceContext;
 import org.apache.doris.nereids.DorisParser.DropCatalogRecycleBinContext;
 import org.apache.doris.nereids.DorisParser.DropConstraintContext;
+import org.apache.doris.nereids.DorisParser.DropEncryptkeyContext;
 import org.apache.doris.nereids.DorisParser.DropMTMVContext;
 import org.apache.doris.nereids.DorisParser.DropProcedureContext;
 import org.apache.doris.nereids.DorisParser.DropRoleContext;
@@ -450,6 +452,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType;
 import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropEncryptkeyCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropProcedureCommand;
@@ -4375,6 +4378,12 @@ public LogicalPlan visitDropRole(DropRoleContext ctx) {
         return new DropRoleCommand(ctx.name.getText(), ctx.EXISTS() != null);
     }
 
+    @Override
+    public LogicalPlan visitDropEncryptkey(DropEncryptkeyContext ctx) {
+        List<String> nameParts = visitMultipartIdentifier(ctx.name);
+        return new DropEncryptkeyCommand(new EncryptKeyName(nameParts), ctx.EXISTS() != null);
+    }
+
     @Override
     public LogicalPlan visitDropSqlBlockRule(DropSqlBlockRuleContext ctx) {
         return new DropSqlBlockRuleCommand(visitIdentifierSeq(ctx.identifierSeq()), ctx.EXISTS() != null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 04551da904bfa9..1f7b838edfd905 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -172,6 +172,7 @@ public enum PlanType {
     ALTER_STORAGE_VAULT,
     ALTER_WORKLOAD_GROUP_COMMAND,
     DROP_CATALOG_RECYCLE_BIN_COMMAND,
+    DROP_ENCRYPTKEY_COMMAND,
     UNSET_VARIABLE_COMMAND,
     UNSET_DEFAULT_STORAGE_VAULT_COMMAND,
     UNSUPPORTED_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropEncryptkeyCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropEncryptkeyCommand.java
new file mode 100644
index 00000000000000..4c163d342d696b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropEncryptkeyCommand.java
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.EncryptKeyName;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.EncryptKeySearchDesc;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+/**
+ * drop encrypt key command
+ */
+public class DropEncryptkeyCommand extends DropCommand {
+    private final boolean ifExists;
+    private final EncryptKeyName encryptKeyName;
+
+    /**
+     * constructor
+     */
+    public DropEncryptkeyCommand(EncryptKeyName encryptKeyName, boolean ifExists) {
+        super(PlanType.DROP_ENCRYPTKEY_COMMAND);
+        this.encryptKeyName = encryptKeyName;
+        this.ifExists = ifExists;
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check operation privilege
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+        // analyze encryptkey name
+        encryptKeyName.analyze(ctx);
+        EncryptKeySearchDesc encryptKeySearchDesc = new EncryptKeySearchDesc(encryptKeyName);
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(encryptKeyName.getDb());
+        db.dropEncryptKey(encryptKeySearchDesc, ifExists);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitDropEncryptKeyCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 13eeef182641fb..6b801524fb295f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -39,6 +39,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DeleteFromUsingCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropEncryptkeyCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropProcedureCommand;
@@ -422,6 +423,10 @@ default R visitDropRoleCommand(DropRoleCommand dropRoleCommand, C context) {
         return visitCommand(dropRoleCommand, context);
     }
 
+    default R visitDropEncryptKeyCommand(DropEncryptkeyCommand dropEncryptkeyCommand, C context) {
+        return visitCommand(dropEncryptkeyCommand, context);
+    }
+
     default R visitDropSqlBlockRuleCommand(DropSqlBlockRuleCommand dropSqlBlockRuleCommand, C context) {
         return visitCommand(dropSqlBlockRuleCommand, context);
     }
diff --git a/regression-test/data/nereids_p0/test_nereids_encrypt_test.out b/regression-test/data/nereids_p0/test_nereids_encrypt_test.out
new file mode 100644
index 00000000000000..7984f572e8f26f
--- /dev/null
+++ b/regression-test/data/nereids_p0/test_nereids_encrypt_test.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !check_encrypt_1 --
+test_nereids_encrypt_test_db.test_nereids_encrypt_test_key	ABCD123456789
+
+-- !check_encrypt_2 --
+
+-- !check_encrypt_3 --
+
diff --git a/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy b/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy
new file mode 100644
index 00000000000000..2fab616580faaa
--- /dev/null
+++ b/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("test_nereids_encrypt_test") {
+    def dbName="test_nereids_encrypt_test_db"
+    def encryptkeyName="test_nereids_encrypt_test_key"
+    sql """ create database IF NOT EXISTS ${dbName}; """
+    sql """ use ${dbName}; """
+    checkNereidsExecute("drop encryptkey if exists ${encryptkeyName}")    
+    sql """CREATE ENCRYPTKEY ${encryptkeyName} AS "ABCD123456789";"""
+    qt_check_encrypt_1("SHOW ENCRYPTKEYS FROM ${dbName}")
+    checkNereidsExecute("drop encryptkey ${encryptkeyName}")
+    qt_check_encrypt_2("SHOW ENCRYPTKEYS FROM ${dbName}")    
+    checkNereidsExecute("drop encryptkey if exists ${encryptkeyName}")
+    qt_check_encrypt_3("SHOW ENCRYPTKEYS FROM ${dbName}")        
+}
\ No newline at end of file

From 8751287549245e5bd6ac8adbe21d75af6c9063ed Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Tue, 26 Nov 2024 17:03:04 +0800
Subject: [PATCH 007/399] [chore](github)Update .asf.yaml (#44614)

---
 .asf.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.asf.yaml b/.asf.yaml
index 3892aca2eddb77..f4c21ce4b0f3c9 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -56,8 +56,7 @@ github:
           - cloud_p0 (Doris Cloud Regression)
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
-          - Build Broker
-          - ShellCheck          
+          - Build Broker      
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - Build Third Party Libraries (macOS-arm64)
@@ -80,7 +79,6 @@ github:
           - Clang Formatter
           - CheckStyle
           - Build Broker
-          - ShellCheck
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - FE UT (Doris FE UT)
@@ -103,7 +101,6 @@ github:
           - Clang Formatter
           - CheckStyle
           - Build Broker
-          - ShellCheck
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - COMPILE (DORIS_COMPILE)

From aa15ca692a1cc010235e15571fac29699d26a367 Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Tue, 26 Nov 2024 17:32:12 +0800
Subject: [PATCH 008/399] [fix](DECIMAL) error DECIMAL cat to BOOLEAN (#44326)

In the past, there were issues with converting `double` and `decimal` to
`boolean`.
For example, a `double` value like 0.13 would first be cast to `uint8`,
resulting in 0.
Then, it would be converted to `bool`, yielding 0 (incorrect, as the
expected result is 1).

Similarly, `decimal` values were directly cast to `uint8`, leading to
non-0/1 values for `bool`.
This issue arises because Doris internally uses `uint8` to represent
`boolean`.

before
```
mysql> select cast(40.123 as BOOLEAN);
+-------------------------+
| cast(40.123 as BOOLEAN) |
+-------------------------+
|                      40 |
+-------------------------+
```

now
```
mysql> select cast(40.123 as BOOLEAN);
+-------------------------+
| cast(40.123 as BOOLEAN) |
+-------------------------+
|                       1 |
+-------------------------+
```
---
 be/src/vec/data_types/data_type_decimal.h     | 19 ++++---
 be/src/vec/functions/function_cast.h          | 43 ++++++++++-----
 .../test_cast_decimalv3_as_bool.out           | 17 ++++++
 .../test_cast_decimalv3_as_bool.groovy        | 55 +++++++++++++++++++
 .../test_case_function_null.groovy            | 18 ++++--
 5 files changed, 124 insertions(+), 28 deletions(-)
 create mode 100644 regression-test/data/correctness/test_cast_decimalv3_as_bool.out
 create mode 100644 regression-test/suites/correctness/test_cast_decimalv3_as_bool.groovy

diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h
index a44390cd58d86a..74655ff6ee8bec 100644
--- a/be/src/vec/data_types/data_type_decimal.h
+++ b/be/src/vec/data_types/data_type_decimal.h
@@ -460,15 +460,20 @@ void convert_from_decimals(RealTo* dst, const RealFrom* src, UInt32 precicion_fr
     MaxFieldType multiplier = DataTypeDecimal<MaxFieldType>::get_scale_multiplier(scale_from);
     FromDataType from_data_type(precicion_from, scale_from);
     for (size_t i = 0; i < size; i++) {
-        auto tmp = static_cast<MaxFieldType>(src[i]).value / multiplier.value;
-        if constexpr (narrow_integral) {
-            if (tmp < min_result.value || tmp > max_result.value) {
-                THROW_DECIMAL_CONVERT_OVERFLOW_EXCEPTION(from_data_type.to_string(src[i]),
-                                                         from_data_type.get_name(),
-                                                         OrigToDataType {}.get_name());
+        // uint8_t now use as boolean in doris
+        if constexpr (std::is_same_v<RealTo, UInt8>) {
+            dst[i] = static_cast<MaxFieldType>(src[i]).value != 0;
+        } else {
+            auto tmp = static_cast<MaxFieldType>(src[i]).value / multiplier.value;
+            if constexpr (narrow_integral) {
+                if (tmp < min_result.value || tmp > max_result.value) {
+                    THROW_DECIMAL_CONVERT_OVERFLOW_EXCEPTION(from_data_type.to_string(src[i]),
+                                                             from_data_type.get_name(),
+                                                             OrigToDataType {}.get_name());
+                }
             }
+            dst[i] = tmp;
         }
-        dst[i] = tmp;
     }
 }
 
diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
index 5de820dfa3a24f..7027a4e9c622bc 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -258,6 +258,21 @@ struct ConvertImpl {
     using FromFieldType = typename FromDataType::FieldType;
     using ToFieldType = typename ToDataType::FieldType;
 
+    // `static_cast_set` is introduced to wrap `static_cast` and handle special cases.
+    // Doris uses `uint8` to represent boolean values internally.
+    // Directly `static_cast` to `uint8` can result in non-0/1 values,
+    // To address this, `static_cast_set` performs an additional check:
+    //  For `uint8` types, it explicitly uses `static_cast<bool>` to ensure
+    //  the result is either 0 or 1.
+    static void static_cast_set(ToFieldType& to, const FromFieldType& from) {
+        // uint8_t now use as boolean in doris
+        if constexpr (std::is_same_v<uint8_t, ToFieldType>) {
+            to = static_cast<bool>(from);
+        } else {
+            to = static_cast<ToFieldType>(from);
+        }
+    }
+
     template <typename Additions = void*>
     static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
                           uint32_t result, size_t input_rows_count,
@@ -377,8 +392,9 @@ struct ConvertImpl {
                     } else if constexpr (IsDateTimeV2Type<ToDataType>) {
                         DataTypeDateTimeV2::cast_from_date(vec_from[i], vec_to[i]);
                     } else {
-                        vec_to[i] =
-                                reinterpret_cast<const VecDateTimeValue&>(vec_from[i]).to_int64();
+                        static_cast_set(
+                                vec_to[i],
+                                reinterpret_cast<const VecDateTimeValue&>(vec_from[i]).to_int64());
                     }
                 }
             } else if constexpr (IsTimeV2Type<FromDataType>) {
@@ -409,13 +425,16 @@ struct ConvertImpl {
                         }
                     } else {
                         if constexpr (IsDateTimeV2Type<FromDataType>) {
-                            vec_to[i] = reinterpret_cast<const DateV2Value<DateTimeV2ValueType>&>(
-                                                vec_from[i])
-                                                .to_int64();
+                            static_cast_set(
+                                    vec_to[i],
+                                    reinterpret_cast<const DateV2Value<DateTimeV2ValueType>&>(
+                                            vec_from[i])
+                                            .to_int64());
                         } else {
-                            vec_to[i] = reinterpret_cast<const DateV2Value<DateV2ValueType>&>(
-                                                vec_from[i])
-                                                .to_int64();
+                            static_cast_set(vec_to[i],
+                                            reinterpret_cast<const DateV2Value<DateV2ValueType>&>(
+                                                    vec_from[i])
+                                                    .to_int64());
                         }
                     }
                 }
@@ -441,16 +460,10 @@ struct ConvertImpl {
                     }
                 } else {
                     for (size_t i = 0; i < size; ++i) {
-                        vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
+                        static_cast_set(vec_to[i], vec_from[i]);
                     }
                 }
             }
-            // TODO: support boolean cast more reasonable
-            if constexpr (std::is_same_v<uint8_t, ToFieldType>) {
-                for (int i = 0; i < size; ++i) {
-                    vec_to[i] = static_cast<bool>(vec_to[i]);
-                }
-            }
 
             block.replace_by_position(result, std::move(col_to));
         } else {
diff --git a/regression-test/data/correctness/test_cast_decimalv3_as_bool.out b/regression-test/data/correctness/test_cast_decimalv3_as_bool.out
new file mode 100644
index 00000000000000..4f41130b00bafd
--- /dev/null
+++ b/regression-test/data/correctness/test_cast_decimalv3_as_bool.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select1 --
+0.000	13131.213132100	0E-16
+0.000	2131231.231000000	2.3323000E-9
+3.141	0E-9	123123.2131231231322130
+
+-- !select2 --
+false	true	false
+false	true	true
+true	false	true
+
+-- !select3 --
+true	1	true	false
+
+-- !select3 --
+true	1	true	false
+
diff --git a/regression-test/suites/correctness/test_cast_decimalv3_as_bool.groovy b/regression-test/suites/correctness/test_cast_decimalv3_as_bool.groovy
new file mode 100644
index 00000000000000..768da493251e6c
--- /dev/null
+++ b/regression-test/suites/correctness/test_cast_decimalv3_as_bool.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cast_decimalv3_as_bool") {
+     sql """ DROP TABLE IF EXISTS cast_decimalv3_as_bool """
+     sql """
+        CREATE TABLE IF NOT EXISTS cast_decimalv3_as_bool (
+            `id` int(11) ,
+            `k1` decimalv3(9,3) ,
+            `k2` decimalv3(18,9) ,
+            `k3` decimalv3(38,16) ,
+        )
+        UNIQUE KEY(`id`)
+        DISTRIBUTED BY HASH(`id`) BUCKETS 10
+        PROPERTIES (
+        "enable_unique_key_merge_on_write" = "true",
+        "replication_num" = "1"
+        );
+    """
+    sql """
+        set enable_nereids_planner=true,enable_fold_constant_by_be = false
+    """
+    sql """
+        INSERT INTO cast_decimalv3_as_bool VALUES
+        (1,0.00001,13131.2131321,0.000000000000000000),
+        (2,0.00000,2131231.231,0.0000000023323),
+        (3,3.141414,0.0000000000,123123.213123123132213);
+    """
+    qt_select1 """
+        select k1,k2,k3 from cast_decimalv3_as_bool order by id
+    """
+    qt_select2 """
+        select cast(k1 as boolean), cast(k2 as boolean) , cast(k3 as boolean) from cast_decimalv3_as_bool order by id
+    """ 
+    qt_select3"""
+        select cast(3.00001 as boolean),  cast(cast(3.00001 as  boolean) as int),cast(0.001 as boolean),cast(0.000 as boolean);
+    """
+        qt_select3"""
+        select cast(cast(3.00001 as double)as boolean),  cast(cast(cast(3.00001 as double) as  boolean) as int),cast(cast(0.001 as double) as boolean),cast(cast(0.000 as double) as boolean);
+    """
+}
\ No newline at end of file
diff --git a/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy b/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
index 5138db6e73b4ad..a91c86b5f48908 100644
--- a/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
+++ b/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
@@ -185,10 +185,11 @@ suite("test_case_function_null", "query,p0,arrow_flight_sql") {
             c2,
             c1;
     """
-
+    // There is a behavior change. The 0.4cast boolean used to be 0 in the past, but now it has changed to 1.
+    // Therefore, we need to update the case accordingly.
     qt_sql_case1 """
         SELECT SUM(
-            CASE (((NULL BETWEEN NULL AND NULL)) and (CAST(0.4716 AS BOOLEAN)))
+            CASE (((NULL BETWEEN NULL AND NULL)) and (CAST(0.0 AS BOOLEAN)))
             WHEN ((CAST('-1530390546' AS VARCHAR)) LIKE ('-1678299490'))
             THEN (- (+ case_null2.c0))
             WHEN CASE (NULL IN (NULL))
@@ -197,9 +198,10 @@ suite("test_case_function_null", "query,p0,arrow_flight_sql") {
             END)
         FROM case_null2;
     """
-
+    // There is a behavior change. The 0.4cast boolean used to be 0 in the past, but now it has changed to 1.
+    // Therefore, we need to update the case accordingly.
     qt_sql_case2 """
-        SELECT SUM(CASE (((NULL BETWEEN NULL AND NULL)) and (CAST(0.4716 AS BOOLEAN)))
+        SELECT SUM(CASE (((NULL BETWEEN NULL AND NULL)) and (CAST(0.0 AS BOOLEAN)))
             WHEN ((CAST('-1530390546' AS VARCHAR)) LIKE ('-1678299490'))
             THEN (- (+ case_null2.c0))
             END)
@@ -209,9 +211,11 @@ suite("test_case_function_null", "query,p0,arrow_flight_sql") {
     sql "SET experimental_enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
 
+    // There is a behavior change. The 0.4cast boolean used to be 0 in the past, but now it has changed to 1.
+    // Therefore, we need to update the case accordingly.
     qt_sql_case1 """
         SELECT SUM(
-            CASE (((NULL BETWEEN NULL AND NULL)) and (CAST(0.4716 AS BOOLEAN)))
+            CASE (((NULL BETWEEN NULL AND NULL)) and (CAST(0.0 AS BOOLEAN)))
             WHEN ((CAST('-1530390546' AS VARCHAR)) LIKE ('-1678299490'))
             THEN (- (+ case_null2.c0))
             WHEN CASE (NULL IN (NULL))
@@ -221,8 +225,10 @@ suite("test_case_function_null", "query,p0,arrow_flight_sql") {
         FROM case_null2;
     """
 
+    // There is a behavior change. The 0.4cast boolean used to be 0 in the past, but now it has changed to 1.
+    // Therefore, we need to update the case accordingly.
     qt_sql_case2 """
-        SELECT SUM(CASE (((NULL BETWEEN NULL AND NULL)) and (CAST(0.4716 AS BOOLEAN)))
+        SELECT SUM(CASE (((NULL BETWEEN NULL AND NULL)) and (CAST(0.0 AS BOOLEAN)))
             WHEN ((CAST('-1530390546' AS VARCHAR)) LIKE ('-1678299490'))
             THEN (- (+ case_null2.c0))
             END)

From 4d48597487b35d9bece381aefae1736a9016c9f1 Mon Sep 17 00:00:00 2001
From: Sun Chenyang <sunchenyang@selectdb.com>
Date: Tue, 26 Nov 2024 17:35:42 +0800
Subject: [PATCH 009/399] [fix](inverted index) fix close error on inverted
 index writer (#44601)

Related PR: #44205
fix close error on inverted index writer
---
 be/src/olap/rowset/segment_v2/inverted_index_writer.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 08cebaa07d6670..86a8f89e4c94e4 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -667,8 +667,13 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
                 FINALLY_CLOSE(meta_out);
                 FINALLY_CLOSE(data_out);
                 FINALLY_CLOSE(index_out);
-                FINALLY_CLOSE(_dir);
-                FINALLY_CLOSE(_index_writer);
+                if constexpr (field_is_numeric_type(field_type)) {
+                    FINALLY_CLOSE(_dir);
+                } else if constexpr (field_is_slice_type(field_type)) {
+                    FINALLY_CLOSE(_index_writer);
+                    // After closing the _index_writer, it needs to be reset to null to prevent issues of not closing it or closing it multiple times.
+                    _index_writer.reset();
+                }
             })
 
             return Status::OK();

From 14a8bac3cad8000fe96e8676e7ef04675c412716 Mon Sep 17 00:00:00 2001
From: qiye <luen@selectdb.com>
Date: Tue, 26 Nov 2024 19:20:47 +0800
Subject: [PATCH 010/399] [fix](build index)Fix non-master nodes cannot see the
 latest build index job status (#44544)

Problem Summary:
Non-master nodes cannot see the latest build index job status, this will
cause the case `test_build_index_with_clone_fault_injection` unstable.

We add forward_to_master in show build index statement to get the latest
job status.

### Release note

Fix non-master nodes cannot see the latest build index job status
---
 .../java/org/apache/doris/alter/IndexChangeJob.java    |  1 +
 .../java/org/apache/doris/alter/SchemaChangeJobV2.java |  2 +-
 .../org/apache/doris/analysis/ShowBuildIndexStmt.java  | 10 ++++++++++
 .../test_build_index_with_clone_fault.groovy           |  1 +
 4 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
index a51cdb01699e23..e00587bee16d98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
@@ -348,6 +348,7 @@ protected void runWaitingTxnJob() throws AlterCancelException {
             olapTable.readUnlock();
         }
         this.jobState = JobState.RUNNING;
+        // DO NOT write edit log here, tasks will be sent again if FE restart or master changed.
         LOG.info("transfer inverted index job {} state to {}", jobId, this.jobState);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 522433dfe898da..58e3e05e214a9d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -556,7 +556,7 @@ protected void runWaitingTxnJob() throws AlterCancelException {
 
         this.jobState = JobState.RUNNING;
 
-        // DO NOT write edit log here, tasks will be send again if FE restart or master changed.
+        // DO NOT write edit log here, tasks will be sent again if FE restart or master changed.
         LOG.info("transfer schema change job {} state to {}", jobId, this.jobState);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java
index 944365fb0d48df..9914bb76400a41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBuildIndexStmt.java
@@ -29,6 +29,7 @@
 import org.apache.doris.common.proc.ProcNodeInterface;
 import org.apache.doris.common.proc.ProcService;
 import org.apache.doris.common.util.OrderByPair;
+import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.ShowResultSetMetaData;
 
 import com.google.common.base.Strings;
@@ -224,4 +225,13 @@ public ShowResultSetMetaData getMetaData() {
 
         return builder.build();
     }
+
+    @Override
+    public RedirectStatus getRedirectStatus() {
+        if (ConnectContext.get().getSessionVariable().getForwardToMaster()) {
+            return RedirectStatus.FORWARD_NO_SYNC;
+        } else {
+            return RedirectStatus.NO_FORWARD;
+        }
+    }
 }
diff --git a/regression-test/suites/fault_injection_p0/test_build_index_with_clone_fault.groovy b/regression-test/suites/fault_injection_p0/test_build_index_with_clone_fault.groovy
index ec175792d48de3..998c8fa98619e0 100644
--- a/regression-test/suites/fault_injection_p0/test_build_index_with_clone_fault.groovy
+++ b/regression-test/suites/fault_injection_p0/test_build_index_with_clone_fault.groovy
@@ -58,6 +58,7 @@ suite("test_build_index_with_clone_fault_injection", "nonConcurrent"){
         while (attempt < maxRetries) {
             def show_build_index = sql_return_maparray("show build index where TableName = \"${tbl}\" ORDER BY JobId DESC LIMIT 1")
             if (show_build_index && show_build_index.size() > 0) {
+                logger.info("show build index result: ${show_build_index}")
                 def currentState = show_build_index[0].State
                 def currentMsg = show_build_index[0].Msg
                 if ((currentState == expectedState && currentMsg == expectedMsg) || currentState == "FINISHED") {

From 0066bcdb26f08e2e5945a4a67a91814aae657814 Mon Sep 17 00:00:00 2001
From: zfr95 <zhangfurong@selectdb.com>
Date: Tue, 26 Nov 2024 20:36:16 +0800
Subject: [PATCH 011/399] [fix](auth)fix stream load information to observer
 (#44632)

---
 .../suites/auth_call/test_dml_stream_load_auth.groovy           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy
index 240fdde6f6926d..30f01c0e72d9c5 100644
--- a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy
@@ -59,7 +59,7 @@ suite("test_dml_stream_load_auth","p0,auth_call") {
 
     def path_file = "${context.file.parent}/../../data/auth_call/stream_load_data.csv"
     def load_path = "${context.file.parent}/../../data/auth_call/stream_load_cm.sh"
-    def cm = """curl --location-trusted -u ${user}:${pwd} -H "column_separator:," -T ${path_file} http://${sql_ip}:${http_port}/api/${dbName}/${tableName}/_stream_load"""
+    def cm = """curl -v --location-trusted -u ${user}:${pwd} -H "column_separator:," -T ${path_file} http://${sql_ip}:${http_port}/api/${dbName}/${tableName}/_stream_load"""
     logger.info("cm: " + cm)
     write_to_file(load_path, cm)
     cm = "bash " + load_path

From bce9b69c7da60d62101516612d34707763b77bc0 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Tue, 26 Nov 2024 20:42:39 +0800
Subject: [PATCH 012/399] [fix](coordinator) Fix wrong bucket assignment
 (#44603)

follow-up : #44459
---
 .../src/main/java/org/apache/doris/qe/Coordinator.java   | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 61ecaf7fc8b359..3a6f6e4f84069d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -2766,17 +2766,18 @@ private void assignScanRanges(PlanFragmentId fragmentId, int parallelExecInstanc
                                 .addAll(nodeScanRange.getValue());
                     }
                 }
-                params.instanceExecParams.add(instanceParam);
+                List<FInstanceExecParam> instanceExecParams = new ArrayList<>();
+                instanceExecParams.add(instanceParam);
                 for (int i = 1; i < parallelExecInstanceNum; i++) {
-                    params.instanceExecParams.add(new FInstanceExecParam(
+                    instanceExecParams.add(new FInstanceExecParam(
                             null, addressScanRange.getKey(), 0, params));
                 }
                 int index = 0;
                 for (Pair<Integer, Map<Integer, List<TScanRangeParams>>> nodeScanRangeMap : scanRange) {
-                    params.instanceExecParams.get(index % params.instanceExecParams.size())
-                            .addBucketSeq(nodeScanRangeMap.first);
+                    instanceExecParams.get(index % instanceExecParams.size()).addBucketSeq(nodeScanRangeMap.first);
                     index++;
                 }
+                params.instanceExecParams.addAll(instanceExecParams);
             } else {
                 int expectedInstanceNum = 1;
                 if (parallelExecInstanceNum > 1) {

From d3714bf55619fffed08250687f614dcaea271458 Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Tue, 26 Nov 2024 22:55:56 +0800
Subject: [PATCH 013/399] [fix](cloud) cloud mode support txn load for mow
 tables (#41932)

---
 .../cloud_engine_calc_delete_bitmap_task.cpp  | 106 ++++++++--
 .../cloud_engine_calc_delete_bitmap_task.h    |   9 +-
 be/src/cloud/cloud_tablet.cpp                 |   6 +-
 be/src/cloud/cloud_tablet.h                   |   3 +-
 be/src/olap/base_tablet.cpp                   |  60 ++++--
 be/src/olap/base_tablet.h                     |  15 +-
 be/src/olap/calc_delete_bitmap_executor.cpp   |   6 +-
 be/src/olap/calc_delete_bitmap_executor.h     |   3 +-
 be/src/olap/tablet.cpp                        |   2 +-
 be/src/olap/tablet.h                          |   3 +-
 be/src/olap/tablet_meta.h                     |   1 +
 be/src/olap/txn_manager.h                     |   6 +
 cloud/src/meta-service/meta_service_txn.cpp   | 181 ++++++++----------
 .../java/org/apache/doris/common/Config.java  |   3 +
 .../CloudGlobalTransactionMgr.java            | 161 +++++++++++-----
 .../doris/transaction/TransactionEntry.java   |   9 -
 gensrc/thrift/AgentService.thrift             |   1 +
 .../insert_p0/transaction/txn_insert.groovy   |  11 +-
 .../txn_insert_concurrent_insert_mow.groovy   | 135 +++++++++++++
 ..._insert_concurrent_insert_aggregate.groovy |   2 +-
 ..._insert_concurrent_insert_duplicate.groovy |   2 +-
 .../txn_insert_concurrent_insert_mor.groovy   |   2 +-
 .../txn_insert_concurrent_insert_mow.groovy   |  13 +-
 .../txn_insert_concurrent_insert_ud.groovy    |   7 +-
 ...txn_insert_concurrent_insert_update.groovy |   2 +-
 .../txn_insert_with_schema_change.groovy      |   2 +-
 26 files changed, 511 insertions(+), 240 deletions(-)
 create mode 100644 regression-test/suites/insert_p0/transaction/txn_insert_concurrent_insert_mow.groovy
 rename regression-test/suites/insert_p2/{ => transaction}/txn_insert_concurrent_insert_aggregate.groovy (99%)
 rename regression-test/suites/insert_p2/{ => transaction}/txn_insert_concurrent_insert_duplicate.groovy (99%)
 rename regression-test/suites/insert_p2/{ => transaction}/txn_insert_concurrent_insert_mor.groovy (99%)
 rename regression-test/suites/insert_p2/{ => transaction}/txn_insert_concurrent_insert_mow.groovy (95%)
 rename regression-test/suites/insert_p2/{ => transaction}/txn_insert_concurrent_insert_ud.groovy (98%)
 rename regression-test/suites/insert_p2/{ => transaction}/txn_insert_concurrent_insert_update.groovy (99%)
 rename regression-test/suites/insert_p2/{ => transaction}/txn_insert_with_schema_change.groovy (99%)

diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
index 6abc3958650ef6..7391449b73f1b1 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
@@ -75,7 +75,7 @@ Status CloudEngineCalcDeleteBitmapTask::execute() {
         for (size_t i = 0; i < partition.tablet_ids.size(); i++) {
             auto tablet_id = partition.tablet_ids[i];
             auto tablet_calc_delete_bitmap_ptr = std::make_shared<CloudTabletCalcDeleteBitmapTask>(
-                    _engine, this, tablet_id, transaction_id, version);
+                    _engine, this, tablet_id, transaction_id, version, partition.sub_txn_ids);
             if (has_compaction_stats) {
                 tablet_calc_delete_bitmap_ptr->set_compaction_stats(
                         partition.base_compaction_cnts[i], partition.cumulative_compaction_cnts[i],
@@ -107,12 +107,13 @@ Status CloudEngineCalcDeleteBitmapTask::execute() {
 
 CloudTabletCalcDeleteBitmapTask::CloudTabletCalcDeleteBitmapTask(
         CloudStorageEngine& engine, CloudEngineCalcDeleteBitmapTask* engine_task, int64_t tablet_id,
-        int64_t transaction_id, int64_t version)
+        int64_t transaction_id, int64_t version, const std::vector<int64_t>& sub_txn_ids)
         : _engine(engine),
           _engine_calc_delete_bitmap_task(engine_task),
           _tablet_id(tablet_id),
           _transaction_id(transaction_id),
-          _version(version) {
+          _version(version),
+          _sub_txn_ids(sub_txn_ids) {
     _mem_tracker = MemTrackerLimiter::create_shared(
             MemTrackerLimiter::Type::OTHER,
             fmt::format("CloudTabletCalcDeleteBitmapTask#_transaction_id={}", _transaction_id));
@@ -189,6 +190,60 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
         return error_st;
     }
 
+    int64_t t3 = MonotonicMicros();
+    Status status;
+    if (_sub_txn_ids.empty()) {
+        status = _handle_rowset(tablet, _version);
+    } else {
+        std::stringstream ss;
+        for (const auto& sub_txn_id : _sub_txn_ids) {
+            ss << sub_txn_id << ", ";
+        }
+        LOG(INFO) << "start calc delete bitmap for txn_id=" << _transaction_id << ", sub_txn_ids=["
+                  << ss.str() << "], table_id=" << tablet->table_id()
+                  << ", partition_id=" << tablet->partition_id() << ", tablet_id=" << _tablet_id
+                  << ", start_version=" << _version;
+        std::vector<RowsetSharedPtr> invisible_rowsets;
+        DeleteBitmapPtr tablet_delete_bitmap =
+                std::make_shared<DeleteBitmap>(tablet->tablet_meta()->delete_bitmap());
+        for (int i = 0; i < _sub_txn_ids.size(); ++i) {
+            int64_t sub_txn_id = _sub_txn_ids[i];
+            int64_t version = _version + i;
+            LOG(INFO) << "start calc delete bitmap for txn_id=" << _transaction_id
+                      << ", sub_txn_id=" << sub_txn_id << ", table_id=" << tablet->table_id()
+                      << ", partition_id=" << tablet->partition_id() << ", tablet_id=" << _tablet_id
+                      << ", start_version=" << _version << ", cur_version=" << version;
+            status = _handle_rowset(tablet, version, sub_txn_id, &invisible_rowsets,
+                                    tablet_delete_bitmap);
+            if (!status.ok()) {
+                LOG(INFO) << "failed to calculate delete bitmap on tablet"
+                          << ", table_id=" << tablet->table_id()
+                          << ", transaction_id=" << _transaction_id << ", sub_txn_id=" << sub_txn_id
+                          << ", tablet_id=" << tablet->tablet_id() << ", start version=" << _version
+                          << ", cur_version=" << version << ", status=" << status;
+                return status;
+            }
+            DCHECK(invisible_rowsets.size() == i + 1);
+        }
+    }
+    auto total_update_delete_bitmap_time_us = MonotonicMicros() - t3;
+    LOG(INFO) << "calculate delete bitmap successfully on tablet"
+              << ", table_id=" << tablet->table_id() << ", transaction_id=" << _transaction_id
+              << ", tablet_id=" << tablet->tablet_id()
+              << ", get_tablet_time_us=" << get_tablet_time_us
+              << ", sync_rowset_time_us=" << sync_rowset_time_us
+              << ", total_update_delete_bitmap_time_us=" << total_update_delete_bitmap_time_us
+              << ", res=" << status;
+    return status;
+}
+
+Status CloudTabletCalcDeleteBitmapTask::_handle_rowset(
+        std::shared_ptr<CloudTablet> tablet, int64_t version, int64_t sub_txn_id,
+        std::vector<RowsetSharedPtr>* invisible_rowsets,
+        DeleteBitmapPtr tablet_delete_bitmap) const {
+    int64_t transaction_id = sub_txn_id == -1 ? _transaction_id : sub_txn_id;
+    std::string txn_str = "txn_id=" + std::to_string(_transaction_id) +
+                          (sub_txn_id == -1 ? "" : ", sub_txn_id=" + std::to_string(sub_txn_id));
     RowsetSharedPtr rowset;
     DeleteBitmapPtr delete_bitmap;
     RowsetIdUnorderedSet rowset_ids;
@@ -197,59 +252,76 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
     int64_t txn_expiration;
     TxnPublishInfo previous_publish_info;
     Status status = _engine.txn_delete_bitmap_cache().get_tablet_txn_info(
-            _transaction_id, _tablet_id, &rowset, &delete_bitmap, &rowset_ids, &txn_expiration,
+            transaction_id, _tablet_id, &rowset, &delete_bitmap, &rowset_ids, &txn_expiration,
             &partial_update_info, &publish_status, &previous_publish_info);
     if (status != Status::OK()) {
-        LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << _tablet_id
-                     << ", txn_id=" << _transaction_id << ", status=" << status;
+        LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << _tablet_id << ", " << txn_str
+                     << ", status=" << status;
         _engine_calc_delete_bitmap_task->add_error_tablet_id(_tablet_id, status);
         return status;
     }
 
     int64_t t3 = MonotonicMicros();
-    rowset->set_version(Version(_version, _version));
+    rowset->set_version(Version(version, version));
     TabletTxnInfo txn_info;
     txn_info.rowset = rowset;
     txn_info.delete_bitmap = delete_bitmap;
     txn_info.rowset_ids = rowset_ids;
     txn_info.partial_update_info = partial_update_info;
     txn_info.publish_status = publish_status;
-    txn_info.publish_info = {.publish_version = _version,
+    txn_info.publish_info = {.publish_version = version,
                              .base_compaction_cnt = _ms_base_compaction_cnt,
                              .cumulative_compaction_cnt = _ms_cumulative_compaction_cnt,
                              .cumulative_point = _ms_cumulative_point};
     auto update_delete_bitmap_time_us = 0;
     if (txn_info.publish_status && (*(txn_info.publish_status) == PublishStatus::SUCCEED) &&
-        _version == previous_publish_info.publish_version &&
+        version == previous_publish_info.publish_version &&
         _ms_base_compaction_cnt == previous_publish_info.base_compaction_cnt &&
         _ms_cumulative_compaction_cnt == previous_publish_info.cumulative_compaction_cnt &&
         _ms_cumulative_point == previous_publish_info.cumulative_point) {
         // if version or compaction stats can't match, it means that this is a retry and there are
         // compaction or other loads finished successfully on the same tablet. So the previous publish
         // is stale and we should re-calculate the delete bitmap
-        LOG(INFO) << "tablet=" << _tablet_id << ",txn=" << _transaction_id
+        LOG(INFO) << "tablet=" << _tablet_id << ", " << txn_str
                   << ",publish_status=SUCCEED,not need to recalculate and update delete_bitmap.";
     } else {
-        status = CloudTablet::update_delete_bitmap(tablet, &txn_info, _transaction_id,
-                                                   txn_expiration);
+        if (invisible_rowsets == nullptr) {
+            status = CloudTablet::update_delete_bitmap(tablet, &txn_info, transaction_id,
+                                                       txn_expiration);
+        } else {
+            txn_info.is_txn_load = true;
+            txn_info.invisible_rowsets = *invisible_rowsets;
+            txn_info.lock_id = _transaction_id;
+            txn_info.next_visible_version = _version;
+            status = CloudTablet::update_delete_bitmap(tablet, &txn_info, transaction_id,
+                                                       txn_expiration, tablet_delete_bitmap);
+        }
         update_delete_bitmap_time_us = MonotonicMicros() - t3;
     }
     if (status != Status::OK()) {
         LOG(WARNING) << "failed to calculate delete bitmap. rowset_id=" << rowset->rowset_id()
-                     << ", tablet_id=" << _tablet_id << ", txn_id=" << _transaction_id
-                     << ", status=" << status;
+                     << ", tablet_id=" << _tablet_id << ", " << txn_str << ", status=" << status;
         _engine_calc_delete_bitmap_task->add_error_tablet_id(_tablet_id, status);
         return status;
     }
 
     _engine_calc_delete_bitmap_task->add_succ_tablet_id(_tablet_id);
     LOG(INFO) << "calculate delete bitmap successfully on tablet"
-              << ", table_id=" << tablet->table_id() << ", transaction_id=" << _transaction_id
+              << ", table_id=" << tablet->table_id() << ", " << txn_str
               << ", tablet_id=" << tablet->tablet_id() << ", num_rows=" << rowset->num_rows()
-              << ", get_tablet_time_us=" << get_tablet_time_us
-              << ", sync_rowset_time_us=" << sync_rowset_time_us
               << ", update_delete_bitmap_time_us=" << update_delete_bitmap_time_us
               << ", res=" << status;
+    if (invisible_rowsets != nullptr) {
+        invisible_rowsets->push_back(rowset);
+        // see CloudTablet::save_delete_bitmap
+        auto dm = txn_info.delete_bitmap->delete_bitmap;
+        for (auto it = dm.begin(); it != dm.end(); ++it) {
+            if (std::get<1>(it->first) != DeleteBitmap::INVALID_SEGMENT_ID) {
+                tablet_delete_bitmap->merge(
+                        {std::get<0>(it->first), std::get<1>(it->first), version}, it->second);
+            }
+        }
+    }
     return status;
 }
 
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
index e3733d3e696ff8..c70a9cfa3903ba 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
@@ -34,7 +34,8 @@ class CloudTabletCalcDeleteBitmapTask {
 public:
     CloudTabletCalcDeleteBitmapTask(CloudStorageEngine& engine,
                                     CloudEngineCalcDeleteBitmapTask* engine_task, int64_t tablet_id,
-                                    int64_t transaction_id, int64_t version);
+                                    int64_t transaction_id, int64_t version,
+                                    const std::vector<int64_t>& sub_txn_ids);
     ~CloudTabletCalcDeleteBitmapTask() = default;
 
     void set_compaction_stats(int64_t ms_base_compaction_cnt, int64_t ms_cumulative_compaction_cnt,
@@ -43,12 +44,18 @@ class CloudTabletCalcDeleteBitmapTask {
     Status handle() const;
 
 private:
+    Status _handle_rowset(std::shared_ptr<CloudTablet> tablet, int64_t version,
+                          int64_t sub_txn_id = -1,
+                          std::vector<RowsetSharedPtr>* invisible_rowsets = nullptr,
+                          DeleteBitmapPtr tablet_delete_bitmap = nullptr) const;
+
     CloudStorageEngine& _engine;
     CloudEngineCalcDeleteBitmapTask* _engine_calc_delete_bitmap_task;
 
     int64_t _tablet_id;
     int64_t _transaction_id;
     int64_t _version;
+    std::vector<int64_t> _sub_txn_ids;
 
     int64_t _ms_base_compaction_cnt {-1};
     int64_t _ms_cumulative_compaction_cnt {-1};
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index c88b073e96494a..267c204c0e608e 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -690,7 +690,8 @@ CalcDeleteBitmapExecutor* CloudTablet::calc_delete_bitmap_executor() {
 
 Status CloudTablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id,
                                        DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
-                                       const RowsetIdUnorderedSet& cur_rowset_ids) {
+                                       const RowsetIdUnorderedSet& cur_rowset_ids,
+                                       int64_t lock_id) {
     RowsetSharedPtr rowset = txn_info->rowset;
     int64_t cur_version = rowset->start_version();
     // update delete bitmap info, in order to avoid recalculation when trying again
@@ -714,8 +715,9 @@ Status CloudTablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t tx
         }
     }
 
+    auto ms_lock_id = lock_id == -1 ? txn_id : lock_id;
     RETURN_IF_ERROR(_engine.meta_mgr().update_delete_bitmap(
-            *this, txn_id, COMPACTION_DELETE_BITMAP_LOCK_ID, new_delete_bitmap.get()));
+            *this, ms_lock_id, COMPACTION_DELETE_BITMAP_LOCK_ID, new_delete_bitmap.get()));
 
     // store the delete bitmap with sentinel marks in txn_delete_bitmap_cache because if the txn is retried for some reason,
     // it will use the delete bitmap from txn_delete_bitmap_cache when re-calculating the delete bitmap, during which it will do
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index 0fde2f5b1d93ff..80038e569ba2fc 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -170,7 +170,8 @@ class CloudTablet final : public BaseTablet {
 
     Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id,
                               DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
-                              const RowsetIdUnorderedSet& cur_rowset_ids) override;
+                              const RowsetIdUnorderedSet& cur_rowset_ids,
+                              int64_t lock_id = -1) override;
 
     Status calc_delete_bitmap_for_compaction(const std::vector<RowsetSharedPtr>& input_rowsets,
                                              const RowsetSharedPtr& output_rowset,
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 2e70e4586cc768..a499a27b07f6e2 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -450,7 +450,8 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest
                                   RowLocation* row_location, uint32_t version,
                                   std::vector<std::unique_ptr<SegmentCacheHandle>>& segment_caches,
                                   RowsetSharedPtr* rowset, bool with_rowid,
-                                  std::string* encoded_seq_value, OlapReaderStatistics* stats) {
+                                  std::string* encoded_seq_value, OlapReaderStatistics* stats,
+                                  DeleteBitmapPtr delete_bitmap) {
     SCOPED_BVAR_LATENCY(g_tablet_lookup_rowkey_latency);
     size_t seq_col_length = 0;
     // use the latest tablet schema to decide if the tablet has sequence column currently
@@ -467,6 +468,8 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest
             Slice(encoded_key.get_data(), encoded_key.get_size() - seq_col_length - rowid_length);
     RowLocation loc;
 
+    auto tablet_delete_bitmap =
+            delete_bitmap == nullptr ? _tablet_meta->delete_bitmap_ptr() : delete_bitmap;
     for (size_t i = 0; i < specified_rowsets.size(); i++) {
         const auto& rs = specified_rowsets[i];
         const auto& segments_key_bounds = rs->rowset_meta()->get_segments_key_bounds();
@@ -501,7 +504,7 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest
             if (!s.ok() && !s.is<KEY_ALREADY_EXISTS>()) {
                 return s;
             }
-            if (s.ok() && _tablet_meta->delete_bitmap().contains_agg_without_cache(
+            if (s.ok() && tablet_delete_bitmap->contains_agg_without_cache(
                                   {loc.rowset_id, loc.segment_id, version}, loc.row_id)) {
                 // if has sequence col, we continue to compare the sequence_id of
                 // all rowsets, util we find an existing key.
@@ -535,7 +538,8 @@ Status BaseTablet::calc_delete_bitmap(const BaseTabletSPtr& tablet, RowsetShared
                                       const std::vector<segment_v2::SegmentSharedPtr>& segments,
                                       const std::vector<RowsetSharedPtr>& specified_rowsets,
                                       DeleteBitmapPtr delete_bitmap, int64_t end_version,
-                                      CalcDeleteBitmapToken* token, RowsetWriter* rowset_writer) {
+                                      CalcDeleteBitmapToken* token, RowsetWriter* rowset_writer,
+                                      DeleteBitmapPtr tablet_delete_bitmap) {
     auto rowset_id = rowset->rowset_id();
     if (specified_rowsets.empty() || segments.empty()) {
         LOG(INFO) << "skip to construct delete bitmap tablet: " << tablet->tablet_id()
@@ -548,10 +552,11 @@ Status BaseTablet::calc_delete_bitmap(const BaseTabletSPtr& tablet, RowsetShared
         const auto& seg = segment;
         if (token != nullptr) {
             RETURN_IF_ERROR(token->submit(tablet, rowset, seg, specified_rowsets, end_version,
-                                          delete_bitmap, rowset_writer));
+                                          delete_bitmap, rowset_writer, tablet_delete_bitmap));
         } else {
             RETURN_IF_ERROR(tablet->calc_segment_delete_bitmap(
-                    rowset, segment, specified_rowsets, delete_bitmap, end_version, rowset_writer));
+                    rowset, segment, specified_rowsets, delete_bitmap, end_version, rowset_writer,
+                    tablet_delete_bitmap));
         }
     }
 
@@ -562,7 +567,8 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
                                               const segment_v2::SegmentSharedPtr& seg,
                                               const std::vector<RowsetSharedPtr>& specified_rowsets,
                                               DeleteBitmapPtr delete_bitmap, int64_t end_version,
-                                              RowsetWriter* rowset_writer) {
+                                              RowsetWriter* rowset_writer,
+                                              DeleteBitmapPtr tablet_delete_bitmap) {
     OlapStopWatch watch;
     auto rowset_id = rowset->rowset_id();
     Version dummy_version(end_version + 1, end_version + 1);
@@ -676,9 +682,16 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
             }
 
             RowsetSharedPtr rowset_find;
-            auto st = lookup_row_key(key, rowset_schema.get(), true, specified_rowsets, &loc,
-                                     cast_set<uint32_t>(dummy_version.first - 1), segment_caches,
-                                     &rowset_find);
+            Status st = Status::OK();
+            if (tablet_delete_bitmap == nullptr) {
+                st = lookup_row_key(key, rowset_schema.get(), true, specified_rowsets, &loc,
+                                    cast_set<uint32_t>(dummy_version.first - 1), segment_caches,
+                                    &rowset_find);
+            } else {
+                st = lookup_row_key(key, rowset_schema.get(), true, specified_rowsets, &loc,
+                                    cast_set<uint32_t>(dummy_version.first - 1), segment_caches,
+                                    &rowset_find, true, nullptr, nullptr, tablet_delete_bitmap);
+            }
             bool expected_st = st.ok() || st.is<KEY_NOT_FOUND>() || st.is<KEY_ALREADY_EXISTS>();
             // It's a defensive DCHECK, we need to exclude some common errors to avoid core-dump
             // while stress test
@@ -1351,7 +1364,8 @@ Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap
 }
 
 Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInfo* txn_info,
-                                        int64_t txn_id, int64_t txn_expiration) {
+                                        int64_t txn_id, int64_t txn_expiration,
+                                        DeleteBitmapPtr tablet_delete_bitmap) {
     SCOPED_BVAR_LATENCY(g_tablet_update_delete_bitmap_latency);
     RowsetIdUnorderedSet cur_rowset_ids;
     RowsetIdUnorderedSet rowset_ids_to_add;
@@ -1380,6 +1394,8 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf
     auto t1 = watch.get_elapse_time_us();
 
     {
+        int64_t next_visible_version = txn_info->is_txn_load ? txn_info->next_visible_version
+                                                             : txn_info->rowset->start_version();
         std::shared_lock meta_rlock(self->_meta_lock);
         // tablet is under alter process. The delete bitmap will be calculated after conversion.
         if (self->tablet_state() == TABLET_NOTREADY) {
@@ -1387,7 +1403,7 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf
                       << self->tablet_id();
             return Status::OK();
         }
-        RETURN_IF_ERROR(self->get_all_rs_id_unlocked(cur_version - 1, &cur_rowset_ids));
+        RETURN_IF_ERROR(self->get_all_rs_id_unlocked(next_visible_version - 1, &cur_rowset_ids));
     }
     auto t2 = watch.get_elapse_time_us();
 
@@ -1402,6 +1418,15 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf
         std::shared_lock meta_rlock(self->_meta_lock);
         specified_rowsets = self->get_rowset_by_ids(&rowset_ids_to_add);
     }
+    if (txn_info->is_txn_load) {
+        for (auto invisible_rowset : txn_info->invisible_rowsets) {
+            specified_rowsets.emplace_back(invisible_rowset);
+        }
+        std::sort(specified_rowsets.begin(), specified_rowsets.end(),
+                  [](RowsetSharedPtr& lhs, RowsetSharedPtr& rhs) {
+                      return lhs->end_version() > rhs->end_version();
+                  });
+    }
     auto t3 = watch.get_elapse_time_us();
 
     // If a rowset is produced by compaction before the commit phase of the partial update load
@@ -1446,7 +1471,8 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf
         auto token = self->calc_delete_bitmap_executor()->create_token();
         // set rowset_writer to nullptr to skip the alignment process
         RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, rowsets_skip_alignment,
-                                           delete_bitmap, cur_version - 1, token.get(), nullptr));
+                                           delete_bitmap, cur_version - 1, token.get(), nullptr,
+                                           tablet_delete_bitmap));
         RETURN_IF_ERROR(token->wait());
     }
 
@@ -1454,13 +1480,14 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf
     // Otherwise, it will be submitted to the thread pool for calculation.
     if (segments.size() <= 1) {
         RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
-                                           cur_version - 1, nullptr, transient_rs_writer.get()));
+                                           cur_version - 1, nullptr, transient_rs_writer.get(),
+                                           tablet_delete_bitmap));
 
     } else {
         auto token = self->calc_delete_bitmap_executor()->create_token();
         RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
-                                           cur_version - 1, token.get(),
-                                           transient_rs_writer.get()));
+                                           cur_version - 1, token.get(), transient_rs_writer.get(),
+                                           tablet_delete_bitmap));
         RETURN_IF_ERROR(token->wait());
     }
 
@@ -1511,8 +1538,9 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf
             segments.begin(), segments.end(), 0,
             [](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); });
     auto t5 = watch.get_elapse_time_us();
+    int64_t lock_id = txn_info->is_txn_load ? txn_info->lock_id : -1;
     RETURN_IF_ERROR(self->save_delete_bitmap(txn_info, txn_id, delete_bitmap,
-                                             transient_rs_writer.get(), cur_rowset_ids));
+                                             transient_rs_writer.get(), cur_rowset_ids, lock_id));
     LOG(INFO) << "[Publish] construct delete bitmap tablet: " << self->tablet_id()
               << ", rowset_ids to add: " << rowset_ids_to_add.size()
               << ", rowset_ids to del: " << rowset_ids_to_del.size()
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index b6fc953e46082e..f961f4c49eedd6 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -156,7 +156,8 @@ class BaseTablet {
                           std::vector<std::unique_ptr<SegmentCacheHandle>>& segment_caches,
                           RowsetSharedPtr* rowset = nullptr, bool with_rowid = true,
                           std::string* encoded_seq_value = nullptr,
-                          OlapReaderStatistics* stats = nullptr);
+                          OlapReaderStatistics* stats = nullptr,
+                          DeleteBitmapPtr tablet_delete_bitmap = nullptr);
 
     // calc delete bitmap when flush memtable, use a fake version to calc
     // For example, cur max version is 5, and we use version 6 to calc but
@@ -169,13 +170,15 @@ class BaseTablet {
                                      const std::vector<RowsetSharedPtr>& specified_rowsets,
                                      DeleteBitmapPtr delete_bitmap, int64_t version,
                                      CalcDeleteBitmapToken* token,
-                                     RowsetWriter* rowset_writer = nullptr);
+                                     RowsetWriter* rowset_writer = nullptr,
+                                     DeleteBitmapPtr tablet_delete_bitmap = nullptr);
 
     Status calc_segment_delete_bitmap(RowsetSharedPtr rowset,
                                       const segment_v2::SegmentSharedPtr& seg,
                                       const std::vector<RowsetSharedPtr>& specified_rowsets,
                                       DeleteBitmapPtr delete_bitmap, int64_t end_version,
-                                      RowsetWriter* rowset_writer);
+                                      RowsetWriter* rowset_writer,
+                                      DeleteBitmapPtr tablet_delete_bitmap = nullptr);
 
     Status calc_delete_bitmap_between_segments(
             RowsetSharedPtr rowset, const std::vector<segment_v2::SegmentSharedPtr>& segments,
@@ -235,11 +238,13 @@ class BaseTablet {
             int64_t txn_expiration = 0) = 0;
 
     static Status update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInfo* txn_info,
-                                       int64_t txn_id, int64_t txn_expiration = 0);
+                                       int64_t txn_id, int64_t txn_expiration = 0,
+                                       DeleteBitmapPtr tablet_delete_bitmap = nullptr);
 
     virtual Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id,
                                       DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
-                                      const RowsetIdUnorderedSet& cur_rowset_ids) = 0;
+                                      const RowsetIdUnorderedSet& cur_rowset_ids,
+                                      int64_t lock_id = -1) = 0;
     virtual CalcDeleteBitmapExecutor* calc_delete_bitmap_executor() = 0;
 
     void calc_compaction_output_rowset_delete_bitmap(
diff --git a/be/src/olap/calc_delete_bitmap_executor.cpp b/be/src/olap/calc_delete_bitmap_executor.cpp
index 3983dc0a98642a..e45f9801f68ba4 100644
--- a/be/src/olap/calc_delete_bitmap_executor.cpp
+++ b/be/src/olap/calc_delete_bitmap_executor.cpp
@@ -34,7 +34,8 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet, RowsetSharedPtr cur_
                                      const segment_v2::SegmentSharedPtr& cur_segment,
                                      const std::vector<RowsetSharedPtr>& target_rowsets,
                                      int64_t end_version, DeleteBitmapPtr delete_bitmap,
-                                     RowsetWriter* rowset_writer) {
+                                     RowsetWriter* rowset_writer,
+                                     DeleteBitmapPtr tablet_delete_bitmap) {
     {
         std::shared_lock rlock(_lock);
         RETURN_IF_ERROR(_status);
@@ -44,7 +45,8 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet, RowsetSharedPtr cur_
     return _thread_token->submit_func([=, this]() {
         SCOPED_ATTACH_TASK(_query_thread_context);
         auto st = tablet->calc_segment_delete_bitmap(cur_rowset, cur_segment, target_rowsets,
-                                                     delete_bitmap, end_version, rowset_writer);
+                                                     delete_bitmap, end_version, rowset_writer,
+                                                     tablet_delete_bitmap);
         if (!st.ok()) {
             LOG(WARNING) << "failed to calc segment delete bitmap, tablet_id: "
                          << tablet->tablet_id() << " rowset: " << cur_rowset->rowset_id()
diff --git a/be/src/olap/calc_delete_bitmap_executor.h b/be/src/olap/calc_delete_bitmap_executor.h
index fa1e79b7feaa19..288108b04971df 100644
--- a/be/src/olap/calc_delete_bitmap_executor.h
+++ b/be/src/olap/calc_delete_bitmap_executor.h
@@ -52,7 +52,8 @@ class CalcDeleteBitmapToken {
     Status submit(BaseTabletSPtr tablet, RowsetSharedPtr cur_rowset,
                   const segment_v2::SegmentSharedPtr& cur_segment,
                   const std::vector<RowsetSharedPtr>& target_rowsets, int64_t end_version,
-                  DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer);
+                  DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
+                  DeleteBitmapPtr tablet_delete_bitmap);
 
     // wait all tasks in token to be completed.
     Status wait();
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 8a0e23e75b86ad..0d04984d0e06ba 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2490,7 +2490,7 @@ CalcDeleteBitmapExecutor* Tablet::calc_delete_bitmap_executor() {
 
 Status Tablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id,
                                   DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
-                                  const RowsetIdUnorderedSet& cur_rowset_ids) {
+                                  const RowsetIdUnorderedSet& cur_rowset_ids, int64_t lock_id) {
     RowsetSharedPtr rowset = txn_info->rowset;
     int64_t cur_version = rowset->start_version();
 
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index f5866c67641581..0b7d758ab8fd88 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -417,7 +417,8 @@ class Tablet final : public BaseTablet {
     CalcDeleteBitmapExecutor* calc_delete_bitmap_executor() override;
     Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id,
                               DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
-                              const RowsetIdUnorderedSet& cur_rowset_ids) override;
+                              const RowsetIdUnorderedSet& cur_rowset_ids,
+                              int64_t lock_id = -1) override;
 
     void merge_delete_bitmap(const DeleteBitmap& delete_bitmap);
     bool check_all_rowset_segment();
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index d56e529e42bf4b..fb0895604a19fe 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -236,6 +236,7 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
     static void init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn,
                                          ColumnPB* column);
 
+    DeleteBitmapPtr delete_bitmap_ptr() { return _delete_bitmap; }
     DeleteBitmap& delete_bitmap() { return *_delete_bitmap; }
 
     bool enable_unique_key_merge_on_write() const { return _enable_unique_key_merge_on_write; }
diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h
index 88ee97c5f6a3b9..1994dec949414f 100644
--- a/be/src/olap/txn_manager.h
+++ b/be/src/olap/txn_manager.h
@@ -87,6 +87,12 @@ struct TabletTxnInfo {
     std::shared_ptr<PublishStatus> publish_status;
     TxnPublishInfo publish_info;
 
+    // for cloud only, used to calculate delete bitmap for txn load
+    bool is_txn_load = false;
+    std::vector<RowsetSharedPtr> invisible_rowsets;
+    int64_t lock_id;
+    int64_t next_visible_version;
+
     TxnState state {TxnState::PREPARED};
     TabletTxnInfo() = default;
 
diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp
index 32f6b56f51af4c..58930f6edfc63b 100644
--- a/cloud/src/meta-service/meta_service_txn.cpp
+++ b/cloud/src/meta-service/meta_service_txn.cpp
@@ -912,6 +912,69 @@ void update_tablet_stats(const StatsTabletKeyInfo& info, const TabletStats& stat
     }
 }
 
+// process mow table, check lock and remove pending key
+void process_mow_when_commit_txn(
+        const CommitTxnRequest* request, const std::string& instance_id, MetaServiceCode& code,
+        std::string& msg, std::unique_ptr<Transaction>& txn,
+        std::unordered_map<int64_t, std::vector<int64_t>>& table_id_tablet_ids) {
+    int64_t txn_id = request->txn_id();
+    std::stringstream ss;
+    std::vector<std::string> lock_keys;
+    lock_keys.reserve(request->mow_table_ids().size());
+    for (auto table_id : request->mow_table_ids()) {
+        lock_keys.push_back(meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}));
+    }
+    std::vector<std::optional<std::string>> lock_values;
+    TxnErrorCode err = txn->batch_get(&lock_values, lock_keys);
+    if (err != TxnErrorCode::TXN_OK) {
+        ss << "failed to get delete bitmap update lock key info, instance_id=" << instance_id
+           << " err=" << err;
+        msg = ss.str();
+        code = cast_as<ErrCategory::READ>(err);
+        LOG(WARNING) << msg << " txn_id=" << txn_id;
+        return;
+    }
+    size_t total_locks = lock_keys.size();
+    for (size_t i = 0; i < total_locks; i++) {
+        int64_t table_id = request->mow_table_ids(i);
+        // When the key does not exist, it means the lock has been acquired
+        // by another transaction and successfully committed.
+        if (!lock_values[i].has_value()) {
+            ss << "get delete bitmap update lock info, lock is expired"
+               << " table_id=" << table_id << " key=" << hex(lock_keys[i]);
+            code = MetaServiceCode::LOCK_EXPIRED;
+            msg = ss.str();
+            LOG(WARNING) << msg << " txn_id=" << txn_id;
+            return;
+        }
+
+        DeleteBitmapUpdateLockPB lock_info;
+        if (!lock_info.ParseFromString(lock_values[i].value())) [[unlikely]] {
+            code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+            msg = "failed to parse DeleteBitmapUpdateLockPB";
+            LOG(WARNING) << msg << " txn_id=" << txn_id;
+            return;
+        }
+        if (lock_info.lock_id() != request->txn_id()) {
+            msg = "lock is expired";
+            code = MetaServiceCode::LOCK_EXPIRED;
+            return;
+        }
+        txn->remove(lock_keys[i]);
+        LOG(INFO) << "xxx remove delete bitmap lock, lock_key=" << hex(lock_keys[i])
+                  << " txn_id=" << txn_id;
+
+        for (auto tablet_id : table_id_tablet_ids[table_id]) {
+            std::string pending_key = meta_pending_delete_bitmap_key({instance_id, tablet_id});
+            txn->remove(pending_key);
+            LOG(INFO) << "xxx remove delete bitmap pending key, pending_key=" << hex(pending_key)
+                      << " txn_id=" << txn_id;
+        }
+    }
+    lock_keys.clear();
+    lock_values.clear();
+}
+
 /**
  * 0. Extract txn_id from request
  * 1. Get db id from TxnKv with txn_id
@@ -1173,61 +1236,11 @@ void commit_txn_immediately(
             stats.num_segs += i.num_segments();
         } // for tmp_rowsets_meta
 
-        // process mow table, check lock and remove pending key
-        std::vector<std::string> lock_keys;
-        lock_keys.reserve(request->mow_table_ids().size());
-        for (auto table_id : request->mow_table_ids()) {
-            lock_keys.push_back(meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}));
-        }
-        std::vector<std::optional<std::string>> lock_values;
-        err = txn->batch_get(&lock_values, lock_keys);
-        if (err != TxnErrorCode::TXN_OK) {
-            ss << "failed to get delete bitmap update lock key info, instance_id=" << instance_id
-               << " err=" << err;
-            msg = ss.str();
-            code = cast_as<ErrCategory::READ>(err);
-            LOG(WARNING) << msg << " txn_id=" << txn_id;
+        process_mow_when_commit_txn(request, instance_id, code, msg, txn, table_id_tablet_ids);
+        if (code != MetaServiceCode::OK) {
+            LOG(WARNING) << "process mow failed, txn_id=" << txn_id << " code=" << code;
             return;
         }
-        size_t total_locks = lock_keys.size();
-        for (size_t i = 0; i < total_locks; i++) {
-            int64_t table_id = request->mow_table_ids(i);
-            // When the key does not exist, it means the lock has been acquired
-            // by another transaction and successfully committed.
-            if (!lock_values[i].has_value()) {
-                ss << "get delete bitmap update lock info, lock is expired"
-                   << " table_id=" << table_id << " key=" << hex(lock_keys[i]);
-                code = MetaServiceCode::LOCK_EXPIRED;
-                msg = ss.str();
-                LOG(WARNING) << msg << " txn_id=" << txn_id;
-                return;
-            }
-
-            DeleteBitmapUpdateLockPB lock_info;
-            if (!lock_info.ParseFromString(lock_values[i].value())) [[unlikely]] {
-                code = MetaServiceCode::PROTOBUF_PARSE_ERR;
-                msg = "failed to parse DeleteBitmapUpdateLockPB";
-                LOG(WARNING) << msg << " txn_id=" << txn_id;
-                return;
-            }
-            if (lock_info.lock_id() != request->txn_id()) {
-                msg = "lock is expired";
-                code = MetaServiceCode::LOCK_EXPIRED;
-                return;
-            }
-            txn->remove(lock_keys[i]);
-            LOG(INFO) << "xxx remove delete bitmap lock, lock_key=" << hex(lock_keys[i])
-                      << " txn_id=" << txn_id;
-
-            for (auto tablet_id : table_id_tablet_ids[table_id]) {
-                std::string pending_key = meta_pending_delete_bitmap_key({instance_id, tablet_id});
-                txn->remove(pending_key);
-                LOG(INFO) << "xxx remove delete bitmap pending key, pending_key="
-                          << hex(pending_key) << " txn_id=" << txn_id;
-            }
-        }
-        lock_keys.clear();
-        lock_values.clear();
 
         // Save rowset meta
         for (auto& i : rowsets) {
@@ -1810,62 +1823,12 @@ void commit_txn_eventually(
             response->add_versions(i.second + 1);
         }
 
-        // process mow table, check lock and remove pending key
-        std::vector<std::string> lock_keys;
-        lock_keys.reserve(request->mow_table_ids().size());
-        for (auto table_id : request->mow_table_ids()) {
-            lock_keys.push_back(meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}));
-        }
-        std::vector<std::optional<std::string>> lock_values;
-        err = txn->batch_get(&lock_values, lock_keys);
-        if (err != TxnErrorCode::TXN_OK) {
-            ss << "failed to get delete bitmap update lock key info, instance_id=" << instance_id
-               << " err=" << err;
-            msg = ss.str();
-            code = cast_as<ErrCategory::READ>(err);
-            LOG(WARNING) << msg << " txn_id=" << txn_id;
+        process_mow_when_commit_txn(request, instance_id, code, msg, txn, table_id_tablet_ids);
+        if (code != MetaServiceCode::OK) {
+            LOG(WARNING) << "process mow failed, txn_id=" << txn_id << " code=" << code;
             return;
         }
 
-        for (size_t i = 0; i < lock_keys.size(); i++) {
-            int64_t table_id = request->mow_table_ids(i);
-            // When the key does not exist, it means the lock has been acquired
-            // by another transaction and successfully committed.
-            if (!lock_values[i].has_value()) {
-                ss << "get delete bitmap update lock info, lock is expired"
-                   << " table_id=" << table_id << " key=" << hex(lock_keys[i]);
-                code = MetaServiceCode::LOCK_EXPIRED;
-                msg = ss.str();
-                LOG(WARNING) << msg << " txn_id=" << txn_id;
-                return;
-            }
-
-            DeleteBitmapUpdateLockPB lock_info;
-            if (!lock_info.ParseFromString(lock_values[i].value())) [[unlikely]] {
-                code = MetaServiceCode::PROTOBUF_PARSE_ERR;
-                msg = "failed to parse DeleteBitmapUpdateLockPB";
-                LOG(WARNING) << msg << " txn_id=" << txn_id;
-                return;
-            }
-            if (lock_info.lock_id() != request->txn_id()) {
-                msg = "lock is expired";
-                code = MetaServiceCode::LOCK_EXPIRED;
-                return;
-            }
-            txn->remove(lock_keys[i]);
-            LOG(INFO) << "xxx remove delete bitmap lock, lock_key=" << hex(lock_keys[i])
-                      << " txn_id=" << txn_id;
-
-            for (auto tablet_id : table_id_tablet_ids[table_id]) {
-                std::string pending_key = meta_pending_delete_bitmap_key({instance_id, tablet_id});
-                txn->remove(pending_key);
-                LOG(INFO) << "xxx remove delete bitmap pending key, pending_key="
-                          << hex(pending_key) << " txn_id=" << txn_id;
-            }
-        }
-        lock_keys.clear();
-        lock_values.clear();
-
         // Save table versions
         for (auto& i : table_id_tablet_ids) {
             std::string ver_key = table_version_key({instance_id, db_id, i.first});
@@ -2282,6 +2245,12 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse*
         } // for tmp_rowsets_meta
     }
 
+    process_mow_when_commit_txn(request, instance_id, code, msg, txn, table_id_tablet_ids);
+    if (code != MetaServiceCode::OK) {
+        LOG(WARNING) << "process mow failed, txn_id=" << txn_id << " code=" << code;
+        return;
+    }
+
     // Save rowset meta
     for (auto& i : rowsets) {
         size_t rowset_size = i.first.size() + i.second.size();
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index be0390db584ca9..72e6438999c396 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3188,6 +3188,9 @@ public static int metaServiceRpcRetryTimes() {
     @ConfField(mutable = true, description = {"存算分离模式下calculate delete bitmap task 超时时间，默认15s"})
     public static int calculate_delete_bitmap_task_timeout_seconds = 15;
 
+    @ConfField(mutable = true, description = {"存算分离模式下事务导入calculate delete bitmap task 超时时间，默认300s"})
+    public static int calculate_delete_bitmap_task_timeout_seconds_for_transaction_load = 300;
+
     @ConfField(mutable = true, description = {"存算分离模式下commit阶段等锁超时时间，默认5s"})
     public static int try_commit_lock_timeout_seconds = 5;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index 1a80058759bba9..3c56f7dc56a180 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -99,6 +99,7 @@
 import org.apache.doris.thrift.TCalcDeleteBitmapPartitionInfo;
 import org.apache.doris.thrift.TStatus;
 import org.apache.doris.thrift.TStatusCode;
+import org.apache.doris.thrift.TTabletCommitInfo;
 import org.apache.doris.thrift.TTaskType;
 import org.apache.doris.thrift.TUniqueId;
 import org.apache.doris.thrift.TWaitingTxnStatusRequest;
@@ -492,7 +493,7 @@ private void commitTransaction(long dbId, List<Table> tableList, long transactio
                             transactionState.getTransactionStatus().toString());
                 }
             }
-            calcDeleteBitmapForMow(dbId, mowTableList, transactionId, tabletCommitInfos);
+            calcDeleteBitmapForMow(dbId, mowTableList, transactionId, tabletCommitInfos, null);
         }
 
         CommitTxnRequest.Builder builder = CommitTxnRequest.newBuilder();
@@ -529,12 +530,17 @@ private void commitTransaction(long dbId, List<Table> tableList, long transactio
         }
 
         final CommitTxnRequest commitTxnRequest = builder.build();
+        executeCommitTxnRequest(commitTxnRequest, transactionId, is2PC, txnCommitAttachment);
+    }
+
+    private void executeCommitTxnRequest(CommitTxnRequest commitTxnRequest, long transactionId, boolean is2PC,
+            TxnCommitAttachment txnCommitAttachment) throws UserException {
         boolean txnOperated = false;
         TransactionState txnState = null;
         TxnStateChangeCallback cb = null;
         long callbackId = 0L;
         try {
-            txnState = commitTxn(commitTxnRequest, transactionId, is2PC, dbId, tableList);
+            txnState = commitTxn(commitTxnRequest, transactionId, is2PC);
             txnOperated = true;
             if (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.commitTransaction.timeout")) {
                 throw new UserException(InternalErrorCode.DELETE_BITMAP_LOCK_ERR,
@@ -558,8 +564,8 @@ private void commitTransaction(long dbId, List<Table> tableList, long transactio
         }
     }
 
-    private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long transactionId, boolean is2PC, long dbId,
-            List<Table> tableList) throws UserException {
+    private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long transactionId, boolean is2PC)
+            throws UserException {
         CommitTxnResponse commitTxnResponse = null;
         TransactionState txnState = null;
         int retryTime = 0;
@@ -648,9 +654,9 @@ private List<OlapTable> getMowTableList(List<Table> tableList, List<TabletCommit
     }
 
     private void calcDeleteBitmapForMow(long dbId, List<OlapTable> tableList, long transactionId,
-            List<TabletCommitInfo> tabletCommitInfos)
+            List<TabletCommitInfo> tabletCommitInfos, List<SubTransactionState> subTransactionStates)
             throws UserException {
-        Map<Long, Map<Long, List<Long>>> backendToPartitionTablets = Maps.newHashMap();
+        Map<Long, Map<Long, Set<Long>>> backendToPartitionTablets = Maps.newHashMap();
         Map<Long, Partition> partitions = Maps.newHashMap();
         Map<Long, Set<Long>> tableToPartitions = Maps.newHashMap();
         Map<Long, List<Long>> tableToTabletList = Maps.newHashMap();
@@ -661,6 +667,8 @@ private void calcDeleteBitmapForMow(long dbId, List<OlapTable> tableList, long t
             throw new UserException("The partition info is empty, table may be dropped, txnid=" + transactionId);
         }
 
+        Map<Long, List<Long>> partitionToSubTxnIds = getPartitionSubTxnIds(subTransactionStates, tableToTabletList,
+                tabletToTabletMeta);
         Map<Long, Long> baseCompactionCnts = Maps.newHashMap();
         Map<Long, Long> cumulativeCompactionCnts = Maps.newHashMap();
         Map<Long, Long> cumulativePoints = Maps.newHashMap();
@@ -670,9 +678,11 @@ private void calcDeleteBitmapForMow(long dbId, List<OlapTable> tableList, long t
 
         Map<Long, List<TCalcDeleteBitmapPartitionInfo>> backendToPartitionInfos = getCalcDeleteBitmapInfo(
                 backendToPartitionTablets, partitionVersions, baseCompactionCnts, cumulativeCompactionCnts,
-                        cumulativePoints);
+                cumulativePoints, partitionToSubTxnIds);
         try {
-            sendCalcDeleteBitmaptask(dbId, transactionId, backendToPartitionInfos);
+            sendCalcDeleteBitmaptask(dbId, transactionId, backendToPartitionInfos,
+                    subTransactionStates == null ? Config.calculate_delete_bitmap_task_timeout_seconds
+                            : Config.calculate_delete_bitmap_task_timeout_seconds_for_transaction_load);
         } catch (UserException e) {
             LOG.warn("failed to sendCalcDeleteBitmaptask for txn=" + transactionId + ",exception=" + e.getMessage());
             removeDeleteBitmapUpdateLock(tableToPartitions, transactionId);
@@ -680,11 +690,34 @@ private void calcDeleteBitmapForMow(long dbId, List<OlapTable> tableList, long t
         }
     }
 
+    private Map<Long, List<Long>> getPartitionSubTxnIds(List<SubTransactionState> subTransactionStates,
+            Map<Long, List<Long>> tableToTabletList, Map<Long, TabletMeta> tabletToTabletMeta) {
+        if (subTransactionStates == null) {
+            return null;
+        }
+        Map<Long, List<Long>> partitionToSubTxnIds = Maps.newHashMap();
+        for (SubTransactionState subTransactionState : subTransactionStates) {
+            if (!tableToTabletList.containsKey(subTransactionState.getTable().getId())) {
+                // skip non mow table
+                continue;
+            }
+            for (TTabletCommitInfo ci : subTransactionState.getTabletCommitInfos()) {
+                TabletMeta tabletMeta = tabletToTabletMeta.get(ci.getTabletId());
+                long partitionId = tabletMeta.getPartitionId();
+                List<Long> subTxnIds = partitionToSubTxnIds.computeIfAbsent(partitionId, k -> Lists.newArrayList());
+                if (!subTxnIds.contains(subTransactionState.getSubTransactionId())) {
+                    subTxnIds.add(subTransactionState.getSubTransactionId());
+                }
+            }
+        }
+        return partitionToSubTxnIds;
+    }
+
     private void getPartitionInfo(List<OlapTable> tableList,
             List<TabletCommitInfo> tabletCommitInfos,
             Map<Long, Set<Long>> tableToParttions,
             Map<Long, Partition> partitions,
-            Map<Long, Map<Long, List<Long>>> backendToPartitionTablets,
+            Map<Long, Map<Long, Set<Long>>> backendToPartitionTablets,
             Map<Long, List<Long>> tableToTabletList,
             Map<Long, TabletMeta> tabletToTabletMeta) {
         Map<Long, OlapTable> tableMap = Maps.newHashMap();
@@ -697,18 +730,22 @@ private void getPartitionInfo(List<OlapTable> tableList,
         TabletInvertedIndex tabletInvertedIndex = Env.getCurrentEnv().getTabletInvertedIndex();
         List<TabletMeta> tabletMetaList = tabletInvertedIndex.getTabletMetaList(tabletIds);
         for (int i = 0; i < tabletMetaList.size(); i++) {
+            long tabletId = tabletIds.get(i);
+            if (tabletToTabletMeta.containsKey(tabletId)) {
+                continue;
+            }
             TabletMeta tabletMeta = tabletMetaList.get(i);
             long tableId = tabletMeta.getTableId();
             if (!tableMap.containsKey(tableId)) {
                 continue;
             }
 
-            tabletToTabletMeta.put(tabletIds.get(i), tabletMeta);
+            tabletToTabletMeta.put(tabletId, tabletMeta);
 
-            if (!tableToTabletList.containsKey(tableId)) {
-                tableToTabletList.put(tableId, Lists.newArrayList());
+            List<Long> tableTabletIds = tableToTabletList.computeIfAbsent(tableId, k -> Lists.newArrayList());
+            if (!tableTabletIds.contains(tabletId)) {
+                tableTabletIds.add(tabletId);
             }
-            tableToTabletList.get(tableId).add(tabletIds.get(i));
 
             long partitionId = tabletMeta.getPartitionId();
             long backendId = tabletCommitInfos.get(i).getBackendId();
@@ -721,11 +758,11 @@ private void getPartitionInfo(List<OlapTable> tableList,
             if (!backendToPartitionTablets.containsKey(backendId)) {
                 backendToPartitionTablets.put(backendId, Maps.newHashMap());
             }
-            Map<Long, List<Long>> partitionToTablets = backendToPartitionTablets.get(backendId);
+            Map<Long, Set<Long>> partitionToTablets = backendToPartitionTablets.get(backendId);
             if (!partitionToTablets.containsKey(partitionId)) {
-                partitionToTablets.put(partitionId, Lists.newArrayList());
+                partitionToTablets.put(partitionId, Sets.newHashSet());
             }
-            partitionToTablets.get(partitionId).add(tabletIds.get(i));
+            partitionToTablets.get(partitionId).add(tabletId);
             partitions.putIfAbsent(partitionId, tableMap.get(tableId).getPartition(partitionId));
         }
     }
@@ -741,18 +778,18 @@ private Map<Long, Long> getPartitionVersions(Map<Long, Partition> partitionMap)
     }
 
     private Map<Long, List<TCalcDeleteBitmapPartitionInfo>> getCalcDeleteBitmapInfo(
-            Map<Long, Map<Long, List<Long>>> backendToPartitionTablets, Map<Long, Long> partitionVersions,
+            Map<Long, Map<Long, Set<Long>>> backendToPartitionTablets, Map<Long, Long> partitionVersions,
                     Map<Long, Long> baseCompactionCnts, Map<Long, Long> cumulativeCompactionCnts,
-                            Map<Long, Long> cumulativePoints) {
+                            Map<Long, Long> cumulativePoints, Map<Long, List<Long>> partitionToSubTxnIds) {
         Map<Long, List<TCalcDeleteBitmapPartitionInfo>> backendToPartitionInfos = Maps.newHashMap();
-        for (Map.Entry<Long, Map<Long, List<Long>>> entry : backendToPartitionTablets.entrySet()) {
+        for (Map.Entry<Long, Map<Long, Set<Long>>> entry : backendToPartitionTablets.entrySet()) {
             List<TCalcDeleteBitmapPartitionInfo> partitionInfos = Lists.newArrayList();
-            for (Map.Entry<Long, List<Long>> partitionToTablets : entry.getValue().entrySet()) {
+            for (Map.Entry<Long, Set<Long>> partitionToTablets : entry.getValue().entrySet()) {
                 Long partitionId = partitionToTablets.getKey();
-                List<Long> tabletList = partitionToTablets.getValue();
+                Set<Long> tabletList = partitionToTablets.getValue();
                 TCalcDeleteBitmapPartitionInfo partitionInfo = new TCalcDeleteBitmapPartitionInfo(partitionId,
                         partitionVersions.get(partitionId),
-                        tabletList);
+                        Lists.newArrayList(tabletList));
                 if (!baseCompactionCnts.isEmpty() && !cumulativeCompactionCnts.isEmpty()
                         && !cumulativePoints.isEmpty()) {
                     List<Long> reqBaseCompactionCnts = Lists.newArrayList();
@@ -766,6 +803,13 @@ private Map<Long, List<TCalcDeleteBitmapPartitionInfo>> getCalcDeleteBitmapInfo(
                     partitionInfo.setBaseCompactionCnts(reqBaseCompactionCnts);
                     partitionInfo.setCumulativeCompactionCnts(reqCumulativeCompactionCnts);
                     partitionInfo.setCumulativePoints(reqCumulativePoints);
+                    if (partitionToSubTxnIds != null) {
+                        List<Long> subTxnIds = partitionToSubTxnIds.get(partitionId);
+                        if (subTxnIds != null && !subTxnIds.isEmpty()) {
+                            partitionInfo.setSubTxnIds(subTxnIds);
+                            LOG.debug("partitionId: {}, subTxnIds: {}", partitionId, subTxnIds);
+                        }
+                    }
                 }
                 partitionInfos.add(partitionInfo);
             }
@@ -926,8 +970,8 @@ private void removeDeleteBitmapUpdateLock(Map<Long, Set<Long>> tableToParttions,
     }
 
     private void sendCalcDeleteBitmaptask(long dbId, long transactionId,
-            Map<Long, List<TCalcDeleteBitmapPartitionInfo>> backendToPartitionInfos)
-            throws UserException {
+            Map<Long, List<TCalcDeleteBitmapPartitionInfo>> backendToPartitionInfos,
+            long calculateDeleteBitmapTaskTimeoutSeconds) throws UserException {
         if (backendToPartitionInfos.isEmpty()) {
             return;
         }
@@ -948,13 +992,14 @@ private void sendCalcDeleteBitmaptask(long dbId, long transactionId,
             // not check return value, because the add will success
             AgentTaskQueue.addTask(task);
             batchTask.addTask(task);
-            LOG.info("send calculate delete bitmap task to be {}, txn_id {}", entry.getKey(), transactionId);
+            LOG.info("send calculate delete bitmap task to be {}, txn_id {}, partitionInfos={}", entry.getKey(),
+                    transactionId, entry.getValue());
         }
         AgentTaskExecutor.submit(batchTask);
 
         boolean ok;
         try {
-            ok = countDownLatch.await(Config.calculate_delete_bitmap_task_timeout_seconds, TimeUnit.SECONDS);
+            ok = countDownLatch.await(calculateDeleteBitmapTaskTimeoutSeconds, TimeUnit.SECONDS);
         } catch (InterruptedException e) {
             LOG.warn("InterruptedException: ", e);
             ok = false;
@@ -1043,14 +1088,42 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId,
                     "disable_load_job is set to true, all load jobs are not allowed");
         }
         LOG.info("try to commit transaction, txnId: {}, subTxnStates: {}", transactionId, subTransactionStates);
+
+        Preconditions.checkState(db instanceof Database);
+        List<Long> tableIdList = subTransactionStates.stream().map(s -> s.getTable().getId()).distinct()
+                .collect(Collectors.toList());
+        List<Table> tableList = ((Database) db).getTablesOnIdOrderOrThrowException(tableIdList);
+        beforeCommitTransaction(tableList, transactionId, timeoutMillis);
+        try {
+            commitTransactionWithSubTxns(db.getId(), tableList, transactionId, subTransactionStates);
+        } finally {
+            decreaseWaitingLockCount(tableList);
+            MetaLockUtils.commitUnlockTables(tableList);
+        }
+        return true;
+    }
+
+    private void commitTransactionWithSubTxns(long dbId, List<Table> tableList, long transactionId,
+            List<SubTransactionState> subTransactionStates) throws UserException {
+        List<TabletCommitInfo> tabletCommitInfos = subTransactionStates.stream().map(
+                        SubTransactionState::getTabletCommitInfos).flatMap(Collection::stream)
+                .map(c -> new TabletCommitInfo(c.getTabletId(), c.getBackendId())).collect(Collectors.toList());
+        List<OlapTable> mowTableList = getMowTableList(tableList, tabletCommitInfos);
+        if (!mowTableList.isEmpty()) {
+            calcDeleteBitmapForMow(dbId, mowTableList, transactionId, tabletCommitInfos, subTransactionStates);
+        }
+
         cleanSubTransactions(transactionId);
         CommitTxnRequest.Builder builder = CommitTxnRequest.newBuilder();
-        builder.setDbId(db.getId())
+        builder.setDbId(dbId)
                 .setTxnId(transactionId)
                 .setIs2Pc(false)
                 .setCloudUniqueId(Config.cloud_unique_id)
                 .setIsTxnLoad(true)
                 .setEnableTxnLazyCommit(Config.enable_cloud_txn_lazy_commit);
+        for (OlapTable olapTable : mowTableList) {
+            builder.addMowTableIds(olapTable.getId());
+        }
         // add sub txn infos
         for (SubTransactionState subTransactionState : subTransactionStates) {
             builder.addSubTxnInfos(SubTxnInfo.newBuilder().setSubTxnId(subTransactionState.getSubTransactionId())
@@ -1064,31 +1137,12 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId,
         }
 
         final CommitTxnRequest commitTxnRequest = builder.build();
-        TransactionState txnState = null;
-        boolean txnOperated = false;
-        try {
-            txnState = commitTxn(commitTxnRequest, transactionId, false, db.getId(),
-                    subTransactionStates.stream().map(SubTransactionState::getTable)
-                        .collect(Collectors.toList()));
-            txnOperated = true;
-        } finally {
-            if (txnState != null) {
-                TxnStateChangeCallback cb = callbackFactory.getCallback(txnState.getCallbackId());
-                if (cb != null) {
-                    LOG.info("commitTxn, run txn callback, transactionId:{} callbackId:{}, txnState:{}",
-                            txnState.getTransactionId(), txnState.getCallbackId(), txnState);
-                    cb.afterCommitted(txnState, txnOperated);
-                    cb.afterVisible(txnState, txnOperated);
-                }
-            }
-        }
-        return true;
+        executeCommitTxnRequest(commitTxnRequest, transactionId, false, null);
     }
 
-    @Override
-    public boolean commitAndPublishTransaction(DatabaseIf db, List<Table> tableList, long transactionId,
-                                               List<TabletCommitInfo> tabletCommitInfos, long timeoutMillis,
-                                               TxnCommitAttachment txnCommitAttachment) throws UserException {
+    // add some log and get commit lock, mainly used for mow tables
+    private void beforeCommitTransaction(List<Table> tableList, long transactionId, long timeoutMillis)
+            throws UserException {
         for (int i = 0; i < tableList.size(); i++) {
             long tableId = tableList.get(i).getId();
             LOG.info("start commit txn=" + transactionId + ",table=" + tableId);
@@ -1107,6 +1161,13 @@ public boolean commitAndPublishTransaction(DatabaseIf db, List<Table> tableList,
                     "get table cloud commit lock timeout, tableList=("
                             + StringUtils.join(tableList, ",") + ")");
         }
+    }
+
+    @Override
+    public boolean commitAndPublishTransaction(DatabaseIf db, List<Table> tableList, long transactionId,
+                                               List<TabletCommitInfo> tabletCommitInfos, long timeoutMillis,
+                                               TxnCommitAttachment txnCommitAttachment) throws UserException {
+        beforeCommitTransaction(tableList, transactionId, timeoutMillis);
         try {
             commitTransaction(db.getId(), tableList, transactionId, tabletCommitInfos, txnCommitAttachment);
         } finally {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java
index 25c4ff4b3b2493..8a75dd0d0c2303 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java
@@ -21,8 +21,6 @@
 import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.DatabaseIf;
 import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.KeysType;
-import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.Table;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.cloud.transaction.CloudGlobalTransactionMgr;
@@ -206,13 +204,6 @@ public long beginTransaction(TableIf table, SubTransactionType subTransactionTyp
             throw new AnalysisException(
                     "Transaction insert can not insert into values and insert into select at the same time");
         }
-        if (Config.isCloudMode()) {
-            OlapTable olapTable = (OlapTable) table;
-            if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite()) {
-                throw new UserException(
-                        "Transaction load is not supported for merge on write unique keys table in cloud mode");
-            }
-        }
         DatabaseIf database = table.getDatabase();
         if (!isTransactionBegan) {
             long timeoutSecond = ConnectContext.get().getExecTimeout();
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index abffd176ef8ea7..fdbf4483bf8c4c 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -440,6 +440,7 @@ struct TCalcDeleteBitmapPartitionInfo {
     4: optional list<i64> base_compaction_cnts
     5: optional list<i64> cumulative_compaction_cnts
     6: optional list<i64> cumulative_points
+    7: optional list<i64> sub_txn_ids
 }
 
 struct TCalcDeleteBitmapRequest {
diff --git a/regression-test/suites/insert_p0/transaction/txn_insert.groovy b/regression-test/suites/insert_p0/transaction/txn_insert.groovy
index 6653c05740e3b1..a4868ca6b11ffe 100644
--- a/regression-test/suites/insert_p0/transaction/txn_insert.groovy
+++ b/regression-test/suites/insert_p0/transaction/txn_insert.groovy
@@ -584,19 +584,10 @@ suite("txn_insert") {
             } catch (Exception e) {
                 logger.info("exception: " + e)
                 sql """ rollback """
-                if (isCloudMode()) {
-                    assertTrue(e.getMessage().contains("Transaction load is not supported for merge on write unique keys table in cloud mode"))
-                } else {
-                    assertTrue(false, "should not reach here")
-                }
+                assertTrue(false, "should not reach here")
             }
         }
 
-        // the following cases are not supported in cloud mode
-        if (isCloudMode()) {
-            break
-        }
-
         // 16. update stmt(mow table)
         if (use_nereids_planner) {
             def ut_table = "txn_insert_ut"
diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_concurrent_insert_mow.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_concurrent_insert_mow.groovy
new file mode 100644
index 00000000000000..8e6de4dd9e9a38
--- /dev/null
+++ b/regression-test/suites/insert_p0/transaction/txn_insert_concurrent_insert_mow.groovy
@@ -0,0 +1,135 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.sql.Connection
+import java.sql.DriverManager
+import java.sql.Statement
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.CompletableFuture
+
+suite("txn_insert_concurrent_insert_mow") {
+    def tableName = "txn_insert_concurrent_insert_mow"
+    List<String> errors = new ArrayList<>()
+
+    for (int i = 0; i < 3; i++) {
+        def table_name = "${tableName}_${i}"
+        sql """ drop table if exists ${table_name} """
+        sql """
+            CREATE TABLE IF NOT EXISTS ${table_name} (
+                L_ORDERKEY    INTEGER NOT NULL,
+                L_PARTKEY     INTEGER NOT NULL,
+                L_SUPPKEY     INTEGER NOT NULL,
+                L_LINENUMBER  INTEGER NOT NULL,
+                L_QUANTITY    DECIMAL(15,2) NOT NULL,
+                L_EXTENDEDPRICE  DECIMAL(15,2) NOT NULL,
+                L_DISCOUNT    DECIMAL(15,2) NOT NULL,
+                L_TAX         DECIMAL(15,2) NOT NULL,
+                L_RETURNFLAG  CHAR(1) NOT NULL,
+                L_LINESTATUS  CHAR(1) NOT NULL,
+                L_SHIPDATE    DATE NOT NULL,
+                L_COMMITDATE  DATE NOT NULL,
+                L_RECEIPTDATE DATE NOT NULL,
+                L_SHIPINSTRUCT CHAR(25) NOT NULL,
+                L_SHIPMODE     CHAR(10) NOT NULL,
+                L_COMMENT      VARCHAR(44) NOT NULL
+            )
+            UNIQUE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+            DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+            PROPERTIES (
+                "replication_num" = "1"
+            );
+        """
+
+        if (i == 0) {
+            continue
+        }
+
+        streamLoad {
+            table table_name
+            set 'column_separator', '|'
+            set 'compress_type', 'GZ'
+            set 'columns', "l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag,l_linestatus, l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment,temp"
+            file """${getS3Url()}/regression/tpch/sf0.1/lineitem.tbl.gz"""
+
+            time 10000 // limit inflight 10s
+            check { result, exception, startTime, endTime ->
+                if (exception != null) {
+                    throw exception
+                }
+                log.info("Stream load result: ${result}".toString())
+                def json = parseJson(result)
+                assertEquals("success", json.Status.toLowerCase())
+                assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+                assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+            }
+        }
+    }
+    sql """ sync """
+
+    def dbName = "regression_test_insert_p0_transaction"
+    def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true"
+    logger.info("url: ${url}")
+
+    def sqls = [
+            "begin",
+            "insert into ${tableName}_0 select * from ${tableName}_1 where L_ORDERKEY < 30000;",
+            "insert into ${tableName}_1 select * from ${tableName}_2 where L_ORDERKEY > 500000;",
+            "insert into ${tableName}_0 select * from ${tableName}_2 where L_ORDERKEY < 30000;",
+            "commit"
+    ]
+    def txn_insert = { ->
+        try (Connection conn = DriverManager.getConnection(url, context.config.jdbcUser, context.config.jdbcPassword);
+             Statement stmt = conn.createStatement()) {
+            for (def sql : sqls) {
+                logger.info(Thread.currentThread().getName() + " execute sql: " + sql)
+                stmt.execute(sql)
+            }
+            logger.info("finish txn insert for " + Thread.currentThread().getName())
+        } catch (Throwable e) {
+            logger.error("txn insert failed", e)
+            errors.add("txn insert failed " + e.getMessage())
+        }
+    }
+
+    List<CompletableFuture<Void>> futures = new ArrayList<>()
+    for (int i = 0; i < 10; i++) {
+        CompletableFuture<Void> future = CompletableFuture.runAsync(txn_insert)
+        futures.add(future)
+    }
+    CompletableFuture<?>[] futuresArray = futures.toArray(new CompletableFuture[0])
+    CompletableFuture.allOf(futuresArray).get(10, TimeUnit.MINUTES)
+    sql """ sync """
+
+    logger.info("error num: " + errors.size() + ", errors: " + errors)
+
+    def result = sql """ select count() from ${tableName}_0 """
+    logger.info("result: ${result}")
+    assertEquals(30209, result[0][0])
+    result = sql """ select count() from ${tableName}_1 """
+    logger.info("result: ${result}")
+    assertEquals(600572, result[0][0])
+
+    def db_name = "regression_test_insert_p0_transaction"
+    def tables = sql """ show tables from $db_name """
+    logger.info("tables: $tables")
+    for (def table_info : tables) {
+        def table_name = table_info[0]
+        if (table_name.startsWith(tableName)) {
+            check_table_version_continuous(db_name, table_name)
+        }
+    }
+}
diff --git a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_aggregate.groovy b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_aggregate.groovy
similarity index 99%
rename from regression-test/suites/insert_p2/txn_insert_concurrent_insert_aggregate.groovy
rename to regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_aggregate.groovy
index 1fbe18bf212c47..fa57c5501187bc 100644
--- a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_aggregate.groovy
+++ b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_aggregate.groovy
@@ -81,7 +81,7 @@ suite("txn_insert_concurrent_insert_aggregate") {
     }
     sql """ sync """
 
-    def dbName = "regression_test_insert_p2"
+    def dbName = "regression_test_insert_p2_transaction"
     def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true"
     logger.info("url: ${url}")
 
diff --git a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_duplicate.groovy b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_duplicate.groovy
similarity index 99%
rename from regression-test/suites/insert_p2/txn_insert_concurrent_insert_duplicate.groovy
rename to regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_duplicate.groovy
index 048a07fb817fff..e771078f1fbecb 100644
--- a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_duplicate.groovy
+++ b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_duplicate.groovy
@@ -81,7 +81,7 @@ suite("txn_insert_concurrent_insert_duplicate") {
     }
     sql """ sync """
 
-    def dbName = "regression_test_insert_p2"
+    def dbName = "regression_test_insert_p2_transaction"
     def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true"
     logger.info("url: ${url}")
 
diff --git a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_mor.groovy b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_mor.groovy
similarity index 99%
rename from regression-test/suites/insert_p2/txn_insert_concurrent_insert_mor.groovy
rename to regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_mor.groovy
index c67119328ef7a6..418992835d0551 100644
--- a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_mor.groovy
+++ b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_mor.groovy
@@ -82,7 +82,7 @@ suite("txn_insert_concurrent_insert_mor") {
     }
     sql """ sync """
 
-    def dbName = "regression_test_insert_p2"
+    def dbName = "regression_test_insert_p2_transaction"
     def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true"
     logger.info("url: ${url}")
 
diff --git a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_mow.groovy b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_mow.groovy
similarity index 95%
rename from regression-test/suites/insert_p2/txn_insert_concurrent_insert_mow.groovy
rename to regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_mow.groovy
index f8a971db75ee2f..4d6e297cac918b 100644
--- a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_mow.groovy
+++ b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_mow.groovy
@@ -22,11 +22,6 @@ import java.util.concurrent.TimeUnit
 import java.util.concurrent.CompletableFuture
 
 suite("txn_insert_concurrent_insert_mow") {
-    if (isCloudMode()) {
-        logger.info("cloud txn load does not support mow")
-        return
-    }
-
     def tableName = "txn_insert_concurrent_insert_mow"
     List<String> errors = new ArrayList<>()
 
@@ -85,8 +80,12 @@ suite("txn_insert_concurrent_insert_mow") {
         }
     }
     sql """ sync """
+    def t2_row_count = 6001215
+    def result = sql """ select count() from ${tableName}_2 """
+    logger.info("${tableName}_2 row count: ${result}, expected: ${t2_row_count}")
+    assertEquals(t2_row_count, result[0][0] as int)
 
-    def dbName = "regression_test_insert_p2"
+    def dbName = "regression_test_insert_p2_transaction"
     def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true"
     logger.info("url: ${url}")
 
@@ -156,7 +155,7 @@ suite("txn_insert_concurrent_insert_mow") {
     logger.info("error num: " + errors.size() + ", errors: " + errors)
 
     def t0_row_count = 6001215
-    def result = sql """ select count() from ${tableName}_0 """
+    result = sql """ select count() from ${tableName}_0 """
     logger.info("${tableName}_0 row count: ${result}, expected: ${t0_row_count}")
 
     def t1_row_count = 2999666
diff --git a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_ud.groovy b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_ud.groovy
similarity index 98%
rename from regression-test/suites/insert_p2/txn_insert_concurrent_insert_ud.groovy
rename to regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_ud.groovy
index a524703f9ef99f..d0b27641c64a99 100644
--- a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_ud.groovy
+++ b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_ud.groovy
@@ -23,11 +23,6 @@ import java.util.concurrent.CompletableFuture
 
 // test update and delete command
 suite("txn_insert_concurrent_insert_ud") {
-    if (isCloudMode()) {
-        logger.info("cloud txn load does not support mow")
-        return
-    }
-
     def tableName = "txn_insert_concurrent_insert_ud"
     List<String> errors = new ArrayList<>()
 
@@ -88,7 +83,7 @@ suite("txn_insert_concurrent_insert_ud") {
     }
     sql """ sync """
 
-    def dbName = "regression_test_insert_p2"
+    def dbName = "regression_test_insert_p2_transaction"
     def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true"
     logger.info("url: ${url}")
 
diff --git a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_update.groovy b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_update.groovy
similarity index 99%
rename from regression-test/suites/insert_p2/txn_insert_concurrent_insert_update.groovy
rename to regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_update.groovy
index b467a87de8201c..eba69918660caf 100644
--- a/regression-test/suites/insert_p2/txn_insert_concurrent_insert_update.groovy
+++ b/regression-test/suites/insert_p2/transaction/txn_insert_concurrent_insert_update.groovy
@@ -84,7 +84,7 @@ suite("txn_insert_concurrent_insert_update") {
     }
     sql """ sync """
 
-    def dbName = "regression_test_insert_p2"
+    def dbName = "regression_test_insert_p2_transaction"
     def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true"
     logger.info("url: ${url}")
 
diff --git a/regression-test/suites/insert_p2/txn_insert_with_schema_change.groovy b/regression-test/suites/insert_p2/transaction/txn_insert_with_schema_change.groovy
similarity index 99%
rename from regression-test/suites/insert_p2/txn_insert_with_schema_change.groovy
rename to regression-test/suites/insert_p2/transaction/txn_insert_with_schema_change.groovy
index 56692b68d3730d..34b859b5e80d0a 100644
--- a/regression-test/suites/insert_p2/txn_insert_with_schema_change.groovy
+++ b/regression-test/suites/insert_p2/transaction/txn_insert_with_schema_change.groovy
@@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit
 // schema change and modify replica num
 suite("txn_insert_with_schema_change") {
     def tableName = "txn_insert_with_schema_change"
-    def dbName = "regression_test_insert_p2"
+    def dbName = "regression_test_insert_p2_transaction"
     def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true"
     logger.info("url: ${url}")
 

From 267a8cc9ea9e25cf66d7e1b0be645cf75e5100ce Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Tue, 26 Nov 2024 23:08:44 +0800
Subject: [PATCH 014/399] [fix](cluster key) add cluster key case (#44488)

---
 .../java/org/apache/doris/catalog/Table.java  |   5 +-
 .../translator/PhysicalPlanTranslator.java    |   2 +-
 .../data/nereids_p0/delete/delete_cte_ck.out  |  29 +++++
 .../nereids_p0/delete/delete_cte_ck.groovy    | 123 ++++++++++++++++++
 4 files changed, 156 insertions(+), 3 deletions(-)
 create mode 100644 regression-test/data/nereids_p0/delete/delete_cte_ck.out
 create mode 100644 regression-test/suites/nereids_p0/delete/delete_cte_ck.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index d85d98a8ea550f..ed37a936e10819 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -40,6 +40,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
@@ -393,7 +394,7 @@ public TableType getType() {
     }
 
     public List<Column> getFullSchema() {
-        return fullSchema;
+        return ImmutableList.copyOf(fullSchema);
     }
 
     // should override in subclass if necessary
@@ -403,7 +404,7 @@ public List<Column> getBaseSchema() {
 
     public List<Column> getBaseSchema(boolean full) {
         if (full) {
-            return fullSchema;
+            return ImmutableList.copyOf(fullSchema);
         } else {
             return fullSchema.stream().filter(Column::isVisible).collect(Collectors.toList());
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 654ccc8ca1155a..56ae65ec722941 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -2695,7 +2695,7 @@ private boolean checkPushSort(SortNode sortNode, OlapTable olapTable) {
         if (sortExprs.size() > olapTable.getDataSortInfo().getColNum()) {
             return false;
         }
-        List<Column> sortKeyColumns = olapTable.getFullSchema();
+        List<Column> sortKeyColumns = new ArrayList<>(olapTable.getFullSchema());
         if (olapTable.getEnableUniqueKeyMergeOnWrite()) {
             Map<Integer, Column> clusterKeyMap = new TreeMap<>();
             for (Column column : olapTable.getFullSchema()) {
diff --git a/regression-test/data/nereids_p0/delete/delete_cte_ck.out b/regression-test/data/nereids_p0/delete/delete_cte_ck.out
new file mode 100644
index 00000000000000..2734e5b4a33b33
--- /dev/null
+++ b/regression-test/data/nereids_p0/delete/delete_cte_ck.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1	\N	2	1	1.0	\N
+1	10	1	1	1.0	2000-01-01
+2	\N	4	2	2.0	\N
+2	20	2	2	2.0	2000-01-02
+3	\N	6	3	3.0	\N
+3	30	3	3	3.0	2000-01-03
+
+-- !sql --
+2	\N	4	2	2.0	\N
+2	20	2	2	2.0	2000-01-02
+3	\N	6	3	3.0	\N
+3	30	3	3	3.0	2000-01-03
+
+-- !sql --
+1	\N	2	1	1.0	\N
+1	10	1	1	1.0	2000-01-01
+2	\N	4	2	2.0	\N
+2	20	2	2	2.0	2000-01-02
+3	\N	6	3	3.0	\N
+3	30	3	3	3.0	2000-01-03
+
+-- !sql --
+2	\N	4	2	2.0	\N
+2	20	2	2	2.0	2000-01-02
+3	\N	6	3	3.0	\N
+3	30	3	3	3.0	2000-01-03
+
diff --git a/regression-test/suites/nereids_p0/delete/delete_cte_ck.groovy b/regression-test/suites/nereids_p0/delete/delete_cte_ck.groovy
new file mode 100644
index 00000000000000..2e9bd1f8fb3fa9
--- /dev/null
+++ b/regression-test/suites/nereids_p0/delete/delete_cte_ck.groovy
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('nereids_delete_cte_ck') {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "select 1;" // to create database
+
+    for (def use_row_store : [false, true]) {
+        logger.info("current params: use_row_store: ${use_row_store}")
+
+        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+            sql "use ${db};"
+
+            def t1 = 't1_cte_ck'
+            def t2 = 't2_cte_ck'
+            def t3 = 't3_cte_ck'
+
+            sql "drop table if exists ${t1}"
+            sql """
+                create table ${t1} (
+                    id int,
+                    id1 int,
+                    c1 bigint,
+                    c2 string,
+                    c3 double,
+                    c4 date
+                ) unique key (id, id1)
+                CLUSTER BY (`id1`, `id`, `c4`)
+                distributed by hash(id, id1)
+                properties(
+                    "replication_num"="1",
+                    "enable_unique_key_merge_on_write" = "true",
+                    "store_row_column" = "${use_row_store}"); """
+
+            sql "drop table if exists ${t2}"
+            sql """
+                create table ${t2} (
+                    id int,
+                    c1 bigint,
+                    c2 string,
+                    c3 double,
+                    c4 date
+                ) unique key (id)
+                CLUSTER BY (`c4`, `c1`)
+                distributed by hash(id)
+                properties(
+                    "replication_num"="1"
+                );
+            """
+
+            sql "drop table if exists ${t3}"
+            sql """
+                create table ${t3} (
+                    id int
+                ) distributed by hash(id)
+                properties(
+                    "replication_num"="1"
+                );
+            """
+
+            sql """
+                INSERT INTO ${t1} VALUES
+                    (1, 10, 1, '1', 1.0, '2000-01-01'),
+                    (2, 20, 2, '2', 2.0, '2000-01-02'),
+                    (3, 30, 3, '3', 3.0, '2000-01-03');
+            """
+
+            sql """
+
+                INSERT INTO ${t2} VALUES
+                    (1, 10, '10', 10.0, '2000-01-10'),
+                    (2, 20, '20', 20.0, '2000-01-20'),
+                    (3, 30, '30', 30.0, '2000-01-30'),
+                    (4, 4, '4', 4.0, '2000-01-04'),
+                    (5, 5, '5', 5.0, '2000-01-05');
+            """
+
+            sql """
+                INSERT INTO ${t3} VALUES
+                    (1),
+                    (4),
+                    (5);
+            """
+
+            sql "set enable_nereids_planner=true"
+            sql "set enable_fallback_to_original_planner=false"
+            sql "set enable_nereids_dml=true"
+
+            sql "insert into ${t1}(id, c1, c2, c3) select id, c1 * 2, c2, c3 from ${t1}"
+            sql "insert into ${t2}(id, c1, c2, c3) select id, c1, c2 * 2, c3 from ${t2}"
+            sql "insert into ${t2}(c1, c3) select c1 + 1, c3 + 1 from (select id, c1, c3 from ${t1} order by id, c1 limit 10) ${t1}, ${t3}"
+
+            qt_sql "select * from ${t1} order by id, id1"
+
+            sql """
+                with cte as (select * from ${t3})
+                delete from ${t1}
+                using ${t2} join cte on ${t2}.id = cte.id
+                where ${t1}.id = ${t2}.id;
+            """
+
+            qt_sql "select * from ${t1} order by id, id1"
+
+            // test cte != null and using == null
+            sql """with cte as (select 1) delete from ${t1} where ${t1}.id = 100"""
+        }
+    }
+}
\ No newline at end of file

From 4d66933819192b664b840f1770099e558066473b Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Tue, 26 Nov 2024 23:25:48 +0800
Subject: [PATCH 015/399] [enhancement](backup) Add config
 backup_handler_update_interval_millis (#44628)

---
 .../src/main/java/org/apache/doris/common/Config.java       | 6 ++++++
 .../main/java/org/apache/doris/backup/BackupHandler.java    | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 72e6438999c396..9650f73be50adc 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1634,6 +1634,12 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true, masterOnly = true)
     public static boolean ignore_backup_not_support_table_type = false;
 
+    /**
+     * A internal config, to control the update interval of backup handler. Only used to speed up tests.
+     */
+    @ConfField(mutable = false)
+    public static long backup_handler_update_interval_millis = 3000;
+
     /**
      * Control the default max num of the instance for a user.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 7cad2f985001bc..d70544add98747 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -122,7 +122,7 @@ public BackupHandler() {
     }
 
     public BackupHandler(Env env) {
-        super("backupHandler", 3000L);
+        super("backupHandler", Config.backup_handler_update_interval_millis);
         this.env = env;
     }
 

From 0a61efaa2fe1efee7ef3a117e62c680695c7be69 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Tue, 26 Nov 2024 23:26:10 +0800
Subject: [PATCH 016/399] [fix](backup) Automatic adapt upload/download
 snapshot batch size (#44560)

The original `backup_upload_task_num_per_be` and
`restore_download_task_num_per_be` would not adapt the different num of
snapshots, a large `UploadTask` or `DownloadTask` might exceed the
threshold of `thrift_max_message_size` and failed forever.

This PR changes these options to adapt the number of snapshots
automatically.
---
 .../java/org/apache/doris/common/Config.java  | 13 ++++---
 .../org/apache/doris/backup/BackupJob.java    | 17 +++------
 .../org/apache/doris/backup/RestoreJob.java   | 37 ++++++-------------
 3 files changed, 25 insertions(+), 42 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 9650f73be50adc..66dbff5abed8e6 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2771,16 +2771,17 @@ public class Config extends ConfigBase {
     public static String nereids_trace_log_dir = System.getenv("LOG_DIR") + "/nereids_trace";
 
     @ConfField(mutable = true, masterOnly = true, description = {
-            "备份过程中，分配给每个be的upload任务最大个数，默认值为3个。",
-            "The max number of upload tasks assigned to each be during the backup process, the default value is 3."
+            "备份过程中，一个 upload 任务上传的快照数量上限，默认值为10个",
+            "The max number of snapshots assigned to a upload task during the backup process, the default value is 10."
     })
-    public static int backup_upload_task_num_per_be = 3;
+    public static int backup_upload_snapshot_batch_size = 10;
 
     @ConfField(mutable = true, masterOnly = true, description = {
-            "恢复过程中，分配给每个be的download任务最大个数，默认值为3个。",
-            "The max number of download tasks assigned to each be during the restore process, the default value is 3."
+            "恢复过程中，一个 download 任务下载的快照数量上限，默认值为10个",
+            "The max number of snapshots assigned to a download task during the restore process, "
+            + "the default value is 10."
     })
-    public static int restore_download_task_num_per_be = 3;
+    public static int restore_download_snapshot_batch_size = 10;
 
     @ConfField(mutable = true, masterOnly = true, description = {
             "备份恢复过程中，单次 RPC 分配给每个be的任务最大个数，默认值为10000个。",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 478e8902d7d8c4..621a2b1d9f7d29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -751,13 +751,10 @@ private void uploadSnapshot() {
         for (Long beId : beToSnapshots.keySet()) {
             List<SnapshotInfo> infos = beToSnapshots.get(beId);
             int totalNum = infos.size();
-            int batchNum = totalNum;
-            if (Config.backup_upload_task_num_per_be > 0) {
-                batchNum = Math.min(totalNum, Config.backup_upload_task_num_per_be);
-            }
             // each task contains several upload sub tasks
-            int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
-            LOG.info("backend {} has {} batch, total {} tasks, {}", beId, batchNum, totalNum, this);
+            int taskNumPerBatch = Config.backup_upload_snapshot_batch_size;
+            LOG.info("backend {} has total {} snapshots, per task batch size {}, {}",
+                    beId, totalNum, taskNumPerBatch, this);
 
             List<FsBroker> brokers = Lists.newArrayList();
             Status st = repo.getBrokerAddress(beId, env, brokers);
@@ -768,12 +765,10 @@ private void uploadSnapshot() {
             Preconditions.checkState(brokers.size() == 1);
 
             // allot tasks
-            int index = 0;
-            for (int batch = 0; batch < batchNum; batch++) {
+            for (int index = 0; index < totalNum; index += taskNumPerBatch) {
                 Map<String, String> srcToDest = Maps.newHashMap();
-                int currentBatchTaskNum = (batch == batchNum - 1) ? totalNum - index : taskNumPerBatch;
-                for (int j = 0; j < currentBatchTaskNum; j++) {
-                    SnapshotInfo info = infos.get(index++);
+                for (int j = 0; j < taskNumPerBatch && index + j < totalNum; j++) {
+                    SnapshotInfo info = infos.get(index + j);
                     String src = info.getTabletPath();
                     String dest = repo.getRepoTabletPathBySnapshotInfo(label, info);
                     if (dest == null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index aef09bd78beb8f..0b5b0cb03390f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1687,16 +1687,10 @@ private void downloadRemoteSnapshots() {
                 for (Long beId : beToSnapshots.keySet()) {
                     List<SnapshotInfo> beSnapshotInfos = beToSnapshots.get(beId);
                     int totalNum = beSnapshotInfos.size();
-                    int batchNum = totalNum;
-                    if (Config.restore_download_task_num_per_be > 0) {
-                        batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be);
-                    }
                     // each task contains several upload sub tasks
-                    int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
-                    if (LOG.isDebugEnabled()) {
-                        LOG.debug("backend {} has {} batch, total {} tasks, {}",
-                                  beId, batchNum, totalNum, this);
-                    }
+                    int taskNumPerBatch = Config.restore_download_snapshot_batch_size;
+                    LOG.info("backend {} has total {} snapshots, per task batch size {}, {}",
+                            beId, totalNum, taskNumPerBatch, this);
 
                     List<FsBroker> brokerAddrs = null;
                     brokerAddrs = Lists.newArrayList();
@@ -1708,12 +1702,10 @@ private void downloadRemoteSnapshots() {
                     Preconditions.checkState(brokerAddrs.size() == 1);
 
                     // allot tasks
-                    int index = 0;
-                    for (int batch = 0; batch < batchNum; batch++) {
+                    for (int index = 0; index < totalNum; index += taskNumPerBatch) {
                         Map<String, String> srcToDest = Maps.newHashMap();
-                        int currentBatchTaskNum = (batch == batchNum - 1) ? totalNum - index : taskNumPerBatch;
-                        for (int j = 0; j < currentBatchTaskNum; j++) {
-                            SnapshotInfo info = beSnapshotInfos.get(index++);
+                        for (int j = 0; j < taskNumPerBatch && index + j < totalNum; j++) {
+                            SnapshotInfo info = beSnapshotInfos.get(index + j);
                             Table tbl = db.getTableNullable(info.getTblId());
                             if (tbl == null) {
                                 status = new Status(ErrCode.NOT_FOUND, "restored table "
@@ -1847,22 +1839,17 @@ private void downloadLocalSnapshots() {
                 for (Long beId : beToSnapshots.keySet()) {
                     List<SnapshotInfo> beSnapshotInfos = beToSnapshots.get(beId);
                     int totalNum = beSnapshotInfos.size();
-                    int batchNum = totalNum;
-                    if (Config.restore_download_task_num_per_be > 0) {
-                        batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be);
-                    }
                     // each task contains several upload sub tasks
-                    int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
+                    int taskNumPerBatch = Config.restore_download_snapshot_batch_size;
+                    LOG.info("backend {} has total {} snapshots, per task batch size {}, {}",
+                            beId, totalNum, taskNumPerBatch, this);
 
                     // allot tasks
-                    int index = 0;
-                    for (int batch = 0; batch < batchNum; batch++) {
+                    for (int index = 0; index < totalNum; index += taskNumPerBatch) {
                         List<TRemoteTabletSnapshot> remoteTabletSnapshots = Lists.newArrayList();
-                        int currentBatchTaskNum = (batch == batchNum - 1) ? totalNum - index : taskNumPerBatch;
-                        for (int j = 0; j < currentBatchTaskNum; j++) {
+                        for (int j = 0; j < taskNumPerBatch && index + j < totalNum; j++) {
                             TRemoteTabletSnapshot remoteTabletSnapshot = new TRemoteTabletSnapshot();
-
-                            SnapshotInfo info = beSnapshotInfos.get(index++);
+                            SnapshotInfo info = beSnapshotInfos.get(index + j);
                             Table tbl = db.getTableNullable(info.getTblId());
                             if (tbl == null) {
                                 status = new Status(ErrCode.NOT_FOUND, "restored table "

From 7560a78fcbc083d35c9b49981133999b0662ba93 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <chenkaijie@selectdb.com>
Date: Tue, 26 Nov 2024 23:38:50 +0800
Subject: [PATCH 017/399] [cleanup](load) remove bvar
 "g_memtable_input_block_allocated_size" (#44304)

### What problem does this PR solve?

Bvar "g_memtable_input_block_allocated_size" is no longer needed after
#40912.
Calling `MutableBlock::allocated_bytes()` in `Memtable::insert()` has
some performance penalties.
So we should remove it.
---
 be/src/olap/memtable.cpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index 233c59f0910871..5db3d89378bcc2 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -43,7 +43,6 @@
 namespace doris {
 
 bvar::Adder<int64_t> g_memtable_cnt("memtable_cnt");
-bvar::Adder<int64_t> g_memtable_input_block_allocated_size("memtable_input_block_allocated_size");
 
 using namespace ErrorCode;
 
@@ -151,7 +150,6 @@ MemTable::~MemTable() {
                        << _mem_tracker->consumption();
         }
     }
-    g_memtable_input_block_allocated_size << -_input_mutable_block.allocated_bytes();
     g_memtable_cnt << -1;
     if (_keys_type != KeysType::DUP_KEYS) {
         for (auto it = _row_in_blocks.begin(); it != _row_in_blocks.end(); it++) {
@@ -222,11 +220,8 @@ Status MemTable::insert(const vectorized::Block* input_block,
 
     auto num_rows = row_idxs.size();
     size_t cursor_in_mutableblock = _input_mutable_block.rows();
-    auto block_size0 = _input_mutable_block.allocated_bytes();
     RETURN_IF_ERROR(_input_mutable_block.add_rows(input_block, row_idxs.data(),
                                                   row_idxs.data() + num_rows, &_column_offset));
-    auto block_size1 = _input_mutable_block.allocated_bytes();
-    g_memtable_input_block_allocated_size << block_size1 - block_size0;
     for (int i = 0; i < num_rows; i++) {
         _row_in_blocks.emplace_back(new RowInBlock {cursor_in_mutableblock + i});
     }
@@ -631,7 +626,6 @@ Status MemTable::_to_block(std::unique_ptr<vectorized::Block>* res) {
         }
         RETURN_IF_ERROR(_sort_by_cluster_keys());
     }
-    g_memtable_input_block_allocated_size << -_input_mutable_block.allocated_bytes();
     _input_mutable_block.clear();
     // After to block, all data in arena is saved in the block
     _arena.reset();

From 053e5935576f017b4cbcdb7425902c7cb329080b Mon Sep 17 00:00:00 2001
From: Kaijie Chen <chenkaijie@selectdb.com>
Date: Tue, 26 Nov 2024 23:40:07 +0800
Subject: [PATCH 018/399] [fix](load) restore "use_new_load_scan_node" in
 property map for compatibility (#44285)

`LoadStmt#checkProperties()` will throw DdlException when encounter
unknown properties.
So we have to keep the deprecated property keys in the map for
compatibility.
---
 .../java/org/apache/doris/analysis/LoadStmt.java    |  9 +++++++++
 .../broker_load/test_s3_load_properties.groovy      | 13 +++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
index 3a2efe3c98c902..9ceec9415e3cfb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
@@ -99,6 +99,9 @@ public class LoadStmt extends DdlStmt implements NotFallbackInParser {
     public static final String PRIORITY = "priority";
     public static final String LOAD_TO_SINGLE_TABLET = "load_to_single_tablet";
 
+    // deprecated, keeping this property to make LoadStmt#checkProperties() happy
+    public static final String USE_NEW_LOAD_SCAN_NODE = "use_new_load_scan_node";
+
     // for load data from Baidu Object Store(BOS)
     public static final String BOS_ENDPOINT = "bos_endpoint";
     public static final String BOS_ACCESSKEY = "bos_accesskey";
@@ -221,6 +224,12 @@ public class LoadStmt extends DdlStmt implements NotFallbackInParser {
                     return Boolean.valueOf(s);
                 }
             })
+            .put(USE_NEW_LOAD_SCAN_NODE, new Function<String, Boolean>() {
+                @Override
+                public @Nullable Boolean apply(@Nullable String s) {
+                    return Boolean.valueOf(s);
+                }
+            })
             .put(KEY_SKIP_LINES, new Function<String, Integer>() {
                 @Override
                 public @Nullable Integer apply(@Nullable String s) {
diff --git a/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy b/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy
index 9d73256d38ccd2..fdc6971f475fac 100644
--- a/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy
+++ b/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy
@@ -187,6 +187,19 @@ suite("test_s3_load_properties", "p2") {
 //                "", "", "", "","").addProperties("skip_lines", "10"))
 //    }
 
+    /* ========================================================== deprecated properties ========================================================== */
+    for (String table : basicTables) {
+        attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
+                "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+                "", "", "", "", "")).addProperties("use_new_load_scan_node", "true")
+    }
+
+    for (String table : basicTables) {
+        attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
+                "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+                "", "", "", "", "")).addProperties("use_new_load_scan_node", "false")
+    }
+
     /* ========================================================== wrong column sep ========================================================== */
     for (String table : basicTables) {
         attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",

From 09daf0684f69758a2b49c17464593aa0e74bd80c Mon Sep 17 00:00:00 2001
From: huanghaibin <huanghaibin@selectdb.com>
Date: Tue, 26 Nov 2024 23:46:31 +0800
Subject: [PATCH 019/399] [fix](cloud-mow)Fix some implicit conversion loses
 when complie mow code (#44410)

Related PR: #40204

Problem Summary:
There is some implicit conversion loses like uint64_t to uint32_t on mow
code, this pr fix it.
---
 be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp | 2 +-
 be/src/cloud/cloud_tablet.cpp                         | 4 ++--
 be/src/olap/tablet_meta.cpp                           | 2 +-
 cloud/src/meta-service/meta_service.cpp               | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
index 7391449b73f1b1..91611d20c6270b 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
@@ -273,7 +273,7 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset(
                              .base_compaction_cnt = _ms_base_compaction_cnt,
                              .cumulative_compaction_cnt = _ms_cumulative_compaction_cnt,
                              .cumulative_point = _ms_cumulative_point};
-    auto update_delete_bitmap_time_us = 0;
+    int64_t update_delete_bitmap_time_us = 0;
     if (txn_info.publish_status && (*(txn_info.publish_status) == PublishStatus::SUCCEED) &&
         version == previous_publish_info.publish_version &&
         _ms_base_compaction_cnt == previous_publish_info.base_compaction_cnt &&
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 267c204c0e608e..ea86f3b40ff1dc 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -397,8 +397,8 @@ int CloudTablet::delete_expired_stale_rowsets() {
         }
 
         for (int64_t path_id : path_ids) {
-            int start_version = -1;
-            int end_version = -1;
+            int64_t start_version = -1;
+            int64_t end_version = -1;
             // delete stale versions in version graph
             auto version_path = _timestamped_version_tracker.fetch_and_delete_path_by_id(path_id);
             for (auto& v_ts : version_path->timestamped_versions()) {
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 9a27b95dbcd446..0570aff349c583 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -668,7 +668,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) {
             RowsetId rst_id;
             rst_id.init(tablet_meta_pb.delete_bitmap().rowset_ids(i));
             auto seg_id = tablet_meta_pb.delete_bitmap().segment_ids(i);
-            uint32_t ver = tablet_meta_pb.delete_bitmap().versions(i);
+            auto ver = tablet_meta_pb.delete_bitmap().versions(i);
             auto bitmap = tablet_meta_pb.delete_bitmap().segment_delete_bitmaps(i).data();
             delete_bitmap().delete_bitmap[{rst_id, seg_id, ver}] = roaring::Roaring::read(bitmap);
         }
diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index 4f374832925dd7..974b3091a73e95 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -1748,7 +1748,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
     }
     RPC_RATE_LIMIT(update_delete_bitmap)
 
-    uint32_t fdb_txn_size = 0;
+    uint64_t fdb_txn_size = 0;
     auto table_id = request->table_id();
     auto tablet_id = request->tablet_id();
 

From 54f87af876d28005d690a95ae48f6a4a7da30863 Mon Sep 17 00:00:00 2001
From: Gavin Chou <gavin@selectdb.com>
Date: Tue, 26 Nov 2024 23:55:10 +0800
Subject: [PATCH 020/399] [opt](log) Detail log for file cache resource limit
 mode (#44094)

---
 be/src/io/cache/block_file_cache.cpp | 31 ++++++++++++++--------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp
index ebcbe9135daa35..528ace90e3e7d7 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -1678,13 +1678,16 @@ void BlockFileCache::check_disk_resource_limit() {
         LOG_ERROR("").tag("file cache path", _cache_base_path).tag("error", strerror(errno));
         return;
     }
-    auto [capacity_percentage, inode_percentage] = percent;
-    auto inode_is_insufficient = [](const int& inode_percentage) {
-        return inode_percentage >= config::file_cache_enter_disk_resource_limit_mode_percent;
+    auto [space_percentage, inode_percentage] = percent;
+    auto is_insufficient = [](const int& percentage) {
+        return percentage >= config::file_cache_enter_disk_resource_limit_mode_percent;
     };
-    DCHECK(capacity_percentage >= 0 && capacity_percentage <= 100);
-    DCHECK(inode_percentage >= 0 && inode_percentage <= 100);
-    // ATTN: due to that can be change, so if its invalid, set it to default value
+    DCHECK_GE(space_percentage, 0);
+    DCHECK_LE(space_percentage, 100);
+    DCHECK_GE(inode_percentage, 0);
+    DCHECK_LE(inode_percentage, 100);
+    // ATTN: due to that can be changed dynamically, set it to default value if it's invalid
+    // FIXME: reject with config validator
     if (config::file_cache_enter_disk_resource_limit_mode_percent <=
         config::file_cache_exit_disk_resource_limit_mode_percent) {
         LOG_WARNING("config error, set to default value")
@@ -1693,23 +1696,21 @@ void BlockFileCache::check_disk_resource_limit() {
         config::file_cache_enter_disk_resource_limit_mode_percent = 90;
         config::file_cache_exit_disk_resource_limit_mode_percent = 80;
     }
-    if (capacity_percentage >= config::file_cache_enter_disk_resource_limit_mode_percent ||
-        inode_is_insufficient(inode_percentage)) {
+    if (is_insufficient(space_percentage) || is_insufficient(inode_percentage)) {
         _disk_resource_limit_mode = true;
         _disk_limit_mode_metrics->set_value(1);
     } else if (_disk_resource_limit_mode &&
-               (capacity_percentage < config::file_cache_exit_disk_resource_limit_mode_percent) &&
+               (space_percentage < config::file_cache_exit_disk_resource_limit_mode_percent) &&
                (inode_percentage < config::file_cache_exit_disk_resource_limit_mode_percent)) {
         _disk_resource_limit_mode = false;
         _disk_limit_mode_metrics->set_value(0);
     }
     if (_disk_resource_limit_mode) {
-        // log per mins
-        LOG_EVERY_N(WARNING, 3) << "file cache background thread space percent="
-                                << capacity_percentage << " inode percent=" << inode_percentage
-                                << " is inode insufficient="
-                                << inode_is_insufficient(inode_percentage)
-                                << " mode run in resource limit";
+        LOG(WARNING) << "file_cache=" << get_base_path() << " space_percent=" << space_percentage
+                     << " inode_percent=" << inode_percentage
+                     << " is_space_insufficient=" << is_insufficient(space_percentage)
+                     << " is_inode_insufficient=" << is_insufficient(inode_percentage)
+                     << " mode run in resource limit";
     }
 }
 

From 95d19858cbd5f953dfbe06cbb47359a6adfdd769 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Wed, 27 Nov 2024 07:05:23 +0530
Subject: [PATCH 021/399] [Enhancement] (nereids)implement
 DropWorkloadPolicyCommand in nereids (#44490)

Issue Number: close #42622
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  7 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/DropWorkloadPolicyCommand.java   | 61 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../WorkloadSchedPolicyMgr.java               |  7 ++-
 .../test_nereids_workload_policy_test.out     |  6 ++
 .../test_nereids_workload_policy_test.groovy  | 32 ++++++++++
 8 files changed, 118 insertions(+), 3 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropWorkloadPolicyCommand.java
 create mode 100644 regression-test/data/workload_manager_p0/test_nereids_workload_policy_test.out
 create mode 100644 regression-test/suites/workload_manager_p0/test_nereids_workload_policy_test.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 932f644fca4e7a..b1fdda33646e13 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -204,6 +204,7 @@ supportedDropStatement
     | DROP SQL_BLOCK_RULE (IF EXISTS)? identifierSeq                            #dropSqlBlockRule
     | DROP USER (IF EXISTS)? userIdentify                                       #dropUser
     | DROP WORKLOAD GROUP (IF EXISTS)? name=identifierOrText                    #dropWorkloadGroup
+    | DROP WORKLOAD POLICY (IF EXISTS)? name=identifierOrText                   #dropWorkloadPolicy
     ;
 
 supportedShowStatement
@@ -673,7 +674,6 @@ unsupportedDropStatement
         ((FROM | IN) database=identifier)? properties=propertyClause            #dropFile
     | DROP INDEX (IF EXISTS)? name=identifier ON tableName=multipartIdentifier  #dropIndex
     | DROP RESOURCE (IF EXISTS)? name=identifierOrText                          #dropResource
-    | DROP WORKLOAD POLICY (IF EXISTS)? name=identifierOrText                   #dropWorkloadPolicy
     | DROP ROW POLICY (IF EXISTS)? policyName=identifier
         ON tableName=multipartIdentifier
         (FOR (userIdentify | ROLE roleName=identifier))?                        #dropRowPolicy
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 320043e12b8b53..216cd60135b423 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -108,6 +108,7 @@
 import org.apache.doris.nereids.DorisParser.DropSqlBlockRuleContext;
 import org.apache.doris.nereids.DorisParser.DropUserContext;
 import org.apache.doris.nereids.DorisParser.DropWorkloadGroupContext;
+import org.apache.doris.nereids.DorisParser.DropWorkloadPolicyContext;
 import org.apache.doris.nereids.DorisParser.ElementAtContext;
 import org.apache.doris.nereids.DorisParser.ExceptContext;
 import org.apache.doris.nereids.DorisParser.ExceptOrReplaceContext;
@@ -460,6 +461,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DropSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropUserCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropWorkloadGroupCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropWorkloadPolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
 import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
 import org.apache.doris.nereids.trees.plans.commands.ExportCommand;
@@ -4400,6 +4402,11 @@ public LogicalPlan visitDropWorkloadGroup(DropWorkloadGroupContext ctx) {
         return new DropWorkloadGroupCommand(ctx.name.getText(), ctx.EXISTS() != null);
     }
 
+    @Override
+    public LogicalPlan visitDropWorkloadPolicy(DropWorkloadPolicyContext ctx) {
+        return new DropWorkloadPolicyCommand(ctx.name.getText(), ctx.EXISTS() != null);
+    }
+
     @Override
     public LogicalPlan visitShowTableId(ShowTableIdContext ctx) {
         long tableId = -1;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 1f7b838edfd905..0b45dde83d288f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -187,6 +187,7 @@ public enum PlanType {
     DROP_SQL_BLOCK_RULE_COMMAND,
     DROP_USER_COMMAND,
     DROP_WORKLOAD_GROUP_NAME,
+    DROP_WORKLOAD_POLICY_COMMAND,
     SHOW_BACKENDS_COMMAND,
     SHOW_BLOCK_RULE_COMMAND,
     SHOW_BROKER_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropWorkloadPolicyCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropWorkloadPolicyCommand.java
new file mode 100644
index 00000000000000..96a3c0c69cf9ae
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropWorkloadPolicyCommand.java
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+/**
+ * drop workload policy command
+ */
+public class DropWorkloadPolicyCommand extends DropCommand {
+    private final boolean ifExists;
+    private final String workloadPolicy;
+
+    /**
+     * constructor
+     */
+    public DropWorkloadPolicyCommand(String workloadPolicy, boolean ifExists) {
+        super(PlanType.DROP_WORKLOAD_POLICY_COMMAND);
+        this.workloadPolicy = workloadPolicy;
+        this.ifExists = ifExists;
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        FeNameFormat.checkWorkloadSchedPolicyName(workloadPolicy);
+        Env.getCurrentEnv().getWorkloadSchedPolicyMgr().dropWorkloadSchedPolicy(workloadPolicy, ifExists);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitDropWorkloadPolicyCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 6b801524fb295f..4893ef41be8e44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -47,6 +47,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DropSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropUserCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropWorkloadGroupCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropWorkloadPolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
 import org.apache.doris.nereids.trees.plans.commands.ExportCommand;
 import org.apache.doris.nereids.trees.plans.commands.LoadCommand;
@@ -439,6 +440,10 @@ default R visitDropWorkloadGroupCommand(DropWorkloadGroupCommand dropWorkloadGro
         return visitCommand(dropWorkloadGroupCommand, context);
     }
 
+    default R visitDropWorkloadPolicyCommand(DropWorkloadPolicyCommand dropWorkloadPolicyCommand, C context) {
+        return visitCommand(dropWorkloadPolicyCommand, context);
+    }
+
     default R visitShowTableIdCommand(ShowTableIdCommand showTableIdCommand, C context) {
         return visitCommand(showTableIdCommand, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
index 3879dd83b9adfb..715bdfc5e1ee6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
@@ -476,12 +476,15 @@ public void alterWorkloadSchedPolicy(AlterWorkloadSchedPolicyStmt alterStmt) thr
     }
 
     public void dropWorkloadSchedPolicy(DropWorkloadSchedPolicyStmt dropStmt) throws UserException {
+        dropWorkloadSchedPolicy(dropStmt.getPolicyName(), dropStmt.isIfExists());
+    }
+
+    public void dropWorkloadSchedPolicy(String policyName, boolean isExists) throws UserException {
         writeLock();
         try {
-            String policyName = dropStmt.getPolicyName();
             WorkloadSchedPolicy schedPolicy = nameToPolicy.get(policyName);
             if (schedPolicy == null) {
-                if (dropStmt.isIfExists()) {
+                if (isExists) {
                     return;
                 } else {
                     throw new UserException("workload schedule policy " + policyName + " not exists");
diff --git a/regression-test/data/workload_manager_p0/test_nereids_workload_policy_test.out b/regression-test/data/workload_manager_p0/test_nereids_workload_policy_test.out
new file mode 100644
index 00000000000000..8eead5743138a6
--- /dev/null
+++ b/regression-test/data/workload_manager_p0/test_nereids_workload_policy_test.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !check_workload_policy_check1 --
+test_nereids_worklod_policy1
+
+-- !check_workload_policy_check2 --
+
diff --git a/regression-test/suites/workload_manager_p0/test_nereids_workload_policy_test.groovy b/regression-test/suites/workload_manager_p0/test_nereids_workload_policy_test.groovy
new file mode 100644
index 00000000000000..c3b93f9ffa28d1
--- /dev/null
+++ b/regression-test/suites/workload_manager_p0/test_nereids_workload_policy_test.groovy
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_nereids_workload_policy_test") {
+    sql "drop workload policy if exists test_nereids_worklod_policy1;"
+    sql "create workload policy test_nereids_worklod_policy1 " +
+            "conditions(username='root') " +
+            "actions(set_session_variable 'workload_group=normal') " +
+            "properties( " +
+            "'enabled' = 'false', " +
+            "'priority'='10' " +
+            ");"
+    qt_check_workload_policy_check1("select NAME from information_schema.workload_policy where NAME='test_nereids_worklod_policy1';")
+    checkNereidsExecute("drop workload policy  test_nereids_worklod_policy1;")
+    checkNereidsExecute("drop workload policy if exists test_nereids_worklod_policy1;")
+    qt_check_workload_policy_check2("select NAME from information_schema.workload_policy where NAME='test_nereids_worklod_policy1';")
+
+}
\ No newline at end of file

From bf67d440f3f34ab35e2095a82944dcece9654f21 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Wed, 27 Nov 2024 07:08:17 +0530
Subject: [PATCH 022/399] [Enhancement] (nereids)implement
 alterWorkloadPolicyCommand in nereids (#44501)

Issue Number: close #42794
---
 .../org/apache/doris/nereids/DorisParser.g4   |  4 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  9 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/AlterWorkloadPolicyCommand.java  | 65 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../WorkloadSchedPolicyMgr.java               |  6 +-
 ...test_nereids_workloadpolicy_alter_test.out |  7 ++
 ...t_nereids_workloadpolicy_alter_test.groovy | 32 +++++++++
 8 files changed, 125 insertions(+), 4 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterWorkloadPolicyCommand.java
 create mode 100644 regression-test/data/workload_manager_p0/test_nereids_workloadpolicy_alter_test.out
 create mode 100644 regression-test/suites/workload_manager_p0/test_nereids_workloadpolicy_alter_test.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index b1fdda33646e13..777a0e2d7d7fa6 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -195,6 +195,8 @@ supportedAlterStatement
     | ALTER ROLE role=identifier commentSpec                                        #alterRole
     | ALTER WORKLOAD GROUP name=identifierOrText
         properties=propertyClause?                                                  #alterWorkloadGroup
+    | ALTER WORKLOAD POLICY name=identifierOrText
+        properties=propertyClause?                                                  #alterWorkloadPolicy
     ;
 
 supportedDropStatement
@@ -562,8 +564,6 @@ unsupportedAlterStatement
     | ALTER RESOURCE name=identifierOrText properties=propertyClause?               #alterResource
     | ALTER COLOCATE GROUP name=multipartIdentifier
         SET LEFT_PAREN propertyItemList RIGHT_PAREN                                 #alterColocateGroup
-    | ALTER WORKLOAD POLICY name=identifierOrText
-        properties=propertyClause?                                                  #alterWorkloadPolicy
     | ALTER ROUTINE LOAD FOR name=multipartIdentifier properties=propertyClause?
             (FROM type=identifier LEFT_PAREN propertyItemList RIGHT_PAREN)?         #alterRoutineLoad
     | ALTER SQL_BLOCK_RULE name=identifier properties=propertyClause?               #alterSqlBlockRule
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 216cd60135b423..94e499277ca221 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -58,6 +58,7 @@
 import org.apache.doris.nereids.DorisParser.AlterStorageVaultContext;
 import org.apache.doris.nereids.DorisParser.AlterViewContext;
 import org.apache.doris.nereids.DorisParser.AlterWorkloadGroupContext;
+import org.apache.doris.nereids.DorisParser.AlterWorkloadPolicyContext;
 import org.apache.doris.nereids.DorisParser.ArithmeticBinaryContext;
 import org.apache.doris.nereids.DorisParser.ArithmeticUnaryContext;
 import org.apache.doris.nereids.DorisParser.ArrayLiteralContext;
@@ -435,6 +436,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AlterStorageVaultCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
+import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadPolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CallCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelJobTaskCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
@@ -4312,6 +4314,13 @@ public LogicalPlan visitAlterWorkloadGroup(AlterWorkloadGroupContext ctx) {
         return new AlterWorkloadGroupCommand(ctx.name.getText(), properties);
     }
 
+    @Override
+    public LogicalPlan visitAlterWorkloadPolicy(AlterWorkloadPolicyContext ctx) {
+        Map<String, String> properties = ctx.propertyClause() != null
+                        ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+        return new AlterWorkloadPolicyCommand(ctx.name.getText(), properties);
+    }
+
     @Override
     public LogicalPlan visitAlterRole(AlterRoleContext ctx) {
         String comment = visitCommentSpec(ctx.commentSpec());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 0b45dde83d288f..7fa4db085848eb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -171,6 +171,7 @@ public enum PlanType {
     ALTER_VIEW_COMMAND,
     ALTER_STORAGE_VAULT,
     ALTER_WORKLOAD_GROUP_COMMAND,
+    ALTER_WORKLOAD_POLICY_COMMAND,
     DROP_CATALOG_RECYCLE_BIN_COMMAND,
     DROP_ENCRYPTKEY_COMMAND,
     UNSET_VARIABLE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterWorkloadPolicyCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterWorkloadPolicyCommand.java
new file mode 100644
index 00000000000000..42e5250d5b62e7
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterWorkloadPolicyCommand.java
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import java.util.Map;
+
+/**
+ * alter workload policy command
+ */
+public class AlterWorkloadPolicyCommand extends AlterCommand {
+    private final String workloadPolicyName;
+    private final Map<String, String> properties;
+
+    /**
+     * constructor
+     */
+    public AlterWorkloadPolicyCommand(String workloadPolicyName, Map<String, String> properties) {
+        super(PlanType.ALTER_WORKLOAD_POLICY_COMMAND);
+        this.workloadPolicyName = workloadPolicyName;
+        this.properties = properties;
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        if (properties == null || properties.isEmpty()) {
+            throw new AnalysisException("properties can't be null when alter workload schedule policy");
+        }
+        Env.getCurrentEnv().getWorkloadSchedPolicyMgr().alterWorkloadSchedPolicy(workloadPolicyName, properties);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAlterWorkloadPolicyCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 4893ef41be8e44..a564cb1c140208 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AlterRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
+import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadPolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CallCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelJobTaskCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
@@ -384,6 +385,10 @@ default R visitAlterWorkloadGroupCommand(AlterWorkloadGroupCommand alterWorkload
         return visitCommand(alterWorkloadGroupCommand, context);
     }
 
+    default R visitAlterWorkloadPolicyCommand(AlterWorkloadPolicyCommand alterWorkloadPolicyCommand, C context) {
+        return visitCommand(alterWorkloadPolicyCommand, context);
+    }
+
     default R visitCleanAllProfileCommand(CleanAllProfileCommand cleanAllProfileCommand, C context) {
         return visitCommand(cleanAllProfileCommand, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
index 715bdfc5e1ee6d..43f9cd2a794b0f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
@@ -456,15 +456,17 @@ private void checkProperties(Map<String, String> properties, List<Long> wgIdList
     }
 
     public void alterWorkloadSchedPolicy(AlterWorkloadSchedPolicyStmt alterStmt) throws UserException {
+        alterWorkloadSchedPolicy(alterStmt.getPolicyName(), alterStmt.getProperties());
+    }
+
+    public void alterWorkloadSchedPolicy(String policyName, Map<String, String> properties) throws UserException {
         writeLock();
         try {
-            String policyName = alterStmt.getPolicyName();
             WorkloadSchedPolicy policy = nameToPolicy.get(policyName);
             if (policy == null) {
                 throw new UserException("can not find workload schedule policy " + policyName);
             }
 
-            Map<String, String> properties = alterStmt.getProperties();
             List<Long> wgIdList = new ArrayList<>();
             checkProperties(properties, wgIdList);
             policy.updatePropertyIfNotNull(properties, wgIdList);
diff --git a/regression-test/data/workload_manager_p0/test_nereids_workloadpolicy_alter_test.out b/regression-test/data/workload_manager_p0/test_nereids_workloadpolicy_alter_test.out
new file mode 100644
index 00000000000000..ca42544b6bd6e4
--- /dev/null
+++ b/regression-test/data/workload_manager_p0/test_nereids_workloadpolicy_alter_test.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !check_workload_policy_check1 --
+test_nereids_alter_worklod_policy1	10
+
+-- !check_workload_policy_check2 --
+test_nereids_alter_worklod_policy1	17
+
diff --git a/regression-test/suites/workload_manager_p0/test_nereids_workloadpolicy_alter_test.groovy b/regression-test/suites/workload_manager_p0/test_nereids_workloadpolicy_alter_test.groovy
new file mode 100644
index 00000000000000..e696f09587fd83
--- /dev/null
+++ b/regression-test/suites/workload_manager_p0/test_nereids_workloadpolicy_alter_test.groovy
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_nereids_workloadpolicy_alter_test") {
+    sql "drop workload policy if exists test_nereids_alter_worklod_policy1;"
+    sql "create workload policy test_nereids_alter_worklod_policy1 " +
+            "conditions(username='root') " +
+            "actions(set_session_variable 'workload_group=normal') " +
+            "properties( " +
+            "'enabled' = 'false', " +
+            "'priority'='10');"
+
+    qt_check_workload_policy_check1("select NAME,PRIORITY from information_schema.workload_policy where NAME='test_nereids_alter_worklod_policy1';")
+    checkNereidsExecute("alter workload policy  test_nereids_alter_worklod_policy1 properties('priority'='17');")
+    qt_check_workload_policy_check2("select NAME,PRIORITY from information_schema.workload_policy where NAME='test_nereids_alter_worklod_policy1';")
+    sql "drop workload policy if exists test_nereids_alter_worklod_policy1;"
+
+}
\ No newline at end of file

From 9c640d89156331995a2ad359ffe5c958a63b488f Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Wed, 27 Nov 2024 07:23:42 +0530
Subject: [PATCH 023/399] [Enhancement] (nereids)implement showEventsCommand in
 nereids (#44499)

Issue Number: close #42732
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  7 ++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/ShowEventsCommand.java     | 70 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../show/test_show_commands_nereids.groovy    |  1 +
 6 files changed, 85 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowEventsCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 777a0e2d7d7fa6..2d68e01b78761a 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -214,6 +214,7 @@ supportedShowStatement
     | SHOW AUTHORS                                                                  #showAuthors
     | SHOW BROKER                                                                   #showBroker
     | SHOW DYNAMIC PARTITION TABLES ((FROM | IN) database=multipartIdentifier)?     #showDynamicPartition
+    | SHOW EVENTS ((FROM | IN) database=multipartIdentifier)? wildWhere?            #showEvents
     | SHOW LAST INSERT                                                              #showLastInsert 
     | SHOW DELETE ((FROM | IN) database=multipartIdentifier)?                       #showDelete
     | SHOW ALL? GRANTS                                                              #showGrants
@@ -287,7 +288,6 @@ unsupportedShowStatement
     | SHOW FULL? VIEWS ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showViews
     | SHOW FULL? PROCESSLIST                                                        #showProcessList
     | SHOW (GLOBAL | SESSION | LOCAL)? STATUS wildWhere?                            #showStatus
-    | SHOW EVENTS ((FROM | IN) database=multipartIdentifier)? wildWhere?            #showEvents
     | SHOW CREATE VIEW name=multipartIdentifier                                     #showCreateView
     | SHOW CREATE MATERIALIZED VIEW name=multipartIdentifier                        #showMaterializedView
     | SHOW CREATE (DATABASE | SCHEMA) name=multipartIdentifier                      #showCreateDatabase
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 94e499277ca221..ae0921a1732419 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -218,6 +218,7 @@
 import org.apache.doris.nereids.DorisParser.ShowCreateTableContext;
 import org.apache.doris.nereids.DorisParser.ShowDeleteContext;
 import org.apache.doris.nereids.DorisParser.ShowDynamicPartitionContext;
+import org.apache.doris.nereids.DorisParser.ShowEventsContext;
 import org.apache.doris.nereids.DorisParser.ShowFrontendsContext;
 import org.apache.doris.nereids.DorisParser.ShowGrantsContext;
 import org.apache.doris.nereids.DorisParser.ShowGrantsForUserContext;
@@ -493,6 +494,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowEventsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowFrontendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowGrantsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowLastInsertCommand;
@@ -4013,6 +4015,11 @@ public LogicalPlan visitShowAuthors(ShowAuthorsContext ctx) {
         return new ShowAuthorsCommand();
     }
 
+    @Override
+    public LogicalPlan visitShowEvents(ShowEventsContext ctx) {
+        return new ShowEventsCommand();
+    }
+
     @Override
     public LogicalPlan visitShowConfig(ShowConfigContext ctx) {
         ShowConfigCommand command;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 7fa4db085848eb..79a4ff31c1fc20 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -198,6 +198,7 @@ public enum PlanType {
     SHOW_CREATE_TABLE_COMMAND,
     SHOW_DELETE_COMMAND,
     SHOW_DYNAMIC_PARTITION_COMMAND,
+    SHOW_EVENTS_COMMAND,
     SHOW_FRONTENDS_COMMAND,
     SHOW_GRANTS_COMMAND,
     SHOW_LAST_INSERT_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowEventsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowEventsCommand.java
new file mode 100644
index 00000000000000..02b516601f40fc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowEventsCommand.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * show events
+ */
+public class ShowEventsCommand extends ShowCommand {
+    private static final ShowResultSetMetaData META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("Db", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Name", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("Definer", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Time", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Type", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Execute at", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Interval value", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("Interval field", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("Status", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("Ends", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("Status", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("Originator", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("character_set_client", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("collation_connection", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("Database Collation", ScalarType.createVarchar(30)))
+                    .build();
+
+    public ShowEventsCommand() {
+        super(PlanType.SHOW_EVENTS_COMMAND);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowEventsCommand(this, context);
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        List<List<String>> rowSet = Lists.newArrayList();
+        return new ShowResultSet(META_DATA, rowSet);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index a564cb1c140208..c0f86d8dfa45ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -77,6 +77,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowEventsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowFrontendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowGrantsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowLastInsertCommand;
@@ -457,6 +458,10 @@ default R visitSyncCommand(SyncCommand syncCommand, C context) {
         return visitCommand(syncCommand, context);
     }
 
+    default R visitShowEventsCommand(ShowEventsCommand showEventsCommand, C context) {
+        return visitCommand(showEventsCommand, context);
+    }
+
     default R visitShowDeleteCommand(ShowDeleteCommand showDeleteCommand, C context) {
         return visitCommand(showDeleteCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
index 998033deff9863..84dcbc2323fa02 100644
--- a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
@@ -26,5 +26,6 @@ suite("test_show_commands_nereids") {
     checkNereidsExecute("""show backends;""")
     checkNereidsExecute("""show whitelist;""")
     checkNereidsExecute("""show triggers;""")
+    checkNereidsExecute("""show events;""")
     checkNereidsExecute("""show load profile \"\\";""")
 }

From aca0eaad6859242aa33a3929e0f901473844ab23 Mon Sep 17 00:00:00 2001
From: xzj7019 <xiongzhongjian@selectdb.com>
Date: Wed, 27 Nov 2024 10:28:01 +0800
Subject: [PATCH 024/399] [fix](nereids) fix ptopN push down under multi
 winexprs with partial forbidden type (#44617)

Related PR: #38393

Problem Summary: In the previous pr which supporting multi win expr
ptopN pushdown, it handled partial forbidden type unexpectly and will
lead some case to push down the pTopN wrongly.
plan before fixing:

explain shape plan select * from (select row_number() over(partition by c1, c2 order by c3) as rn, sum(c2) over(order by c2 range between unbounded preceding and unbounded following) as sw from push_down_multi_predicate_through_window_t) t where rn <= 1 and sw <= 1;

+------------------------------------------------------------------------------------+
| Explain String(Nereids Planner)                                                    |
+------------------------------------------------------------------------------------+
| PhysicalResultSink                                                                 |
| --PhysicalProject                                                                  |
| ----filter((rn <= 1) and (sw <= 1))                                                |
| ------PhysicalWindow                                                               |
| --------PhysicalQuickSort[MERGE_SORT]                                              |
| ----------PhysicalDistribute[DistributionSpecGather]                               |
| ------------PhysicalQuickSort[LOCAL_SORT]                                          |
| --------------PhysicalWindow                                                       |
| ----------------PhysicalQuickSort[LOCAL_SORT]                                      |
| ------------------PhysicalDistribute[DistributionSpecHash]                         |
| --------------------PhysicalPartitionTopN                                          |
| ----------------------PhysicalOlapScan[push_down_multi_predicate_through_window_t] |
+------------------------------------------------------------------------------------+

plan after fixing:

explain shape plan select * from (select row_number() over(partition by c1, c2 order by c3) as rn, sum(c2) over(order by c2 range between unbounded preceding and unbounded following) as sw from push_down_multi_predicate_through_window_t) t where rn <= 1 and sw <= 1;

+----------------------------------------------------------------------------------+
| Explain String(Nereids Planner)                                                  |
+----------------------------------------------------------------------------------+
| PhysicalResultSink                                                               |
| --PhysicalProject                                                                |
| ----filter((rn <= 1) and (sw <= 1))                                              |
| ------PhysicalWindow                                                             |
| --------PhysicalQuickSort[MERGE_SORT]                                            |
| ----------PhysicalDistribute[DistributionSpecGather]                             |
| ------------PhysicalQuickSort[LOCAL_SORT]                                        |
| --------------PhysicalWindow                                                     |
| ----------------PhysicalQuickSort[LOCAL_SORT]                                    |
| ------------------PhysicalDistribute[DistributionSpecHash]                       |
| --------------------PhysicalOlapScan[push_down_multi_predicate_through_window_t] |
+----------------------------------------------------------------------------------+
---
 .../trees/plans/logical/LogicalWindow.java    | 10 ++---
 .../nereids_syntax_p0/window_function.out     |  6 +++
 ...sh_down_multi_filter_through_window.groovy | 20 ++++++++++
 .../nereids_syntax_p0/window_function.groovy  | 39 +++++++++++++++++++
 4 files changed, 70 insertions(+), 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
index 6cd8389349a007..5ee70afbf5ff86 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
@@ -218,7 +218,7 @@ && child(0).child(0) instanceof LogicalPartitionTopN)) {
         for (NamedExpression windowExpr : windowExpressions) {
             if (windowExpr == null || windowExpr.children().size() != 1
                     || !(windowExpr.child(0) instanceof WindowExpression)) {
-                continue;
+                return null;
             }
             WindowExpression windowFunc = (WindowExpression) windowExpr.child(0);
 
@@ -226,12 +226,12 @@ && child(0).child(0) instanceof LogicalPartitionTopN)) {
             if (!(windowFunc.getFunction() instanceof RowNumber
                     || windowFunc.getFunction() instanceof Rank
                     || windowFunc.getFunction() instanceof DenseRank)) {
-                continue;
+                return null;
             }
 
             // Check the partition key and order key.
             if (windowFunc.getPartitionKeys().isEmpty() && windowFunc.getOrderKeys().isEmpty()) {
-                continue;
+                return null;
             }
 
             // Check the window type and window frame.
@@ -240,10 +240,10 @@ && child(0).child(0) instanceof LogicalPartitionTopN)) {
                 WindowFrame frame = windowFrame.get();
                 if (!(frame.getLeftBoundary().getFrameBoundType() == WindowFrame.FrameBoundType.UNBOUNDED_PRECEDING
                         && frame.getRightBoundary().getFrameBoundType() == WindowFrame.FrameBoundType.CURRENT_ROW)) {
-                    continue;
+                    return null;
                 }
             } else {
-                continue;
+                return null;
             }
 
             // Check filter conditions.
diff --git a/regression-test/data/nereids_syntax_p0/window_function.out b/regression-test/data/nereids_syntax_p0/window_function.out
index 4ec92fc61ad46c..38eba68274e18f 100644
--- a/regression-test/data/nereids_syntax_p0/window_function.out
+++ b/regression-test/data/nereids_syntax_p0/window_function.out
@@ -561,3 +561,9 @@
 \N
 \N
 
+-- !multi_winf1 --
+1	c
+
+-- !multi_winf2 --
+1	35
+
diff --git a/regression-test/suites/nereids_rules_p0/push_down_filter_through_window/push_down_multi_filter_through_window.groovy b/regression-test/suites/nereids_rules_p0/push_down_filter_through_window/push_down_multi_filter_through_window.groovy
index d808d30f8ebdfe..015a6e7fae7d47 100644
--- a/regression-test/suites/nereids_rules_p0/push_down_filter_through_window/push_down_multi_filter_through_window.groovy
+++ b/regression-test/suites/nereids_rules_p0/push_down_filter_through_window/push_down_multi_filter_through_window.groovy
@@ -157,4 +157,24 @@ suite("push_down_multi_filter_through_window") {
         sql ("select * from (select row_number() over(partition by c1, c2 order by c3) as rn, rank() over(partition by c1 order by c3) as rk from push_down_multi_predicate_through_window_t) t where rn <= 1 or rk <= 1;")
         notContains "VPartitionTopN"
     }
+
+    explain {
+        sql ("select * from (select row_number() over(partition by c1, c2 order by c3) as rn, sum(c2) over(order by c2 range between unbounded preceding and unbounded following) as sw from push_down_multi_predicate_through_window_t) t where rn <= 1 and sw <= 1;")
+        notContains "VPartitionTopN"
+    }
+
+    explain {
+        sql ("select * from (select sum(c2) over(order by c2 range between unbounded preceding and unbounded following) as sw, row_number() over(partition by c1, c2 order by c3) as rn from push_down_multi_predicate_through_window_t) t where rn <= 1 and sw <= 1;")
+        notContains "VPartitionTopN"
+    }
+
+    explain {
+        sql ("select * from (select row_number() over(partition by c1, c2 order by c3 rows between unbounded preceding and current row) as rn, sum(c2) over(order by c2) as sw from push_down_multi_predicate_through_window_t) t where rn <= 1 and sw <= 1;")
+        notContains "VPartitionTopN"
+    }
+
+    explain {
+        sql ("select * from (select sum(c2) over(order by c2) as sw, row_number() over(partition by c1, c2 order by c3  rows between unbounded preceding and current row) as rn from push_down_multi_predicate_through_window_t) t where rn <= 1 and sw <= 1;")
+        notContains "VPartitionTopN"
+    }
 }
diff --git a/regression-test/suites/nereids_syntax_p0/window_function.groovy b/regression-test/suites/nereids_syntax_p0/window_function.groovy
index 9a427d1019887d..31cb425fcf7a44 100644
--- a/regression-test/suites/nereids_syntax_p0/window_function.groovy
+++ b/regression-test/suites/nereids_syntax_p0/window_function.groovy
@@ -240,4 +240,43 @@ suite("window_function") {
     """
 
     qt_sql """ select LAST_VALUE(col_tinyint_undef_signed_not_null) over (partition by col_double_undef_signed_not_null, col_int_undef_signed, (col_float_undef_signed_not_null - col_int_undef_signed), round_bankers(col_int_undef_signed) order by pk rows between unbounded preceding and 4 preceding) AS col_alias56089 from table_200_undef_partitions2_keys3_properties4_distributed_by53 order by col_alias56089;  """
+
+    order_qt_multi_winf1 """
+        select *
+            from (
+                select
+                row_number() over(partition by c1 order by c2) rn,
+                lead(c2, 2, '') over(partition by c1 order by c2)
+                from (
+                  select 1 as c1, 'a' as c2
+                  union all
+                  select 1 as c1, 'b' as c2
+                  union all
+                  select 1 as c1, 'c' as c2
+                  union all
+                  select 1 as c1, 'd' as c2
+                  union all
+                  select 1 as c1, 'e' as c2
+                )t
+            )a where rn=1
+    """
+    order_qt_multi_winf2 """
+        select *
+            from (
+                select
+                row_number() over(partition by c1 order by c2) rn,
+                sum(c2) over(order by c2 range between unbounded preceding and unbounded following)
+                from (
+                  select 1 as c1, 5 as c2
+                  union all
+                  select 1 as c1, 6 as c2
+                  union all
+                  select 1 as c1, 7 as c2
+                  union all
+                  select 1 as c1, 8 as c2
+                  union all
+                  select 1 as c1, 9 as c2
+                )t
+            )a where rn=1
+    """
 }

From df5bfe8f66b2fb50285c1c0253abca207f4d7801 Mon Sep 17 00:00:00 2001
From: zhiqiang <hezhiqiang@selectdb.com>
Date: Wed, 27 Nov 2024 10:37:29 +0800
Subject: [PATCH 025/399] [fix](hyperscan) Fix hyper scan fall back to re2
 (#44547)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Core modification
When hyper scan failed, we should not set_error in FunctionContext.
Since set_error will try cancel query, but actually we want to fall back
to re2 in this case.

* Some refactor
Rename FunctionRegexp so that we can distinguish regexp match with
regexp_extract.

* Reproduce
```cpp
SELECT * FROM regexp_test_chinese WHERE city REGEXP "^上海｜^北京" ORDER BY id;
```
Note, the `|` in above sql is a Chinese character.
---
 be/src/vec/functions/function_regexp.cpp      | 15 ++++----
 be/src/vec/functions/like.cpp                 | 21 +++++------
 be/src/vec/functions/like.h                   |  4 +--
 .../string_functions/test_regexp_chinese.out  |  3 ++
 .../test_regexp_chinese.groovy                | 35 +++++++++++++++++++
 5 files changed, 57 insertions(+), 21 deletions(-)
 create mode 100644 regression-test/data/query_p0/sql_functions/string_functions/test_regexp_chinese.out
 create mode 100644 regression-test/suites/query_p0/sql_functions/string_functions/test_regexp_chinese.groovy

diff --git a/be/src/vec/functions/function_regexp.cpp b/be/src/vec/functions/function_regexp.cpp
index ae508120cf9ee8..f740cac273c7cd 100644
--- a/be/src/vec/functions/function_regexp.cpp
+++ b/be/src/vec/functions/function_regexp.cpp
@@ -383,12 +383,13 @@ struct RegexpExtractAllImpl {
     }
 };
 
+// template FunctionRegexpFunctionality is used for regexp_xxxx series functions, not for regexp match.
 template <typename Impl>
-class FunctionRegexp : public IFunction {
+class FunctionRegexpFunctionality : public IFunction {
 public:
     static constexpr auto name = Impl::name;
 
-    static FunctionPtr create() { return std::make_shared<FunctionRegexp>(); }
+    static FunctionPtr create() { return std::make_shared<FunctionRegexpFunctionality>(); }
 
     String get_name() const override { return name; }
 
@@ -486,11 +487,11 @@ class FunctionRegexp : public IFunction {
 };
 
 void register_function_regexp_extract(SimpleFunctionFactory& factory) {
-    factory.register_function<FunctionRegexp<RegexpReplaceImpl>>();
-    factory.register_function<FunctionRegexp<RegexpExtractImpl<true>>>();
-    factory.register_function<FunctionRegexp<RegexpExtractImpl<false>>>();
-    factory.register_function<FunctionRegexp<RegexpReplaceOneImpl>>();
-    factory.register_function<FunctionRegexp<RegexpExtractAllImpl>>();
+    factory.register_function<FunctionRegexpFunctionality<RegexpReplaceImpl>>();
+    factory.register_function<FunctionRegexpFunctionality<RegexpExtractImpl<true>>>();
+    factory.register_function<FunctionRegexpFunctionality<RegexpExtractImpl<false>>>();
+    factory.register_function<FunctionRegexpFunctionality<RegexpReplaceOneImpl>>();
+    factory.register_function<FunctionRegexpFunctionality<RegexpExtractAllImpl>>();
 }
 
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/like.cpp b/be/src/vec/functions/like.cpp
index 631ebb8dbe65f2..d727ba6e850f04 100644
--- a/be/src/vec/functions/like.cpp
+++ b/be/src/vec/functions/like.cpp
@@ -486,12 +486,10 @@ Status FunctionLikeBase::hs_prepare(FunctionContext* context, const char* expres
 
     if (res != HS_SUCCESS) {
         *database = nullptr;
-        if (context) {
-            context->set_error("hs_compile regex pattern error");
-        }
-        return Status::RuntimeError("hs_compile regex pattern error:" +
-                                    std::string(compile_err->message));
+        std::string error_message = compile_err->message;
         hs_free_compile_error(compile_err);
+        // Do not call FunctionContext::set_error here, since we do not want to cancel the query here.
+        return Status::RuntimeError<false>("hs_compile regex pattern error:" + error_message);
     }
     hs_free_compile_error(compile_err);
 
@@ -499,10 +497,8 @@ Status FunctionLikeBase::hs_prepare(FunctionContext* context, const char* expres
         hs_free_database(*database);
         *database = nullptr;
         *scratch = nullptr;
-        if (context) {
-            context->set_error("hs_alloc_scratch allocate scratch space error");
-        }
-        return Status::RuntimeError("hs_alloc_scratch allocate scratch space error");
+        // Do not call FunctionContext::set_error here, since we do not want to cancel the query here.
+        return Status::RuntimeError<false>("hs_alloc_scratch allocate scratch space error");
     }
 
     return Status::OK();
@@ -934,7 +930,8 @@ Status FunctionLike::open(FunctionContext* context, FunctionContext::FunctionSta
     return Status::OK();
 }
 
-Status FunctionRegexp::open(FunctionContext* context, FunctionContext::FunctionStateScope scope) {
+Status FunctionRegexpLike::open(FunctionContext* context,
+                                FunctionContext::FunctionStateScope scope) {
     if (scope != FunctionContext::THREAD_LOCAL) {
         return Status::OK();
     }
@@ -1001,8 +998,8 @@ void register_function_like(SimpleFunctionFactory& factory) {
 }
 
 void register_function_regexp(SimpleFunctionFactory& factory) {
-    factory.register_function<FunctionRegexp>();
-    factory.register_alias(FunctionRegexp::name, FunctionRegexp::alias);
+    factory.register_function<FunctionRegexpLike>();
+    factory.register_alias(FunctionRegexpLike::name, FunctionRegexpLike::alias);
 }
 
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/like.h b/be/src/vec/functions/like.h
index 229341e7bd13d1..fed26718603575 100644
--- a/be/src/vec/functions/like.h
+++ b/be/src/vec/functions/like.h
@@ -280,12 +280,12 @@ class FunctionLike : public FunctionLikeBase {
     static void remove_escape_character(std::string* search_string);
 };
 
-class FunctionRegexp : public FunctionLikeBase {
+class FunctionRegexpLike : public FunctionLikeBase {
 public:
     static constexpr auto name = "regexp";
     static constexpr auto alias = "rlike";
 
-    static FunctionPtr create() { return std::make_shared<FunctionRegexp>(); }
+    static FunctionPtr create() { return std::make_shared<FunctionRegexpLike>(); }
 
     String get_name() const override { return name; }
 
diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_regexp_chinese.out b/regression-test/data/query_p0/sql_functions/string_functions/test_regexp_chinese.out
new file mode 100644
index 00000000000000..27bb4af4d8a44d
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/string_functions/test_regexp_chinese.out
@@ -0,0 +1,3 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql_regexp --
+
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_regexp_chinese.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_regexp_chinese.groovy
new file mode 100644
index 00000000000000..900a0a04610f9f
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_regexp_chinese.groovy
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_regexp_chinese") {
+    sql "DROP TABLE IF EXISTS regexp_test_chinese;"
+    sql """
+        CREATE TABLE regexp_test_chinese (
+            id int NULL DEFAULT "0",
+            city varchar(50) NOT NULL DEFAULT ""
+        ) DISTRIBUTED BY HASH(id) BUCKETS 5 properties("replication_num" = "1");
+    """
+
+    sql """
+        INSERT INTO regexp_test_chinese VALUES(1, "上海"),(2, "深圳"),(3, "上海测试"), (4, "北京测试");
+    """
+
+    qt_sql_regexp """
+        SELECT * FROM regexp_test_chinese WHERE city REGEXP "^上海｜^北京" ORDER BY id;
+    """
+}
+

From 7e5a380e1e9f8cb1ef552e25e293c4459e663abb Mon Sep 17 00:00:00 2001
From: zhiqiang <hezhiqiang@selectdb.com>
Date: Wed, 27 Nov 2024 10:40:18 +0800
Subject: [PATCH 026/399] [chore](regression-test) Remove some flaky profile
 test. (#42396)

As title
---
 ..._pipeline_task_serial_read_on_limit.groovy | 172 ------------------
 .../query_profile/scanner_profile.groovy      | 108 -----------
 .../test_execute_by_frontend.groovy           |  90 ---------
 3 files changed, 370 deletions(-)
 delete mode 100644 regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy
 delete mode 100644 regression-test/suites/query_profile/scanner_profile.groovy
 delete mode 100644 regression-test/suites/query_profile/test_execute_by_frontend.groovy

diff --git a/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy b/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy
deleted file mode 100644
index 87279f65157409..00000000000000
--- a/regression-test/suites/query_profile/adaptive_pipeline_task_serial_read_on_limit.groovy
+++ /dev/null
@@ -1,172 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import groovy.json.JsonOutput
-import groovy.json.JsonSlurper
-import groovy.json.StringEscapeUtils
-
-
-def getProfileList = {
-    def dst = 'http://' + context.config.feHttpAddress
-    def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
-    conn.setRequestMethod("GET")
-    def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + 
-            (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
-    conn.setRequestProperty("Authorization", "Basic ${encoding}")
-    return conn.getInputStream().getText()
-}
-
-
-def getProfile = { id ->
-        def dst = 'http://' + context.config.feHttpAddress
-        def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection()
-        conn.setRequestMethod("GET")
-        def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + 
-                (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
-        conn.setRequestProperty("Authorization", "Basic ${encoding}")
-        return conn.getInputStream().getText()
-}
-
-suite('adaptive_pipeline_task_serial_read_on_limit') {
-    sql """
-        DROP TABLE IF EXISTS adaptive_pipeline_task_serial_read_on_limit;
-    """
-    sql """
-        CREATE TABLE if not exists `adaptive_pipeline_task_serial_read_on_limit` (
-            `id` INT,
-            `name` varchar(32)
-        ) ENGINE=OLAP
-        DISTRIBUTED BY HASH(`id`) BUCKETS 5
-        PROPERTIES (
-            "replication_allocation" = "tag.location.default: 1"
-        );
-    """
-
-    // Insert data to table
-    sql """
-        insert into adaptive_pipeline_task_serial_read_on_limit values 
-        (1, "A"),(2, "B"),(3, "C"),(4, "D"),(5,"E"),(6,"F"),(7,"G"),(8,"H"),(9,"K");
-    """
-    sql """
-        insert into adaptive_pipeline_task_serial_read_on_limit values 
-        (10, "A"),(20, "B"),(30, "C"),(40, "D"),(50,"E"),(60,"F"),(70,"G"),(80,"H"),(90,"K");
-    """
-    sql """
-        insert into adaptive_pipeline_task_serial_read_on_limit values 
-        (101, "A"),(201, "B"),(301, "C"),(401, "D"),(501,"E"),(601,"F"),(701,"G"),(801,"H"),(901,"K");
-    """
-    sql """
-        insert into adaptive_pipeline_task_serial_read_on_limit values 
-        (1010, "A"),(2010, "B"),(3010, "C"),(4010, "D"),(5010,"E"),(6010,"F"),(7010,"G"),(8010,"H"),(9010,"K");
-    """
-    
-    def uuidString = UUID.randomUUID().toString()
-    sql "set enable_profile=true"
-    // set parallel_pipeline_task_num to 1 so that only one scan node,
-    // and we can check MaxScannerThreadNum in profile.
-    sql "set parallel_pipeline_task_num=1;"
-    // no limit, MaxScannerThreadNum = TabletNum
-    sql """
-        select "no_limit_1_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit;
-    """
-    sql "set parallel_pipeline_task_num=0;"
-    // With Limit, MaxScannerThreadNum = 1
-    sql """
-        select "with_limit_1_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit limit 10000;
-    """
-    // With Limit, but bigger then adaptive_pipeline_task_serial_read_on_limit,  MaxScannerThreadNum = TabletNum
-    sql """
-        select "with_limit_2_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit limit 10001;
-    """
-    sql """
-        set enable_adaptive_pipeline_task_serial_read_on_limit=false;
-    """
-    sql "set parallel_pipeline_task_num=1;"
-    // Forbid the strategy, with limit, MaxScannerThreadNum = TabletNum
-    sql """
-        select "not_enable_limit_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit limit 100;
-    """
-
-    sql "set parallel_pipeline_task_num=0;"
-
-    // Enable the strategy, with limit 20, MaxScannerThreadNum = 1
-    sql """
-        set enable_adaptive_pipeline_task_serial_read_on_limit=true;
-    """
-    sql """
-        set adaptive_pipeline_task_serial_read_on_limit=20;
-    """
-    sql """
-        select "modify_to_20_${uuidString}", * from adaptive_pipeline_task_serial_read_on_limit limit 15;
-    """
-
-    sql "set enable_profile=false"
-
-    Thread.sleep(5)
-
-    def wholeString = getProfileList()
-    List profileData = new JsonSlurper().parseText(wholeString).data.rows
-    String queryIdNoLimit1 = "";
-    String queryIdWithLimit1 = "";
-    String queryIdWithLimit2 = "";
-    String queryIDNotEnableLimit = "";
-    String queryIdModifyTo20 = "";
-
-    logger.info("{}", uuidString)
-
-    for (def profileItem in profileData) {
-        if (profileItem["Sql Statement"].toString().contains("no_limit_1_${uuidString}")) {
-            queryIdNoLimit1 = profileItem["Profile ID"].toString()
-            logger.info("profileItem: {}", profileItem)
-        }
-        if (profileItem["Sql Statement"].toString().contains("with_limit_1_${uuidString}")) {
-            queryIdWithLimit1 = profileItem["Profile ID"].toString()
-            logger.info("profileItem: {}", profileItem)
-        }
-        if (profileItem["Sql Statement"].toString().contains("with_limit_2_${uuidString}")) {
-            queryIdWithLimit2 = profileItem["Profile ID"].toString()
-            logger.info("profileItem: {}", profileItem)
-        }
-        if (profileItem["Sql Statement"].toString().contains("not_enable_limit_${uuidString}")) {
-            queryIDNotEnableLimit = profileItem["Profile ID"].toString()
-            logger.info("profileItem: {}", profileItem)
-        }
-        if (profileItem["Sql Statement"].toString().contains("modify_to_20_${uuidString}")) {
-            queryIdModifyTo20 = profileItem["Profile ID"].toString()
-            logger.info("profileItem: {}", profileItem)
-        }
-    }
-    
-    logger.info("queryIdWithLimit1_${uuidString}: {}", queryIdWithLimit1)
-    logger.info("queryIdModifyTo20_${uuidString}: {}", queryIdModifyTo20)
-
-    assertTrue(queryIdWithLimit1 != "")
-    assertTrue(queryIdModifyTo20 != "")
-
-    def String profileWithLimit1 = getProfile(queryIdWithLimit1).toString()
-    def String profileModifyTo20 = getProfile(queryIdModifyTo20).toString()
-    
-    if (!profileWithLimit1.contains("- MaxScannerThreadNum: 1")) {
-        logger.info("profileWithLimit1:\n{}", profileWithLimit1)
-    }
-    assertTrue(profileWithLimit1.contains("- MaxScannerThreadNum: 1"))
-
-    if (!profileModifyTo20.contains("- MaxScannerThreadNum: 1")) {
-        logger.info("profileModifyTo20:\n{}", profileModifyTo20)
-    }
-    assertTrue(profileModifyTo20.contains("- MaxScannerThreadNum: 1"))
-}
\ No newline at end of file
diff --git a/regression-test/suites/query_profile/scanner_profile.groovy b/regression-test/suites/query_profile/scanner_profile.groovy
deleted file mode 100644
index 75ae6a5ab65a9b..00000000000000
--- a/regression-test/suites/query_profile/scanner_profile.groovy
+++ /dev/null
@@ -1,108 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import groovy.json.JsonOutput
-import groovy.json.JsonSlurper
-import groovy.json.StringEscapeUtils
-
-
-def getProfileList = {
-    def dst = 'http://' + context.config.feHttpAddress
-    def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
-    conn.setRequestMethod("GET")
-    def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + 
-            (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
-    conn.setRequestProperty("Authorization", "Basic ${encoding}")
-    return conn.getInputStream().getText()
-}
-
-
-def getProfile = { id ->
-        def dst = 'http://' + context.config.feHttpAddress
-        def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection()
-        conn.setRequestMethod("GET")
-        def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + 
-                (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
-        conn.setRequestProperty("Authorization", "Basic ${encoding}")
-        return conn.getInputStream().getText()
-}
-
-suite('scanner_profile') {
-    sql """
-        DROP TABLE IF EXISTS scanner_profile;
-    """
-    sql """
-        CREATE TABLE if not exists `scanner_profile` (
-            `id` INT,
-            `name` varchar(32)
-        ) ENGINE=OLAP
-        DISTRIBUTED BY HASH(`id`) BUCKETS 10
-        PROPERTIES (
-            "replication_allocation" = "tag.location.default: 1"
-        );
-    """
-
-    // Insert data to table
-    sql """
-        insert into scanner_profile values 
-        (1, "A"),(2, "B"),(3, "C"),(4, "D"),(5,"E"),(6,"F"),(7,"G"),(8,"H"),(9,"K");
-    """
-    sql """
-        insert into scanner_profile values 
-        (10, "A"),(20, "B"),(30, "C"),(40, "D"),(50,"E"),(60,"F"),(70,"G"),(80,"H"),(90,"K");
-    """
-    sql """
-        insert into scanner_profile values 
-        (101, "A"),(201, "B"),(301, "C"),(401, "D"),(501,"E"),(601,"F"),(701,"G"),(801,"H"),(901,"K");
-    """
-    sql """
-        insert into scanner_profile values 
-        (1010, "A"),(2010, "B"),(3010, "C"),(4010, "D"),(5010,"E"),(6010,"F"),(7010,"G"),(8010,"H"),(9010,"K");
-    """
-
-    def uuidString = UUID.randomUUID().toString()
-    sql "set enable_profile=true"
-    // With Limit, MaxScannerThreadNum = 1
-    sql """
-        select "with_limit_1_${uuidString}", * from scanner_profile limit 10;
-    """
-    
-    def wholeString = getProfileList()
-    List profileData = new JsonSlurper().parseText(wholeString).data.rows
-    String queryIdWithLimit1 = "";
-    
-
-    logger.info("{}", uuidString)
-
-    for (def profileItem in profileData) {
-        if (profileItem["Sql Statement"].toString().contains("with_limit_1_${uuidString}")) {
-            queryIdWithLimit1 = profileItem["Profile ID"].toString()
-            logger.info("profileItem: {}", profileItem)
-        }
-    }
-
-    logger.info("queryIdWithLimit1_${uuidString}: {}", queryIdWithLimit1)
-
-    assertTrue(queryIdWithLimit1 != "")
-
-    // Sleep 5 seconds to make sure profile collection is done
-    Thread.sleep(5000)
-
-    def String profileWithLimit1 = getProfile(queryIdWithLimit1).toString()
-    logger.info("query profile {}", profileWithLimit1)
-    assertTrue(profileWithLimit1.contains("- PeakRunningScanner: 1"))
-}
\ No newline at end of file
diff --git a/regression-test/suites/query_profile/test_execute_by_frontend.groovy b/regression-test/suites/query_profile/test_execute_by_frontend.groovy
deleted file mode 100644
index 40435e02d99870..00000000000000
--- a/regression-test/suites/query_profile/test_execute_by_frontend.groovy
+++ /dev/null
@@ -1,90 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import groovy.json.JsonOutput
-import groovy.json.JsonSlurper
-import groovy.json.StringEscapeUtils
-
-def getProfileList = {
-    def dst = 'http://' + context.config.feHttpAddress
-    def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
-    conn.setRequestMethod("GET")
-    def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + 
-            (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
-    conn.setRequestProperty("Authorization", "Basic ${encoding}")
-    return conn.getInputStream().getText()
-}
-
-def getProfile = { id ->
-        def dst = 'http://' + context.config.feHttpAddress
-        def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection()
-        conn.setRequestMethod("GET")
-        def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + 
-                (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
-        conn.setRequestProperty("Authorization", "Basic ${encoding}")
-        // set conn parameters
-        
-        return conn.getInputStream().getText()
-    }
-
-suite('test_execute_by_frontend') {
-    sql """
-        CREATE TABLE if not exists `test_execute_by_frontend` (
-          `id` INT,
-          `name` varchar(32)
-        )ENGINE=OLAP
-        UNIQUE KEY(`id`)
-        DISTRIBUTED BY HASH(`id`) BUCKETS 1
-        PROPERTIES (
-            "replication_allocation" = "tag.location.default: 1"
-        );
-    """
-
-    sql "set enable_profile=true"
-    def simpleSql1 = "select * from test_execute_by_frontend"
-    sql "${simpleSql1}"
-    simpleSql2 = """select  cast("1"  as  Int)"""
-    sql "${simpleSql2}"
-    def isRecorded = false
-    def wholeString = getProfileList()
-    List profileData = new JsonSlurper().parseText(wholeString).data.rows
-    String queryId1 = "";
-    String queryId2 = "";
-
-    for (final def profileItem in profileData) {
-        if (profileItem["Sql Statement"].toString() == simpleSql1) {
-            isRecorded = true
-            queryId1 = profileItem["Profile ID"].toString()
-            assertEquals("internal", profileItem["Default Catalog"].toString())
-        }
-        if (profileItem["Sql Statement"].toString() == simpleSql2) {
-            queryId2 = profileItem["Profile ID"].toString()
-        }
-    }
-
-    assertTrue(isRecorded)
-
-    String profileContent1 = getProfile(queryId1)
-    def executionProfileIdx1 = profileContent1.indexOf("Executed By Frontend: true")
-    assertTrue(executionProfileIdx1 > 0)
-    String profileContent2 = getProfile(queryId2)
-    def executionProfileIdx2 = profileContent2.indexOf("Executed By Frontend: true")
-    assertTrue(executionProfileIdx2 > 0)
-
-    sql """ SET enable_profile = false """
-    sql """ DROP TABLE IF EXISTS test_execute_by_frontend """
-}
\ No newline at end of file

From db05c2860aa81ab1ee8f08f838859b73d00213f7 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Wed, 27 Nov 2024 10:43:41 +0800
Subject: [PATCH 027/399] [chore](github)Update .asf.yaml (#44651)

---
 .asf.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.asf.yaml b/.asf.yaml
index f4c21ce4b0f3c9..7a7d845e4c9bb0 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -125,7 +125,6 @@ github:
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
           - Build Broker
-          - ShellCheck
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - COMPILE (DORIS_COMPILE)

From 23df40b73a10078f4fa5cb5bfc8d66763b514451 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Wed, 27 Nov 2024 10:44:07 +0800
Subject: [PATCH 028/399] [fix](memory) Fix compatibility with CgroupV2
 (#44579)

### What problem does this PR solve?

Problem Summary:

1. Fix `memory.max` value is `max`.
2. Fix `memory_usage` minus `slab_reclaimable`, part of "slab" that
might be reclaimed.

Note, CgroupV2 MemAvailable is usually a little smaller than Process
MemAvailable.
Process `MemAvailable = MemFree - LowWaterMark + (PageCache -
min(PageCache / 2, LowWaterMark))`, from `MemAvailable` in
`/proc/meminfo`, calculated by OS.
CgroupV2 `MemAvailable = cgroup_mem_limit - cgroup_mem_usage`,
`cgroup_mem_usage = memory.current - inactive_file - slab_reclaimable`,
in fact, there seems to be some memory that can be reused in
`cgroup_mem_usage`.

ref: https://arthurchiao.art/blog/cgroupv2-zh/
---
 be/src/common/cgroup_memory_ctl.cpp | 23 +++++++++++++++++++----
 be/src/util/mem_info.cpp            | 18 +++++++++++++++---
 2 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/be/src/common/cgroup_memory_ctl.cpp b/be/src/common/cgroup_memory_ctl.cpp
index a29432bdb4ede5..dddcbd50338d82 100644
--- a/be/src/common/cgroup_memory_ctl.cpp
+++ b/be/src/common/cgroup_memory_ctl.cpp
@@ -27,6 +27,7 @@
 
 #include "common/status.h"
 #include "util/cgroup_util.h"
+#include "util/error_util.h"
 
 namespace doris {
 
@@ -84,14 +85,23 @@ struct CgroupsV2Reader : CGroupMemoryCtl::ICgroupsReader {
             : _mount_file_dir(std::move(mount_file_dir)) {}
 
     Status read_memory_limit(int64_t* value) override {
-        RETURN_IF_ERROR(CGroupUtil::read_int_line_from_cgroup_file((_mount_file_dir / "memory.max"),
-                                                                   value));
+        std::filesystem::path file_path = _mount_file_dir / "memory.max";
+        std::string line;
+        std::ifstream file_stream(file_path, std::ios::in);
+        getline(file_stream, line);
+        if (file_stream.fail() || file_stream.bad()) {
+            return Status::CgroupError("Error reading {}: {}", file_path.string(),
+                                       get_str_err_msg());
+        }
+        if (line == "max") {
+            *value = std::numeric_limits<int64_t>::max();
+            return Status::OK();
+        }
+        RETURN_IF_ERROR(CGroupUtil::read_int_line_from_cgroup_file(file_path, value));
         return Status::OK();
     }
 
     Status read_memory_usage(int64_t* value) override {
-        // memory.current contains a single number
-        // the reason why we subtract it described here: https://github.com/ClickHouse/ClickHouse/issues/64652#issuecomment-2149630667
         RETURN_IF_ERROR(CGroupUtil::read_int_line_from_cgroup_file(
                 (_mount_file_dir / "memory.current"), value));
         std::unordered_map<std::string, int64_t> metrics_map;
@@ -100,7 +110,12 @@ struct CgroupsV2Reader : CGroupMemoryCtl::ICgroupsReader {
         if (*value < metrics_map["inactive_file"]) {
             return Status::CgroupError("CgroupsV2Reader read_memory_usage negative memory usage");
         }
+        // the reason why we subtract inactive_file described here:
+        // https://github.com/ClickHouse/ClickHouse/issues/64652#issuecomment-2149630667
         *value -= metrics_map["inactive_file"];
+        // Part of "slab" that might be reclaimed, such as dentries and inodes.
+        // https://arthurchiao.art/blog/cgroupv2-zh/
+        *value -= metrics_map["slab_reclaimable"];
         return Status::OK();
     }
 
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index 36579452db3f85..fe9cf84b2aed54 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -197,9 +197,10 @@ void MemInfo::refresh_proc_meminfo() {
                 _s_cgroup_mem_limit = std::numeric_limits<int64_t>::max();
                 // find cgroup limit failed, wait 300s, 1000 * 100ms.
                 _s_cgroup_mem_refresh_wait_times = -3000;
-                LOG(INFO) << "Refresh cgroup memory limit failed, refresh again after 300s, cgroup "
-                             "mem limit: "
-                          << _s_cgroup_mem_limit;
+                LOG(WARNING)
+                        << "Refresh cgroup memory limit failed, refresh again after 300s, cgroup "
+                           "mem limit: "
+                        << _s_cgroup_mem_limit << ", " << status;
             } else {
                 _s_cgroup_mem_limit = cgroup_mem_limit;
                 // wait 10s, 100 * 100ms, avoid too frequently.
@@ -209,12 +210,17 @@ void MemInfo::refresh_proc_meminfo() {
             _s_cgroup_mem_refresh_wait_times++;
         }
 
+        // cgroup mem limit is refreshed every 10 seconds,
+        // cgroup mem usage is refreshed together with memInfo every time, which is very frequent.
         if (_s_cgroup_mem_limit != std::numeric_limits<int64_t>::max()) {
             int64_t cgroup_mem_usage;
             auto status = CGroupMemoryCtl::find_cgroup_mem_usage(&cgroup_mem_usage);
             if (!status.ok()) {
                 _s_cgroup_mem_usage = std::numeric_limits<int64_t>::min();
                 _s_cgroup_mem_refresh_state = false;
+                LOG_EVERY_N(WARNING, 500)
+                        << "Refresh cgroup memory usage failed, cgroup mem limit: "
+                        << _s_cgroup_mem_limit << ", " << status;
             } else {
                 _s_cgroup_mem_usage = cgroup_mem_usage;
                 _s_cgroup_mem_refresh_state = true;
@@ -279,6 +285,12 @@ void MemInfo::refresh_proc_meminfo() {
         mem_available = _mem_info_bytes["MemAvailable"];
     }
     if (_s_cgroup_mem_refresh_state) {
+        // Note, CgroupV2 MemAvailable is usually a little smaller than Process MemAvailable.
+        // Process `MemAvailable = MemFree - LowWaterMark + (PageCache - min(PageCache / 2, LowWaterMark))`,
+        // from `MemAvailable` in `/proc/meminfo`, calculated by OS.
+        // CgroupV2 `MemAvailable = cgroup_mem_limit - cgroup_mem_usage`,
+        // `cgroup_mem_usage = memory.current - inactive_file - slab_reclaimable`, in fact,
+        // there seems to be some memory that can be reused in `cgroup_mem_usage`.
         if (mem_available < 0) {
             mem_available = _s_cgroup_mem_limit - _s_cgroup_mem_usage;
         } else {

From 6d57e3866c538801355eea26a7259db58b464699 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Wed, 27 Nov 2024 10:51:35 +0800
Subject: [PATCH 029/399] [opt](nereids) optimize push limit to agg (#44042)

### What problem does this PR solve?
Pr #34853 introduced PushTopnToAgg rule.
But there is a limitation that Topn(limit) should output all group by
keys.
This pr removes this limitation by using the first group by key as order
key.
---
 .../rules/rewrite/LimitAggToTopNAgg.java      | 61 ++++++++++--
 .../plans/physical/PhysicalHashAggregate.java |  4 +-
 .../rules/rewrite/EliminateSortTest.java      |  7 +-
 .../nereids_hint_tpcds_p0/shape/query23.out   | 63 ++++++------
 .../nereids_hint_tpcds_p0/shape/query32.out   | 43 ++++----
 .../nereids_hint_tpcds_p0/shape/query38.out   | 91 ++++++++---------
 .../nereids_hint_tpcds_p0/shape/query97.out   | 61 ++++++------
 .../shape/query23.out                         | 63 ++++++------
 .../shape/query32.out                         | 43 ++++----
 .../shape/query38.out                         | 91 ++++++++---------
 .../shape/query97.out                         | 61 ++++++------
 .../constraints/query23.out                   | 63 ++++++------
 .../noStatsRfPrune/query23.out                | 63 ++++++------
 .../noStatsRfPrune/query32.out                | 43 ++++----
 .../noStatsRfPrune/query38.out                | 91 ++++++++---------
 .../noStatsRfPrune/query97.out                | 57 +++++------
 .../no_stats_shape/query23.out                | 63 ++++++------
 .../no_stats_shape/query32.out                | 43 ++++----
 .../no_stats_shape/query38.out                | 91 ++++++++---------
 .../no_stats_shape/query97.out                | 57 +++++------
 .../rf_prune/query23.out                      | 63 ++++++------
 .../rf_prune/query32.out                      | 43 ++++----
 .../rf_prune/query38.out                      | 91 ++++++++---------
 .../rf_prune/query97.out                      | 57 +++++------
 .../shape/query23.out                         | 63 ++++++------
 .../shape/query32.out                         | 43 ++++----
 .../shape/query38.out                         | 91 ++++++++---------
 .../shape/query97.out                         | 57 +++++------
 .../shape/query23.out                         | 63 ++++++------
 .../shape/query32.out                         | 43 ++++----
 .../shape/query38.out                         | 97 ++++++++++---------
 .../shape/query97.out                         | 57 +++++------
 .../tpcds_sf100/constraints/query23.out       | 63 ++++++------
 .../tpcds_sf100/noStatsRfPrune/query23.out    | 63 ++++++------
 .../tpcds_sf100/noStatsRfPrune/query32.out    | 43 ++++----
 .../tpcds_sf100/noStatsRfPrune/query38.out    | 91 ++++++++---------
 .../tpcds_sf100/noStatsRfPrune/query97.out    | 57 +++++------
 .../tpcds_sf100/no_stats_shape/query23.out    | 63 ++++++------
 .../tpcds_sf100/no_stats_shape/query32.out    | 43 ++++----
 .../tpcds_sf100/no_stats_shape/query38.out    | 91 ++++++++---------
 .../tpcds_sf100/no_stats_shape/query97.out    | 57 +++++------
 .../tpcds_sf100/rf_prune/query23.out          | 63 ++++++------
 .../tpcds_sf100/rf_prune/query32.out          | 43 ++++----
 .../tpcds_sf100/rf_prune/query38.out          | 91 ++++++++---------
 .../tpcds_sf100/rf_prune/query97.out          | 57 +++++------
 .../tpcds_sf100/shape/query23.out             | 63 ++++++------
 .../tpcds_sf100/shape/query32.out             | 43 ++++----
 .../tpcds_sf100/shape/query38.out             | 91 ++++++++---------
 .../tpcds_sf100/shape/query97.out             | 57 +++++------
 .../tpcds_sf1000/shape/query23.out            | 63 ++++++------
 .../tpcds_sf1000/shape/query32.out            | 43 ++++----
 .../tpcds_sf1000/shape/query38.out            | 91 ++++++++---------
 .../tpcds_sf1000/shape/query97.out            | 61 ++++++------
 .../tpch/push_topn_to_agg.groovy              |  4 +-
 54 files changed, 1679 insertions(+), 1589 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java
index 040bd9c10f3f3e..dfa1230a8f8f0e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java
@@ -21,6 +21,8 @@
 import org.apache.doris.nereids.rules.Rule;
 import org.apache.doris.nereids.rules.RuleType;
 import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
@@ -32,6 +34,7 @@
 import com.google.common.collect.Lists;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
@@ -53,7 +56,11 @@ public List<Rule> buildRules() {
                                 >= limit.getLimit() + limit.getOffset())
                         .then(limit -> {
                             LogicalAggregate<? extends Plan> agg = limit.child();
-                            List<OrderKey> orderKeys = generateOrderKeyByGroupKey(agg);
+                            Optional<OrderKey> orderKeysOpt = tryGenerateOrderKeyByTheFirstGroupKey(agg);
+                            if (!orderKeysOpt.isPresent()) {
+                                return null;
+                            }
+                            List<OrderKey> orderKeys = Lists.newArrayList(orderKeysOpt.get());
                             return new LogicalTopN<>(orderKeys, limit.getLimit(), limit.getOffset(), agg);
                         }).toRule(RuleType.LIMIT_AGG_TO_TOPN_AGG),
                 //limit->project->agg to topn->project->agg
@@ -62,12 +69,47 @@ public List<Rule> buildRules() {
                                 && ConnectContext.get().getSessionVariable().pushTopnToAgg
                                 && ConnectContext.get().getSessionVariable().topnOptLimitThreshold
                                 >= limit.getLimit() + limit.getOffset())
-                        .when(limit -> outputAllGroupKeys(limit, limit.child().child()))
                         .then(limit -> {
                             LogicalProject<? extends Plan> project = limit.child();
-                            LogicalAggregate<? extends Plan> agg = (LogicalAggregate<? extends Plan>) project.child();
-                            List<OrderKey> orderKeys = generateOrderKeyByGroupKey(agg);
-                            return new LogicalTopN<>(orderKeys, limit.getLimit(), limit.getOffset(), project);
+                            LogicalAggregate<? extends Plan> agg
+                                    = (LogicalAggregate<? extends Plan>) project.child();
+                            Optional<OrderKey> orderKeysOpt = tryGenerateOrderKeyByTheFirstGroupKey(agg);
+                            if (!orderKeysOpt.isPresent()) {
+                                return null;
+                            }
+                            List<OrderKey> orderKeys = Lists.newArrayList(orderKeysOpt.get());
+                            Plan result;
+
+                            if (outputAllGroupKeys(limit, agg)) {
+                                result = new LogicalTopN<>(orderKeys, limit.getLimit(),
+                                        limit.getOffset(), project);
+                            } else {
+                                // add the first group by key to topn, and prune this key by upper project
+                                // topn order keys are prefix of group by keys
+                                // refer to PushTopnToAgg.tryGenerateOrderKeyByGroupKeyAndTopnKey()
+                                Expression firstGroupByKey = agg.getGroupByExpressions().get(0);
+                                if (!(firstGroupByKey instanceof SlotReference)) {
+                                    return null;
+                                }
+                                boolean shouldPruneFirstGroupByKey = true;
+                                if (project.getOutputs().contains(firstGroupByKey)) {
+                                    shouldPruneFirstGroupByKey = false;
+                                } else {
+                                    List<NamedExpression> bottomProjections = Lists.newArrayList(project.getProjects());
+                                    bottomProjections.add((SlotReference) firstGroupByKey);
+                                    project = project.withProjects(bottomProjections);
+                                }
+                                LogicalTopN topn = new LogicalTopN<>(orderKeys, limit.getLimit(),
+                                        limit.getOffset(), project);
+                                if (shouldPruneFirstGroupByKey) {
+                                    List<NamedExpression> limitOutput = limit.getOutput().stream()
+                                            .map(e -> (NamedExpression) e).collect(Collectors.toList());
+                                    result = new LogicalProject<>(limitOutput, topn);
+                                } else {
+                                    result = topn;
+                                }
+                            }
+                            return result;
                         }).toRule(RuleType.LIMIT_AGG_TO_TOPN_AGG),
                 // topn -> agg: add all group key to sort key, if sort key is prefix of group key
                 logicalTopN(logicalAggregate())
@@ -111,9 +153,10 @@ private boolean outputAllGroupKeys(LogicalLimit limit, LogicalAggregate agg) {
         return limit.getOutputSet().containsAll(agg.getGroupByExpressions());
     }
 
-    private List<OrderKey> generateOrderKeyByGroupKey(LogicalAggregate<? extends Plan> agg) {
-        return agg.getGroupByExpressions().stream()
-                .map(key -> new OrderKey(key, true, false))
-                .collect(Collectors.toList());
+    private Optional<OrderKey> tryGenerateOrderKeyByTheFirstGroupKey(LogicalAggregate<? extends Plan> agg) {
+        if (agg.getGroupByExpressions().isEmpty()) {
+            return Optional.empty();
+        }
+        return Optional.of(new OrderKey(agg.getGroupByExpressions().get(0), true, false));
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
index 2a78b063a973f3..11baf2137ab5d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
@@ -206,8 +206,8 @@ public String toString() {
                 "groupByExpr", groupByExpressions,
                 "outputExpr", outputExpressions,
                 "partitionExpr", partitionExpressions,
-                "requireProperties", requireProperties,
-                "topnOpt", topnPushInfo != null
+                "topnFilter", topnPushInfo != null,
+                "topnPushDown", getMutableState(MutableState.KEY_PUSH_TOPN_TO_AGG).isPresent()
         );
     }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateSortTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateSortTest.java
index ac1ffa97ef9e7d..cef9bac4ed90a7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateSortTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateSortTest.java
@@ -165,8 +165,7 @@ void testSortLimit() {
         PlanChecker.from(connectContext).disableNereidsRules("PRUNE_EMPTY_PARTITION")
                 .analyze("select count(*) from (select * from student order by id) t limit 1")
                 .rewrite()
-                // there is no topn below agg
-                .matches(logicalTopN(logicalAggregate(logicalProject(logicalOlapScan()))));
+                .nonMatch(logicalTopN());
         PlanChecker.from(connectContext)
                 .disableNereidsRules("PRUNE_EMPTY_PARTITION")
                 .analyze("select count(*) from (select * from student order by id limit 1) t")
@@ -184,8 +183,6 @@ void testSortLimit() {
                 .analyze("select count(*) from "
                         + "(select * from student order by id) t1 left join student t2 on t1.id = t2.id limit 1")
                 .rewrite()
-                .matches(logicalTopN(logicalAggregate(logicalProject(logicalJoin(
-                        logicalProject(logicalOlapScan()),
-                        logicalProject(logicalOlapScan()))))));
+                .nonMatch(logicalTopN());
     }
 }
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query23.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query23.out
index 5e2eafb60ffe8d..45d7a47eb7e46a 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query23.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
 
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query32.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query32.out
index 152d2f05e77185..52b5030de53cbc 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query32.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query32.out
@@ -1,27 +1,28 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 722))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '2001-06-07') and (date_dim.d_date >= '2001-03-09'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 722))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '2001-06-07') and (date_dim.d_date >= '2001-03-09'))
+------------------------------PhysicalOlapScan[date_dim]
 
 Hint log:
 Used: leading(catalog_sales item date_dim )
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query38.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query38.out
index 5ae805bdffe325..dc794c95f97b2f 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query38.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query97.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query97.out
index ffa14a9727df17..66b55c629d658a 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query97.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query97.out
@@ -1,36 +1,37 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------filter(( not ss_sold_date_sk IS NULL))
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------filter(( not cs_sold_date_sk IS NULL))
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------filter(( not ss_sold_date_sk IS NULL))
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------filter(( not cs_sold_date_sk IS NULL))
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
+--------------------------------PhysicalOlapScan[date_dim]
 
 Hint log:
 Used: leading(store_sales date_dim ) leading(ssci csci )
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out
index 5e2eafb60ffe8d..45d7a47eb7e46a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query32.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query32.out
index 0389bfeea82673..c413940debc6e7 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query32.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 722))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '2001-06-07') and (date_dim.d_date >= '2001-03-09'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 722))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '2001-06-07') and (date_dim.d_date >= '2001-03-09'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query38.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query38.out
index 5ae805bdffe325..dc794c95f97b2f 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query38.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query97.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query97.out
index ca6f63d5e6bdf0..d3a845763241f7 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query97.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query97.out
@@ -1,34 +1,35 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------filter(( not ss_sold_date_sk IS NULL))
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------filter(( not cs_sold_date_sk IS NULL))
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------filter(( not ss_sold_date_sk IS NULL))
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------filter(( not cs_sold_date_sk IS NULL))
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/constraints/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/constraints/query23.out
index 85217462d1a5ed..923ee0e2966746 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/constraints/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/constraints/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 item_sk->[cs_item_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4 RF5
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalCteConsumer ( cteId=CTEId#0 )
-----------------PhysicalProject
-------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 item_sk->[ws_item_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7 RF8
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 item_sk->[cs_item_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4 RF5
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------------PhysicalProject
+--------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 item_sk->[ws_item_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7 RF8
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out
index aa0bb3fa70d0e7..c5d202bfee2bc3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=()
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5
-----------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF7 ws_item_sk->[item_sk]
-------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=()
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8
-----------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=()
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5
+------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+--------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF7 ws_item_sk->[item_sk]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8
+------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query32.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query32.out
index 15393efb582c73..7992f57d1c87b3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query32.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 29))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 29))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query38.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query38.out
index 304fa6dd35c5f6..b0bda7f927c138 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query38.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query97.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query97.out
index 404a9e7e9e92a1..4ebfd5abc0eb1c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query97.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query97.out
@@ -1,32 +1,33 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out
index 26e1f5089288a1..a65c74fa0b81dd 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF4 item_sk->[cs_item_sk]
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4 RF5
-----------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF7 ws_item_sk->[item_sk]
-------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[ws_bill_customer_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF8
-----------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF4 item_sk->[cs_item_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4 RF5
+------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+--------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF7 ws_item_sk->[item_sk]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[ws_bill_customer_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF8
+------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query32.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query32.out
index 15393efb582c73..7992f57d1c87b3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query32.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 29))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 29))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query38.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query38.out
index fbc5dd43162046..2bc9d9fecbca40 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query38.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query97.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query97.out
index 404a9e7e9e92a1..4ebfd5abc0eb1c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query97.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query97.out
@@ -1,32 +1,33 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out
index 1343c4f4fb0214..6103f4eb74d6ab 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query32.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query32.out
index 15393efb582c73..7992f57d1c87b3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query32.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 29))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 29))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query38.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query38.out
index 802779104ed84f..26d52e9fae9c40 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query38.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query97.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query97.out
index 404a9e7e9e92a1..4ebfd5abc0eb1c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query97.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query97.out
@@ -1,32 +1,33 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out
index 91ab82284bc473..0d8b21c19639e9 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query32.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query32.out
index 15393efb582c73..7992f57d1c87b3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query32.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 29))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 29))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query38.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query38.out
index 9a6c130acd1eaf..ef2051eb2d000c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query38.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query97.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query97.out
index 404a9e7e9e92a1..4ebfd5abc0eb1c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query97.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query97.out
@@ -1,32 +1,33 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query23.out
index 5b81b4629a48e0..eb873d6b08706a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query23.out
@@ -48,35 +48,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF7 d_date_sk->[cs_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6 RF7
-----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ws_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF9 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 item_sk->[ws_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8 RF9 RF10
-----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF7 d_date_sk->[cs_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6 RF7
+------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ws_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF9 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 item_sk->[ws_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8 RF9 RF10
+------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query32.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query32.out
index a9acbffc271bf6..0a251a341c24df 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query32.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 66))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '2002-06-27') and (date_dim.d_date >= '2002-03-29'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 66))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '2002-06-27') and (date_dim.d_date >= '2002-03-29'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query38.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query38.out
index 08d22c95f4ff40..ce87ec57d57863 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query38.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query38.out
@@ -1,52 +1,53 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------hashAgg[GLOBAL]
-----------------PhysicalDistribute[DistributionSpecHash]
-------------------hashAgg[LOCAL]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF0 c_customer_sk->[ws_bill_customer_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[customer]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_month_seq <= 1197) and (date_dim.d_month_seq >= 1186))
-----------------------------PhysicalOlapScan[date_dim]
---------------hashAgg[GLOBAL]
-----------------PhysicalDistribute[DistributionSpecHash]
-------------------hashAgg[LOCAL]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[customer]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_month_seq <= 1197) and (date_dim.d_month_seq >= 1186))
-----------------------------PhysicalOlapScan[date_dim]
---------------hashAgg[GLOBAL]
-----------------PhysicalDistribute[DistributionSpecHash]
-------------------hashAgg[LOCAL]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ss_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[customer]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_month_seq <= 1197) and (date_dim.d_month_seq >= 1186))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------hashAgg[GLOBAL]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------hashAgg[LOCAL]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF0 c_customer_sk->[ws_bill_customer_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[customer]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_month_seq <= 1197) and (date_dim.d_month_seq >= 1186))
+------------------------------PhysicalOlapScan[date_dim]
+----------------hashAgg[GLOBAL]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------hashAgg[LOCAL]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[customer]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_month_seq <= 1197) and (date_dim.d_month_seq >= 1186))
+------------------------------PhysicalOlapScan[date_dim]
+----------------hashAgg[GLOBAL]
+------------------PhysicalDistribute[DistributionSpecHash]
+--------------------hashAgg[LOCAL]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ss_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[customer]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_month_seq <= 1197) and (date_dim.d_month_seq >= 1186))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query97.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query97.out
index 199f8d20f62131..0960d491884815 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query97.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query97.out
@@ -1,32 +1,33 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1201) and (date_dim.d_month_seq >= 1190))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1201) and (date_dim.d_month_seq >= 1190))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1201) and (date_dim.d_month_seq >= 1190))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1201) and (date_dim.d_month_seq >= 1190))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/constraints/query23.out b/regression-test/data/new_shapes_p0/tpcds_sf100/constraints/query23.out
index 85217462d1a5ed..923ee0e2966746 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/constraints/query23.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/constraints/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 item_sk->[cs_item_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4 RF5
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalCteConsumer ( cteId=CTEId#0 )
-----------------PhysicalProject
-------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 item_sk->[ws_item_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7 RF8
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 item_sk->[cs_item_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4 RF5
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------------PhysicalProject
+--------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 item_sk->[ws_item_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7 RF8
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query23.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query23.out
index aa0bb3fa70d0e7..c5d202bfee2bc3 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query23.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=()
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5
-----------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF7 ws_item_sk->[item_sk]
-------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=()
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8
-----------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=()
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5
+------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+--------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF7 ws_item_sk->[item_sk]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=()
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8
+------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query32.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query32.out
index 15393efb582c73..7992f57d1c87b3 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query32.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 29))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 29))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query38.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query38.out
index 304fa6dd35c5f6..b0bda7f927c138 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query38.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query97.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query97.out
index 404a9e7e9e92a1..4ebfd5abc0eb1c 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query97.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query97.out
@@ -1,32 +1,33 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query23.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query23.out
index 26e1f5089288a1..a65c74fa0b81dd 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query23.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF4 item_sk->[cs_item_sk]
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4 RF5
-----------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
-----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF7 ws_item_sk->[item_sk]
-------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[ws_bill_customer_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF8
-----------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
---------------------PhysicalProject
-----------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
-------------------------PhysicalOlapScan[date_dim]
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF4 item_sk->[cs_item_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4 RF5
+------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+--------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
+----------------------PhysicalProject
+------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF7 ws_item_sk->[item_sk]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7
+--------------------------PhysicalProject
+----------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[ws_bill_customer_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF8
+------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+----------------------PhysicalProject
+------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+--------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query32.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query32.out
index 15393efb582c73..7992f57d1c87b3 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query32.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 29))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 29))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query38.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query38.out
index fbc5dd43162046..2bc9d9fecbca40 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query38.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query97.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query97.out
index 404a9e7e9e92a1..4ebfd5abc0eb1c 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query97.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query97.out
@@ -1,32 +1,33 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query23.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query23.out
index 1343c4f4fb0214..6103f4eb74d6ab 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query23.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query32.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query32.out
index 15393efb582c73..7992f57d1c87b3 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query32.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 29))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 29))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query38.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query38.out
index 802779104ed84f..26d52e9fae9c40 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query38.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query97.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query97.out
index 404a9e7e9e92a1..4ebfd5abc0eb1c 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query97.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query97.out
@@ -1,32 +1,33 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query23.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query23.out
index 91ab82284bc473..0d8b21c19639e9 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query23.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query32.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query32.out
index 15393efb582c73..7992f57d1c87b3 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query32.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 29))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 29))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '1999-04-07') and (date_dim.d_date >= '1999-01-07'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query38.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query38.out
index 9a6c130acd1eaf..ef2051eb2d000c 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query38.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1194) and (date_dim.d_month_seq >= 1183))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query97.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query97.out
index 404a9e7e9e92a1..4ebfd5abc0eb1c 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query97.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query97.out
@@ -1,32 +1,33 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query23.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query23.out
index 5e2eafb60ffe8d..45d7a47eb7e46a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query23.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query23.out
@@ -46,35 +46,36 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 ----------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
-------PhysicalTopN[GATHER_SORT]
---------hashAgg[GLOBAL]
-----------PhysicalDistribute[DistributionSpecGather]
-------------hashAgg[LOCAL]
---------------PhysicalUnion
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
-----------------PhysicalProject
-------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
---------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
---------------------PhysicalProject
-----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
---------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------PhysicalLimit[GLOBAL]
+--------PhysicalLimit[LOCAL]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------hashAgg[LOCAL]
+----------------PhysicalUnion
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF8 ws_item_sk->[item_sk]
+----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8
+----------------------PhysicalProject
+------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF6 RF7
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query32.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query32.out
index 0389bfeea82673..c413940debc6e7 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query32.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query32.out
@@ -1,25 +1,26 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_32 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
---------------PhysicalWindow
-----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((item.i_manufact_id = 722))
---------------------------------PhysicalOlapScan[item]
-------------------------PhysicalProject
---------------------------filter((date_dim.d_date <= '2001-06-07') and (date_dim.d_date >= '2001-03-09'))
-----------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cast(cs_ext_discount_amt as DECIMALV3(9, 4))) OVER(PARTITION BY i_item_sk))))
+----------------PhysicalWindow
+------------------PhysicalQuickSort[LOCAL_SORT]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = catalog_sales.cs_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((item.i_manufact_id = 722))
+----------------------------------PhysicalOlapScan[item]
+--------------------------PhysicalProject
+----------------------------filter((date_dim.d_date <= '2001-06-07') and (date_dim.d_date >= '2001-03-09'))
+------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query38.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query38.out
index 5ae805bdffe325..dc794c95f97b2f 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query38.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query38.out
@@ -1,49 +1,50 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_38 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------PhysicalIntersect
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
---------------PhysicalDistribute[DistributionSpecHash]
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
-------------------hashAgg[GLOBAL]
---------------------PhysicalDistribute[DistributionSpecHash]
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
-----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
-----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
---------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalOlapScan[customer]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------PhysicalIntersect
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ws_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
+----------------PhysicalDistribute[DistributionSpecHash]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk]
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute[DistributionSpecHash]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF5
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1200) and (date_dim.d_month_seq >= 1189))
+----------------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query97.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query97.out
index ca6f63d5e6bdf0..d3a845763241f7 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query97.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query97.out
@@ -1,34 +1,35 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !ds_shape_97 --
 PhysicalResultSink
---PhysicalTopN[GATHER_SORT]
-----hashAgg[GLOBAL]
-------PhysicalDistribute[DistributionSpecGather]
---------hashAgg[LOCAL]
-----------PhysicalProject
-------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
---------------------------PhysicalProject
-----------------------------filter(( not ss_sold_date_sk IS NULL))
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
-------------------------------PhysicalOlapScan[date_dim]
---------------PhysicalProject
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute[DistributionSpecHash]
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
---------------------------PhysicalProject
-----------------------------filter(( not cs_sold_date_sk IS NULL))
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
-------------------------------PhysicalOlapScan[date_dim]
+--PhysicalLimit[GLOBAL]
+----PhysicalLimit[LOCAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecGather]
+----------hashAgg[LOCAL]
+------------PhysicalProject
+--------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((ssci.customer_sk = csci.customer_sk) and (ssci.item_sk = csci.item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------filter(( not ss_sold_date_sk IS NULL))
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
+--------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute[DistributionSpecHash]
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
+----------------------------PhysicalProject
+------------------------------filter(( not cs_sold_date_sk IS NULL))
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
+----------------------------PhysicalProject
+------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
+--------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy b/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy
index 36460f06e32d56..631656a6b1921b 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy
@@ -68,10 +68,10 @@ suite("push_topn_to_agg") {
         contains("sortByGroupKey:true")
     }
 
-    // group key is not output of limit, deny opt
+    // use group key as sort key to enable topn-push opt
     explain {
         sql "select sum(o_shippriority) from orders group by o_clerk limit 14; "
-        contains("sortByGroupKey:false")
+        contains("sortByGroupKey:true")
     }
 
     // group key is part of output of limit, apply opt

From d3276d99cefeb2f2310dcd613ee563547d40ab22 Mon Sep 17 00:00:00 2001
From: LiBinfeng <libinfeng@selectdb.com>
Date: Wed, 27 Nov 2024 10:51:44 +0800
Subject: [PATCH 030/399] [Improvement](Nereids) Support create routine load
 command (#43930)

Issue Number: close #42805
---
 .../org/apache/doris/nereids/DorisParser.g4   |  25 +-
 .../doris/analysis/CreateRoutineLoadStmt.java |  46 ++
 .../org/apache/doris/analysis/Separator.java  |   5 +
 .../nereids/parser/LogicalPlanBuilder.java    | 122 +++++
 .../doris/nereids/trees/plans/PlanType.java   |   3 +-
 .../commands/info/CreateRoutineLoadInfo.java  | 517 ++++++++++++++++++
 .../plans/commands/info/LabelNameInfo.java    | 108 ++++
 .../load/CreateRoutineLoadCommand.java        |  90 +++
 .../plans/commands/load/LoadColumnClause.java |  35 ++
 .../plans/commands/load/LoadColumnDesc.java   |  45 ++
 .../commands/load/LoadDeleteOnClause.java     |  35 ++
 .../commands/load/LoadPartitionNames.java     |  70 +++
 .../plans/commands/load/LoadProperty.java     |  36 ++
 .../plans/commands/load/LoadSeparator.java    |  33 ++
 .../commands/load/LoadSequenceClause.java     |  33 ++
 .../plans/commands/load/LoadWhereClause.java  |  42 ++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../test_routine_load_condition.groovy        |   4 +-
 .../routine_load/test_routine_load.groovy     |   6 +-
 19 files changed, 1240 insertions(+), 20 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/LabelNameInfo.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/CreateRoutineLoadCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadColumnClause.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadColumnDesc.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadDeleteOnClause.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadPartitionNames.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadProperty.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadSeparator.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadSequenceClause.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadWhereClause.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 2d68e01b78761a..d7ed0598d19ddb 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -58,13 +58,11 @@ statementBase
     | supportedUnsetStatement           #supportedUnsetStatementAlias
     | supportedRefreshStatement         #supportedRefreshStatementAlias
     | supportedShowStatement            #supportedShowStatementAlias
+    | supportedLoadStatement            #supportedLoadStatementAlias
     | supportedRecoverStatement         #supportedRecoverStatementAlias
-    | supportedLoadStatement            #supportedLoadfStatementAlias
     | unsupportedStatement              #unsupported
     ;
 
-
-
 unsupportedStatement
     : unsupportedUseStatement
     | unsupportedDmlStatement
@@ -242,11 +240,12 @@ supportedShowStatement
     | SHOW TABLE tableId=INTEGER_VALUE                                              #showTableId
     | SHOW WHITELIST                                                                #showWhitelist
     | SHOW TABLETS BELONG
-        tabletIds+=INTEGER_VALUE (COMMA tabletIds+=INTEGER_VALUE)*                  #showTabletsBelong    
+        tabletIds+=INTEGER_VALUE (COMMA tabletIds+=INTEGER_VALUE)*                  #showTabletsBelong
     ;
 
 supportedLoadStatement
     : SYNC                                                                          #sync
+    | createRoutineLoad                                                             #createRoutineLoadAlias
     ;
 
 unsupportedOtherStatement
@@ -358,6 +357,14 @@ unsupportedShowStatement
     | SHOW WARM UP JOB wildWhere?                                                   #showWarmUpJob
     ;
 
+createRoutineLoad
+    : CREATE ROUTINE LOAD label=multipartIdentifier (ON table=identifier)?
+              (WITH (APPEND | DELETE | MERGE))?
+              (loadProperty (COMMA loadProperty)*)? propertyClause? FROM type=identifier
+              LEFT_PAREN customProperties=propertyItemList RIGHT_PAREN
+              commentSpec?
+    ;
+
 unsupportedLoadStatement
     : LOAD mysqlDataDesc
         (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)?
@@ -369,11 +376,6 @@ unsupportedLoadStatement
     | STOP SYNC JOB name=multipartIdentifier                                        #stopDataSyncJob
     | RESUME SYNC JOB name=multipartIdentifier                                      #resumeDataSyncJob
     | PAUSE SYNC JOB name=multipartIdentifier                                       #pauseDataSyncJob
-    | CREATE ROUTINE LOAD label=multipartIdentifier (ON table=identifier)?
-        (WITH (APPEND | DELETE | MERGE))?
-        (loadProperty (COMMA loadProperty)*)? propertyClause? FROM type=identifier
-        LEFT_PAREN customProperties=propertyItemList RIGHT_PAREN
-        commentSpec?                                                                #createRoutineLoadJob
     | PAUSE ROUTINE LOAD FOR label=multipartIdentifier                              #pauseRoutineLoad
     | PAUSE ALL ROUTINE LOAD                                                        #pauseAllRoutineLoad
     | RESUME ROUTINE LOAD FOR label=multipartIdentifier                             #resumeRoutineLoad
@@ -383,11 +385,6 @@ unsupportedLoadStatement
     | SHOW ROUTINE LOAD TASK ((FROM | IN) database=identifier)? wildWhere?          #showRoutineLoadTask
     | SHOW ALL? CREATE ROUTINE LOAD FOR label=multipartIdentifier                   #showCreateRoutineLoad
     | SHOW CREATE LOAD FOR label=multipartIdentifier                                #showCreateLoad
-    | importSequenceStatement                                                       #importSequenceStatementAlias
-    | importPrecedingFilterStatement                                                #importPrecedingFilterStatementAlias
-    | importWhereStatement                                                          #importWhereStatementAlias
-    | importDeleteOnStatement                                                       #importDeleteOnStatementAlias
-    | importColumnsStatement                                                        #importColumnsStatementAlias
     ;
 
 loadProperty
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
index a16259dcdb98d2..27379ccd28d098 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
@@ -34,6 +34,7 @@
 import org.apache.doris.load.routineload.RoutineLoadDataSourcePropertyFactory;
 import org.apache.doris.load.routineload.RoutineLoadJob;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.OriginStatement;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
@@ -229,6 +230,51 @@ public CreateRoutineLoadStmt(LabelName labelName, String tableName, List<ParseNo
         }
     }
 
+    /*
+     * make stmt by nereids
+     */
+    public CreateRoutineLoadStmt(LabelName labelName, String dbName, String name, String tableName,
+            List<ParseNode> loadPropertyList, OriginStatement origStmt, UserIdentity userIdentity,
+            Map<String, String> jobProperties, String typeName, RoutineLoadDesc routineLoadDesc,
+            int desireTaskConcurrentNum, long maxErrorNum, double maxFilterRatio, long maxBatchIntervalS,
+            long maxBatchRows, long maxBatchSizeBytes, long execMemLimit, int sendBatchParallelism, String timezone,
+            String format, String jsonPaths, String jsonRoot, byte enclose, byte escape, long workloadGroupId,
+            boolean loadToSingleTablet, boolean strictMode, boolean isPartialUpdate, boolean stripOuterArray,
+            boolean numAsString, boolean fuzzyParse, AbstractDataSourceProperties dataSourceProperties) {
+        this.labelName = labelName;
+        this.dbName = dbName;
+        this.name = name;
+        this.tableName = tableName;
+        this.loadPropertyList = loadPropertyList;
+        this.setOrigStmt(origStmt);
+        this.setUserInfo(userIdentity);
+        this.jobProperties = jobProperties;
+        this.typeName = typeName;
+        this.routineLoadDesc = routineLoadDesc;
+        this.desiredConcurrentNum = desireTaskConcurrentNum;
+        this.maxErrorNum = maxErrorNum;
+        this.maxFilterRatio = maxFilterRatio;
+        this.maxBatchIntervalS = maxBatchIntervalS;
+        this.maxBatchRows = maxBatchRows;
+        this.maxBatchSizeBytes = maxBatchSizeBytes;
+        this.execMemLimit = execMemLimit;
+        this.sendBatchParallelism = sendBatchParallelism;
+        this.timezone = timezone;
+        this.format = format;
+        this.jsonPaths = jsonPaths;
+        this.jsonRoot = jsonRoot;
+        this.enclose = enclose;
+        this.escape = escape;
+        this.workloadGroupId = workloadGroupId;
+        this.loadToSingleTablet = loadToSingleTablet;
+        this.strictMode = strictMode;
+        this.isPartialUpdate = isPartialUpdate;
+        this.stripOuterArray = stripOuterArray;
+        this.numAsString = numAsString;
+        this.fuzzyParse = fuzzyParse;
+        this.dataSourceProperties = dataSourceProperties;
+    }
+
     public String getName() {
         return name;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java
index 3a5731944a6b95..86027ea9ceac5b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java
@@ -34,6 +34,11 @@ public Separator(String separator) {
         this.separator = null;
     }
 
+    public Separator(String separator, String oriSeparator) {
+        this.oriSeparator = oriSeparator;
+        this.separator = separator;
+    }
+
     public String getOriSeparator() {
         return oriSeparator;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index ae0921a1732419..6e86b4073f5792 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -87,6 +87,7 @@
 import org.apache.doris.nereids.DorisParser.ConstantContext;
 import org.apache.doris.nereids.DorisParser.CreateMTMVContext;
 import org.apache.doris.nereids.DorisParser.CreateProcedureContext;
+import org.apache.doris.nereids.DorisParser.CreateRoutineLoadContext;
 import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext;
 import org.apache.doris.nereids.DorisParser.CreateTableContext;
 import org.apache.doris.nereids.DorisParser.CreateTableLikeContext;
@@ -126,6 +127,12 @@
 import org.apache.doris.nereids.DorisParser.IdentifierContext;
 import org.apache.doris.nereids.DorisParser.IdentifierListContext;
 import org.apache.doris.nereids.DorisParser.IdentifierSeqContext;
+import org.apache.doris.nereids.DorisParser.ImportColumnsContext;
+import org.apache.doris.nereids.DorisParser.ImportDeleteOnContext;
+import org.apache.doris.nereids.DorisParser.ImportPartitionsContext;
+import org.apache.doris.nereids.DorisParser.ImportPrecedingFilterContext;
+import org.apache.doris.nereids.DorisParser.ImportSequenceContext;
+import org.apache.doris.nereids.DorisParser.ImportWhereContext;
 import org.apache.doris.nereids.DorisParser.InPartitionDefContext;
 import org.apache.doris.nereids.DorisParser.IndexDefContext;
 import org.apache.doris.nereids.DorisParser.IndexDefsContext;
@@ -141,6 +148,7 @@
 import org.apache.doris.nereids.DorisParser.LateralViewContext;
 import org.apache.doris.nereids.DorisParser.LessThanPartitionDefContext;
 import org.apache.doris.nereids.DorisParser.LimitClauseContext;
+import org.apache.doris.nereids.DorisParser.LoadPropertyContext;
 import org.apache.doris.nereids.DorisParser.LogicalBinaryContext;
 import org.apache.doris.nereids.DorisParser.LogicalNotContext;
 import org.apache.doris.nereids.DorisParser.MapLiteralContext;
@@ -193,6 +201,7 @@
 import org.apache.doris.nereids.DorisParser.SelectClauseContext;
 import org.apache.doris.nereids.DorisParser.SelectColumnClauseContext;
 import org.apache.doris.nereids.DorisParser.SelectHintContext;
+import org.apache.doris.nereids.DorisParser.SeparatorContext;
 import org.apache.doris.nereids.DorisParser.SetCharsetContext;
 import org.apache.doris.nereids.DorisParser.SetCollateContext;
 import org.apache.doris.nereids.DorisParser.SetDefaultStorageVaultContext;
@@ -532,6 +541,7 @@
 import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
 import org.apache.doris.nereids.trees.plans.commands.info.CreateJobInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo;
+import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.CreateTableLikeInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.CreateViewInfo;
@@ -544,6 +554,7 @@
 import org.apache.doris.nereids.trees.plans.commands.info.GeneratedColumnDesc;
 import org.apache.doris.nereids.trees.plans.commands.info.InPartition;
 import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition;
+import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.LessThanPartition;
 import org.apache.doris.nereids.trees.plans.commands.info.MTMVPartitionDefinition;
 import org.apache.doris.nereids.trees.plans.commands.info.PartitionDefinition;
@@ -568,6 +579,15 @@
 import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.load.CreateRoutineLoadCommand;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnClause;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnDesc;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadDeleteOnClause;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadPartitionNames;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadSequenceClause;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
@@ -1451,6 +1471,108 @@ public LogicalSubQueryAlias<Plan> visitAliasQuery(AliasQueryContext ctx) {
         });
     }
 
+    /**
+     * process LoadProperty in routine load
+     */
+    public LoadProperty visitLoadProperty(LoadPropertyContext ctx) {
+        LoadProperty loadProperty = null;
+        if (ctx instanceof SeparatorContext) {
+            String separator = stripQuotes(((SeparatorContext) ctx).STRING_LITERAL().getText());
+            loadProperty = new LoadSeparator(separator);
+        } else if (ctx instanceof ImportColumnsContext) {
+            List<LoadColumnDesc> descList = new ArrayList<>();
+            for (DorisParser.ImportColumnDescContext loadColumnDescCtx : ((ImportColumnsContext) ctx)
+                    .importColumnsStatement().importColumnDesc()) {
+                LoadColumnDesc desc;
+                if (loadColumnDescCtx.booleanExpression() != null) {
+                    desc = new LoadColumnDesc(loadColumnDescCtx.name.getText(),
+                        getExpression(loadColumnDescCtx.booleanExpression()));
+                } else {
+                    desc = new LoadColumnDesc(loadColumnDescCtx.name.getText());
+                }
+                descList.add(desc);
+            }
+            loadProperty = new LoadColumnClause(descList);
+        } else if (ctx instanceof ImportDeleteOnContext) {
+            loadProperty = new LoadDeleteOnClause(getExpression(((ImportDeleteOnContext) ctx)
+                    .importDeleteOnStatement().booleanExpression()));
+        } else if (ctx instanceof ImportPartitionsContext) {
+            Pair<Boolean, List<String>> partitionSpec = visitPartitionSpec(
+                    ((ImportPartitionsContext) ctx).partitionSpec());
+            loadProperty = new LoadPartitionNames(partitionSpec.first, partitionSpec.second);
+        } else if (ctx instanceof ImportPrecedingFilterContext) {
+            loadProperty = new LoadWhereClause(getExpression(((ImportPrecedingFilterContext) ctx)
+                    .importPrecedingFilterStatement().booleanExpression()), true);
+        } else if (ctx instanceof ImportSequenceContext) {
+            loadProperty = new LoadSequenceClause(((ImportSequenceContext) ctx)
+                    .importSequenceStatement().identifier().getText());
+        } else if (ctx instanceof ImportWhereContext) {
+            loadProperty = new LoadWhereClause(getExpression(((ImportWhereContext) ctx)
+                    .importWhereStatement().booleanExpression()), false);
+        }
+        return loadProperty;
+    }
+
+    @Override
+    public LogicalPlan visitCreateRoutineLoad(CreateRoutineLoadContext ctx) {
+        List<String> labelParts = visitMultipartIdentifier(ctx.label);
+        String labelName = null;
+        String labelDbName = null;
+        if (ConnectContext.get().getDatabase().isEmpty() && labelParts.size() == 1) {
+            throw new AnalysisException("Current database is not set.");
+        } else if (labelParts.size() == 1) {
+            labelName = labelParts.get(0);
+        } else if (labelParts.size() == 2) {
+            labelDbName = labelParts.get(0);
+            labelName = labelParts.get(1);
+        } else if (labelParts.size() == 3) {
+            labelDbName = labelParts.get(1);
+            labelName = labelParts.get(2);
+        } else {
+            throw new AnalysisException("labelParts in load should be [ctl.][db.]label");
+        }
+        LabelNameInfo jobLabelInfo = new LabelNameInfo(labelDbName, labelName);
+        String tableName = null;
+        if (ctx.table != null) {
+            tableName = ctx.table.getText();
+        }
+        Map<String, String> properties = ctx.propertyClause() != null
+                // NOTICE: we should not generate immutable map here, because it will be modified when analyzing.
+                ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause()))
+                : Maps.newHashMap();
+        String type = ctx.type.getText();
+        Map<String, String> customProperties = ctx.customProperties != null
+                // NOTICE: we should not generate immutable map here, because it will be modified when analyzing.
+                ? Maps.newHashMap(visitPropertyItemList(ctx.customProperties))
+                : Maps.newHashMap();
+        LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND;
+        if (ctx.WITH() != null) {
+            if (ctx.DELETE() != null) {
+                mergeType = LoadTask.MergeType.DELETE;
+            } else if (ctx.MERGE() != null) {
+                mergeType = LoadTask.MergeType.MERGE;
+            }
+        }
+        String comment = visitCommentSpec(ctx.commentSpec());
+        Map<String, LoadProperty> loadPropertyMap = new HashMap<>();
+        for (DorisParser.LoadPropertyContext oneLoadPropertyCOntext : ctx.loadProperty()) {
+            LoadProperty loadProperty = visitLoadProperty(oneLoadPropertyCOntext);
+            if (loadProperty == null) {
+                throw new AnalysisException("invalid clause of routine load");
+            }
+            if (loadPropertyMap.get(loadProperty.getClass().getName()) != null) {
+                throw new AnalysisException("repeat setting of clause load property: "
+                    + loadProperty.getClass().getName());
+            } else {
+                loadPropertyMap.put(loadProperty.getClass().getName(), loadProperty);
+            }
+        }
+        CreateRoutineLoadInfo createRoutineLoadInfo = new CreateRoutineLoadInfo(jobLabelInfo, tableName,
+                loadPropertyMap, properties, type, customProperties, mergeType, comment);
+        return new CreateRoutineLoadCommand(createRoutineLoadInfo);
+
+    }
+
     @Override
     public Command visitCreateRowPolicy(CreateRowPolicyContext ctx) {
         FilterType filterType = FilterType.of(ctx.type.getText());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 79a4ff31c1fc20..57eb2bbb6b42c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -222,5 +222,6 @@ public enum PlanType {
     RECOVER_DATABASE_COMMAND,
     RECOVER_TABLE_COMMAND,
     RECOVER_PARTITION_COMMAND,
-    REPLAY_COMMAND
+    REPLAY_COMMAND,
+    CREATE_ROUTINE_LOAD_COMMAND
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
new file mode 100644
index 00000000000000..3e0fd0d34fb091
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java
@@ -0,0 +1,517 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.info;
+
+import org.apache.doris.analysis.CreateRoutineLoadStmt;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.ImportColumnDesc;
+import org.apache.doris.analysis.ImportColumnsStmt;
+import org.apache.doris.analysis.ImportDeleteOnStmt;
+import org.apache.doris.analysis.ImportSequenceStmt;
+import org.apache.doris.analysis.ImportWhereStmt;
+import org.apache.doris.analysis.LoadStmt;
+import org.apache.doris.analysis.PartitionNames;
+import org.apache.doris.analysis.Separator;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Table;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.load.RoutineLoadDesc;
+import org.apache.doris.load.loadv2.LoadTask;
+import org.apache.doris.load.routineload.AbstractDataSourceProperties;
+import org.apache.doris.load.routineload.RoutineLoadDataSourcePropertyFactory;
+import org.apache.doris.load.routineload.RoutineLoadJob;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.analyzer.Scope;
+import org.apache.doris.nereids.analyzer.UnboundRelation;
+import org.apache.doris.nereids.glue.translator.ExpressionTranslator;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.nereids.jobs.executor.Rewriter;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.rules.analysis.BindRelation;
+import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.OlapScan;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnClause;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnDesc;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadDeleteOnClause;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadPartitionNames;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadSequenceClause;
+import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Predicate;
+
+/**
+ * info in creating routine load.
+ */
+public class CreateRoutineLoadInfo {
+    // routine load properties
+    public static final String DESIRED_CONCURRENT_NUMBER_PROPERTY = "desired_concurrent_number";
+    public static final String CURRENT_CONCURRENT_NUMBER_PROPERTY = "current_concurrent_number";
+    // max error number in ten thousand records
+    public static final String MAX_ERROR_NUMBER_PROPERTY = "max_error_number";
+    public static final String MAX_FILTER_RATIO_PROPERTY = "max_filter_ratio";
+    // the following 3 properties limit the time and batch size of a single routine load task
+    public static final String MAX_BATCH_INTERVAL_SEC_PROPERTY = "max_batch_interval";
+    public static final String MAX_BATCH_ROWS_PROPERTY = "max_batch_rows";
+    public static final String MAX_BATCH_SIZE_PROPERTY = "max_batch_size";
+    public static final String EXEC_MEM_LIMIT_PROPERTY = "exec_mem_limit";
+
+    public static final String FORMAT = "format"; // the value is csv or json, default is csv
+    public static final String STRIP_OUTER_ARRAY = "strip_outer_array";
+    public static final String JSONPATHS = "jsonpaths";
+    public static final String JSONROOT = "json_root";
+    public static final String NUM_AS_STRING = "num_as_string";
+    public static final String FUZZY_PARSE = "fuzzy_parse";
+    public static final String PARTIAL_COLUMNS = "partial_columns";
+    public static final String WORKLOAD_GROUP = "workload_group";
+    public static final String ENDPOINT_REGEX = "[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]";
+    public static final String SEND_BATCH_PARALLELISM = "send_batch_parallelism";
+    public static final String LOAD_TO_SINGLE_TABLET = "load_to_single_tablet";
+    public static final java.util.function.Predicate<Long> DESIRED_CONCURRENT_NUMBER_PRED = (v) -> v > 0L;
+    public static final java.util.function.Predicate<Long> MAX_ERROR_NUMBER_PRED = (v) -> v >= 0L;
+    public static final java.util.function.Predicate<Double> MAX_FILTER_RATIO_PRED = (v) -> v >= 0 && v <= 1;
+    public static final java.util.function.Predicate<Long> MAX_BATCH_INTERVAL_PRED = (v) -> v >= 1;
+    public static final java.util.function.Predicate<Long> MAX_BATCH_ROWS_PRED = (v) -> v >= 200000;
+    public static final java.util.function.Predicate<Long> MAX_BATCH_SIZE_PRED = (v) -> v >= 100 * 1024 * 1024
+            && v <= (long) (1024 * 1024 * 1024) * 10;
+    public static final java.util.function.Predicate<Long> EXEC_MEM_LIMIT_PRED = (v) -> v >= 0L;
+    public static final Predicate<Long> SEND_BATCH_PARALLELISM_PRED = (v) -> v > 0L;
+
+    private static final String NAME_TYPE = "ROUTINE LOAD NAME";
+
+    private static final ImmutableSet<String> PROPERTIES_SET = new ImmutableSet.Builder<String>()
+            .add(DESIRED_CONCURRENT_NUMBER_PROPERTY)
+            .add(MAX_ERROR_NUMBER_PROPERTY)
+            .add(MAX_FILTER_RATIO_PROPERTY)
+            .add(MAX_BATCH_INTERVAL_SEC_PROPERTY)
+            .add(MAX_BATCH_ROWS_PROPERTY)
+            .add(MAX_BATCH_SIZE_PROPERTY)
+            .add(FORMAT)
+            .add(JSONPATHS)
+            .add(STRIP_OUTER_ARRAY)
+            .add(NUM_AS_STRING)
+            .add(FUZZY_PARSE)
+            .add(JSONROOT)
+            .add(LoadStmt.STRICT_MODE)
+            .add(LoadStmt.TIMEZONE)
+            .add(EXEC_MEM_LIMIT_PROPERTY)
+            .add(SEND_BATCH_PARALLELISM)
+            .add(LOAD_TO_SINGLE_TABLET)
+            .add(PARTIAL_COLUMNS)
+            .add(WORKLOAD_GROUP)
+            .add(LoadStmt.KEY_ENCLOSE)
+            .add(LoadStmt.KEY_ESCAPE)
+            .build();
+
+    private final LabelNameInfo labelNameInfo;
+    private String tableName;
+    private final Map<String, LoadProperty> loadPropertyMap;
+    private final Map<String, String> jobProperties;
+    private final String typeName;
+
+    // the following variables will be initialized after analyze
+    // -1 as unset, the default value will set in RoutineLoadJob
+    private String name;
+    private String dbName;
+    private RoutineLoadDesc routineLoadDesc;
+    private int desiredConcurrentNum = 1;
+    private long maxErrorNum = -1;
+    private double maxFilterRatio = -1;
+    private long maxBatchIntervalS = -1;
+    private long maxBatchRows = -1;
+    private long maxBatchSizeBytes = -1;
+    private boolean strictMode = true;
+    private long execMemLimit = 2 * 1024 * 1024 * 1024L;
+    private String timezone = TimeUtils.DEFAULT_TIME_ZONE;
+    private int sendBatchParallelism = 1;
+    private boolean loadToSingleTablet = false;
+    /**
+     * RoutineLoad support json data.
+     * Require Params:
+     * 1) dataFormat = "json"
+     * 2) jsonPaths = "$.XXX.xxx"
+     */
+    private String format = ""; //default is csv.
+    private String jsonPaths = "";
+    private String jsonRoot = ""; // MUST be a jsonpath string
+    private boolean stripOuterArray = false;
+    private boolean numAsString = false;
+    private boolean fuzzyParse = false;
+
+    private byte enclose;
+
+    private byte escape;
+
+    private long workloadGroupId = -1;
+
+    /**
+     * support partial columns load(Only Unique Key Columns)
+     */
+    private boolean isPartialUpdate = false;
+
+    private String comment = "";
+
+    private LoadTask.MergeType mergeType;
+
+    private boolean isMultiTable = false;
+
+    private AbstractDataSourceProperties dataSourceProperties;
+
+    /**
+     * constructor for create table
+     */
+    public CreateRoutineLoadInfo(LabelNameInfo labelNameInfo, String tableName,
+                                 Map<String, LoadProperty> loadPropertyMap,
+                                 Map<String, String> jobProperties, String typeName,
+                                 Map<String, String> dataSourceProperties, LoadTask.MergeType mergeType,
+                                 String comment) {
+        this.labelNameInfo = labelNameInfo;
+        if (StringUtils.isBlank(tableName)) {
+            this.isMultiTable = true;
+        }
+        this.tableName = tableName;
+        this.loadPropertyMap = loadPropertyMap;
+        this.jobProperties = jobProperties == null ? Maps.newHashMap() : jobProperties;
+        this.typeName = typeName.toUpperCase();
+        this.dataSourceProperties = RoutineLoadDataSourcePropertyFactory
+            .createDataSource(typeName, dataSourceProperties, this.isMultiTable);
+        this.mergeType = mergeType;
+        this.isPartialUpdate = this.jobProperties.getOrDefault(PARTIAL_COLUMNS, "false").equalsIgnoreCase("true");
+        if (comment != null) {
+            this.comment = comment;
+        }
+    }
+
+    /**
+     * analyze create table info
+     */
+    public void validate(ConnectContext ctx) throws UserException {
+        // check dbName and tableName
+        checkDBTable(ctx);
+        // check name
+        try {
+            FeNameFormat.checkCommonName(NAME_TYPE, name);
+        } catch (org.apache.doris.common.AnalysisException e) {
+            // 64 is the length of regular expression matching
+            // (FeNameFormat.COMMON_NAME_REGEX/UNDERSCORE_COMMON_NAME_REGEX)
+            throw new AnalysisException(e.getMessage()
+                + " Maybe routine load job name is longer than 64 or contains illegal characters");
+        }
+        // check load properties include column separator etc.
+        checkLoadProperties(ctx);
+        // check routine load job properties include desired concurrent number etc.
+        checkJobProperties();
+        // check data source properties
+        checkDataSourceProperties();
+        // analyze merge type
+        if (routineLoadDesc != null) {
+            if (mergeType != LoadTask.MergeType.MERGE && routineLoadDesc.getDeleteCondition() != null) {
+                throw new AnalysisException("not support DELETE ON clause when merge type is not MERGE.");
+            }
+            if (mergeType == LoadTask.MergeType.MERGE && routineLoadDesc.getDeleteCondition() == null) {
+                throw new AnalysisException("Excepted DELETE ON clause when merge type is MERGE.");
+            }
+        } else if (mergeType == LoadTask.MergeType.MERGE) {
+            throw new AnalysisException("Excepted DELETE ON clause when merge type is MERGE.");
+        }
+    }
+
+    private void checkDBTable(ConnectContext ctx) throws AnalysisException {
+        labelNameInfo.validate(ctx);
+        dbName = labelNameInfo.getDb();
+        name = labelNameInfo.getLabel();
+
+        Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(dbName);
+        if (isPartialUpdate && isMultiTable) {
+            throw new AnalysisException("Partial update is not supported in multi-table load.");
+        }
+        if (isMultiTable) {
+            return;
+        }
+        if (Strings.isNullOrEmpty(tableName)) {
+            throw new AnalysisException("Table name should not be null");
+        }
+        Table table = db.getTableOrAnalysisException(tableName);
+        if (mergeType != LoadTask.MergeType.APPEND
+                && (table.getType() != Table.TableType.OLAP
+                || ((OlapTable) table).getKeysType() != KeysType.UNIQUE_KEYS)) {
+            throw new AnalysisException("load by MERGE or DELETE is only supported in unique tables.");
+        }
+        if (mergeType != LoadTask.MergeType.APPEND
+                && !(table.getType() == Table.TableType.OLAP && ((OlapTable) table).hasDeleteSign())) {
+            throw new AnalysisException("load by MERGE or DELETE need to upgrade table to support batch delete.");
+        }
+        if (isPartialUpdate && !((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) {
+            throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW");
+        }
+    }
+
+    private void checkLoadProperties(ConnectContext ctx) throws UserException {
+        Separator columnSeparator = null;
+        // TODO(yangzhengguo01): add line delimiter to properties
+        Separator lineDelimiter = null;
+        ImportColumnsStmt importColumnsStmt = null;
+        ImportWhereStmt precedingImportWhereStmt = null;
+        ImportWhereStmt importWhereStmt = null;
+        ImportSequenceStmt importSequenceStmt = null;
+        PartitionNames partitionNames = null;
+        ImportDeleteOnStmt importDeleteOnStmt = null;
+        CascadesContext cascadesContext = null;
+        ExpressionAnalyzer analyzer = null;
+        PlanTranslatorContext context = null;
+        if (!isMultiTable) {
+            List<String> nameParts = Lists.newArrayList();
+            nameParts.add(dbName);
+            nameParts.add(tableName);
+            Plan unboundRelation = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), nameParts);
+            cascadesContext = CascadesContext.initContext(ctx.getStatementContext(), unboundRelation,
+                PhysicalProperties.ANY);
+            Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext,
+                ImmutableList.of(Rewriter.bottomUp(new BindRelation()))).execute();
+            Plan boundRelation = cascadesContext.getRewritePlan();
+            // table could have delete sign in LogicalFilter above
+            if (cascadesContext.getRewritePlan() instanceof LogicalFilter) {
+                boundRelation = (Plan) ((LogicalFilter) cascadesContext.getRewritePlan()).child();
+            }
+            context = new PlanTranslatorContext(cascadesContext);
+            List<Slot> slots = boundRelation.getOutput();
+            Scope scope = new Scope(slots);
+            analyzer = new ExpressionAnalyzer(null, scope, cascadesContext, false, false);
+
+            Map<SlotReference, SlotRef> translateMap = Maps.newHashMap();
+
+            TupleDescriptor tupleDescriptor = context.generateTupleDesc();
+            tupleDescriptor.setTable(((OlapScan) boundRelation).getTable());
+            for (int i = 0; i < boundRelation.getOutput().size(); i++) {
+                SlotReference slotReference = (SlotReference) boundRelation.getOutput().get(i);
+                SlotRef slotRef = new SlotRef(null, slotReference.getName());
+                translateMap.put(slotReference, slotRef);
+                context.createSlotDesc(tupleDescriptor, slotReference, ((OlapScan) boundRelation).getTable());
+            }
+        }
+
+        if (loadPropertyMap != null) {
+            for (LoadProperty loadProperty : loadPropertyMap.values()) {
+                loadProperty.validate();
+                if (loadProperty instanceof LoadSeparator) {
+                    String oriSeparator = ((LoadSeparator) loadProperty).getOriSeparator();
+                    String separator = Separator.convertSeparator(oriSeparator);
+                    columnSeparator = new Separator(separator, oriSeparator);
+                } else if (loadProperty instanceof LoadColumnClause) {
+                    if (isMultiTable) {
+                        throw new AnalysisException("Multi-table load does not support setting columns info");
+                    }
+                    List<ImportColumnDesc> importColumnDescList = new ArrayList<>();
+                    for (LoadColumnDesc columnDesc : ((LoadColumnClause) loadProperty).getColumns()) {
+                        if (columnDesc.getExpression() != null) {
+                            Expr expr = translateToLegacyExpr(columnDesc.getExpression(), analyzer,
+                                    context, cascadesContext);
+                            importColumnDescList.add(new ImportColumnDesc(columnDesc.getColumnName(), expr));
+                        } else {
+                            importColumnDescList.add(new ImportColumnDesc(columnDesc.getColumnName(), null));
+                        }
+                    }
+                    importColumnsStmt = new ImportColumnsStmt(importColumnDescList);
+                } else if (loadProperty instanceof LoadWhereClause) {
+                    if (isMultiTable) {
+                        throw new AnalysisException("Multi-table load does not support setting columns info");
+                    }
+                    Expr expr = translateToLegacyExpr(((LoadWhereClause) loadProperty).getExpression(),
+                            analyzer, context, cascadesContext);
+                    if (((LoadWhereClause) loadProperty).isPreceding()) {
+                        precedingImportWhereStmt = new ImportWhereStmt(expr,
+                                ((LoadWhereClause) loadProperty).isPreceding());
+                    } else {
+                        importWhereStmt = new ImportWhereStmt(expr, ((LoadWhereClause) loadProperty).isPreceding());
+                    }
+                } else if (loadProperty instanceof LoadPartitionNames) {
+                    partitionNames = new PartitionNames(((LoadPartitionNames) loadProperty).isTemp(),
+                            ((LoadPartitionNames) loadProperty).getPartitionNames());
+                } else if (loadProperty instanceof LoadDeleteOnClause) {
+                    Expr expr = translateToLegacyExpr(((LoadDeleteOnClause) loadProperty).getExpression(),
+                            analyzer, context, cascadesContext);
+                    importDeleteOnStmt = new ImportDeleteOnStmt(expr);
+                } else if (loadProperty instanceof LoadSequenceClause) {
+                    importSequenceStmt = new ImportSequenceStmt(
+                            ((LoadSequenceClause) loadProperty).getSequenceColName());
+                }
+            }
+        }
+        routineLoadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, importColumnsStmt,
+            precedingImportWhereStmt, importWhereStmt,
+            partitionNames, importDeleteOnStmt == null ? null : importDeleteOnStmt.getExpr(), mergeType,
+            importSequenceStmt == null ? null : importSequenceStmt.getSequenceColName());
+    }
+
+    private Expr translateToLegacyExpr(Expression expr, ExpressionAnalyzer analyzer, PlanTranslatorContext context,
+                                       CascadesContext cascadesContext) {
+        Expression expression;
+        try {
+            expression = analyzer.analyze(expr, new ExpressionRewriteContext(cascadesContext));
+        } catch (org.apache.doris.nereids.exceptions.AnalysisException e) {
+            throw new org.apache.doris.nereids.exceptions.AnalysisException("In where clause '"
+                + expr.toSql() + "', "
+                + Utils.convertFirstChar(e.getMessage()));
+        }
+        return ExpressionTranslator.translate(expression, context);
+    }
+
+    private void checkJobProperties() throws UserException {
+        Optional<String> optional = jobProperties.keySet().stream().filter(
+                entity -> !PROPERTIES_SET.contains(entity)).findFirst();
+        if (optional.isPresent()) {
+            throw new AnalysisException(optional.get() + " is invalid property");
+        }
+
+        desiredConcurrentNum = ((Long) Util.getLongPropertyOrDefault(
+            jobProperties.get(DESIRED_CONCURRENT_NUMBER_PROPERTY),
+            Config.max_routine_load_task_concurrent_num, DESIRED_CONCURRENT_NUMBER_PRED,
+            DESIRED_CONCURRENT_NUMBER_PROPERTY + " must be greater than 0")).intValue();
+
+        maxErrorNum = Util.getLongPropertyOrDefault(jobProperties.get(MAX_ERROR_NUMBER_PROPERTY),
+            RoutineLoadJob.DEFAULT_MAX_ERROR_NUM, MAX_ERROR_NUMBER_PRED,
+            MAX_ERROR_NUMBER_PROPERTY + " should >= 0");
+
+        maxFilterRatio = Util.getDoublePropertyOrDefault(jobProperties.get(MAX_FILTER_RATIO_PROPERTY),
+            RoutineLoadJob.DEFAULT_MAX_FILTER_RATIO, MAX_FILTER_RATIO_PRED,
+            MAX_FILTER_RATIO_PROPERTY + " should between 0 and 1");
+
+        maxBatchIntervalS = Util.getLongPropertyOrDefault(jobProperties.get(MAX_BATCH_INTERVAL_SEC_PROPERTY),
+            RoutineLoadJob.DEFAULT_MAX_INTERVAL_SECOND, MAX_BATCH_INTERVAL_PRED,
+            MAX_BATCH_INTERVAL_SEC_PROPERTY + " should >= 1");
+
+        maxBatchRows = Util.getLongPropertyOrDefault(jobProperties.get(MAX_BATCH_ROWS_PROPERTY),
+            RoutineLoadJob.DEFAULT_MAX_BATCH_ROWS, MAX_BATCH_ROWS_PRED,
+            MAX_BATCH_ROWS_PROPERTY + " should > 200000");
+
+        maxBatchSizeBytes = Util.getLongPropertyOrDefault(jobProperties.get(MAX_BATCH_SIZE_PROPERTY),
+            RoutineLoadJob.DEFAULT_MAX_BATCH_SIZE, MAX_BATCH_SIZE_PRED,
+            MAX_BATCH_SIZE_PROPERTY + " should between 100MB and 10GB");
+
+        strictMode = Util.getBooleanPropertyOrDefault(jobProperties.get(LoadStmt.STRICT_MODE),
+            RoutineLoadJob.DEFAULT_STRICT_MODE,
+            LoadStmt.STRICT_MODE + " should be a boolean");
+        execMemLimit = Util.getLongPropertyOrDefault(jobProperties.get(EXEC_MEM_LIMIT_PROPERTY),
+            RoutineLoadJob.DEFAULT_EXEC_MEM_LIMIT, EXEC_MEM_LIMIT_PRED,
+            EXEC_MEM_LIMIT_PROPERTY + " must be greater than 0");
+
+        sendBatchParallelism = ((Long) Util.getLongPropertyOrDefault(jobProperties.get(SEND_BATCH_PARALLELISM),
+            ConnectContext.get().getSessionVariable().getSendBatchParallelism(), SEND_BATCH_PARALLELISM_PRED,
+            SEND_BATCH_PARALLELISM + " must be greater than 0")).intValue();
+        loadToSingleTablet = Util.getBooleanPropertyOrDefault(jobProperties.get(LoadStmt.LOAD_TO_SINGLE_TABLET),
+            RoutineLoadJob.DEFAULT_LOAD_TO_SINGLE_TABLET,
+            LoadStmt.LOAD_TO_SINGLE_TABLET + " should be a boolean");
+
+        String encloseStr = jobProperties.get(LoadStmt.KEY_ENCLOSE);
+        if (encloseStr != null) {
+            if (encloseStr.length() != 1) {
+                throw new AnalysisException("enclose must be single-char");
+            } else {
+                enclose = encloseStr.getBytes()[0];
+            }
+        }
+        String escapeStr = jobProperties.get(LoadStmt.KEY_ESCAPE);
+        if (escapeStr != null) {
+            if (escapeStr.length() != 1) {
+                throw new AnalysisException("enclose must be single-char");
+            } else {
+                escape = escapeStr.getBytes()[0];
+            }
+        }
+
+        String inputWorkloadGroupStr = jobProperties.get(WORKLOAD_GROUP);
+        if (!StringUtils.isEmpty(inputWorkloadGroupStr)) {
+            this.workloadGroupId = Env.getCurrentEnv().getWorkloadGroupMgr()
+                .getWorkloadGroup(ConnectContext.get().getCurrentUserIdentity(), inputWorkloadGroupStr);
+        }
+
+        if (ConnectContext.get() != null) {
+            timezone = ConnectContext.get().getSessionVariable().getTimeZone();
+        }
+        timezone = TimeUtils.checkTimeZoneValidAndStandardize(jobProperties.getOrDefault(LoadStmt.TIMEZONE, timezone));
+
+        format = jobProperties.get(FORMAT);
+        if (format != null) {
+            if (format.equalsIgnoreCase("csv")) {
+                format = ""; // if it's not json, then it's mean csv and set empty
+            } else if (format.equalsIgnoreCase("json")) {
+                format = "json";
+                jsonPaths = jobProperties.getOrDefault(JSONPATHS, "");
+                jsonRoot = jobProperties.getOrDefault(JSONROOT, "");
+                stripOuterArray = Boolean.parseBoolean(jobProperties.getOrDefault(STRIP_OUTER_ARRAY, "false"));
+                numAsString = Boolean.parseBoolean(jobProperties.getOrDefault(NUM_AS_STRING, "false"));
+                fuzzyParse = Boolean.parseBoolean(jobProperties.getOrDefault(FUZZY_PARSE, "false"));
+            } else {
+                throw new UserException("Format type is invalid. format=`" + format + "`");
+            }
+        } else {
+            format = "csv"; // default csv
+        }
+    }
+
+    private void checkDataSourceProperties() throws UserException {
+        this.dataSourceProperties.setTimezone(this.timezone);
+        this.dataSourceProperties.analyze();
+    }
+
+    /**
+     * make legacy create routine load statement after validate by nereids
+     * @return legacy create routine load statement
+     */
+    public CreateRoutineLoadStmt translateToLegacyStmt(ConnectContext ctx) {
+        return new CreateRoutineLoadStmt(labelNameInfo.transferToLabelName(), dbName, name, tableName, null,
+            ctx.getStatementContext().getOriginStatement(), ctx.getUserIdentity(),
+            jobProperties, typeName, routineLoadDesc,
+            desiredConcurrentNum, maxErrorNum, maxFilterRatio, maxBatchIntervalS, maxBatchRows, maxBatchSizeBytes,
+            execMemLimit, sendBatchParallelism, timezone, format, jsonPaths, jsonRoot, enclose, escape, workloadGroupId,
+            loadToSingleTablet, strictMode, isPartialUpdate, stripOuterArray, numAsString, fuzzyParse,
+            dataSourceProperties
+        );
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/LabelNameInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/LabelNameInfo.java
new file mode 100644
index 00000000000000..c314b5bce0defe
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/LabelNameInfo.java
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/TableName.java
+// and modified by Doris
+
+package org.apache.doris.nereids.trees.plans.commands.info;
+
+import org.apache.doris.analysis.LabelName;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Strings;
+
+import java.util.Objects;
+
+/**
+ * Label name info
+ */
+public class LabelNameInfo {
+    private String label;
+    private String db;
+
+    public LabelNameInfo() {
+
+    }
+
+    /**
+     * TableNameInfo
+     * @param db dbName
+     * @param label tblName
+     */
+    public LabelNameInfo(String db, String label) {
+        Objects.requireNonNull(label, "require label object");
+        this.label = label;
+        if (Env.isStoredTableNamesLowerCase()) {
+            this.label = label.toLowerCase();
+        }
+        this.db = db;
+    }
+
+    /**
+     * validate labelNameInfo
+     * @param ctx ctx
+     */
+    public void validate(ConnectContext ctx) throws org.apache.doris.common.AnalysisException {
+        if (Strings.isNullOrEmpty(db)) {
+            db = ctx.getDatabase();
+            if (Strings.isNullOrEmpty(db)) {
+                throw new AnalysisException("No database selected");
+            }
+        }
+
+        if (Strings.isNullOrEmpty(label)) {
+            throw new AnalysisException("Table name is null");
+        }
+
+        FeNameFormat.checkLabel(label);
+    }
+
+    /**
+     * get db name
+     * @return dbName
+     */
+    public String getDb() {
+        return db;
+    }
+
+    /**
+     * set a new database name
+     * @param db new database name
+     */
+    public void setDb(String db) {
+        this.db = db;
+    }
+
+    /**
+     * get label name
+     * @return labelName
+     */
+    public String getLabel() {
+        return label;
+    }
+
+    /**
+     * transferToLabelName
+     * @return LabelName
+     */
+    public LabelName transferToLabelName() {
+        return new LabelName(db, label);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/CreateRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/CreateRoutineLoadCommand.java
new file mode 100644
index 00000000000000..da7ab86c17abac
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/CreateRoutineLoadCommand.java
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.load;
+
+import org.apache.doris.analysis.CreateRoutineLoadStmt;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.NoForward;
+import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import java.util.Objects;
+
+/**
+ Create routine Load statement,  continually load data from a streaming app
+
+ syntax:
+      CREATE ROUTINE LOAD [database.]name on table
+      [load properties]
+      [PROPERTIES
+      (
+          desired_concurrent_number = xxx,
+          max_error_number = xxx,
+          k1 = v1,
+          ...
+          kn = vn
+      )]
+      FROM type of routine load
+      [(
+          k1 = v1,
+          ...
+          kn = vn
+      )]
+
+      load properties:
+          load property [[,] load property] ...
+
+      load property:
+          column separator | columns_mapping | partitions | where
+
+      column separator:
+          COLUMNS TERMINATED BY xxx
+      columns_mapping:
+          COLUMNS (c1, c2, c3 = c1 + c2)
+      partitions:
+          PARTITIONS (p1, p2, p3)
+      where:
+          WHERE c1 > 1
+
+      type of routine load:
+          KAFKA
+*/
+public class CreateRoutineLoadCommand extends Command implements NoForward {
+    CreateRoutineLoadInfo createRoutineLoadInfo;
+
+    public CreateRoutineLoadCommand(CreateRoutineLoadInfo createRoutineLoadInfo) {
+        super(PlanType.CREATE_ROUTINE_LOAD_COMMAND);
+        this.createRoutineLoadInfo = Objects.requireNonNull(createRoutineLoadInfo, "require CreateTableInfo object");
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        createRoutineLoadInfo.validate(ctx);
+        CreateRoutineLoadStmt createRoutineLoadStmt = createRoutineLoadInfo.translateToLegacyStmt(ctx);
+        Env.getCurrentEnv().getRoutineLoadManager().createRoutineLoadJob(createRoutineLoadStmt);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCreateRoutineLoadCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadColumnClause.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadColumnClause.java
new file mode 100644
index 00000000000000..a2bc7acbabe076
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadColumnClause.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.load;
+
+import java.util.List;
+
+/**
+ * load LoadColumnClause for nereids
+ */
+public class LoadColumnClause implements LoadProperty {
+    private List<LoadColumnDesc> columns;
+
+    public LoadColumnClause(List<LoadColumnDesc> columns) {
+        this.columns = columns;
+    }
+
+    public List<LoadColumnDesc> getColumns() {
+        return columns;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadColumnDesc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadColumnDesc.java
new file mode 100644
index 00000000000000..46b8013b159fa4
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadColumnDesc.java
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.load;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+
+/**
+ * load LoadColumnDesc for nereids
+ */
+public class LoadColumnDesc {
+    private final String columnName;
+    private Expression expression;
+
+    public LoadColumnDesc(String column, Expression expression) {
+        this.columnName = column;
+        this.expression = expression;
+    }
+
+    public LoadColumnDesc(String column) {
+        this.columnName = column;
+    }
+
+    public String getColumnName() {
+        return columnName;
+    }
+
+    public Expression getExpression() {
+        return expression;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadDeleteOnClause.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadDeleteOnClause.java
new file mode 100644
index 00000000000000..4ab77f8e9bb300
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadDeleteOnClause.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.load;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+
+/**
+ * load LoadDeleteOnClause for nereids
+ */
+public class LoadDeleteOnClause implements LoadProperty {
+    private final Expression expression;
+
+    public LoadDeleteOnClause(Expression expression) {
+        this.expression = expression;
+    }
+
+    public Expression getExpression() {
+        return expression;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadPartitionNames.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadPartitionNames.java
new file mode 100644
index 00000000000000..77b06bc695c75b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadPartitionNames.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.load;
+
+import org.apache.doris.common.AnalysisException;
+
+import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
+
+import java.util.List;
+
+/**
+ * load LoadPartitionNames for nereids
+ */
+public class LoadPartitionNames implements LoadProperty {
+    @SerializedName(value = "partitionNames")
+    private final List<String> partitionNames;
+    // true if these partitions are temp partitions
+    @SerializedName(value = "isTemp")
+    private final boolean isTemp;
+    private final boolean isStar;
+    private final long count;
+
+    public LoadPartitionNames(boolean isTemp, List<String> partitionNames) {
+        this.isTemp = isTemp;
+        this.partitionNames = partitionNames;
+        this.isStar = false;
+        this.count = 0;
+    }
+
+    public List<String> getPartitionNames() {
+        return partitionNames;
+    }
+
+    public boolean isTemp() {
+        return isTemp;
+    }
+
+    @Override
+    public void validate() throws AnalysisException {
+        if (isStar && count > 0) {
+            throw new AnalysisException("All partition and partition count couldn't be set at the same time.");
+        }
+        if (isStar || count > 0) {
+            return;
+        }
+        if (partitionNames == null || partitionNames.isEmpty()) {
+            throw new AnalysisException("No partition specified in partition lists");
+        }
+        // check if partition name is not empty string
+        if (partitionNames.stream().anyMatch(Strings::isNullOrEmpty)) {
+            throw new AnalysisException("there are empty partition name");
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadProperty.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadProperty.java
new file mode 100644
index 00000000000000..c21ba1c5774bbb
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadProperty.java
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/ParseNode.java
+// and modified by Doris
+
+package org.apache.doris.nereids.trees.plans.commands.load;
+
+import org.apache.doris.common.AnalysisException;
+
+/**
+ * load LoadProperty for nereids
+ */
+public interface LoadProperty {
+
+    /**
+     * Perform semantic validate of node and all of its children.
+     * Throws exception if any errors found.
+     */
+    default void validate() throws AnalysisException {
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadSeparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadSeparator.java
new file mode 100644
index 00000000000000..a1b31cb721638a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadSeparator.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.load;
+
+/**
+ * load seperator for nereids
+ */
+public class LoadSeparator implements LoadProperty {
+    private final String oriSeparator;
+
+    public LoadSeparator(String oriSeparator) {
+        this.oriSeparator = oriSeparator;
+    }
+
+    public String getOriSeparator() {
+        return oriSeparator;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadSequenceClause.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadSequenceClause.java
new file mode 100644
index 00000000000000..e9545382fe7dc3
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadSequenceClause.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.load;
+
+/**
+ * load LoadSequenceClause for nereids
+ */
+public class LoadSequenceClause implements LoadProperty {
+    private final String sequenceColName;
+
+    public LoadSequenceClause(String sequenceColName) {
+        this.sequenceColName = sequenceColName;
+    }
+
+    public String getSequenceColName() {
+        return sequenceColName;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadWhereClause.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadWhereClause.java
new file mode 100644
index 00000000000000..627f03e89e5c9b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/LoadWhereClause.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.load;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+
+/**
+ * load LoadWhereClause for nereids
+ */
+public class LoadWhereClause implements LoadProperty {
+    private final Expression expression;
+
+    private final boolean isPreceding;
+
+    public LoadWhereClause(Expression expression, boolean isPreceding) {
+        this.expression = expression;
+        this.isPreceding = isPreceding;
+    }
+
+    public Expression getExpression() {
+        return expression;
+    }
+
+    public boolean isPreceding() {
+        return isPreceding;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index c0f86d8dfa45ba..85633cb798b06c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -106,6 +106,7 @@
 import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.load.CreateRoutineLoadCommand;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
 
 /** CommandVisitor. */
@@ -473,4 +474,8 @@ default R visitShowPrivilegesCommand(ShowPrivilegesCommand showPrivilegesCommand
     default R visitShowTabletsBelongCommand(ShowTabletsBelongCommand showTabletBelongCommand, C context) {
         return visitCommand(showTabletBelongCommand, context);
     }
+
+    default R visitCreateRoutineLoadCommand(CreateRoutineLoadCommand createRoutineLoadCommand, C context) {
+        return visitCommand(createRoutineLoadCommand, context);
+    }
 }
diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_condition.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_condition.groovy
index 7735867c749049..d971a298ffb39d 100644
--- a/regression-test/suites/load_p0/routine_load/test_routine_load_condition.groovy
+++ b/regression-test/suites/load_p0/routine_load/test_routine_load_condition.groovy
@@ -157,7 +157,7 @@ suite("test_routine_load_condition","p0") {
                 }
                 log.info("reason of state changed: ${res[0][11].toString()}".toString())
                 def json = parseJson(res[0][11])
-                assertEquals("(`k12` >= date_sub(curdate(), INTERVAL 2 DAY))", json.whereExpr.toString())
+                assertEquals("(k12 >= CAST(days_sub(current_date(), 2) AS datetimev2(0)))", json.whereExpr.toString())
                 break;
             }
             while (true) {
@@ -180,4 +180,4 @@ suite("test_routine_load_condition","p0") {
             sql "DROP TABLE IF EXISTS ${tableName}"
         }
     }
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/load_p2/routine_load/test_routine_load.groovy b/regression-test/suites/load_p2/routine_load/test_routine_load.groovy
index 2b59c0f8e60d77..24cb5627f7f666 100644
--- a/regression-test/suites/load_p2/routine_load/test_routine_load.groovy
+++ b/regression-test/suites/load_p2/routine_load/test_routine_load.groovy
@@ -221,7 +221,7 @@ suite("test_routine_load_p2","p2,nonConcurrent") {
                 sql new File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
 
                 def name = "routine_load_" + tableName
-                sql """
+                checkNereidsExecute("""
                     CREATE ROUTINE LOAD ${jobs[i]} ON ${name}
                     COLUMNS(${columns[i]}),
                     COLUMNS TERMINATED BY "|"
@@ -239,7 +239,7 @@ suite("test_routine_load_p2","p2,nonConcurrent") {
                         "kafka_topic" = "${topics[i]}",
                         "property.kafka_default_offsets" = "OFFSET_BEGINNING"
                     );
-                """
+                """)
                 sql "sync"
                 i++
             }
@@ -2380,4 +2380,4 @@ suite("test_routine_load_p2","p2,nonConcurrent") {
             }
         }
     }
-}
\ No newline at end of file
+}

From 45dc1c91a0dba0ee5f2663167d5dee24a409bfae Mon Sep 17 00:00:00 2001
From: abmdocrt <Yukang.Lian2022@gmail.com>
Date: Wed, 27 Nov 2024 10:57:44 +0800
Subject: [PATCH 031/399] [Fix](group commit) Fix Group Commit Selecting
 Non-Alive Backend Instances (#44577)

---
 .../main/java/org/apache/doris/load/GroupCommitManager.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
index 60394f943654c6..1844ec5bf2b427 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
@@ -336,7 +336,7 @@ private Long getCachedBackend(String cluster, long tableId) {
                     return null;
                 }
                 Backend backend = Env.getCurrentSystemInfo().getBackend(backendId);
-                if (backend != null && backend.isActive() && !backend.isDecommissioned()) {
+                if (backend != null && backend.isAlive() && !backend.isDecommissioned()) {
                     return backend.getId();
                 } else {
                     tableToBeMap.remove(encode(cluster, tableId));
@@ -353,7 +353,7 @@ private Long getRandomBackend(String cluster, long tableId, List<Backend> backen
         OlapTable table = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getTableByTableId(tableId);
         Collections.shuffle(backends);
         for (Backend backend : backends) {
-            if (backend.isActive() && !backend.isDecommissioned()) {
+            if (backend.isAlive() && !backend.isDecommissioned()) {
                 tableToBeMap.put(encode(cluster, tableId), backend.getId());
                 tableToPressureMap.put(tableId,
                         new SlidingWindowCounter(table.getGroupCommitIntervalMs() / 1000 + 1));

From 7a6bb51ec4e30019efb1dcd2298bd0dd9772f0cc Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Wed, 27 Nov 2024 11:09:17 +0800
Subject: [PATCH 032/399] [fix](arrow-flight-sql) Remove `arrow_flight_sql`
 group from regression test (#44489)

### What problem does this PR solve?

Problem Summary:

Add `./run-regression-test.sh --run ... -g arrow_flight_sql` to the
pipeline. then add the group `arrow_flight_sql` again, which is safer.
---
 .../nereids_p0/show/test_show_create_table_nereids.groovy       | 2 +-
 .../suites/query_p0/aggregate/aggregate_count1.groovy           | 2 +-
 .../suites/query_p0/aggregate/select_distinct.groovy            | 2 +-
 .../casesensetive_column/join_with_column_casesensetive.groovy  | 2 +-
 regression-test/suites/query_p0/cast/test_cast.groovy           | 2 +-
 regression-test/suites/query_p0/except/test_query_except.groovy | 2 +-
 .../suites/query_p0/group_concat/test_group_concat.groovy       | 2 +-
 .../suites/query_p0/grouping_sets/test_grouping_sets1.groovy    | 2 +-
 regression-test/suites/query_p0/having/having.groovy            | 2 +-
 regression-test/suites/query_p0/intersect/test_intersect.groovy | 2 +-
 regression-test/suites/query_p0/join/test_join2.groovy          | 2 +-
 regression-test/suites/query_p0/join/test_left_join1.groovy     | 2 +-
 .../suites/query_p0/join/test_nestedloop_outer_join.groovy      | 2 +-
 .../suites/query_p0/join/test_partitioned_hash_join.groovy      | 2 +-
 .../suites/query_p0/lateral_view/lateral_view.groovy            | 2 +-
 .../suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy       | 2 +-
 .../suites/query_p0/literal_view/lietral_test.groovy            | 2 +-
 .../suites/query_p0/operator/test_set_operator.groovy           | 2 +-
 .../suites/query_p0/operator/test_sort_operator.groovy          | 2 +-
 .../suites/query_p0/session_variable/test_default_limit.groovy  | 2 +-
 .../query_p0/show/test_show_create_materialized_view.groovy     | 2 +-
 .../suites/query_p0/show/test_show_create_table.groovy          | 2 +-
 .../aggregate_functions/test_aggregate_all_functions.groovy     | 2 +-
 .../sql_functions/case_function/test_case_function_null.groovy  | 2 +-
 .../sql_functions/hash_functions/test_hash_function.groovy      | 2 +-
 .../sql_functions/ip_functions/test_ip_functions.groovy         | 2 +-
 .../sql_functions/json_function/test_query_json_insert.groovy   | 2 +-
 .../sql_functions/json_functions/test_json_function.groovy      | 2 +-
 .../query_p0/sql_functions/math_functions/test_conv.groovy      | 2 +-
 .../search_functions/test_multi_string_search.groovy            | 2 +-
 .../sql_functions/spatial_functions/test_gis_function.groovy    | 2 +-
 .../sql_functions/string_functions/test_string_function.groovy  | 2 +-
 .../query_p0/sql_functions/table_function/explode_split.groovy  | 2 +-
 .../suites/query_p0/sql_functions/test_alias_function.groovy    | 2 +-
 .../suites/query_p0/sql_functions/test_predicate.groovy         | 2 +-
 .../width_bucket_fuctions/test_width_bucket_function.groovy     | 2 +-
 regression-test/suites/query_p0/subquery/test_subquery2.groovy  | 2 +-
 regression-test/suites/query_p0/test_data_type_marks.groovy     | 2 +-
 regression-test/suites/query_p0/test_dict_with_null.groovy      | 2 +-
 regression-test/suites/query_p0/test_orderby_nullliteral.groovy | 2 +-
 regression-test/suites/query_p0/test_select_constant.groovy     | 2 +-
 .../suites/query_p0/test_select_with_predicate_like.groovy      | 2 +-
 .../suites/query_p0/test_select_with_predicate_prune.groovy     | 2 +-
 .../suites/query_p0/type_inference/test_largeint.groovy         | 2 +-
 .../suites/query_p0/with/test_with_and_two_phase_agg.groovy     | 2 +-
 45 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/regression-test/suites/nereids_p0/show/test_show_create_table_nereids.groovy b/regression-test/suites/nereids_p0/show/test_show_create_table_nereids.groovy
index 53f2242d1fa7f6..8c9b42af41a8a6 100644
--- a/regression-test/suites/nereids_p0/show/test_show_create_table_nereids.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_create_table_nereids.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_show_create_table_nereids", "query,arrow_flight_sql") {
+suite("test_show_create_table_nereids", "query") {
     String tb_name = "tb_show_create_table";
     try {  
         sql """drop table if exists ${tb_name} """
diff --git a/regression-test/suites/query_p0/aggregate/aggregate_count1.groovy b/regression-test/suites/query_p0/aggregate/aggregate_count1.groovy
index 3971f304e38646..cf657cc8ef3e3d 100644
--- a/regression-test/suites/query_p0/aggregate/aggregate_count1.groovy
+++ b/regression-test/suites/query_p0/aggregate/aggregate_count1.groovy
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-suite("aggregate_count1", "query,arrow_flight_sql") {
+suite("aggregate_count1", "query") {
     sql """ DROP TABLE IF EXISTS aggregate_count1 """
     sql """create table if not exists aggregate_count1 (
                 name varchar(128),
diff --git a/regression-test/suites/query_p0/aggregate/select_distinct.groovy b/regression-test/suites/query_p0/aggregate/select_distinct.groovy
index 2d6a8679d87ed8..6456158bdadb0d 100644
--- a/regression-test/suites/query_p0/aggregate/select_distinct.groovy
+++ b/regression-test/suites/query_p0/aggregate/select_distinct.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("select_distinct", "arrow_flight_sql") {
+suite("select_distinct") {
     sql """DROP TABLE IF EXISTS decimal_a;"""
     sql """DROP TABLE IF EXISTS decimal_b;"""
     sql """DROP TABLE IF EXISTS decimal_c;"""
diff --git a/regression-test/suites/query_p0/casesensetive_column/join_with_column_casesensetive.groovy b/regression-test/suites/query_p0/casesensetive_column/join_with_column_casesensetive.groovy
index 8bd3b19088a486..45499fc6f248a2 100644
--- a/regression-test/suites/query_p0/casesensetive_column/join_with_column_casesensetive.groovy
+++ b/regression-test/suites/query_p0/casesensetive_column/join_with_column_casesensetive.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("join_with_column_casesensetive", "arrow_flight_sql") {
+suite("join_with_column_casesensetive") {
     def tables=["ad_order_data_v1","ad_order_data"]
 
     for (String table in tables) {
diff --git a/regression-test/suites/query_p0/cast/test_cast.groovy b/regression-test/suites/query_p0/cast/test_cast.groovy
index dae669e2965f04..947d61bc828861 100644
--- a/regression-test/suites/query_p0/cast/test_cast.groovy
+++ b/regression-test/suites/query_p0/cast/test_cast.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite('test_cast', "arrow_flight_sql") {
+suite('test_cast') {
     def date = "date '2020-01-01'"
     def datev2 = "datev2 '2020-01-01'"
     def datetime = "timestamp '2020-01-01 12:34:45'"
diff --git a/regression-test/suites/query_p0/except/test_query_except.groovy b/regression-test/suites/query_p0/except/test_query_except.groovy
index 410e24f89b92de..1a2aa742d2910d 100644
--- a/regression-test/suites/query_p0/except/test_query_except.groovy
+++ b/regression-test/suites/query_p0/except/test_query_except.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_query_except", "arrow_flight_sql") {
+suite("test_query_except") {
     // test query except, depend on query_test_data_load.groovy
     sql "use test_query_db"
     qt_select_except1 """
diff --git a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
index 522d66ed64b30b..5054dc2ee3a91d 100644
--- a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
+++ b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_group_concat", "query,p0,arrow_flight_sql") {
+suite("test_group_concat", "query,p0") {
     qt_select """
                 SELECT group_concat(k6) FROM test_query_db.test where k6='false'
               """
diff --git a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets1.groovy b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets1.groovy
index f8180b0ab43846..1f12de6628a5eb 100644
--- a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets1.groovy
+++ b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets1.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_grouping_sets1", "arrow_flight_sql") {
+suite("test_grouping_sets1") {
     qt_select """
         select 
             col1
diff --git a/regression-test/suites/query_p0/having/having.groovy b/regression-test/suites/query_p0/having/having.groovy
index bbad236c973870..fb32b3834af4d6 100644
--- a/regression-test/suites/query_p0/having/having.groovy
+++ b/regression-test/suites/query_p0/having/having.groovy
@@ -19,7 +19,7 @@
 // /testing/trino-product-tests/src/main/resources/sql-tests/testcases/aggregate
 // and modified by Doris.
 
-suite("having", "query,p0,arrow_flight_sql") {
+suite("having", "query,p0") {
     sql """DROP TABLE IF EXISTS supplier"""
     sql """CREATE TABLE `supplier` (
             `s_suppkey` int(11) NOT NULL,
diff --git a/regression-test/suites/query_p0/intersect/test_intersect.groovy b/regression-test/suites/query_p0/intersect/test_intersect.groovy
index 7919bec324b876..1c007b95d7d07d 100644
--- a/regression-test/suites/query_p0/intersect/test_intersect.groovy
+++ b/regression-test/suites/query_p0/intersect/test_intersect.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_intersect", "arrow_flight_sql") {
+suite("test_intersect") {
     qt_select """
                 SELECT * FROM (SELECT k1 FROM test_query_db.baseall
                     INTERSECT SELECT k1 FROM test_query_db.test) a ORDER BY k1
diff --git a/regression-test/suites/query_p0/join/test_join2.groovy b/regression-test/suites/query_p0/join/test_join2.groovy
index 9158133948f754..6125b9a873f77e 100644
--- a/regression-test/suites/query_p0/join/test_join2.groovy
+++ b/regression-test/suites/query_p0/join/test_join2.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_join2", "query,p0,arrow_flight_sql") {
+suite("test_join2", "query,p0") {
     def DBname = "regression_test_join2"
     def TBname1 = "J1_TBL"
     def TBname2 = "J2_TBL"
diff --git a/regression-test/suites/query_p0/join/test_left_join1.groovy b/regression-test/suites/query_p0/join/test_left_join1.groovy
index 104adab4a850d0..d4cbeeee65eda2 100644
--- a/regression-test/suites/query_p0/join/test_left_join1.groovy
+++ b/regression-test/suites/query_p0/join/test_left_join1.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_left_join1", "query,p0,arrow_flight_sql") {
+suite("test_left_join1", "query,p0") {
 
     def tableName = "test_left_join1"
     sql """drop table if exists ${tableName}"""
diff --git a/regression-test/suites/query_p0/join/test_nestedloop_outer_join.groovy b/regression-test/suites/query_p0/join/test_nestedloop_outer_join.groovy
index f99dfa042446e9..ad19e554690ee7 100644
--- a/regression-test/suites/query_p0/join/test_nestedloop_outer_join.groovy
+++ b/regression-test/suites/query_p0/join/test_nestedloop_outer_join.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_nestedloop_outer_join", "query_p0,arrow_flight_sql") {
+suite("test_nestedloop_outer_join", "query_p0") {
     def tbl1 = "test_nestedloop_outer_join1"
     def tbl2 = "test_nestedloop_outer_join2"
 
diff --git a/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy b/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
index 676cdd06274a68..cbe09ec527ffbc 100644
--- a/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
+++ b/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_partitioned_hash_join", "query,p0,arrow_flight_sql") {
+suite("test_partitioned_hash_join", "query,p0") {
     sql "drop table if exists test_partitioned_hash_join_l"
     sql "drop table if exists test_partitioned_hash_join_r"
     sql """ create table test_partitioned_hash_join_l (
diff --git a/regression-test/suites/query_p0/lateral_view/lateral_view.groovy b/regression-test/suites/query_p0/lateral_view/lateral_view.groovy
index bfe6ca76872ea8..a24623590cd0af 100644
--- a/regression-test/suites/query_p0/lateral_view/lateral_view.groovy
+++ b/regression-test/suites/query_p0/lateral_view/lateral_view.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("lateral_view", "arrow_flight_sql") {
+suite("lateral_view") {
     sql """ DROP TABLE IF EXISTS `test_explode_bitmap` """
 	sql """
 		CREATE TABLE `test_explode_bitmap` (
diff --git a/regression-test/suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy b/regression-test/suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy
index caa75ac7be38ed..da0c7231f425d1 100644
--- a/regression-test/suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy
+++ b/regression-test/suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_offset_in_subquery_with_join", "query,arrow_flight_sql") {
+suite("test_offset_in_subquery_with_join", "query") {
     // define a sql table
     def testTable = "test_offset_in_subquery_with_join"
 
diff --git a/regression-test/suites/query_p0/literal_view/lietral_test.groovy b/regression-test/suites/query_p0/literal_view/lietral_test.groovy
index 27b82c16247a73..6e9d51f0a0d610 100644
--- a/regression-test/suites/query_p0/literal_view/lietral_test.groovy
+++ b/regression-test/suites/query_p0/literal_view/lietral_test.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("literal_view_test", "arrow_flight_sql") {
+suite("literal_view_test") {
 
     sql """DROP TABLE IF EXISTS table1"""
 
diff --git a/regression-test/suites/query_p0/operator/test_set_operator.groovy b/regression-test/suites/query_p0/operator/test_set_operator.groovy
index 7d6219585e4c4c..cb05e18b3e870b 100644
--- a/regression-test/suites/query_p0/operator/test_set_operator.groovy
+++ b/regression-test/suites/query_p0/operator/test_set_operator.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_set_operators", "query,p0,arrow_flight_sql") {
+suite("test_set_operators", "query,p0") {
 
     sql """
         DROP TABLE IF EXISTS t1;
diff --git a/regression-test/suites/query_p0/operator/test_sort_operator.groovy b/regression-test/suites/query_p0/operator/test_sort_operator.groovy
index d76daff01f6fcc..24a2b8ef73a424 100644
--- a/regression-test/suites/query_p0/operator/test_sort_operator.groovy
+++ b/regression-test/suites/query_p0/operator/test_sort_operator.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_sort_operator", "query,p0,arrow_flight_sql") {
+suite("test_sort_operator", "query,p0") {
 
     sql """
         DROP TABLE IF EXISTS dim_org_ful;
diff --git a/regression-test/suites/query_p0/session_variable/test_default_limit.groovy b/regression-test/suites/query_p0/session_variable/test_default_limit.groovy
index 2ce3b647142ae4..edda5d51790c56 100644
--- a/regression-test/suites/query_p0/session_variable/test_default_limit.groovy
+++ b/regression-test/suites/query_p0/session_variable/test_default_limit.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite('test_default_limit', "arrow_flight_sql") {
+suite('test_default_limit') {
     sql 'drop table if exists baseall'
     sql 'drop table if exists bigtable'
 
diff --git a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
index 9550a7fec3dbd2..7c5edcc81a2e8e 100644
--- a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
+++ b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_show_create_materialized_view", "query,arrow_flight_sql") {
+suite("test_show_create_materialized_view", "query") {
     String tableName = "table_for_mv_test";
     String mvName = "mv_show_create_materialized_view";
     try {  
diff --git a/regression-test/suites/query_p0/show/test_show_create_table.groovy b/regression-test/suites/query_p0/show/test_show_create_table.groovy
index 1e3fc7ff5cb527..6325cbe319fd88 100644
--- a/regression-test/suites/query_p0/show/test_show_create_table.groovy
+++ b/regression-test/suites/query_p0/show/test_show_create_table.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_show_create_table", "query,arrow_flight_sql") {
+suite("test_show_create_table", "query") {
     String tb_name = "tb_show_create_table";
     try {  
         sql """drop table if exists ${tb_name} """
diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy
index cdab9472e27dbd..86a951d7ac33c6 100644
--- a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy
+++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_aggregate_all_functions", "arrow_flight_sql") {
+suite("test_aggregate_all_functions") {
 
     sql "set batch_size = 4096"
     
diff --git a/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy b/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
index a91c86b5f48908..41ef1625080806 100644
--- a/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
+++ b/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_case_function_null", "query,p0,arrow_flight_sql") {
+suite("test_case_function_null", "query,p0") {
     sql """ drop table if exists case_null0 """
     sql """ create table case_null0 (
                 `c0` decimalv3(17, 1) NULL,
diff --git a/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy b/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy
index d547e9fb287d71..590ccd10821f61 100644
--- a/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_hash_function", "arrow_flight_sql") {
+suite("test_hash_function") {
     sql "set batch_size = 4096;"
     sql "set enable_profile = true;"
 
diff --git a/regression-test/suites/query_p0/sql_functions/ip_functions/test_ip_functions.groovy b/regression-test/suites/query_p0/sql_functions/ip_functions/test_ip_functions.groovy
index 5373217503a018..03e9788a58a3b8 100644
--- a/regression-test/suites/query_p0/sql_functions/ip_functions/test_ip_functions.groovy
+++ b/regression-test/suites/query_p0/sql_functions/ip_functions/test_ip_functions.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_ip_functions", "arrow_flight_sql") {
+suite("test_ip_functions") {
     sql "set batch_size = 4096;"
 
     qt_sql "SELECT ipv4_num_to_string(-1);"
diff --git a/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_insert.groovy b/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_insert.groovy
index b5865034538a11..c885e3ae3431f3 100644
--- a/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_insert.groovy
+++ b/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_insert.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_query_json_insert", "query,arrow_flight_sql") {
+suite("test_query_json_insert", "query") {
     qt_sql "select json_insert('{\"a\": 1, \"b\": [2, 3]}', '\$', null);"
     qt_sql "select json_insert('{\"k\": [1, 2]}', '\$.k[0]', null, '\$.[1]', null);"
     def tableName = "test_query_json_insert"
diff --git a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy
index 4bd88bf131e727..aa0deec96f46a2 100644
--- a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_json_function", "arrow_flight_sql") {
+suite("test_json_function") {
     sql "set batch_size = 4096;"
 
     qt_sql "SELECT get_json_double('{\"k1\":1.3, \"k2\":\"2\"}', \"\$.k1\");"
diff --git a/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy b/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy
index 3a74abfe9c8b22..6c4867174d11ac 100644
--- a/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy
+++ b/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_conv", "arrow_flight_sql") {
+suite("test_conv") {
     qt_select "SELECT CONV(15,10,2)"
 
     sql """ drop table if exists test_conv; """
diff --git a/regression-test/suites/query_p0/sql_functions/search_functions/test_multi_string_search.groovy b/regression-test/suites/query_p0/sql_functions/search_functions/test_multi_string_search.groovy
index f1487d283dfcdf..061665d3b9da6e 100644
--- a/regression-test/suites/query_p0/sql_functions/search_functions/test_multi_string_search.groovy
+++ b/regression-test/suites/query_p0/sql_functions/search_functions/test_multi_string_search.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_multi_string_search", "arrow_flight_sql") {
+suite("test_multi_string_search") {
     def table_name = "test_multi_string_search_strings"
 
     sql """ DROP TABLE IF EXISTS ${table_name} """
diff --git a/regression-test/suites/query_p0/sql_functions/spatial_functions/test_gis_function.groovy b/regression-test/suites/query_p0/sql_functions/spatial_functions/test_gis_function.groovy
index f76cb44cb4ad4b..e98e11ba7e6888 100644
--- a/regression-test/suites/query_p0/sql_functions/spatial_functions/test_gis_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/spatial_functions/test_gis_function.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_gis_function", "arrow_flight_sql") {
+suite("test_gis_function") {
     sql "set batch_size = 4096;"
 
     qt_sql "SELECT ST_AsText(ST_Point(24.7, 56.7));"
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
index 6e18fb57eeb4cf..f5d32653c818b5 100644
--- a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_string_function", "arrow_flight_sql") {
+suite("test_string_function") {
     sql "set batch_size = 4096;"
 
     qt_sql "select elt(0, \"hello\", \"doris\");"
diff --git a/regression-test/suites/query_p0/sql_functions/table_function/explode_split.groovy b/regression-test/suites/query_p0/sql_functions/table_function/explode_split.groovy
index 53db931c03bb03..b7dd4d640799fb 100644
--- a/regression-test/suites/query_p0/sql_functions/table_function/explode_split.groovy
+++ b/regression-test/suites/query_p0/sql_functions/table_function/explode_split.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("explode_split", "arrow_flight_sql") {
+suite("explode_split") {
     def tableName = "test_lv_str"
 
     sql """ DROP TABLE IF EXISTS ${tableName} """
diff --git a/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy b/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy
index 095ec89e220f1b..8e0e94fa2df805 100644
--- a/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite('test_alias_function', "arrow_flight_sql") {
+suite('test_alias_function') {
     sql '''
         CREATE ALIAS FUNCTION IF NOT EXISTS f1(DATETIMEV2(3), INT)
             with PARAMETER (datetime1, int1) as date_trunc(days_sub(datetime1, int1), 'day')'''
diff --git a/regression-test/suites/query_p0/sql_functions/test_predicate.groovy b/regression-test/suites/query_p0/sql_functions/test_predicate.groovy
index 6cca6b62c9960b..20b3c179ad5c01 100644
--- a/regression-test/suites/query_p0/sql_functions/test_predicate.groovy
+++ b/regression-test/suites/query_p0/sql_functions/test_predicate.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_predicate", "arrow_flight_sql") {
+suite("test_predicate") {
     sql """drop table if exists t1;"""
     sql """
             create table t1 (
diff --git a/regression-test/suites/query_p0/sql_functions/width_bucket_fuctions/test_width_bucket_function.groovy b/regression-test/suites/query_p0/sql_functions/width_bucket_fuctions/test_width_bucket_function.groovy
index 1a455da92446f8..d0862a580ca600 100644
--- a/regression-test/suites/query_p0/sql_functions/width_bucket_fuctions/test_width_bucket_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/width_bucket_fuctions/test_width_bucket_function.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_width_bucket_function", "arrow_flight_sql") {
+suite("test_width_bucket_function") {
     qt_sql "select width_bucket(1, 2, 3, 2)"
     qt_sql "select width_bucket(null, 2, 3, 2)"
     qt_sql "select width_bucket(6, 2, 6, 4)"
diff --git a/regression-test/suites/query_p0/subquery/test_subquery2.groovy b/regression-test/suites/query_p0/subquery/test_subquery2.groovy
index a14a44fa152b97..e572459cc72fe3 100644
--- a/regression-test/suites/query_p0/subquery/test_subquery2.groovy
+++ b/regression-test/suites/query_p0/subquery/test_subquery2.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_subquery2", "arrow_flight_sql") {
+suite("test_subquery2") {
     
     sql """DROP TABLE IF EXISTS subquerytest2"""
     
diff --git a/regression-test/suites/query_p0/test_data_type_marks.groovy b/regression-test/suites/query_p0/test_data_type_marks.groovy
index 51fb7c9614e488..79803d98723313 100644
--- a/regression-test/suites/query_p0/test_data_type_marks.groovy
+++ b/regression-test/suites/query_p0/test_data_type_marks.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_data_type_marks", "arrow_flight_sql") {
+suite("test_data_type_marks") {
     def tbName = "org"
     sql "DROP TABLE IF EXISTS ${tbName}"
     sql """
diff --git a/regression-test/suites/query_p0/test_dict_with_null.groovy b/regression-test/suites/query_p0/test_dict_with_null.groovy
index 83d253fa4d1b04..b3738bb68aa1ba 100644
--- a/regression-test/suites/query_p0/test_dict_with_null.groovy
+++ b/regression-test/suites/query_p0/test_dict_with_null.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("dict_with_null", "query,arrow_flight_sql") {
+suite("dict_with_null", "query") {
     def tableName = "test_dict_with_null"
     sql "DROP TABLE IF EXISTS ${tableName}"
     sql """
diff --git a/regression-test/suites/query_p0/test_orderby_nullliteral.groovy b/regression-test/suites/query_p0/test_orderby_nullliteral.groovy
index e806060c8bcb1c..fe11c778af0b98 100644
--- a/regression-test/suites/query_p0/test_orderby_nullliteral.groovy
+++ b/regression-test/suites/query_p0/test_orderby_nullliteral.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("orderby_nullliteral", "query,arrow_flight_sql") {
+suite("orderby_nullliteral", "query") {
 
     def tableName = "test_orderby_nullliteral"
     sql "DROP TABLE IF EXISTS ${tableName}"
diff --git a/regression-test/suites/query_p0/test_select_constant.groovy b/regression-test/suites/query_p0/test_select_constant.groovy
index 68f0a28a20e853..6015e19576c690 100644
--- a/regression-test/suites/query_p0/test_select_constant.groovy
+++ b/regression-test/suites/query_p0/test_select_constant.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_select_constant", "arrow_flight_sql") {
+suite("test_select_constant") {
     qt_select1 'select 100, "test", date("2021-01-02");'
     qt_select_geo1 'SELECT ST_AsText(ST_Point(123.12345678901234567890,89.1234567890));'
 }
diff --git a/regression-test/suites/query_p0/test_select_with_predicate_like.groovy b/regression-test/suites/query_p0/test_select_with_predicate_like.groovy
index 0d01f1b958a11c..9491c4271ca530 100644
--- a/regression-test/suites/query_p0/test_select_with_predicate_like.groovy
+++ b/regression-test/suites/query_p0/test_select_with_predicate_like.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_select_with_predicate_like", "arrow_flight_sql") {
+suite("test_select_with_predicate_like") {
     def tables=["test_basic_agg"]
 
     for (String table in tables) {
diff --git a/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy b/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy
index ccd1b9160fb148..768e04b4c327b5 100644
--- a/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy
+++ b/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_select_with_predicate_prune", "arrow_flight_sql") {
+suite("test_select_with_predicate_prune") {
     sql """
         drop table if exists `test_select_with_predicate_prune`;
     """
diff --git a/regression-test/suites/query_p0/type_inference/test_largeint.groovy b/regression-test/suites/query_p0/type_inference/test_largeint.groovy
index 161359cfa97e72..d5cbfa4b479838 100644
--- a/regression-test/suites/query_p0/type_inference/test_largeint.groovy
+++ b/regression-test/suites/query_p0/type_inference/test_largeint.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_largeint", "arrow_flight_sql") {
+suite("test_largeint") {
     def tbName = "test_largeint"
     sql "DROP TABLE IF EXISTS ${tbName}"
     sql """
diff --git a/regression-test/suites/query_p0/with/test_with_and_two_phase_agg.groovy b/regression-test/suites/query_p0/with/test_with_and_two_phase_agg.groovy
index d563ef1630517d..99164a999c557e 100644
--- a/regression-test/suites/query_p0/with/test_with_and_two_phase_agg.groovy
+++ b/regression-test/suites/query_p0/with/test_with_and_two_phase_agg.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_with_and_two_phase_agg", "arrow_flight_sql") {
+suite("test_with_and_two_phase_agg") {
     def tableName = "test_with_and_two_phase_agg_table"
     sql """ DROP TABLE IF EXISTS ${tableName} """
     sql """

From 805520bfe4d90965bbd844945bd302eff38dc8f3 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Wed, 27 Nov 2024 11:22:57 +0800
Subject: [PATCH 033/399] [Bug](join) fix single null eq get wrong result
 (#44613)

introduced by https://github.com/apache/doris/pull/42398
In order to make the null keys equal when using single null eq, all null
keys need to be set to default value.
---
 be/src/pipeline/exec/hashjoin_build_sink.h    |  5 ++++-
 .../exec/join/process_hash_table_probe_impl.h |  4 ++++
 .../data/nereids_p0/join/test_mark_join.out   |  7 +++++++
 .../data/query_p0/join/test_join.out          |  6 ++++++
 .../nereids_p0/join/test_mark_join.groovy     | 21 +++++++++++++++++++
 .../suites/query_p0/join/test_join.groovy     | 13 +++++++++++-
 6 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/be/src/pipeline/exec/hashjoin_build_sink.h b/be/src/pipeline/exec/hashjoin_build_sink.h
index d905afa27582fb..cc78e6a769f3c3 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.h
+++ b/be/src/pipeline/exec/hashjoin_build_sink.h
@@ -197,7 +197,10 @@ struct ProcessHashTableBuild {
         SCOPED_TIMER(_parent->_build_table_insert_timer);
         hash_table_ctx.hash_table->template prepare_build<JoinOpType>(_rows, _batch_size,
                                                                       *has_null_key);
-
+        // In order to make the null keys equal when using single null eq, all null keys need to be set to default value.
+        if (_build_raw_ptrs.size() == 1 && null_map) {
+            _build_raw_ptrs[0]->assume_mutable()->replace_column_null_data(null_map->data());
+        }
         hash_table_ctx.init_serialized_keys(_build_raw_ptrs, _rows,
                                             null_map ? null_map->data() : nullptr, true, true,
                                             hash_table_ctx.hash_table->get_bucket_size());
diff --git a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
index 05cd3d7d9e0590..fc1153b3419d3e 100644
--- a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
+++ b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
@@ -173,6 +173,10 @@ typename HashTableType::State ProcessHashTableProbe<JoinOpType>::_init_probe_sid
     if (!_parent->_ready_probe) {
         _parent->_ready_probe = true;
         hash_table_ctx.reset();
+        // In order to make the null keys equal when using single null eq, all null keys need to be set to default value.
+        if (_parent->_probe_columns.size() == 1 && null_map) {
+            _parent->_probe_columns[0]->assume_mutable()->replace_column_null_data(null_map);
+        }
         hash_table_ctx.init_serialized_keys(_parent->_probe_columns, probe_rows, null_map, true,
                                             false, hash_table_ctx.hash_table->get_bucket_size());
         hash_table_ctx.hash_table->pre_build_idxs(hash_table_ctx.bucket_nums,
diff --git a/regression-test/data/nereids_p0/join/test_mark_join.out b/regression-test/data/nereids_p0/join/test_mark_join.out
index 59fc7d651ad5e9..a603555667416b 100644
--- a/regression-test/data/nereids_p0/join/test_mark_join.out
+++ b/regression-test/data/nereids_p0/join/test_mark_join.out
@@ -53,3 +53,10 @@
 -- !mark_join_null_conjunct --
 \N
 
+-- !mark_join8 --
+1	1	1	false
+2	2	2	false
+3	\N	\N	\N
+3	\N	3	\N
+4	\N	4	true
+
diff --git a/regression-test/data/query_p0/join/test_join.out b/regression-test/data/query_p0/join/test_join.out
index 3814530e0c634d..7cbaf4f77508e8 100644
--- a/regression-test/data/query_p0/join/test_join.out
+++ b/regression-test/data/query_p0/join/test_join.out
@@ -3243,3 +3243,9 @@ false	true	true	false	false
 -- !sql --
 4
 
+-- !sql --
+2	\N
+
+-- !sql --
+2	\N
+
diff --git a/regression-test/suites/nereids_p0/join/test_mark_join.groovy b/regression-test/suites/nereids_p0/join/test_mark_join.groovy
index 3b07c357b5dc04..034900f81acb08 100644
--- a/regression-test/suites/nereids_p0/join/test_mark_join.groovy
+++ b/regression-test/suites/nereids_p0/join/test_mark_join.groovy
@@ -142,4 +142,25 @@ suite("test_mark_join", "nereids_p0") {
     """
 
     qt_mark_join_null_conjunct """select null in ( select k1 from test_mark_join_t1);"""
+
+    qt_mark_join8 """
+            select
+                k1,
+                k2,
+                k3,
+                k1 not in (
+                    select
+                        test_mark_join_t2.k2
+                    from
+                        test_mark_join_t2
+                    where
+                        test_mark_join_t2.k3 <=> test_mark_join_t1.k3
+                ) vv
+            from
+                test_mark_join_t1
+            order by
+                1,
+                2,
+                3;
+    """
 }
diff --git a/regression-test/suites/query_p0/join/test_join.groovy b/regression-test/suites/query_p0/join/test_join.groovy
index e75878cb032aa3..fa83fddbd41482 100644
--- a/regression-test/suites/query_p0/join/test_join.groovy
+++ b/regression-test/suites/query_p0/join/test_join.groovy
@@ -720,7 +720,7 @@ suite("test_join", "query,p0") {
     }
 
     qt_left_anti_join_with_other_pred "select b.k1 from baseall b left anti join test t on b.k1 = t.k1 and 1 = 2 order by b.k1"
-
+    // null not in (1,2,3,null) = true
     qt_left_anti_join_null_1 "select b.k1 from baseall b left anti join test t on b.k1 = t.k1 order by b.k1"
 
     qt_left_anti_join_null_2 "select b.k1 from baseall b left anti join test_join_empty_view t on b.k1 = t.k1 order by b.k1"
@@ -931,6 +931,7 @@ suite("test_join", "query,p0") {
     // https://github.com/apache/doris/issues/4210
     qt_join_bug3"""select * from baseall t1 where k1 = (select min(k1) from test t2 where t2.k1 = t1.k1 and t2.k2=t1.k2)
            order by k1"""
+    // null not in (1,2,3) = false
     qt_join_bug4"""select b.k1 from baseall b where b.k1 not in( select k1 from baseall where k1 is not null )"""
 
 
@@ -1330,4 +1331,14 @@ suite("test_join", "query,p0") {
     qt_sql """ select /*+SET_VAR(batch_size=1, disable_join_reorder=true)*/ count(DISTINCT dcqewrt.engineer)  as active_person_count from tbl1 dcqewrt left join [broadcast] tbl2 dd on dd.data_dt = dcqewrt.data_dt; """
     sql """ DROP TABLE IF EXISTS tbl2; """
     sql """ DROP TABLE IF EXISTS tbl1; """
+
+
+    sql "drop table if exists t01;"
+    sql "drop table if exists t02;"
+    sql"""create table t01 (id int, a varchar(10)) properties ("replication_num" = "1");"""
+    sql"""create table t02 (id int, b varchar(10)) properties ("replication_num" = "1");"""
+    sql"insert into t01 values (1, 'a'), (2, null), (3, 'c');"
+    sql"insert into t02 values (1, 'b');"
+    qt_sql"select * from t01 where (not like (a, 'a%')) <=> 'b';"
+    qt_sql"select * from t01 where (not like (a, 'a%')) <=> (select max(b) from t02); "
 }

From d592e279c1058de37994135a4cc20fd35698a9c1 Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Wed, 27 Nov 2024 12:18:56 +0800
Subject: [PATCH 034/399] [fix](sql_cache) fix sql cache result wrong of
 from_unixtime(col, 'yyyy-MM-dd HH:mm:ss') (#44631)

fix sql cache result wrong of from_unixtime(col, 'yyyy-MM-dd HH:mm:ss')
which introduced by #33262

the wrong result: is `yyyy-MM-dd HH:mm:ss`
---
 .../rules/expression/rules/SupportJavaDateFormatter.java | 3 ++-
 .../executable/DateTimeExtractAndTransform.java          | 9 +++++++++
 .../main/java/org/apache/doris/qe/SessionVariable.java   | 2 +-
 .../nereids_p0/cache/parse_sql_from_sql_cache.groovy     | 7 +++++++
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SupportJavaDateFormatter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SupportJavaDateFormatter.java
index 27b929a2b9f865..f5b442a3989cd5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SupportJavaDateFormatter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SupportJavaDateFormatter.java
@@ -76,7 +76,8 @@ private static Expression translateJavaFormatter(Expression function, int format
         return function;
     }
 
-    private static Expression translateJavaFormatter(Expression formatterExpr) {
+    /** translateJavaFormatter */
+    public static Expression translateJavaFormatter(Expression formatterExpr) {
         if (formatterExpr.isLiteral() && formatterExpr.getDataType().isStringLikeType()) {
             Literal literal = (Literal) formatterExpr;
             String originFormatter = literal.getStringValue();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
index 440f93cac598a2..c908271531884f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
@@ -20,6 +20,7 @@
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.rules.expression.rules.SupportJavaDateFormatter;
 import org.apache.doris.nereids.trees.expressions.ExecFunction;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
@@ -293,12 +294,14 @@ private static LocalDateTime firstDayOfWeek(LocalDateTime dateTime) {
      */
     @ExecFunction(name = "date_format")
     public static Expression dateFormat(DateLiteral date, StringLikeLiteral format) {
+        format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format);
         return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter().format(
                 java.time.LocalDate.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()))));
     }
 
     @ExecFunction(name = "date_format")
     public static Expression dateFormat(DateTimeLiteral date, StringLikeLiteral format) {
+        format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format);
         return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter().format(
                 java.time.LocalDateTime.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()),
                         ((int) date.getHour()), ((int) date.getMinute()), ((int) date.getSecond()))));
@@ -306,12 +309,14 @@ public static Expression dateFormat(DateTimeLiteral date, StringLikeLiteral form
 
     @ExecFunction(name = "date_format")
     public static Expression dateFormat(DateV2Literal date, StringLikeLiteral format) {
+        format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format);
         return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter().format(
                 java.time.LocalDate.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()))));
     }
 
     @ExecFunction(name = "date_format")
     public static Expression dateFormat(DateTimeV2Literal date, StringLikeLiteral format) {
+        format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format);
         return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter().format(
                 java.time.LocalDateTime.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()),
                         ((int) date.getHour()), ((int) date.getMinute()), ((int) date.getSecond()))));
@@ -479,6 +484,8 @@ public static Expression fromUnixTime(BigIntLiteral second) {
      */
     @ExecFunction(name = "from_unixtime")
     public static Expression fromUnixTime(BigIntLiteral second, StringLikeLiteral format) {
+        format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format);
+
         // 32536771199L is max valid timestamp of mysql from_unix_time
         if (second.getValue() < 0 || second.getValue() > 32536771199L) {
             return new NullLiteral(VarcharType.SYSTEM_DEFAULT);
@@ -531,6 +538,7 @@ public static Expression unixTimestamp(DateTimeV2Literal date) {
      */
     @ExecFunction(name = "unix_timestamp")
     public static Expression unixTimestamp(StringLikeLiteral date, StringLikeLiteral format) {
+        format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format);
         DateTimeFormatter formatter = DateUtils.formatBuilder(format.getValue()).toFormatter();
         LocalDateTime dateObj;
         try {
@@ -616,6 +624,7 @@ public static Expression makeDate(IntegerLiteral year, IntegerLiteral dayOfYear)
      */
     @ExecFunction(name = "str_to_date")
     public static Expression strToDate(StringLikeLiteral str, StringLikeLiteral format) {
+        format = (StringLikeLiteral) SupportJavaDateFormatter.translateJavaFormatter(format);
         if (org.apache.doris.analysis.DateLiteral.hasTimePart(format.getStringValue())) {
             DataType returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATETIME));
             if (returnType instanceof DateTimeV2Type) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 2c09ee73f57168..71b746c7907262 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1011,7 +1011,7 @@ public enum IgnoreSplitType {
     @VariableMgr.VarAttr(name = ENABLE_ODBC_TRANSCATION)
     public boolean enableOdbcTransaction = false;
 
-    @VariableMgr.VarAttr(name = ENABLE_SQL_CACHE)
+    @VariableMgr.VarAttr(name = ENABLE_SQL_CACHE, fuzzy = true)
     public boolean enableSqlCache = false;
 
     @VariableMgr.VarAttr(name = ENABLE_QUERY_CACHE)
diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
index 6aeeeed0ead602..54ab702888888d 100644
--- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
+++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
@@ -820,6 +820,13 @@ suite("parse_sql_from_sql_cache") {
                     assertTrue(profileString.contains("Is  Cached:  Yes"))
                 }
             }
+        }),
+        extraThread("sql_cache_with_date_format", {
+            sql "set enable_sql_cache=true"
+            for (def i in 0..3) {
+                def result = sql "select FROM_UNIXTIME(UNIX_TIMESTAMP(), 'yyyy-MM-dd HH:mm:ss')"
+                assertNotEquals("yyyy-MM-dd HH:mm:ss", result[0][0])
+            }
         })
     ).get()
 }

From f6ff621a416224b705e35dc456c3d4a08df40fb2 Mon Sep 17 00:00:00 2001
From: zzzxl <yangsiyu@selectdb.com>
Date: Wed, 27 Nov 2024 15:59:10 +0800
Subject: [PATCH 035/399] [fix](inverted index) Modify Error Handling for File
 Open Failure (#44551)

Problem Summary:

Modify Error Handling for File Open Failure
---
 be/src/olap/compaction.cpp                    |  26 +-
 ...ip_index_compaction_fault_injection.groovy | 240 ++++++++++++++++++
 2 files changed, 261 insertions(+), 5 deletions(-)
 create mode 100644 regression-test/suites/fault_injection_p0/test_skip_index_compaction_fault_injection.groovy

diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 68ed0322a9ec92..e71e1862dc8dbb 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -662,15 +662,28 @@ Status Compaction::do_inverted_index_compaction() {
         try {
             std::vector<std::unique_ptr<DorisCompoundReader>> src_idx_dirs(src_segment_num);
             for (int src_segment_id = 0; src_segment_id < src_segment_num; src_segment_id++) {
-                src_idx_dirs[src_segment_id] =
-                        DORIS_TRY(inverted_index_file_readers[src_segment_id]->open(index_meta));
+                auto res = inverted_index_file_readers[src_segment_id]->open(index_meta);
+                DBUG_EXECUTE_IF("Compaction::open_inverted_index_file_reader", {
+                    res = ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+                            "debug point: Compaction::open_index_file_reader error"));
+                })
+                if (!res.has_value()) {
+                    throw Exception(ErrorCode::INVERTED_INDEX_COMPACTION_ERROR, res.error().msg());
+                }
+                src_idx_dirs[src_segment_id] = std::move(res.value());
             }
             for (int dest_segment_id = 0; dest_segment_id < dest_segment_num; dest_segment_id++) {
-                auto dest_dir =
-                        DORIS_TRY(inverted_index_file_writers[dest_segment_id]->open(index_meta));
+                auto res = inverted_index_file_writers[dest_segment_id]->open(index_meta);
+                DBUG_EXECUTE_IF("Compaction::open_inverted_index_file_writer", {
+                    res = ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+                            "debug point: Compaction::open_inverted_index_file_writer error"));
+                })
+                if (!res.has_value()) {
+                    throw Exception(ErrorCode::INVERTED_INDEX_COMPACTION_ERROR, res.error().msg());
+                }
                 // Destination directories in dest_index_dirs do not need to be deconstructed,
                 // but their lifecycle must be managed by inverted_index_file_writers.
-                dest_index_dirs[dest_segment_id] = dest_dir.get();
+                dest_index_dirs[dest_segment_id] = res.value().get();
             }
             auto st = compact_column(index_meta->index_id(), src_idx_dirs, dest_index_dirs,
                                      index_tmp_path.native(), trans_vec, dest_segment_num_rows);
@@ -681,6 +694,9 @@ Status Compaction::do_inverted_index_compaction() {
         } catch (CLuceneError& e) {
             error_handler(index_meta->index_id(), column_uniq_id);
             status = Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(e.what());
+        } catch (const Exception& e) {
+            error_handler(index_meta->index_id(), column_uniq_id);
+            status = Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(e.what());
         }
     }
 
diff --git a/regression-test/suites/fault_injection_p0/test_skip_index_compaction_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_skip_index_compaction_fault_injection.groovy
new file mode 100644
index 00000000000000..cdc2bdda41716d
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_skip_index_compaction_fault_injection.groovy
@@ -0,0 +1,240 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_skip_index_compaction_fault_injection", "nonConcurrent") {
+  def isCloudMode = isCloudMode()
+  def tableName1 = "test_skip_index_compaction_fault_injection_1"
+  def tableName2 = "test_skip_index_compaction_fault_injection_2"
+  def backendId_to_backendIP = [:]
+  def backendId_to_backendHttpPort = [:]
+  getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+  sql "DROP TABLE IF EXISTS ${tableName1}"
+  sql """
+    CREATE TABLE ${tableName1} (
+      `@timestamp` int(11) NULL COMMENT "",
+      `clientip` varchar(20) NULL COMMENT "",
+      `request` text NULL COMMENT "",
+      `status` int(11) NULL COMMENT "",
+      `size` int(11) NULL COMMENT "",
+      INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '',
+      INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") COMMENT ''
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`@timestamp`)
+    COMMENT "OLAP"
+    DISTRIBUTED BY RANDOM BUCKETS 1
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1",
+      "disable_auto_compaction" = "true",
+      "inverted_index_storage_format" = "V1"
+    );
+  """
+
+  sql "DROP TABLE IF EXISTS ${tableName2}"
+  sql """
+    CREATE TABLE ${tableName2} (
+      `@timestamp` int(11) NULL COMMENT "",
+      `clientip` varchar(20) NULL COMMENT "",
+      `request` text NULL COMMENT "",
+      `status` int(11) NULL COMMENT "",
+      `size` int(11) NULL COMMENT "",
+      INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '',
+      INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") COMMENT ''
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`@timestamp`)
+    COMMENT "OLAP"
+    DISTRIBUTED BY RANDOM BUCKETS 1
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1",
+      "disable_auto_compaction" = "true",
+      "inverted_index_storage_format" = "V2"
+    );
+  """
+
+  boolean disableAutoCompaction = false
+  
+  def set_be_config = { key, value ->
+    for (String backend_id: backendId_to_backendIP.keySet()) {
+      def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+      logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+    }
+  }
+
+  def trigger_full_compaction_on_tablets = { tablets ->
+    for (def tablet : tablets) {
+      String tablet_id = tablet.TabletId
+      String backend_id = tablet.BackendId
+      int times = 1
+
+      String compactionStatus;
+      do{
+        def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+        logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+        ++times
+        sleep(2000)
+        compactionStatus = parseJson(out.trim()).status.toLowerCase();
+      } while (compactionStatus!="success" && times<=10 && compactionStatus!="e-6010")
+
+
+      if (compactionStatus == "fail") {
+        assertEquals(disableAutoCompaction, false)
+        logger.info("Compaction was done automatically!")
+      }
+      if (disableAutoCompaction && compactionStatus!="e-6010") {
+        assertEquals("success", compactionStatus)
+      }
+    }
+  }
+
+  def wait_full_compaction_done = { tablets ->
+    for (def tablet in tablets) {
+      boolean running = true
+      do {
+        Thread.sleep(1000)
+        String tablet_id = tablet.TabletId
+        String backend_id = tablet.BackendId
+        def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+        logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+        assertEquals(code, 0)
+        def compactionStatus = parseJson(out.trim())
+        assertEquals("success", compactionStatus.status.toLowerCase())
+        running = compactionStatus.run_status
+      } while (running)
+    }
+  }
+
+  def get_rowset_count = { tablets ->
+    int rowsetCount = 0
+    for (def tablet in tablets) {
+      def (code, out, err) = curl("GET", tablet.CompactionStatus)
+      logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+      assertEquals(code, 0)
+      def tabletJson = parseJson(out.trim())
+      assert tabletJson.rowsets instanceof List
+      rowsetCount +=((List<String>) tabletJson.rowsets).size()
+    }
+    return rowsetCount
+  }
+
+  def check_config = { String key, String value ->
+    for (String backend_id: backendId_to_backendIP.keySet()) {
+      def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+      logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+      assertEquals(code, 0)
+      def configList = parseJson(out.trim())
+      assert configList instanceof List
+      for (Object ele in (List) configList) {
+        assert ele instanceof List<String>
+        if (((List<String>) ele)[0] == key) {
+          assertEquals(value, ((List<String>) ele)[2])
+        }
+      }
+    }
+  }
+
+  def run_test = { tableName -> 
+    sql """ INSERT INTO ${tableName} VALUES (1, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+    sql """ INSERT INTO ${tableName} VALUES (2, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+    sql """ INSERT INTO ${tableName} VALUES (3, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+    sql """ INSERT INTO ${tableName} VALUES (4, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+    sql """ INSERT INTO ${tableName} VALUES (5, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+    sql """ INSERT INTO ${tableName} VALUES (6, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+    sql """ INSERT INTO ${tableName} VALUES (7, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+    sql """ INSERT INTO ${tableName} VALUES (8, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+    sql """ INSERT INTO ${tableName} VALUES (9, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+    sql """ INSERT INTO ${tableName} VALUES (10, "40.135.0.0", "GET /images/hm_bg.jpg HTTP/1.0", 1, 2); """
+
+    sql "sync"
+
+    def tablets = sql_return_maparray """ show tablets from ${tableName}; """
+    logger.info("tablets: {}", tablets)
+
+    int replicaNum = 1
+    def dedup_tablets = deduplicate_tablets(tablets)
+    if (dedup_tablets.size() > 0) {
+      replicaNum = Math.round(tablets.size() / dedup_tablets.size())
+      if (replicaNum != 1 && replicaNum != 3) {
+        assert(false)
+      }
+    }
+
+    int rowsetCount = get_rowset_count.call(tablets);
+    assert (rowsetCount == 11 * replicaNum)
+
+    // first
+    trigger_full_compaction_on_tablets.call(tablets)
+    wait_full_compaction_done.call(tablets)
+
+    rowsetCount = get_rowset_count.call(tablets);
+    assert (rowsetCount == 11 * replicaNum)
+
+    // second
+    trigger_full_compaction_on_tablets.call(tablets)
+    wait_full_compaction_done.call(tablets)
+
+    rowsetCount = get_rowset_count.call(tablets);
+    if (isCloudMode) {
+      assert (rowsetCount == (1 + 1) * replicaNum)
+    } else {
+      assert (rowsetCount == 1 * replicaNum)
+    }
+  }
+
+  boolean invertedIndexCompactionEnable = false
+  boolean has_update_be_config = false
+  try {
+    String backend_id;
+    backend_id = backendId_to_backendIP.keySet()[0]
+    def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+    
+    logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+    assertEquals(code, 0)
+    def configList = parseJson(out.trim())
+    assert configList instanceof List
+
+    for (Object ele in (List) configList) {
+      assert ele instanceof List<String>
+      if (((List<String>) ele)[0] == "inverted_index_compaction_enable") {
+        invertedIndexCompactionEnable = Boolean.parseBoolean(((List<String>) ele)[2])
+        logger.info("inverted_index_compaction_enable: ${((List<String>) ele)[2]}")
+      }
+    }
+    set_be_config.call("inverted_index_compaction_enable", "true")
+    has_update_be_config = true
+    check_config.call("inverted_index_compaction_enable", "true");
+
+    try {
+      GetDebugPoint().enableDebugPointForAllBEs("Compaction::open_inverted_index_file_reader")
+      run_test.call(tableName1)
+    } finally {
+      GetDebugPoint().disableDebugPointForAllBEs("Compaction::open_inverted_index_file_reader")
+    }
+
+    try {
+      GetDebugPoint().enableDebugPointForAllBEs("Compaction::open_inverted_index_file_writer")
+      run_test.call(tableName2)
+    } finally {
+      GetDebugPoint().disableDebugPointForAllBEs("Compaction::open_inverted_index_file_writer")
+    }
+  } finally {
+    if (has_update_be_config) {
+      set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString())
+    }
+  }
+}
\ No newline at end of file

From 3ef3c0f428da5c649f04837319d4361999db8aae Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Wed, 27 Nov 2024 14:35:50 +0530
Subject: [PATCH 036/399] [Enhancement] (nereids)implement
 showReplicaDistributionCommand in nereids (#44498)

Issue Number: close #42782
#42837
---
 .../org/apache/doris/nereids/DorisParser.g4   |   8 +-
 .../apache/doris/analysis/PartitionNames.java |   8 ++
 .../apache/doris/catalog/MetadataViewer.java  |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  93 +++++++++++++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../nereids/trees/plans/commands/Command.java |   6 +-
 .../ShowReplicaDistributionCommand.java       | 120 ++++++++++++++++
 .../commands/info/PartitionNamesInfo.java     | 131 ++++++++++++++++++
 .../plans/commands/info/TableRefInfo.java     |  78 +++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   6 +
 .../org/apache/doris/qe/StmtExecutor.java     |   2 +-
 .../test_database_management_auth.groovy      |   5 +-
 .../test_nereids_show_replica_dist.groovy     |  84 +++++++++++
 13 files changed, 535 insertions(+), 9 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionNamesInfo.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java
 create mode 100644 regression-test/suites/nereids_p0/show/test_nereids_show_replica_dist.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index d7ed0598d19ddb..f1b17a1b9fb344 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -60,6 +60,7 @@ statementBase
     | supportedShowStatement            #supportedShowStatementAlias
     | supportedLoadStatement            #supportedLoadStatementAlias
     | supportedRecoverStatement         #supportedRecoverStatementAlias
+    | supportedAdminStatement           #supportedAdminStatementAlias
     | unsupportedStatement              #unsupported
     ;
 
@@ -235,6 +236,7 @@ supportedShowStatement
     | SHOW CREATE MATERIALIZED VIEW mvName=identifier
         ON tableName=multipartIdentifier                                            #showCreateMaterializedView   
     | SHOW BACKENDS                                                                 #showBackends
+    | SHOW REPLICA DISTRIBUTION FROM baseTableRef                                   #showReplicaDistribution
     | SHOW FULL? TRIGGERS ((FROM | IN) database=multipartIdentifier)? wildWhere?    #showTriggers    
     | SHOW FRONTENDS name=identifier?                                               #showFrontends 
     | SHOW TABLE tableId=INTEGER_VALUE                                              #showTableId
@@ -349,7 +351,6 @@ unsupportedShowStatement
     | SHOW (CLUSTERS | (COMPUTE GROUPS))                                            #showClusters
     | SHOW CONVERT_LSC ((FROM | IN) database=multipartIdentifier)?                  #showConvertLsc
     | SHOW REPLICA STATUS FROM baseTableRef wildWhere?                              #showReplicaStatus
-    | SHOW REPLICA DISTRIBUTION FROM baseTableRef                                   #showREplicaDistribution
     | SHOW TABLET STORAGE FORMAT VERBOSE?                                           #showTabletStorageFormat
     | SHOW TABLET DIAGNOSIS tabletId=INTEGER_VALUE                                  #showDiagnoseTablet
     | SHOW COPY ((FROM | IN) database=multipartIdentifier)?
@@ -468,6 +469,10 @@ unsupportedCancelStatement
     | CANCEL WARM UP JOB wildWhere?                                                 #cancelWarmUp
     ;
 
+supportedAdminStatement
+    : ADMIN SHOW REPLICA DISTRIBUTION FROM baseTableRef                             #adminShowReplicaDistribution
+    ;
+
 supportedRecoverStatement
     : RECOVER DATABASE name=identifier id=INTEGER_VALUE? (AS alias=identifier)?     #recoverDatabase
     | RECOVER TABLE name=multipartIdentifier
@@ -478,7 +483,6 @@ supportedRecoverStatement
 
 unsupportedAdminStatement
     : ADMIN SHOW REPLICA STATUS FROM baseTableRef wildWhere?                        #adminShowReplicaStatus
-    | ADMIN SHOW REPLICA DISTRIBUTION FROM baseTableRef                             #adminShowReplicaDistribution
     | ADMIN SET REPLICA STATUS PROPERTIES LEFT_PAREN propertyItemList RIGHT_PAREN   #adminSetReplicaStatus
     | ADMIN SET REPLICA VERSION PROPERTIES LEFT_PAREN propertyItemList RIGHT_PAREN  #adminSetReplicaVersion
     | ADMIN REPAIR TABLE baseTableRef                                               #adminRepairTable
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java
index f82f497176bec2..d49043c4cd5c0b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionNames.java
@@ -81,6 +81,14 @@ public PartitionNames(long partitionCount) {
         this.count = partitionCount;
     }
 
+    // for nereids
+    public PartitionNames(boolean isTemp, List<String> partitionNames, boolean isStar, long partitionCount) {
+        this.partitionNames = partitionNames;
+        this.isTemp = isTemp;
+        this.isStar = isStar;
+        this.count = partitionCount;
+    }
+
     public List<String> getPartitionNames() {
         return partitionNames;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
index fd4a530f4369e1..4da77267857f67 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
@@ -167,7 +167,7 @@ public static List<List<String>> getTabletDistribution(ShowReplicaDistributionSt
         return getTabletDistribution(stmt.getDbName(), stmt.getTblName(), stmt.getPartitionNames());
     }
 
-    private static List<List<String>> getTabletDistribution(
+    public static List<List<String>> getTabletDistribution(
             String dbName, String tblName, PartitionNames partitionNames)
             throws DdlException {
         DecimalFormat df = new DecimalFormat("00.00 %");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 6e86b4073f5792..d553d93da9ca0e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -49,6 +49,7 @@
 import org.apache.doris.mtmv.MTMVRefreshTriggerInfo;
 import org.apache.doris.nereids.DorisParser;
 import org.apache.doris.nereids.DorisParser.AddConstraintContext;
+import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext;
 import org.apache.doris.nereids.DorisParser.AggClauseContext;
 import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext;
 import org.apache.doris.nereids.DorisParser.AliasQueryContext;
@@ -64,6 +65,7 @@
 import org.apache.doris.nereids.DorisParser.ArrayLiteralContext;
 import org.apache.doris.nereids.DorisParser.ArrayRangeContext;
 import org.apache.doris.nereids.DorisParser.ArraySliceContext;
+import org.apache.doris.nereids.DorisParser.BaseTableRefContext;
 import org.apache.doris.nereids.DorisParser.BitOperationContext;
 import org.apache.doris.nereids.DorisParser.BooleanExpressionContext;
 import org.apache.doris.nereids.DorisParser.BooleanLiteralContext;
@@ -158,6 +160,7 @@
 import org.apache.doris.nereids.DorisParser.NamedExpressionContext;
 import org.apache.doris.nereids.DorisParser.NamedExpressionSeqContext;
 import org.apache.doris.nereids.DorisParser.NullLiteralContext;
+import org.apache.doris.nereids.DorisParser.OptScanParamsContext;
 import org.apache.doris.nereids.DorisParser.OutFileClauseContext;
 import org.apache.doris.nereids.DorisParser.ParenthesizedExpressionContext;
 import org.apache.doris.nereids.DorisParser.PartitionSpecContext;
@@ -189,6 +192,7 @@
 import org.apache.doris.nereids.DorisParser.RefreshTriggerContext;
 import org.apache.doris.nereids.DorisParser.RegularQuerySpecificationContext;
 import org.apache.doris.nereids.DorisParser.RelationContext;
+import org.apache.doris.nereids.DorisParser.RelationHintContext;
 import org.apache.doris.nereids.DorisParser.ReplaceContext;
 import org.apache.doris.nereids.DorisParser.ResumeMTMVContext;
 import org.apache.doris.nereids.DorisParser.RollupDefContext;
@@ -238,6 +242,7 @@
 import org.apache.doris.nereids.DorisParser.ShowPrivilegesContext;
 import org.apache.doris.nereids.DorisParser.ShowProcContext;
 import org.apache.doris.nereids.DorisParser.ShowProcedureStatusContext;
+import org.apache.doris.nereids.DorisParser.ShowReplicaDistributionContext;
 import org.apache.doris.nereids.DorisParser.ShowRepositoriesContext;
 import org.apache.doris.nereids.DorisParser.ShowRolesContext;
 import org.apache.doris.nereids.DorisParser.ShowSmallFilesContext;
@@ -254,6 +259,7 @@
 import org.apache.doris.nereids.DorisParser.SingleStatementContext;
 import org.apache.doris.nereids.DorisParser.SortClauseContext;
 import org.apache.doris.nereids.DorisParser.SortItemContext;
+import org.apache.doris.nereids.DorisParser.SpecifiedPartitionContext;
 import org.apache.doris.nereids.DorisParser.StarContext;
 import org.apache.doris.nereids.DorisParser.StatementDefaultContext;
 import org.apache.doris.nereids.DorisParser.StepPartitionDefContext;
@@ -266,7 +272,9 @@
 import org.apache.doris.nereids.DorisParser.SystemVariableContext;
 import org.apache.doris.nereids.DorisParser.TableAliasContext;
 import org.apache.doris.nereids.DorisParser.TableNameContext;
+import org.apache.doris.nereids.DorisParser.TableSnapshotContext;
 import org.apache.doris.nereids.DorisParser.TableValuedFunctionContext;
+import org.apache.doris.nereids.DorisParser.TabletListContext;
 import org.apache.doris.nereids.DorisParser.TimestampaddContext;
 import org.apache.doris.nereids.DorisParser.TimestampdiffContext;
 import org.apache.doris.nereids.DorisParser.TypeConstructorContext;
@@ -513,6 +521,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowReplicaDistributionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand;
@@ -559,6 +568,7 @@
 import org.apache.doris.nereids.trees.plans.commands.info.MTMVPartitionDefinition;
 import org.apache.doris.nereids.trees.plans.commands.info.PartitionDefinition;
 import org.apache.doris.nereids.trees.plans.commands.info.PartitionDefinition.MaxValue;
+import org.apache.doris.nereids.trees.plans.commands.info.PartitionNamesInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.PartitionTableInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo;
@@ -576,6 +586,7 @@
 import org.apache.doris.nereids.trees.plans.commands.info.SimpleColumnDefinition;
 import org.apache.doris.nereids.trees.plans.commands.info.StepPartition;
 import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
+import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo;
 import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
@@ -4414,6 +4425,88 @@ public LogicalPlan visitShowProc(ShowProcContext ctx) {
         return new ShowProcCommand(path);
     }
 
+    private TableScanParams visitOptScanParamsContex(OptScanParamsContext ctx) {
+        if (ctx != null) {
+            Map<String, String> map = visitPropertyItemList(ctx.properties);
+            return new TableScanParams(ctx.funcName.getText(), map);
+        }
+        return null;
+    }
+
+    private TableSnapshot visitTableSnapshotContext(TableSnapshotContext ctx) {
+        if (ctx != null) {
+            if (ctx.TIME() != null) {
+                return new TableSnapshot(stripQuotes(ctx.time.getText()));
+            } else {
+                return new TableSnapshot(Long.parseLong(ctx.version.getText()));
+            }
+        }
+        return null;
+    }
+
+    private List<String> visitRelationHintContext(RelationHintContext ctx) {
+        final List<String> relationHints;
+        if (ctx != null) {
+            relationHints = typedVisit(ctx);
+        } else {
+            relationHints = ImmutableList.of();
+        }
+        return relationHints;
+    }
+
+    private PartitionNamesInfo visitSpecifiedPartitionContext(SpecifiedPartitionContext ctx) {
+        if (ctx != null) {
+            List<String> partitions = new ArrayList<>();
+            boolean isTempPart = ctx.TEMPORARY() != null;
+            if (ctx.identifier() != null) {
+                partitions.add(ctx.identifier().getText());
+            } else {
+                partitions.addAll(visitIdentifierList(ctx.identifierList()));
+            }
+            return new PartitionNamesInfo(isTempPart, partitions);
+        }
+        return null;
+    }
+
+    private List<Long> visitTabletListContext(TabletListContext ctx) {
+        List<Long> tabletIdList = new ArrayList<>();
+        if (ctx != null && ctx.tabletIdList != null) {
+            ctx.tabletIdList.stream().forEach(tabletToken -> {
+                tabletIdList.add(Long.parseLong(tabletToken.getText()));
+            });
+        }
+        return tabletIdList;
+    }
+
+    private TableRefInfo visitBaseTableRefContext(BaseTableRefContext ctx) {
+        List<String> nameParts = visitMultipartIdentifier(ctx.multipartIdentifier());
+        TableScanParams scanParams = visitOptScanParamsContex(ctx.optScanParams());
+        TableSnapshot tableSnapShot = visitTableSnapshotContext(ctx.tableSnapshot());
+        PartitionNamesInfo partitionNameInfo = visitSpecifiedPartitionContext(ctx.specifiedPartition());
+        List<Long> tabletIdList = visitTabletListContext(ctx.tabletList());
+
+        String tableAlias = null;
+        if (ctx.tableAlias().strictIdentifier() != null) {
+            tableAlias = ctx.tableAlias().getText();
+        }
+        TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample());
+        List<String> hints = visitRelationHintContext(ctx.relationHint());
+        return new TableRefInfo(new TableNameInfo(nameParts), scanParams, tableSnapShot, partitionNameInfo,
+                                    tabletIdList, tableAlias, tableSample, hints);
+    }
+
+    @Override
+    public LogicalPlan visitShowReplicaDistribution(ShowReplicaDistributionContext ctx) {
+        TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef());
+        return new ShowReplicaDistributionCommand(tableRefInfo);
+    }
+
+    @Override
+    public LogicalPlan visitAdminShowReplicaDistribution(AdminShowReplicaDistributionContext ctx) {
+        TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef());
+        return new ShowReplicaDistributionCommand(tableRefInfo);
+    }
+
     @Override
     public LogicalPlan visitShowCreateCatalog(ShowCreateCatalogContext ctx) {
         return new ShowCreateCatalogCommand(ctx.identifier().getText());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 57eb2bbb6b42c2..b5f511d8fe38f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -207,6 +207,7 @@ public enum PlanType {
     SHOW_PROC_COMMAND,
     SHOW_PLUGINS_COMMAND,
     SHOW_PRIVILEGES_COMMAND,
+    SHOW_REPLICA_DISTRIBUTION_COMMAND,
     SHOW_REPOSITORIES_COMMAND,
     SHOW_ROLE_COMMAND,
     SHOW_SMALL_FILES_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java
index d1b8e5a263c503..2048dc00f3e976 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java
@@ -119,15 +119,15 @@ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
         throw new RuntimeException("Command do not implement withGroupExpression");
     }
 
-    public void verifyCommandSupported() throws DdlException {
+    public void verifyCommandSupported(ConnectContext ctx) throws DdlException {
         // check command has been supported in cloud mode
         if (Config.isCloudMode()) {
-            checkSupportedInCloudMode();
+            checkSupportedInCloudMode(ctx);
         }
     }
 
     // check if the command is supported in cloud mode
     // see checkStmtSupported() in fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
     // override this method if the command is not supported in cloud mode
-    protected void checkSupportedInCloudMode() throws DdlException {}
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {}
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java
new file mode 100644
index 00000000000000..2a073b59fbccbd
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.PartitionNames;
+import org.apache.doris.analysis.RedirectStatus;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MetadataViewer;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.mysql.privilege.Auth;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+
+/**
+ * show replica distribution command
+ */
+public class ShowReplicaDistributionCommand extends ShowCommand {
+    public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
+            .add("BackendId").add("ReplicaNum").add("ReplicaSize")
+            .add("NumGraph").add("NumPercent")
+            .add("SizeGraph").add("SizePercent")
+            .add("CloudClusterName").add("CloudClusterId")
+            .build();
+    private static final Logger LOG = LogManager.getLogger(ShowReplicaDistributionCommand.class);
+    private final TableRefInfo tableRefInfo;
+
+    /**
+     * constructor
+     */
+
+    public ShowReplicaDistributionCommand(TableRefInfo tableRefInfo) {
+        super(PlanType.SHOW_REPLICA_DISTRIBUTION_COMMAND);
+        this.tableRefInfo = tableRefInfo;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+        tableRefInfo.analyze(ctx);
+        Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName());
+
+        List<List<String>> results;
+        try {
+            PartitionNames partitionNames = (tableRefInfo.getPartitionNamesInfo() != null)
+                                    ? tableRefInfo.getPartitionNamesInfo().translateToLegacyPartitionNames() : null;
+            results = MetadataViewer.getTabletDistribution(tableRefInfo.getTableNameInfo().getDb(),
+                                    tableRefInfo.getTableNameInfo().getTbl(),
+                                    partitionNames);
+        } catch (DdlException e) {
+            throw new AnalysisException(e.getMessage());
+        }
+        return new ShowResultSet(getMetaData(), results);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowReplicaDistributionCommand(this, context);
+    }
+
+    public ShowResultSetMetaData getMetaData() {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        for (String title : TITLE_NAMES) {
+            builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
+        }
+        return builder.build();
+    }
+
+    @Override
+    public RedirectStatus toRedirectStatus() {
+        if (ConnectContext.get().getSessionVariable().getForwardToMaster()) {
+            return RedirectStatus.FORWARD_NO_SYNC;
+        } else {
+            return RedirectStatus.NO_FORWARD;
+        }
+    }
+
+    @Override
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
+        if (!ctx.getCurrentUserIdentity().getUser().equals(Auth.ROOT_USER)) {
+            LOG.info("ShowReplicaDistributionCommand not supported in cloud mode");
+            throw new DdlException("Unsupported operation");
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionNamesInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionNamesInfo.java
new file mode 100644
index 00000000000000..a31058c8271f0b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionNamesInfo.java
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.info;
+
+import org.apache.doris.analysis.PartitionNames;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.UserException;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * PartitionNamesInfo
+ */
+public class PartitionNamesInfo {
+    // Default partition count to collect statistic for external table.
+    private static final long DEFAULT_PARTITION_COUNT = 100;
+    private final List<String> partitionNames;
+    // true if these partitions are temp partitions
+    private final boolean isTemp;
+    private final boolean isStar;
+    private final long count;
+
+    public PartitionNamesInfo(boolean isTemp, List<String> partitionNames) {
+        this.partitionNames = partitionNames;
+        this.isTemp = isTemp;
+        this.isStar = false;
+        this.count = 0;
+    }
+
+    public PartitionNamesInfo(PartitionNamesInfo other) {
+        this.partitionNames = Lists.newArrayList(other.partitionNames);
+        this.isTemp = other.isTemp;
+        this.isStar = other.isStar;
+        this.count = 0;
+    }
+
+    public PartitionNamesInfo(boolean isStar) {
+        this.partitionNames = null;
+        this.isTemp = false;
+        this.isStar = isStar;
+        this.count = 0;
+    }
+
+    public PartitionNamesInfo(long partitionCount) {
+        this.partitionNames = null;
+        this.isTemp = false;
+        this.isStar = false;
+        this.count = partitionCount;
+    }
+
+    public List<String> getPartitionNames() {
+        return partitionNames;
+    }
+
+    public boolean isTemp() {
+        return isTemp;
+    }
+
+    public boolean isStar() {
+        return isStar;
+    }
+
+    public long getCount() {
+        return count;
+    }
+
+    /**
+     * validate
+     */
+    public void validate(ConnectContext ctx) throws UserException {
+        if (isStar && count > 0) {
+            throw new AnalysisException("All partition and partition count couldn't be set at the same time.");
+        }
+        if (isStar || count > 0) {
+            return;
+        }
+        if (partitionNames == null || partitionNames.isEmpty()) {
+            throw new AnalysisException("No partition specified in partition lists");
+        }
+        // check if partition name is not empty string
+        if (partitionNames.stream().anyMatch(Strings::isNullOrEmpty)) {
+            throw new AnalysisException("there are empty partition name");
+        }
+    }
+
+    /**
+     * toSql
+     */
+    public String toSql() {
+        if (partitionNames == null || partitionNames.isEmpty()) {
+            return "";
+        }
+        StringBuilder sb = new StringBuilder();
+        if (isTemp) {
+            sb.append("TEMPORARY ");
+        }
+        sb.append("PARTITIONS (");
+        sb.append(Joiner.on(", ").join(partitionNames));
+        sb.append(")");
+        return sb.toString();
+    }
+
+    public PartitionNames translateToLegacyPartitionNames() {
+        return new PartitionNames(isTemp, partitionNames, isStar, count);
+    }
+
+    @Override
+    public String toString() {
+        return toSql();
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java
new file mode 100644
index 00000000000000..7171eef841dabe
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is modified copy of TableRef
+
+package org.apache.doris.nereids.trees.plans.commands.info;
+
+import org.apache.doris.analysis.TableScanParams;
+import org.apache.doris.analysis.TableSnapshot;
+import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.trees.TableSample;
+import org.apache.doris.qe.ConnectContext;
+
+import java.util.List;
+
+/**
+ * table ref info
+ * baseTableRef
+    : multipartIdentifier optScanParams? tableSnapshot? specifiedPartition?
+        tabletList? tableAlias sample? relationHint?
+    currently  tableNameInfo(multipartIdentifier) and specifiedPartition API exposed.
+    if you need write one and use.
+ */
+public class TableRefInfo {
+    private final TableNameInfo tableNameInfo;
+    private final TableScanParams scanParams;
+    private final TableSnapshot tableSnapShot;
+    private final PartitionNamesInfo partitionNamesInfo;
+    private final List<Long> tabletIdList;
+    private final String tableAlias;
+    private final TableSample tableSample;
+    private final List<String> relationHints;
+
+    /**
+     * constructor
+     */
+    public TableRefInfo(TableNameInfo tableNameInfo, TableScanParams scanParams,
+                        TableSnapshot tableSnapShot, PartitionNamesInfo partitionNamesInfo,
+                        List<Long> tabletIdList, String tableAlias,
+                        TableSample tableSample, List<String> relationHints) {
+        this.tableNameInfo = tableNameInfo;
+        this.scanParams = scanParams;
+        this.tableSnapShot = tableSnapShot;
+        this.partitionNamesInfo = partitionNamesInfo;
+        this.tabletIdList = tabletIdList;
+        this.tableAlias = tableAlias;
+        this.tableSample = tableSample;
+        this.relationHints = relationHints;
+    }
+
+    public TableNameInfo getTableNameInfo() {
+        return tableNameInfo;
+    }
+
+    public PartitionNamesInfo getPartitionNamesInfo() {
+        return partitionNamesInfo;
+    }
+
+    public void analyze(ConnectContext ctx) throws UserException {
+        tableNameInfo.analyze(ctx);
+        if (partitionNamesInfo != null) {
+            partitionNamesInfo.validate(ctx);
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 85633cb798b06c..8cdd9fa3278c70 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -87,6 +87,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowReplicaDistributionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand;
@@ -447,6 +448,11 @@ default R visitDropWorkloadGroupCommand(DropWorkloadGroupCommand dropWorkloadGro
         return visitCommand(dropWorkloadGroupCommand, context);
     }
 
+    default R visitShowReplicaDistributionCommand(ShowReplicaDistributionCommand showReplicaDistributedCommand,
+                                                    C context) {
+        return visitCommand(showReplicaDistributedCommand, context);
+    }
+
     default R visitDropWorkloadPolicyCommand(DropWorkloadPolicyCommand dropWorkloadPolicyCommand, C context) {
         return visitCommand(dropWorkloadPolicyCommand, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 05c7853ca469df..05df53ed6796cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -756,7 +756,7 @@ private void executeByNereids(TUniqueId queryId) throws Exception {
             // t3: observer fe receive editlog creating the table from the master fe
             syncJournalIfNeeded();
             try {
-                ((Command) logicalPlan).verifyCommandSupported();
+                ((Command) logicalPlan).verifyCommandSupported(context);
                 ((Command) logicalPlan).run(context, this);
             } catch (MustFallbackException e) {
                 if (LOG.isDebugEnabled()) {
diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy
index 33a26959619e5f..0e1805e20d6840 100644
--- a/regression-test/suites/auth_call/test_database_management_auth.groovy
+++ b/regression-test/suites/auth_call/test_database_management_auth.groovy
@@ -23,13 +23,14 @@ suite("test_database_management_auth","p0,auth_call") {
     String user = 'test_database_management_auth_user'
     String pwd = 'C123_567p'
     String dbName = 'test_database_management_auth_db'
-
+    def String show_dis_error_msg = "denied"
     //cloud-mode
     if (isCloudMode()) {
         def clusters = sql " SHOW CLUSTERS; "
         assertTrue(!clusters.isEmpty())
         def validCluster = clusters[0][0]
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}""";
+        show_dis_error_msg = "Unsupported"
     }
 
     try_sql("DROP USER ${user}")
@@ -78,7 +79,7 @@ suite("test_database_management_auth","p0,auth_call") {
         }
         test {
             sql """SHOW REPLICA DISTRIBUTION FROM tbl;"""
-            exception "denied"
+            exception "${show_dis_error_msg}"
         }
         test {
             sql """SHOW REPLICA STATUS FROM db1.tbl1;"""
diff --git a/regression-test/suites/nereids_p0/show/test_nereids_show_replica_dist.groovy b/regression-test/suites/nereids_p0/show/test_nereids_show_replica_dist.groovy
new file mode 100644
index 00000000000000..1cd21d1ea0169c
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_nereids_show_replica_dist.groovy
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_nereids_show_replica_dist") {
+    def tableName = "test_nereids_show_replica_dist"
+    // create table and insert data
+    sql """ drop table if exists ${tableName}"""
+    sql """
+    create table ${tableName} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 1
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+
+    def queryReplicaCount = { partitionName ->
+        def result = sql "SHOW REPLICA DISTRIBUTION FROM ${tableName} PARTITION ${partitionName}"
+        logger.info("${result}")
+        int sum = 0
+        for (row in result) {
+            sum += row[1].toInteger()
+        }
+        sum
+    }
+
+    def queryAdminReplicaCount = { partitionName ->
+        def result = sql "ADMIN SHOW REPLICA DISTRIBUTION FROM ${tableName} PARTITION ${partitionName}"
+        logger.info("${result}")
+        int sum = 0
+        for (row in result) {
+            sum += row[1].toInteger()
+        }
+        sum
+    }
+    
+    def replication_num = 1
+    def forceReplicaNum = getFeConfig('force_olap_table_replication_num').toInteger()
+    if (forceReplicaNum > 0) {
+        replication_num = forceReplicaNum
+    }
+
+    assertEquals(replication_num, queryReplicaCount("p3"))   
+    assertEquals(replication_num, queryReplicaCount("p4"))   
+    assertEquals(replication_num, queryReplicaCount("p5"))           
+
+    assertEquals(replication_num, queryAdminReplicaCount("p3"))   
+    assertEquals(replication_num, queryAdminReplicaCount("p4"))   
+    assertEquals(replication_num, queryAdminReplicaCount("p5"))  
+
+    checkNereidsExecute("SHOW REPLICA DISTRIBUTION FROM ${tableName}")
+    sql "SHOW REPLICA DISTRIBUTION FROM ${tableName}"
+    checkNereidsExecute("ADMIN SHOW REPLICA DISTRIBUTION FROM ${tableName}")
+    sql "ADMIN SHOW REPLICA DISTRIBUTION FROM ${tableName}"    
+    sql """ drop table if exists ${tableName}"""
+
+}
+

From f3e9d70ab9558bf8df5145d0432b4d43cceef24b Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Wed, 27 Nov 2024 14:48:58 +0530
Subject: [PATCH 037/399] [Enhancement] (nereids)implement DropFileCommand in
 nereids (#44504)

Issue Number: close #42619
---
 .../org/apache/doris/nereids/DorisParser.g4   |  4 +-
 .../nereids/parser/LogicalPlanBuilder.java    | 13 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../trees/plans/commands/DropFileCommand.java | 93 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 +
 .../auth_call/test_ddl_file_auth.groovy       |  2 +-
 6 files changed, 115 insertions(+), 3 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFileCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index f1b17a1b9fb344..cee972a63d210b 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -205,6 +205,8 @@ supportedDropStatement
     | DROP SQL_BLOCK_RULE (IF EXISTS)? identifierSeq                            #dropSqlBlockRule
     | DROP USER (IF EXISTS)? userIdentify                                       #dropUser
     | DROP WORKLOAD GROUP (IF EXISTS)? name=identifierOrText                    #dropWorkloadGroup
+    | DROP FILE name=STRING_LITERAL
+        ((FROM | IN) database=identifier)? properties=propertyClause            #dropFile
     | DROP WORKLOAD POLICY (IF EXISTS)? name=identifierOrText                   #dropWorkloadPolicy
     ;
 
@@ -671,8 +673,6 @@ unsupportedDropStatement
     | DROP TABLE (IF EXISTS)? name=multipartIdentifier FORCE?                   #dropTable
     | DROP VIEW (IF EXISTS)? name=multipartIdentifier                           #dropView
     | DROP REPOSITORY name=identifier                                           #dropRepository
-    | DROP FILE name=STRING_LITERAL
-        ((FROM | IN) database=identifier)? properties=propertyClause            #dropFile
     | DROP INDEX (IF EXISTS)? name=identifier ON tableName=multipartIdentifier  #dropIndex
     | DROP RESOURCE (IF EXISTS)? name=identifierOrText                          #dropResource
     | DROP ROW POLICY (IF EXISTS)? policyName=identifier
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index d553d93da9ca0e..1a49c7ba65fa56 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -106,6 +106,7 @@
 import org.apache.doris.nereids.DorisParser.DropCatalogRecycleBinContext;
 import org.apache.doris.nereids.DorisParser.DropConstraintContext;
 import org.apache.doris.nereids.DorisParser.DropEncryptkeyContext;
+import org.apache.doris.nereids.DorisParser.DropFileContext;
 import org.apache.doris.nereids.DorisParser.DropMTMVContext;
 import org.apache.doris.nereids.DorisParser.DropProcedureContext;
 import org.apache.doris.nereids.DorisParser.DropRoleContext;
@@ -474,6 +475,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType;
 import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropEncryptkeyCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropFileCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropProcedureCommand;
@@ -4617,6 +4619,17 @@ public LogicalPlan visitDropEncryptkey(DropEncryptkeyContext ctx) {
         return new DropEncryptkeyCommand(new EncryptKeyName(nameParts), ctx.EXISTS() != null);
     }
 
+    @Override
+    public LogicalPlan visitDropFile(DropFileContext ctx) {
+        String dbName = null;
+        if (ctx.database != null) {
+            dbName = ctx.database.getText();
+        }
+        Map<String, String> properties = ctx.propertyClause() != null
+                                    ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+        return new DropFileCommand(stripQuotes(ctx.name.getText()), dbName, properties);
+    }
+
     @Override
     public LogicalPlan visitDropSqlBlockRule(DropSqlBlockRuleContext ctx) {
         return new DropSqlBlockRuleCommand(visitIdentifierSeq(ctx.identifierSeq()), ctx.EXISTS() != null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index b5f511d8fe38f1..20d46d612d02ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -174,6 +174,7 @@ public enum PlanType {
     ALTER_WORKLOAD_POLICY_COMMAND,
     DROP_CATALOG_RECYCLE_BIN_COMMAND,
     DROP_ENCRYPTKEY_COMMAND,
+    DROP_FILE_COMMAND,
     UNSET_VARIABLE_COMMAND,
     UNSET_DEFAULT_STORAGE_VAULT_COMMAND,
     UNSUPPORTED_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFileCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFileCommand.java
new file mode 100644
index 00000000000000..92e516238a589b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFileCommand.java
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * drop file command command
+ */
+public class DropFileCommand extends DropCommand {
+    public static final String PROP_CATALOG = "catalog";
+    private final String fileName;
+    private String dbName; // update based on current db if not present.
+    private final Map<String, String> properties;
+    private String catalogName = null;
+
+    /**
+     * constructor
+     */
+    public DropFileCommand(String fileName, String dbName, Map<String, String> properties) {
+        super(PlanType.DROP_FILE_COMMAND);
+        this.fileName = fileName;
+        this.dbName = dbName;
+        this.properties = properties;
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check operation privilege
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        if (Strings.isNullOrEmpty(dbName)) {
+            dbName = ctx.getDatabase();
+            if (Strings.isNullOrEmpty(dbName)) {
+                ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
+            }
+        }
+
+        if (Strings.isNullOrEmpty(fileName)) {
+            throw new AnalysisException("File name is not specified");
+        }
+
+        Optional<String> optional = properties.keySet().stream().filter(
+                entity -> !PROP_CATALOG.equals(entity)).findFirst();
+        if (optional.isPresent()) {
+            throw new AnalysisException(optional.get() + " is invalid property");
+        }
+
+        catalogName = properties.get(PROP_CATALOG);
+        if (Strings.isNullOrEmpty(catalogName)) {
+            throw new AnalysisException("catalog name is missing");
+        }
+
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
+        Env.getCurrentEnv().getSmallFileMgr().removeFile(db.getId(), catalogName, fileName, false);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitDropFileCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 8cdd9fa3278c70..0af9ce4dc9c4cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -41,6 +41,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropEncryptkeyCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropFileCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropProcedureCommand;
@@ -436,6 +437,10 @@ default R visitDropEncryptKeyCommand(DropEncryptkeyCommand dropEncryptkeyCommand
         return visitCommand(dropEncryptkeyCommand, context);
     }
 
+    default R visitDropFileCommand(DropFileCommand dropFileCommand, C context) {
+        return visitCommand(dropFileCommand, context);
+    }
+
     default R visitDropSqlBlockRuleCommand(DropSqlBlockRuleCommand dropSqlBlockRuleCommand, C context) {
         return visitCommand(dropSqlBlockRuleCommand, context);
     }
diff --git a/regression-test/suites/auth_call/test_ddl_file_auth.groovy b/regression-test/suites/auth_call/test_ddl_file_auth.groovy
index 99b71eabddf140..53cab5b3687ba5 100644
--- a/regression-test/suites/auth_call/test_ddl_file_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_file_auth.groovy
@@ -81,7 +81,7 @@ suite("test_ddl_file_auth","p0,auth_call") {
         def res = sql """SHOW FILE FROM ${dbName};"""
         assertTrue(res.size() == 1)
 
-        sql """DROP FILE "${fileName}" from ${dbName} properties("catalog" = "internal");"""
+        checkNereidsExecute("""DROP FILE "${fileName}" from ${dbName} properties("catalog" = "internal");""")
         res = sql """SHOW FILE FROM ${dbName};"""
         assertTrue(res.size() == 0)
     }

From 5e54970d26eb0db803cfe14b7bac5a831218d8b0 Mon Sep 17 00:00:00 2001
From: deardeng <dengxin@selectdb.com>
Date: Wed, 27 Nov 2024 18:59:49 +0800
Subject: [PATCH 038/399] (fix)[db] Fix create database and create table data
 race (#44600)

There may be a race condition between CREATE DATABASE and CREATE TABLE.
When CREATE DATABASE is being executed, the edit log may get stuck,
while fullNameToDb in getDbNullable can still return the database. As a
result, the CREATE TABLE process continues, which may ultimately lead to
a strange order in the edit log of BDBJE, where CREATE TABLE appears
before CREATE DATABASE.
---
 .../java/org/apache/doris/catalog/Database.java     |  2 +-
 .../apache/doris/datasource/InternalCatalog.java    | 13 +++++++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 09e2ae350c5e98..4f205be0381b37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -93,7 +93,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>,
     private final Map<Long, Table> idToTable;
     private ConcurrentMap<String, Table> nameToTable;
     // table name lower case -> table name
-    private final Map<String, String> lowerCaseToTableName;
+    private final ConcurrentMap<String, String> lowerCaseToTableName;
 
     // user define function
     @SerializedName(value = "name2Function")
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 3a9e96bade67f5..18f2e333e9119b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -441,8 +441,17 @@ public void createDb(CreateDbStmt stmt) throws DdlException {
                     ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, fullDbName);
                 }
             } else {
-                unprotectCreateDb(db);
-                Env.getCurrentEnv().getEditLog().logCreateDb(db);
+                if (!db.tryWriteLock(100, TimeUnit.SECONDS)) {
+                    LOG.warn("try lock failed, create database failed {}", fullDbName);
+                    ErrorReport.reportDdlException(ErrorCode.ERR_EXECUTE_TIMEOUT,
+                            "create database " + fullDbName + " time out");
+                }
+                try {
+                    unprotectCreateDb(db);
+                    Env.getCurrentEnv().getEditLog().logCreateDb(db);
+                } finally {
+                    db.writeUnlock();
+                }
             }
         } finally {
             unlock();

From c56abeed7b6812b072771a9cb7ef427044d86272 Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Wed, 27 Nov 2024 20:07:59 +0800
Subject: [PATCH 039/399] [case](sql_block_rule) fix case return list should
 order (#44562)

---
 .../suites/sql_block_rule_p0/test_sql_block_rule.groovy         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy b/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
index 13b83d272decdb..001c64fbc0d0ad 100644
--- a/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
+++ b/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
@@ -107,7 +107,7 @@ suite("test_sql_block_rule", "nonConcurrent") {
                 PROPERTIES("sql"="SELECT \\\\* FROM table_2", "global"= "true", "enable"= "true")
               """              
 
-    qt_select4_exist """
+    order_qt_select4_exist """
                 SHOW SQL_BLOCK_RULE
               """
 

From da79ceb9bb08c871e92cb346ee67582137e06e72 Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Wed, 27 Nov 2024 20:08:43 +0800
Subject: [PATCH 040/399] [fix](auth)Fix the need for low-level table
 permissions when querying views in certain situations (#44621)

### What problem does this PR solve?

fix when `create view v1 as select * from table1 union select * from
table2` and user has select_priv of v1,but he can not `select * from v1`

### Release note

Fix the need for low-level table permissions when querying views in
certain situations
---
 .../doris/nereids/StatementContext.java       | 10 +++
 .../rules/rewrite/CheckPrivileges.java        |  6 +-
 .../auth_p0/test_select_view_auth.groovy      | 89 +++++++++++++++++++
 3 files changed, 104 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/suites/auth_p0/test_select_view_auth.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 69848ca8f04da2..6b6e335b74a676 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -174,6 +174,8 @@ public class StatementContext implements Closeable {
 
     private Backend groupCommitMergeBackend;
 
+    private boolean privChecked;
+
     public StatementContext() {
         this(ConnectContext.get(), null, 0);
     }
@@ -580,4 +582,12 @@ public void setGroupCommitMergeBackend(
             Backend groupCommitMergeBackend) {
         this.groupCommitMergeBackend = groupCommitMergeBackend;
     }
+
+    public boolean isPrivChecked() {
+        return privChecked;
+    }
+
+    public void setPrivChecked(boolean privChecked) {
+        this.privChecked = privChecked;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckPrivileges.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckPrivileges.java
index 74609694431e33..ebef2ecea21207 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckPrivileges.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckPrivileges.java
@@ -49,9 +49,13 @@ public class CheckPrivileges extends ColumnPruning {
 
     @Override
     public Plan rewriteRoot(Plan plan, JobContext jobContext) {
+        // Only enter once, if repeated, the permissions of the table in the view will be checked
+        if (jobContext.getCascadesContext().getStatementContext().isPrivChecked()) {
+            return plan;
+        }
         this.jobContext = jobContext;
         super.rewriteRoot(plan, jobContext);
-
+        jobContext.getCascadesContext().getStatementContext().setPrivChecked(true);
         // don't rewrite plan
         return plan;
     }
diff --git a/regression-test/suites/auth_p0/test_select_view_auth.groovy b/regression-test/suites/auth_p0/test_select_view_auth.groovy
new file mode 100644
index 00000000000000..87ec8cf0aeb758
--- /dev/null
+++ b/regression-test/suites/auth_p0/test_select_view_auth.groovy
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_select_view_auth","p0,auth") {
+    String suiteName = "test_select_view_auth"
+    String user = "${suiteName}_user"
+    String pwd = 'C123_567p'
+    String dbName = "${suiteName}_db"
+    String tableName1 = "${suiteName}_table1"
+    String tableName2 = "${suiteName}_table2"
+    String viewName = "${suiteName}_view"
+
+    try_sql("drop user ${user}")
+    try_sql """drop table if exists ${dbName}.${tableName1}"""
+    try_sql """drop table if exists ${dbName}.${tableName2}"""
+    try_sql """drop view if exists ${dbName}.${viewName}"""
+    sql """drop database if exists ${dbName}"""
+
+    sql """create user '${user}' IDENTIFIED by '${pwd}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}""";
+    }
+    sql """create database ${dbName}"""
+    sql("""use ${dbName}""")
+    sql """
+        CREATE TABLE IF NOT EXISTS ${dbName}.`${tableName1}` (
+            id BIGINT,
+            username VARCHAR(20)
+        )
+        DISTRIBUTED BY HASH(id) BUCKETS 2
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+        """
+
+    sql """
+        CREATE TABLE IF NOT EXISTS ${dbName}.`${tableName2}` (
+            id BIGINT,
+            username VARCHAR(20)
+        )
+        DISTRIBUTED BY HASH(id) BUCKETS 2
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+        """
+
+    sql """create view ${dbName}.${viewName} as select * from ${dbName}.${tableName1} union select * from ${dbName}.${tableName2};"""
+
+    sql """grant select_priv on regression_test to ${user}"""
+
+    // table column
+    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        try {
+            sql "select * from ${dbName}.${viewName}"
+        } catch (Exception e) {
+            log.info(e.getMessage())
+            assertTrue(e.getMessage().contains("denied"))
+        }
+    }
+    sql """grant select_priv on ${dbName}.${viewName} to ${user}"""
+    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        sql "select * from ${dbName}.${viewName}"
+    }
+
+    try_sql("drop user ${user}")
+    try_sql """drop table if exists ${dbName}.${tableName1}"""
+    try_sql """drop table if exists ${dbName}.${tableName2}"""
+    try_sql """drop view if exists ${dbName}.${viewName}"""
+    sql """drop database if exists ${dbName}"""
+}

From de8cfd163b881237631963499e9bd8d569980217 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Wed, 27 Nov 2024 21:19:17 +0800
Subject: [PATCH 041/399] [opt](memory) Process profile add metadata profile
 (#44647)

### What problem does this PR solve?

Problem Summary:

http://127.0.0.1:8040/profile
```
ObjectHeapDump:
   - AllRowsetsMemBytes: 16.33 MB
   - AllRowsetsNum: 76.45K
   - AllSegmentsMemBytes: 4.16 KB
   - AllSegmentsNum: 1
   - BitmapIndexReaderMemBytes: 0
   - BitmapIndexReaderNum: 0
   - BloomFilterIndexReaderMemBytes: 0
   - BloomFilterIndexReaderNum: 0
   - ColumnReaderMemBytes: 0
   - ColumnReaderNum: 0
   - IndexPageReaderMemBytes: 0
   - IndexPageReaderNum: 0
   - IndexedColumnReaderMemBytes: 0
   - IndexedColumnReaderNum: 0
   - InvertedIndexReaderMemBytes: 0
   - InvertedIndexReaderNum: 0
   - OrdinalIndexReaderMemBytes: 0
   - OrdinalIndexReaderNum: 0
   - RowsetMetaMemBytes: 45.47 MB
   - RowsetMetaNum: 76.45K
   - TabletColumnMemBytes: 253.28 MB
   - TabletColumnNum: 922.18K
   - TabletIndexMemBytes: 16.78 MB
   - TabletIndexNum: 104.74K
   - TabletMetaMemBytes: 17.50 MB
   - TabletMetaNum: 45.89K
   - TabletSchemaMemBytes: 60.15 MB
   - TabletSchemaNum: 49.19K
   - ZoneMapIndexReaderMemBytes: 0
   - ZoneMapIndexReaderNum: 0
 ```
---
 be/src/olap/metadata_adder.h       | 140 +++++++++++++++++++++++++++--
 be/src/olap/rowset/rowset.cpp      |   7 --
 be/src/olap/rowset/rowset.h        |   5 +-
 be/src/runtime/process_profile.cpp |  10 ++-
 4 files changed, 146 insertions(+), 16 deletions(-)

diff --git a/be/src/olap/metadata_adder.h b/be/src/olap/metadata_adder.h
index bdc9e7a398d8a3..559c5db873b2f6 100644
--- a/be/src/olap/metadata_adder.h
+++ b/be/src/olap/metadata_adder.h
@@ -20,11 +20,16 @@
 #include <bvar/bvar.h>
 #include <stdint.h>
 
+#include "util/runtime_profile.h"
+
 namespace doris {
 
 inline bvar::Adder<int64_t> g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");
 
+inline bvar::Adder<int64_t> g_all_rowsets_mem_bytes("doris_all_rowsets_mem_bytes");
+inline bvar::Adder<int64_t> g_all_rowsets_num("doris_all_rowsets_num");
+
 inline bvar::Adder<int64_t> g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");
 
@@ -37,8 +42,8 @@ inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
 inline bvar::Adder<int64_t> g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");
 
-inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
-inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
+inline bvar::Adder<int64_t> g_all_segments_mem_bytes("doris_all_segments_mem_bytes");
+inline bvar::Adder<int64_t> g_all_segments_num("doris_all_segments_num");
 
 inline bvar::Adder<int64_t> g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
 inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
@@ -104,6 +109,8 @@ class MetadataAdder {
 public:
     MetadataAdder();
 
+    static void dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot);
+
 protected:
     MetadataAdder(const MetadataAdder& other);
 
@@ -159,6 +166,8 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     }
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_mem_bytes << val;
+    } else if constexpr (std::is_same_v<T, Rowset>) {
+        g_all_rowsets_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -168,7 +177,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_segment_mem_bytes << val;
+        g_all_segments_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -185,6 +194,9 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
         g_ordinal_index_reader_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
         g_zone_map_index_reader_mem_bytes << val;
+    } else {
+        LOG(FATAL) << "add_mem_size not match class type: " << typeid(T).name() << ", " << val;
+        __builtin_unreachable();
     }
 }
 
@@ -195,6 +207,8 @@ void MetadataAdder<T>::add_num(int64_t val) {
     }
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_num << val;
+    } else if constexpr (std::is_same_v<T, Rowset>) {
+        g_all_rowsets_num << val;
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_num << val;
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -204,7 +218,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_segment_num << val;
+        g_all_segments_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -221,7 +235,123 @@ void MetadataAdder<T>::add_num(int64_t val) {
         g_ordinal_index_reader_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
         g_zone_map_index_reader_num << val;
+    } else {
+        LOG(FATAL) << "add_num not match class type: " << typeid(T).name() << ", " << val;
+        __builtin_unreachable();
     }
 }
 
-}; // namespace doris
\ No newline at end of file
+template <typename T>
+void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot) {
+    RuntimeProfile::Counter* rowset_meta_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* rowset_meta_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaNum", TUnit::UNIT);
+    COUNTER_SET(rowset_meta_mem_bytes_counter, g_rowset_meta_mem_bytes.get_value());
+    COUNTER_SET(rowset_meta_num_counter, g_rowset_meta_num.get_value());
+
+    RuntimeProfile::Counter* all_rowsets_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* all_rowsets_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsNum", TUnit::UNIT);
+    COUNTER_SET(all_rowsets_mem_bytes_counter, g_all_rowsets_mem_bytes.get_value());
+    COUNTER_SET(all_rowsets_num_counter, g_all_rowsets_num.get_value());
+
+    RuntimeProfile::Counter* tablet_meta_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* tablet_meta_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaNum", TUnit::UNIT);
+    COUNTER_SET(tablet_meta_mem_bytes_counter, g_tablet_meta_mem_bytes.get_value());
+    COUNTER_SET(tablet_meta_num_counter, g_tablet_meta_num.get_value());
+
+    RuntimeProfile::Counter* tablet_column_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* tablet_column_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnNum", TUnit::UNIT);
+    COUNTER_SET(tablet_column_mem_bytes_counter, g_tablet_column_mem_bytes.get_value());
+    COUNTER_SET(tablet_column_num_counter, g_tablet_column_num.get_value());
+
+    RuntimeProfile::Counter* tablet_index_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* tablet_index_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexNum", TUnit::UNIT);
+    COUNTER_SET(tablet_index_mem_bytes_counter, g_tablet_index_mem_bytes.get_value());
+    COUNTER_SET(tablet_index_num_counter, g_tablet_index_num.get_value());
+
+    RuntimeProfile::Counter* tablet_schema_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* tablet_schema_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaNum", TUnit::UNIT);
+    COUNTER_SET(tablet_schema_mem_bytes_counter, g_tablet_schema_mem_bytes.get_value());
+    COUNTER_SET(tablet_schema_num_counter, g_tablet_schema_num.get_value());
+
+    RuntimeProfile::Counter* all_segments_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* all_segments_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsNum", TUnit::UNIT);
+    COUNTER_SET(all_segments_mem_bytes_counter, g_all_segments_mem_bytes.get_value());
+    COUNTER_SET(all_segments_num_counter, g_all_segments_num.get_value());
+
+    RuntimeProfile::Counter* column_reader_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* column_reader_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderNum", TUnit::UNIT);
+    COUNTER_SET(column_reader_mem_bytes_counter, g_column_reader_mem_bytes.get_value());
+    COUNTER_SET(column_reader_num_counter, g_column_reader_num.get_value());
+
+    RuntimeProfile::Counter* bitmap_index_reader_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "BitmapIndexReaderMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* bitmap_index_reader_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "BitmapIndexReaderNum", TUnit::UNIT);
+    COUNTER_SET(bitmap_index_reader_mem_bytes_counter, g_bitmap_index_reader_mem_bytes.get_value());
+    COUNTER_SET(bitmap_index_reader_num_counter, g_bitmap_index_reader_num.get_value());
+
+    RuntimeProfile::Counter* bloom_filter_index_reader_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "BloomFilterIndexReaderMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* filter_index_reader_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "BloomFilterIndexReaderNum", TUnit::UNIT);
+    COUNTER_SET(bloom_filter_index_reader_mem_bytes_counter,
+                g_bloom_filter_index_reader_mem_bytes.get_value());
+    COUNTER_SET(filter_index_reader_num_counter, g_bloom_filter_index_reader_num.get_value());
+
+    RuntimeProfile::Counter* index_page_reader_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* index_page_reader_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderNum", TUnit::UNIT);
+    COUNTER_SET(index_page_reader_mem_bytes_counter, g_index_page_reader_mem_bytes.get_value());
+    COUNTER_SET(index_page_reader_num_counter, g_index_page_reader_num.get_value());
+
+    RuntimeProfile::Counter* indexed_column_reader_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "IndexedColumnReaderMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* indexed_column_reader_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "IndexedColumnReaderNum", TUnit::UNIT);
+    COUNTER_SET(indexed_column_reader_mem_bytes_counter,
+                g_indexed_column_reader_mem_bytes.get_value());
+    COUNTER_SET(indexed_column_reader_num_counter, g_indexed_column_reader_num.get_value());
+
+    RuntimeProfile::Counter* inverted_index_reader_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "InvertedIndexReaderMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* inverted_index_reader_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "InvertedIndexReaderNum", TUnit::UNIT);
+    COUNTER_SET(inverted_index_reader_mem_bytes_counter,
+                g_inverted_index_reader_mem_bytes.get_value());
+    COUNTER_SET(inverted_index_reader_num_counter, g_inverted_index_reader_num.get_value());
+
+    RuntimeProfile::Counter* ordinal_index_reader_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "OrdinalIndexReaderMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* ordinal_index_reader_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "OrdinalIndexReaderNum", TUnit::UNIT);
+    COUNTER_SET(ordinal_index_reader_mem_bytes_counter,
+                g_ordinal_index_reader_mem_bytes.get_value());
+    COUNTER_SET(ordinal_index_reader_num_counter, g_ordinal_index_reader_num.get_value());
+
+    RuntimeProfile::Counter* zone_map_index_reader_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "ZoneMapIndexReaderMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* zone_map_index_reader_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "ZoneMapIndexReaderNum", TUnit::UNIT);
+    COUNTER_SET(zone_map_index_reader_mem_bytes_counter,
+                g_zone_map_index_reader_mem_bytes.get_value());
+    COUNTER_SET(zone_map_index_reader_num_counter, g_zone_map_index_reader_num.get_value());
+}
+
+}; // namespace doris
diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp
index 256f4d35313d13..ac3a2a7a1dc5c2 100644
--- a/be/src/olap/rowset/rowset.cpp
+++ b/be/src/olap/rowset/rowset.cpp
@@ -27,8 +27,6 @@
 
 namespace doris {
 
-static bvar::Adder<size_t> g_total_rowset_num("doris_total_rowset_num");
-
 Rowset::Rowset(const TabletSchemaSPtr& schema, RowsetMetaSharedPtr rowset_meta,
                std::string tablet_path)
         : _rowset_meta(std::move(rowset_meta)),
@@ -56,11 +54,6 @@ Rowset::Rowset(const TabletSchemaSPtr& schema, RowsetMetaSharedPtr rowset_meta,
     }
     // build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema
     _schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() : schema;
-    g_total_rowset_num << 1;
-}
-
-Rowset::~Rowset() {
-    g_total_rowset_num << -1;
 }
 
 Status Rowset::load(bool use_cache) {
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index e1a2347f6aeaa8..98d88ba19f2068 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -33,6 +33,7 @@
 
 #include "common/logging.h"
 #include "common/status.h"
+#include "olap/metadata_adder.h"
 #include "olap/olap_common.h"
 #include "olap/rowset/rowset_meta.h"
 #include "olap/tablet_schema.h"
@@ -116,10 +117,8 @@ class RowsetStateMachine {
     RowsetState _rowset_state;
 };
 
-class Rowset : public std::enable_shared_from_this<Rowset> {
+class Rowset : public std::enable_shared_from_this<Rowset>, public MetadataAdder<Rowset> {
 public:
-    virtual ~Rowset();
-
     // Open all segment files in this rowset and load necessary metadata.
     // - `use_cache` : whether to use fd cache, only applicable to alpha rowset now
     //
diff --git a/be/src/runtime/process_profile.cpp b/be/src/runtime/process_profile.cpp
index d91aedbeac2025..60b381e9c31578 100644
--- a/be/src/runtime/process_profile.cpp
+++ b/be/src/runtime/process_profile.cpp
@@ -19,6 +19,7 @@
 
 #include <memory>
 
+#include "olap/metadata_adder.h"
 #include "runtime/memory/memory_profile.h"
 
 namespace doris {
@@ -37,8 +38,15 @@ void ProcessProfile::refresh_profile() {
     std::unique_ptr<RuntimeProfile> process_profile =
             std::make_unique<RuntimeProfile>("ProcessProfile");
     _memory_profile->make_memory_profile(process_profile.get());
-    _process_profile.set(std::move(process_profile));
     // TODO make other profile
+
+    // 3. dump object heap
+    RuntimeProfile* object_heap_dump_snapshot =
+            process_profile->create_child("ObjectHeapDump", true, false);
+    MetadataAdder<ProcessProfile>::dump_metadata_object(object_heap_dump_snapshot);
+    // TODO dump other object (block, column, etc.)
+
+    _process_profile.set(std::move(process_profile));
 }
 
 } // namespace doris

From db3a3ec49d20019b1cb871c11aa10d13b974ec22 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Wed, 27 Nov 2024 22:13:44 +0800
Subject: [PATCH 042/399] [env](compile)open compile check in columns class
 (#44425)

### What problem does this PR solve?

Problem Summary:

open compile check in columns class
---
 be/src/util/hash_util.hpp              | 16 +++----
 be/src/vec/columns/column_const.cpp    |  5 ++-
 be/src/vec/columns/column_const.h      |  5 ++-
 be/src/vec/columns/column_decimal.h    |  4 +-
 be/src/vec/columns/column_dictionary.h | 22 ++++++----
 be/src/vec/columns/column_nullable.cpp |  5 ++-
 be/src/vec/columns/column_nullable.h   |  7 ++-
 be/src/vec/columns/column_object.cpp   | 19 +++++---
 be/src/vec/columns/column_object.h     |  6 +--
 be/src/vec/columns/column_string.cpp   | 61 +++++++++++++++-----------
 be/src/vec/columns/column_string.h     | 29 ++++++++----
 be/src/vec/columns/column_vector.cpp   |  3 +-
 be/src/vec/columns/column_vector.h     |  9 ++--
 be/src/vec/core/field.h                | 11 +++--
 14 files changed, 125 insertions(+), 77 deletions(-)

diff --git a/be/src/util/hash_util.hpp b/be/src/util/hash_util.hpp
index e9ac72c5ccdcb4..d444daa8c68d11 100644
--- a/be/src/util/hash_util.hpp
+++ b/be/src/util/hash_util.hpp
@@ -46,7 +46,7 @@ class HashUtil {
         return std::hash<T>()(value);
     }
 
-    static uint32_t zlib_crc_hash(const void* data, int32_t bytes, uint32_t hash) {
+    static uint32_t zlib_crc_hash(const void* data, uint32_t bytes, uint32_t hash) {
         return crc32(hash, (const unsigned char*)data, bytes);
     }
 
@@ -66,7 +66,7 @@ class HashUtil {
     // NOTE: Any changes made to this function need to be reflected in Codegen::GetHashFn.
     // TODO: crc32 hashes with different seeds do not result in different hash functions.
     // The resulting hashes are correlated.
-    static uint32_t crc_hash(const void* data, int32_t bytes, uint32_t hash) {
+    static uint32_t crc_hash(const void* data, uint32_t bytes, uint32_t hash) {
         if (!CpuInfo::is_supported(CpuInfo::SSE4_2)) {
             return zlib_crc_hash(data, bytes, hash);
         }
@@ -93,7 +93,7 @@ class HashUtil {
         return hash;
     }
 
-    static uint64_t crc_hash64(const void* data, int32_t bytes, uint64_t hash) {
+    static uint64_t crc_hash64(const void* data, uint32_t bytes, uint64_t hash) {
         uint32_t words = bytes / sizeof(uint32_t);
         bytes = bytes % sizeof(uint32_t);
 
@@ -125,7 +125,7 @@ class HashUtil {
         return converter.u64;
     }
 #else
-    static uint32_t crc_hash(const void* data, int32_t bytes, uint32_t hash) {
+    static uint32_t crc_hash(const void* data, uint32_t bytes, uint32_t hash) {
         return zlib_crc_hash(data, bytes, hash);
     }
 #endif
@@ -202,7 +202,7 @@ class HashUtil {
     // For example, if the data is <1000, 2000, 3000, 4000, ..> and then the mod of 1000
     // is taken on the hash, all values will collide to the same bucket.
     // For string values, Fnv is slightly faster than boost.
-    static uint32_t fnv_hash(const void* data, int32_t bytes, uint32_t hash) {
+    static uint32_t fnv_hash(const void* data, uint32_t bytes, uint32_t hash) {
         const uint8_t* ptr = reinterpret_cast<const uint8_t*>(data);
 
         while (bytes--) {
@@ -213,7 +213,7 @@ class HashUtil {
         return hash;
     }
 
-    static uint64_t fnv_hash64(const void* data, int32_t bytes, uint64_t hash) {
+    static uint64_t fnv_hash64(const void* data, uint32_t bytes, uint64_t hash) {
         const uint8_t* ptr = reinterpret_cast<const uint8_t*>(data);
 
         while (bytes--) {
@@ -291,7 +291,7 @@ class HashUtil {
     // depending on hardware capabilities.
     // Seed values for different steps of the query execution should use different seeds
     // to prevent accidental key collisions. (See IMPALA-219 for more details).
-    static uint32_t hash(const void* data, int32_t bytes, uint32_t seed) {
+    static uint32_t hash(const void* data, uint32_t bytes, uint32_t seed) {
 #ifdef __SSE4_2__
 
         if (LIKELY(CpuInfo::is_supported(CpuInfo::SSE4_2))) {
@@ -305,7 +305,7 @@ class HashUtil {
 #endif
     }
 
-    static uint64_t hash64(const void* data, int32_t bytes, uint64_t seed) {
+    static uint64_t hash64(const void* data, uint32_t bytes, uint64_t seed) {
 #ifdef _SSE4_2_
         if (LIKELY(CpuInfo::is_supported(CpuInfo::SSE4_2))) {
             return crc_hash64(data, bytes, seed);
diff --git a/be/src/vec/columns/column_const.cpp b/be/src/vec/columns/column_const.cpp
index f751f1d8d3e1b8..fd05127f6d72aa 100644
--- a/be/src/vec/columns/column_const.cpp
+++ b/be/src/vec/columns/column_const.cpp
@@ -35,6 +35,7 @@
 #include "vec/core/column_with_type_and_name.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 ColumnConst::ColumnConst(const ColumnPtr& data_, size_t s_) : data(data_), s(s_) {
     /// Squash Const of Const.
@@ -66,7 +67,9 @@ ColumnConst::ColumnConst(const ColumnPtr& data_, size_t s_, bool create_with_emp
 }
 
 ColumnPtr ColumnConst::convert_to_full_column() const {
-    return data->replicate(Offsets(1, s));
+    // Assuming the number of replicate rows will not exceed Offset(UInt32),
+    // currently Column::replicate only supports Uint32 Offsets
+    return data->replicate(Offsets(1, cast_set<Offset>(s)));
 }
 
 ColumnPtr ColumnConst::remove_low_cardinality() const {
diff --git a/be/src/vec/columns/column_const.h b/be/src/vec/columns/column_const.h
index ee3860f0635c93..0fa22ca23bf20b 100644
--- a/be/src/vec/columns/column_const.h
+++ b/be/src/vec/columns/column_const.h
@@ -48,6 +48,7 @@
 class SipHash;
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 class Arena;
 class Block;
@@ -267,7 +268,8 @@ class ColumnConst final : public COWHelper<IColumn, ColumnConst> {
 
     template <typename T>
     T get_value() const {
-        return get_field().safe_get<NearestFieldType<T>>();
+        // Here the cast is correct, relevant code is rather tricky.
+        return static_cast<T>(get_field().safe_get<NearestFieldType<T>>());
     }
 
     void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override {
@@ -276,3 +278,4 @@ class ColumnConst final : public COWHelper<IColumn, ColumnConst> {
     }
 };
 } // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h
index 4c2f69d5ef3fb6..946b268436ef6c 100644
--- a/be/src/vec/columns/column_decimal.h
+++ b/be/src/vec/columns/column_decimal.h
@@ -53,6 +53,7 @@ class ColumnSorter;
 } // namespace doris
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 /// PaddedPODArray extended by Decimal scale
 template <typename T>
@@ -261,7 +262,7 @@ class ColumnDecimal final : public COWHelper<IColumn, ColumnDecimal<T>> {
         for (U i = 0; i < s; ++i) res[i] = i;
 
         auto sort_end = res.end();
-        if (limit && limit < s / 8.0) {
+        if (limit && limit < static_cast<double>(s) / 8.0L) {
             sort_end = res.begin() + limit;
             if (reverse)
                 std::partial_sort(res.begin(), sort_end, res.end(),
@@ -305,3 +306,4 @@ template <typename T>
 using ColumnVectorOrDecimal = typename ColumnVectorOrDecimalT<T, IsDecimalNumber<T>>::Col;
 
 } // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/columns/column_dictionary.h b/be/src/vec/columns/column_dictionary.h
index ae7d001a31d78c..c4b1f3e27e0cc9 100644
--- a/be/src/vec/columns/column_dictionary.h
+++ b/be/src/vec/columns/column_dictionary.h
@@ -29,6 +29,7 @@
 #include "vec/core/types.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 /**
  * For low cardinality string columns, using ColumnDictionary can reduce memory
@@ -265,9 +266,9 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
         }
     }
 
-    int32_t find_code(const StringRef& value) const { return _dict.find_code(value); }
+    T find_code(const StringRef& value) const { return _dict.find_code(value); }
 
-    int32_t find_code_by_bound(const StringRef& value, bool greater, bool eq) const {
+    T find_code_by_bound(const StringRef& value, bool greater, bool eq) const {
         return _dict.find_code_by_bound(value, greater, eq);
     }
 
@@ -346,8 +347,9 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
             _total_str_len += value.size;
         }
 
-        int32_t find_code(const StringRef& value) const {
-            for (size_t i = 0; i < _dict_data->size(); i++) {
+        T find_code(const StringRef& value) const {
+            // _dict_data->size will not exceed the range of T.
+            for (T i = 0; i < _dict_data->size(); i++) {
                 if ((*_dict_data)[i] == value) {
                     return i;
                 }
@@ -384,11 +386,11 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
 
                 // For dictionary data of char type, sv.size is the schema length,
                 // so use strnlen to remove the 0 at the end to get the actual length.
-                int32_t len = sv.size;
+                size_t len = sv.size;
                 if (type == FieldType::OLAP_FIELD_TYPE_CHAR) {
                     len = strnlen(sv.data, sv.size);
                 }
-                uint32_t hash_val = HashUtil::crc_hash(sv.data, len, 0);
+                uint32_t hash_val = HashUtil::crc_hash(sv.data, static_cast<uint32_t>(len), 0);
                 _hash_values[code] = hash_val;
                 _compute_hash_value_flags[code] = 1;
                 return _hash_values[code];
@@ -412,13 +414,14 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
         //  so upper_bound is the code 0 of b, then evaluate code < 0 and returns empty
         // If the predicate is col <= 'a' and upper_bound-1 is -1,
         //  then evaluate code <= -1 and returns empty
-        int32_t find_code_by_bound(const StringRef& value, bool greater, bool eq) const {
+        T find_code_by_bound(const StringRef& value, bool greater, bool eq) const {
             auto code = find_code(value);
             if (code >= 0) {
                 return code;
             }
-            auto bound = std::upper_bound(_dict_data->begin(), _dict_data->end(), value) -
-                         _dict_data->begin();
+            auto bound =
+                    static_cast<T>(std::upper_bound(_dict_data->begin(), _dict_data->end(), value) -
+                                   _dict_data->begin());
             return greater ? bound - greater + eq : bound - eq;
         }
 
@@ -536,3 +539,4 @@ template class ColumnDictionary<int32_t>;
 using ColumnDictI32 = vectorized::ColumnDictionary<doris::vectorized::Int32>;
 
 } // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/columns/column_nullable.cpp b/be/src/vec/columns/column_nullable.cpp
index 5e34ad4d8d4d1d..c58c78f5611d02 100644
--- a/be/src/vec/columns/column_nullable.cpp
+++ b/be/src/vec/columns/column_nullable.cpp
@@ -29,6 +29,7 @@
 #include "vec/utils/util.hpp"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 ColumnNullable::ColumnNullable(MutableColumnPtr&& nested_column_, MutableColumnPtr&& null_map_)
         : NullMapProvider(std::move(null_map_)), nested_column(std::move(nested_column_)) {
@@ -62,7 +63,7 @@ void ColumnNullable::update_xxHash_with_value(size_t start, size_t end, uint64_t
     } else {
         const auto* __restrict real_null_data =
                 assert_cast<const ColumnUInt8&>(get_null_map_column()).get_data().data();
-        for (int i = start; i < end; ++i) {
+        for (size_t i = start; i < end; ++i) {
             if (real_null_data[i] != 0) {
                 hash = HashUtil::xxHash64NullWithSeed(hash);
             }
@@ -78,7 +79,7 @@ void ColumnNullable::update_crc_with_value(size_t start, size_t end, uint32_t& h
     } else {
         const auto* __restrict real_null_data =
                 assert_cast<const ColumnUInt8&>(get_null_map_column()).get_data().data();
-        for (int i = start; i < end; ++i) {
+        for (size_t i = start; i < end; ++i) {
             if (real_null_data[i] != 0) {
                 hash = HashUtil::zlib_crc_hash_null(hash);
             }
diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h
index 252144fbc5f612..83d5e6af35aeb4 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -43,6 +43,7 @@
 class SipHash;
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 class Arena;
 class ColumnSorter;
 
@@ -404,7 +405,8 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable>, public N
         }
         static constexpr auto MAX_NUMBER_OF_ROWS_FOR_FULL_SEARCH = 1000;
         size_t num_rows = size();
-        size_t num_sampled_rows = std::min(static_cast<size_t>(num_rows * sample_ratio), num_rows);
+        size_t num_sampled_rows = std::min(
+                static_cast<size_t>(static_cast<double>(num_rows) * sample_ratio), num_rows);
         size_t num_checked_rows = 0;
         size_t res = 0;
         if (num_sampled_rows == num_rows || num_rows <= MAX_NUMBER_OF_ROWS_FOR_FULL_SEARCH) {
@@ -423,7 +425,7 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable>, public N
         if (num_checked_rows == 0) {
             return 0.0;
         }
-        return static_cast<double>(res) / num_checked_rows;
+        return static_cast<double>(res) / static_cast<double>(num_checked_rows);
     }
 
     void convert_dict_codes_if_necessary() override {
@@ -460,3 +462,4 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable>, public N
 ColumnPtr make_nullable(const ColumnPtr& column, bool is_nullable = false);
 ColumnPtr remove_nullable(const ColumnPtr& column);
 } // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
index d5e52d07bcf788..3e8d3722305e8d 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -82,6 +82,7 @@
 #endif
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 namespace {
 
 DataTypePtr create_array_of_type(TypeIndex type, size_t num_dimensions, bool is_nullable) {
@@ -653,7 +654,7 @@ bool ColumnObject::Subcolumn::check_if_sparse_column(size_t num_rows) {
         defaults_ratio.push_back(data[i]->get_ratio_of_default_rows());
     }
     double default_ratio = std::accumulate(defaults_ratio.begin(), defaults_ratio.end(), 0.0) /
-                           defaults_ratio.size();
+                           static_cast<double>(defaults_ratio.size());
     return default_ratio >= config::variant_ratio_of_defaults_as_sparse_column;
 }
 
@@ -1294,7 +1295,11 @@ rapidjson::Value* find_leaf_node_by_path(rapidjson::Value& json, const PathInDat
     if (!json.IsObject()) {
         return nullptr;
     }
-    rapidjson::Value name(current_key.data(), current_key.size());
+    /*! RapidJSON uses 32-bit array/string indices even on 64-bit platforms,
+    instead of using \c size_t. Users may override the SizeType by defining
+    \ref RAPIDJSON_NO_SIZETYPEDEFINE.
+    */
+    rapidjson::Value name(current_key.data(), cast_set<unsigned>(current_key.size()));
     auto it = json.FindMember(name);
     if (it == json.MemberEnd()) {
         return nullptr;
@@ -1312,7 +1317,7 @@ rapidjson::Value* find_leaf_node_by_path(rapidjson::Value& json, const PathInDat
 // 3. empty root jsonb value(not null)
 // 4. type is nothing
 bool skip_empty_json(const ColumnNullable* nullable, const DataTypePtr& type,
-                     TypeIndex base_type_id, int row, const PathInData& path) {
+                     TypeIndex base_type_id, size_t row, const PathInData& path) {
     // skip nulls
     if (nullable && nullable->is_null_at(row)) {
         return true;
@@ -1348,7 +1353,7 @@ Status find_and_set_leave_value(const IColumn* column, const PathInData& path,
                                 const DataTypeSerDeSPtr& type_serde, const DataTypePtr& type,
                                 TypeIndex base_type_index, rapidjson::Value& root,
                                 rapidjson::Document::AllocatorType& allocator, Arena& mem_pool,
-                                int row) {
+                                size_t row) {
 #ifndef NDEBUG
     // sanitize type and column
     if (column->get_name() != type->create_column()->get_name()) {
@@ -1416,7 +1421,7 @@ void get_json_by_column_tree(rapidjson::Value& root, rapidjson::Document::Alloca
     }
 }
 
-Status ColumnObject::serialize_one_row_to_string(int64_t row, std::string* output) const {
+Status ColumnObject::serialize_one_row_to_string(size_t row, std::string* output) const {
     if (!is_finalized()) {
         const_cast<ColumnObject*>(this)->finalize(FinalizeMode::READ_MODE);
     }
@@ -1432,7 +1437,7 @@ Status ColumnObject::serialize_one_row_to_string(int64_t row, std::string* outpu
     return Status::OK();
 }
 
-Status ColumnObject::serialize_one_row_to_string(int64_t row, BufferWritable& output) const {
+Status ColumnObject::serialize_one_row_to_string(size_t row, BufferWritable& output) const {
     if (!is_finalized()) {
         const_cast<ColumnObject*>(this)->finalize(FinalizeMode::READ_MODE);
     }
@@ -1447,7 +1452,7 @@ Status ColumnObject::serialize_one_row_to_string(int64_t row, BufferWritable& ou
     return Status::OK();
 }
 
-Status ColumnObject::serialize_one_row_to_json_format(int64_t row, rapidjson::StringBuffer* output,
+Status ColumnObject::serialize_one_row_to_json_format(size_t row, rapidjson::StringBuffer* output,
                                                       bool* is_null) const {
     CHECK(is_finalized());
     if (subcolumns.empty()) {
diff --git a/be/src/vec/columns/column_object.h b/be/src/vec/columns/column_object.h
index 21bb44691151b0..e4127197a22b02 100644
--- a/be/src/vec/columns/column_object.h
+++ b/be/src/vec/columns/column_object.h
@@ -272,12 +272,12 @@ class ColumnObject final : public COWHelper<IColumn, ColumnObject> {
         return subcolumns.get_mutable_root()->data.get_finalized_column_ptr()->assume_mutable();
     }
 
-    Status serialize_one_row_to_string(int64_t row, std::string* output) const;
+    Status serialize_one_row_to_string(size_t row, std::string* output) const;
 
-    Status serialize_one_row_to_string(int64_t row, BufferWritable& output) const;
+    Status serialize_one_row_to_string(size_t row, BufferWritable& output) const;
 
     // serialize one row to json format
-    Status serialize_one_row_to_json_format(int64_t row, rapidjson::StringBuffer* output,
+    Status serialize_one_row_to_json_format(size_t row, rapidjson::StringBuffer* output,
                                             bool* is_null) const;
 
     // merge multiple sub sparse columns into root
diff --git a/be/src/vec/columns/column_string.cpp b/be/src/vec/columns/column_string.cpp
index 3caa194551bf79..cb83a29bbada2c 100644
--- a/be/src/vec/columns/column_string.cpp
+++ b/be/src/vec/columns/column_string.cpp
@@ -34,6 +34,7 @@
 #include "vec/core/sort_block.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 void ColumnStr<T>::sanity_check() const {
@@ -74,8 +75,8 @@ MutableColumnPtr ColumnStr<T>::clone_resized(size_t to_size) const {
             res->offsets.assign(offsets.begin(), offsets.end());
             res->chars.assign(chars.begin(), chars.end());
         }
-
-        res->offsets.resize_fill(to_size, chars.size());
+        // If offset is uint32, size will not exceed, check the size when inserting data into ColumnStr<T>.
+        res->offsets.resize_fill(to_size, static_cast<T>(chars.size()));
     }
 
     return res;
@@ -92,14 +93,14 @@ void ColumnStr<T>::shrink_padding_chars() {
 
     // deal the 0-th element. no need to move.
     auto next_start = offset[0];
-    offset[0] = strnlen(data, size_at(0));
+    offset[0] = static_cast<T>(strnlen(data, size_at(0)));
     for (size_t i = 1; i < size; i++) {
         // get the i-th length and whole move it to cover the last's trailing void
         auto length = strnlen(data + next_start, offset[i] - next_start);
         memmove(data + offset[i - 1], data + next_start, length);
         // offset i will be changed. so save the old value for (i+1)-th to get its length.
         next_start = offset[i];
-        offset[i] = offset[i - 1] + length;
+        offset[i] = offset[i - 1] + static_cast<T>(length);
     }
     chars.resize_fill(offsets.back()); // just call it to shrink memory here. no possible to expand.
 }
@@ -125,8 +126,8 @@ void ColumnStr<T>::insert_range_from_ignore_overflow(const doris::vectorized::IC
                 "Parameter out of bound in IColumnStr<T>::insert_range_from method.");
     }
 
-    size_t nested_offset = src_concrete.offset_at(start);
-    size_t nested_length = src_concrete.offsets[start + length - 1] - nested_offset;
+    auto nested_offset = src_concrete.offset_at(start);
+    auto nested_length = src_concrete.offsets[start + length - 1] - nested_offset;
 
     size_t old_chars_size = chars.size();
     chars.resize(old_chars_size + nested_length);
@@ -136,7 +137,7 @@ void ColumnStr<T>::insert_range_from_ignore_overflow(const doris::vectorized::IC
         offsets.assign(src_concrete.offsets.begin(), src_concrete.offsets.begin() + length);
     } else {
         size_t old_size = offsets.size();
-        size_t prev_max_offset = offsets.back(); /// -1th index is Ok, see PaddedPODArray
+        auto prev_max_offset = offsets.back(); /// -1th index is Ok, see PaddedPODArray
         offsets.resize(old_size + length);
 
         for (size_t i = 0; i < length; ++i) {
@@ -161,8 +162,8 @@ void ColumnStr<T>::insert_range_from(const IColumn& src, size_t start, size_t le
                     doris::ErrorCode::INTERNAL_ERROR,
                     "Parameter out of bound in IColumnStr<T>::insert_range_from method.");
         }
-        size_t nested_offset = src_offsets[static_cast<ssize_t>(start) - 1];
-        size_t nested_length = src_offsets[start + length - 1] - nested_offset;
+        auto nested_offset = src_offsets[static_cast<ssize_t>(start) - 1];
+        auto nested_length = src_offsets[start + length - 1] - nested_offset;
 
         size_t old_chars_size = chars.size();
         check_chars_length(old_chars_size + nested_length, offsets.size() + length);
@@ -174,11 +175,13 @@ void ColumnStr<T>::insert_range_from(const IColumn& src, size_t start, size_t le
             offsets.assign(src_offsets.begin(), src_offsets.begin() + length);
         } else {
             size_t old_size = offsets.size();
-            size_t prev_max_offset = offsets.back(); /// -1th index is Ok, see PaddedPODArray
+            auto prev_max_offset = offsets.back(); /// -1th index is Ok, see PaddedPODArray
             offsets.resize(old_size + length);
 
             for (size_t i = 0; i < length; ++i) {
-                offsets[old_size + i] = src_offsets[start + i] - nested_offset + prev_max_offset;
+                // if Offsets is uint32, size will not exceed range of uint32, cast is OK.
+                offsets[old_size + i] =
+                        static_cast<T>(src_offsets[start + i] - nested_offset) + prev_max_offset;
             }
         }
     };
@@ -208,7 +211,7 @@ void ColumnStr<T>::insert_many_from(const IColumn& src, size_t position, size_t
     auto prev_pos = old_chars_size;
     for (; start_pos < end_pos; ++start_pos) {
         memcpy(&chars[prev_pos], data_val, data_length);
-        offsets[start_pos] = prev_pos + data_length;
+        offsets[start_pos] = static_cast<T>(prev_pos + data_length);
         prev_pos = prev_pos + data_length;
     }
 }
@@ -229,7 +232,8 @@ void ColumnStr<T>::insert_indices_from(const IColumn& src, const uint32_t* indic
         for (const auto* x = indices_begin; x != indices_end; ++x) {
             int64_t src_offset = *x;
             total_chars_size += src_offset_data[src_offset] - src_offset_data[src_offset - 1];
-            dst_offsets_data[dst_offsets_pos++] = total_chars_size;
+            // if Offsets is uint32, size will not exceed range of uint32, cast is OK.
+            dst_offsets_data[dst_offsets_pos++] = static_cast<T>(total_chars_size);
         }
         check_chars_length(total_chars_size, offsets.size());
 
@@ -267,13 +271,16 @@ void ColumnStr<T>::update_crcs_with_value(uint32_t* __restrict hashes, doris::Pr
     if (null_data == nullptr) {
         for (size_t i = 0; i < s; i++) {
             auto data_ref = get_data_at(i);
-            hashes[i] = HashUtil::zlib_crc_hash(data_ref.data, data_ref.size, hashes[i]);
+            // If offset is uint32, size will not exceed, check the size when inserting data into ColumnStr<T>.
+            hashes[i] = HashUtil::zlib_crc_hash(data_ref.data, static_cast<uint32_t>(data_ref.size),
+                                                hashes[i]);
         }
     } else {
         for (size_t i = 0; i < s; i++) {
             if (null_data[i] == 0) {
                 auto data_ref = get_data_at(i);
-                hashes[i] = HashUtil::zlib_crc_hash(data_ref.data, data_ref.size, hashes[i]);
+                hashes[i] = HashUtil::zlib_crc_hash(
+                        data_ref.data, static_cast<uint32_t>(data_ref.size), hashes[i]);
             }
         }
     }
@@ -391,8 +398,9 @@ ColumnPtr ColumnStr<T>::permute(const IColumn::Permutation& perm, size_t limit)
 template <typename T>
 StringRef ColumnStr<T>::serialize_value_into_arena(size_t n, Arena& arena,
                                                    char const*& begin) const {
-    uint32_t string_size(size_at(n));
-    uint32_t offset(offset_at(n));
+    // Use uint32 instead of size_t to reduce agg key's length.
+    auto string_size(static_cast<uint32_t>(size_at(n)));
+    auto offset(static_cast<uint32_t>(offset_at(n)));
 
     StringRef res;
     res.size = sizeof(string_size) + string_size;
@@ -421,7 +429,7 @@ const char* ColumnStr<T>::deserialize_and_insert_from_arena(const char* pos) {
 
 template <typename T>
 size_t ColumnStr<T>::get_max_row_byte_size() const {
-    size_t max_size = 0;
+    T max_size = 0;
     size_t num_rows = offsets.size();
     for (size_t i = 0; i < num_rows; ++i) {
         max_size = std::max(max_size, size_at(i));
@@ -434,8 +442,9 @@ template <typename T>
 void ColumnStr<T>::serialize_vec(std::vector<StringRef>& keys, size_t num_rows,
                                  size_t max_row_byte_size) const {
     for (size_t i = 0; i < num_rows; ++i) {
-        uint32_t offset(offset_at(i));
-        uint32_t string_size(size_at(i));
+        // Use uint32 instead of size_t to reduce agg key's length.
+        auto offset(static_cast<uint32_t>(offset_at(i)));
+        auto string_size(static_cast<uint32_t>(size_at(i)));
 
         auto* ptr = const_cast<char*>(keys[i].data + keys[i].size);
         memcpy_fixed<uint32_t>(ptr, (char*)&string_size);
@@ -458,8 +467,8 @@ void ColumnStr<T>::serialize_vec_with_null_map(std::vector<StringRef>& keys, siz
             memcpy(dest, null_map + i, sizeof(uint8_t));
 
             if (null_map[i] == 0) {
-                UInt32 offset(offset_at(i));
-                UInt32 string_size(size_at(i));
+                auto offset(offset_at(i));
+                auto string_size(size_at(i));
 
                 memcpy_fixed<UInt32>(dest + 1, (char*)&string_size);
                 memcpy(dest + 1 + sizeof(string_size), &chars[offset], string_size);
@@ -475,8 +484,8 @@ void ColumnStr<T>::serialize_vec_with_null_map(std::vector<StringRef>& keys, siz
             // serialize null first
             memcpy(dest, null_map + i, sizeof(uint8_t));
 
-            UInt32 offset(offset_at(i));
-            UInt32 string_size(size_at(i));
+            auto offset(offset_at(i));
+            auto string_size(size_at(i));
 
             memcpy_fixed<UInt32>(dest + 1, (char*)&string_size);
             memcpy(dest + 1 + sizeof(string_size), &chars[offset], string_size);
@@ -559,8 +568,8 @@ ColumnPtr ColumnStr<T>::replicate(const IColumn::Offsets& replicate_offsets) con
     T current_new_offset = 0;
 
     for (size_t i = 0; i < col_size; ++i) {
-        size_t size_to_replicate = replicate_offsets[i] - prev_replicate_offset;
-        size_t string_size = offsets[i] - prev_string_offset;
+        T size_to_replicate = replicate_offsets[i] - prev_replicate_offset;
+        T string_size = offsets[i] - prev_string_offset;
 
         for (size_t j = 0; j < size_to_replicate; ++j) {
             current_new_offset += string_size;
diff --git a/be/src/vec/columns/column_string.h b/be/src/vec/columns/column_string.h
index f116d4ce1f17cf..1674fd90933dbe 100644
--- a/be/src/vec/columns/column_string.h
+++ b/be/src/vec/columns/column_string.h
@@ -47,6 +47,7 @@
 #include "vec/core/types.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 class Arena;
 class ColumnSorter;
 
@@ -86,10 +87,10 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
     Chars chars;
 
     // Start position of i-th element.
-    size_t ALWAYS_INLINE offset_at(ssize_t i) const { return offsets[i - 1]; }
+    T ALWAYS_INLINE offset_at(ssize_t i) const { return offsets[i - 1]; }
 
     /// Size of i-th element, including terminating zero.
-    size_t ALWAYS_INLINE size_at(ssize_t i) const { return offsets[i] - offsets[i - 1]; }
+    T ALWAYS_INLINE size_at(ssize_t i) const { return offsets[i] - offsets[i - 1]; }
 
     template <bool positive>
     struct less;
@@ -220,7 +221,7 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
 
         const char* ptr = strings[0].data;
         for (size_t i = 0; i != num; i++) {
-            uint32_t len = strings[i].size;
+            size_t len = strings[i].size;
             length += len;
             offset += len;
             offsets.push_back(offset);
@@ -282,7 +283,7 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
         Char* data = chars.data();
         size_t offset = old_size;
         for (size_t i = 0; i < num; i++) {
-            uint32_t len = strings[i].size;
+            size_t len = strings[i].size;
             if (len) {
                 memcpy(data + offset, strings[i].data, len);
                 offset += len;
@@ -305,7 +306,7 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
         Char* data = chars.data();
         size_t offset = old_size;
         for (size_t i = 0; i < num; i++) {
-            uint32_t len = strings[i].size;
+            size_t len = strings[i].size;
             if (len) {
                 memcpy(data + offset, strings[i].data, copy_length);
                 offset += len;
@@ -342,9 +343,15 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
         for (size_t i = 0; i < num; i++) {
             int32_t codeword = data_array[i + start_index];
             new_size += dict[codeword].size;
-            offsets[offset_size + i] = new_size;
+            offsets[offset_size + i] = static_cast<T>(new_size);
         }
 
+        if (new_size > std::numeric_limits<T>::max()) {
+            throw doris::Exception(ErrorCode::INTERNAL_ERROR,
+                                   "ColumnString insert size out of range type {} [{},{}]",
+                                   typeid(T).name(), std::numeric_limits<T>::min(),
+                                   std::numeric_limits<T>::max());
+        }
         check_chars_length(new_size, offsets.size());
         chars.resize(new_size);
 
@@ -406,13 +413,16 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
             for (size_t i = start; i < end; ++i) {
                 if (null_data[i] == 0) {
                     auto data_ref = get_data_at(i);
-                    hash = HashUtil::zlib_crc_hash(data_ref.data, data_ref.size, hash);
+                    // If offset is uint32, size will not exceed, check the size when inserting data into ColumnStr<T>.
+                    hash = HashUtil::zlib_crc_hash(data_ref.data,
+                                                   static_cast<uint32_t>(data_ref.size), hash);
                 }
             }
         } else {
             for (size_t i = start; i < end; ++i) {
                 auto data_ref = get_data_at(i);
-                hash = HashUtil::zlib_crc_hash(data_ref.data, data_ref.size, hash);
+                hash = HashUtil::zlib_crc_hash(data_ref.data, static_cast<uint32_t>(data_ref.size),
+                                               hash);
             }
         }
     }
@@ -473,7 +483,7 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
     void insert_default() override { offsets.push_back(chars.size()); }
 
     void insert_many_defaults(size_t length) override {
-        offsets.resize_fill(offsets.size() + length, chars.size());
+        offsets.resize_fill(offsets.size() + length, static_cast<T>(chars.size()));
     }
 
     int compare_at(size_t n, size_t m, const IColumn& rhs_,
@@ -525,3 +535,4 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
 using ColumnString = ColumnStr<UInt32>;
 using ColumnString64 = ColumnStr<UInt64>;
 } // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/columns/column_vector.cpp b/be/src/vec/columns/column_vector.cpp
index 0e24446a5cdb92..f0f0bec8b99bb1 100644
--- a/be/src/vec/columns/column_vector.cpp
+++ b/be/src/vec/columns/column_vector.cpp
@@ -42,6 +42,7 @@
 #include "vec/data_types/data_type.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 StringRef ColumnVector<T>::serialize_value_into_arena(size_t n, Arena& arena,
@@ -242,7 +243,7 @@ void ColumnVector<T>::get_permutation(bool reverse, size_t limit, int nan_direct
     if (s == 0) return;
 
     // std::partial_sort need limit << s can get performance benefit
-    if (limit > (s / 8.0)) limit = 0;
+    if (limit > (s / 8.0L)) limit = 0;
 
     if (limit) {
         for (size_t i = 0; i < s; ++i) res[i] = i;
diff --git a/be/src/vec/columns/column_vector.h b/be/src/vec/columns/column_vector.h
index 2cb320b6992095..970997a91864cc 100644
--- a/be/src/vec/columns/column_vector.h
+++ b/be/src/vec/columns/column_vector.h
@@ -61,6 +61,7 @@ class ColumnSorter;
 } // namespace doris
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 /** Stuff for comparing numbers.
   * Integer values are compared as usual.
@@ -178,10 +179,9 @@ class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
     void insert_range_of_integer(T begin, T end) {
         if constexpr (std::is_integral_v<T>) {
             auto old_size = data.size();
-            data.resize(old_size + (end - begin));
-            for (int i = 0; i < end - begin; i++) {
-                data[old_size + i] = begin + i;
-            }
+            auto new_size = old_size + static_cast<size_t>(end - begin);
+            data.resize(new_size);
+            std::iota(data.begin() + old_size, data.begin() + new_size, begin);
         } else {
             throw doris::Exception(ErrorCode::INTERNAL_ERROR,
                                    "double column not support insert_range_of_integer");
@@ -409,3 +409,4 @@ class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
 };
 
 } // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h
index 8113dc602fbd4e..341f65e075ed11 100644
--- a/be/src/vec/core/field.h
+++ b/be/src/vec/core/field.h
@@ -165,7 +165,7 @@ class JsonbField {
 public:
     JsonbField() = default;
 
-    JsonbField(const char* ptr, uint32_t len) : size(len) {
+    JsonbField(const char* ptr, size_t len) : size(len) {
         data = new char[size];
         if (!data) {
             LOG(FATAL) << "new data buffer failed, size: " << size;
@@ -213,7 +213,7 @@ class JsonbField {
     }
 
     const char* get_value() const { return data; }
-    uint32_t get_size() const { return size; }
+    size_t get_size() const { return size; }
 
     bool operator<(const JsonbField& r) const {
         LOG(FATAL) << "comparing between JsonbField is not supported";
@@ -252,7 +252,7 @@ class JsonbField {
 
 private:
     char* data = nullptr;
-    uint32_t size = 0;
+    size_t size = 0;
 };
 
 template <typename T>
@@ -498,6 +498,9 @@ class Field {
 
     bool is_null() const { return which == Types::Null; }
 
+    // The template parameter T needs to be consistent with `which`.
+    // If not, use NearestFieldType<> externally.
+    // Maybe modify this in the future, reference: https://github.com/ClickHouse/ClickHouse/pull/22003
     template <typename T>
     T& get() {
         using TWithoutRef = std::remove_reference_t<T>;
@@ -520,6 +523,8 @@ class Field {
         return true;
     }
 
+    // The template parameter T needs to be consistent with `which`.
+    // If not, use NearestFieldType<> externally.
     template <typename T>
     bool try_get(T& result) const {
         const Types::Which requested = TypeToEnum<std::decay_t<T>>::value;

From 9b9bd9ee0e152aaa84d4933d2973dfc2ce11319d Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@selectdb.com>
Date: Thu, 28 Nov 2024 10:09:07 +0800
Subject: [PATCH 043/399] [Improment]log more detail when query hits policy
 (#44685)

---
 .../workload_management/workload_action.cpp   |  2 +-
 .../workload_management/workload_condition.h  | 20 +++++++++++++++++++
 .../workload_management/workload_query_info.h |  3 ++-
 .../workload_sched_policy.cpp                 |  5 +++++
 4 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/be/src/runtime/workload_management/workload_action.cpp b/be/src/runtime/workload_management/workload_action.cpp
index 8e6e3b19e2c385..77042b074fd624 100644
--- a/be/src/runtime/workload_management/workload_action.cpp
+++ b/be/src/runtime/workload_management/workload_action.cpp
@@ -25,7 +25,7 @@ void WorkloadActionCancelQuery::exec(WorkloadQueryInfo* query_info) {
     std::stringstream msg;
     msg << "query " << query_info->query_id
         << " cancelled by workload policy: " << query_info->policy_name
-        << ", id:" << query_info->policy_id;
+        << ", id:" << query_info->policy_id << ", " << query_info->cond_eval_msg;
     std::string msg_str = msg.str();
     LOG(INFO) << "[workload_schedule]" << msg_str;
     ExecEnv::GetInstance()->fragment_mgr()->cancel_query(query_info->tquery_id,
diff --git a/be/src/runtime/workload_management/workload_condition.h b/be/src/runtime/workload_management/workload_condition.h
index a85268a8dc3a6a..cf53a5f07ddf9d 100644
--- a/be/src/runtime/workload_management/workload_condition.h
+++ b/be/src/runtime/workload_management/workload_condition.h
@@ -33,6 +33,10 @@ class WorkloadCondition {
     virtual bool eval(std::string str_val) = 0;
 
     virtual WorkloadMetricType get_workload_metric_type() = 0;
+
+    virtual std::string get_metric_string() = 0;
+
+    virtual std::string get_metric_value_string() = 0;
 };
 
 class WorkloadConditionQueryTime : public WorkloadCondition {
@@ -45,6 +49,10 @@ class WorkloadConditionQueryTime : public WorkloadCondition {
         return WorkloadMetricType::QUERY_TIME;
     }
 
+    std::string get_metric_string() override { return "query_time"; }
+
+    std::string get_metric_value_string() override { return std::to_string(_query_time); }
+
 private:
     int64_t _query_time;
     WorkloadCompareOperator _op;
@@ -56,6 +64,10 @@ class WorkloadConditionScanRows : public WorkloadCondition {
     bool eval(std::string str_val) override;
     WorkloadMetricType get_workload_metric_type() override { return WorkloadMetricType::SCAN_ROWS; }
 
+    std::string get_metric_string() override { return "scan_rows"; }
+
+    std::string get_metric_value_string() override { return std::to_string(_scan_rows); }
+
 private:
     int64_t _scan_rows;
     WorkloadCompareOperator _op;
@@ -69,6 +81,10 @@ class WorkloadConditionScanBytes : public WorkloadCondition {
         return WorkloadMetricType::SCAN_BYTES;
     }
 
+    std::string get_metric_string() override { return "scan_bytes"; }
+
+    std::string get_metric_value_string() override { return std::to_string(_scan_bytes); }
+
 private:
     int64_t _scan_bytes;
     WorkloadCompareOperator _op;
@@ -82,6 +98,10 @@ class WorkloadConditionQueryMemory : public WorkloadCondition {
         return WorkloadMetricType::QUERY_MEMORY_BYTES;
     }
 
+    std::string get_metric_string() override { return "query_memory"; }
+
+    std::string get_metric_value_string() override { return std::to_string(_query_memory_bytes); }
+
 private:
     int64_t _query_memory_bytes;
     WorkloadCompareOperator _op;
diff --git a/be/src/runtime/workload_management/workload_query_info.h b/be/src/runtime/workload_management/workload_query_info.h
index e544668e1039ed..16151eec390746 100644
--- a/be/src/runtime/workload_management/workload_query_info.h
+++ b/be/src/runtime/workload_management/workload_query_info.h
@@ -30,7 +30,8 @@ class WorkloadQueryInfo {
     std::string query_id;
     int64_t wg_id;
     int64_t policy_id;
-    std::string policy_name;
+    std::string policy_name {""};
+    std::string cond_eval_msg {""};
 };
 
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_sched_policy.cpp b/be/src/runtime/workload_management/workload_sched_policy.cpp
index efa8965dd77121..63b9362bc217be 100644
--- a/be/src/runtime/workload_management/workload_sched_policy.cpp
+++ b/be/src/runtime/workload_management/workload_sched_policy.cpp
@@ -60,6 +60,7 @@ bool WorkloadSchedPolicy::is_match(WorkloadQueryInfo* query_info_ptr) {
     }
 
     auto& metric_val_map = query_info_ptr->metric_map;
+    std::string cond_eval_msg = "";
     for (auto& cond : _condition_list) {
         if (metric_val_map.find(cond->get_workload_metric_type()) == metric_val_map.end()) {
             return false;
@@ -69,7 +70,11 @@ bool WorkloadSchedPolicy::is_match(WorkloadQueryInfo* query_info_ptr) {
         if (!cond->eval(val)) {
             return false;
         }
+        cond_eval_msg += cond->get_metric_string() + ":" + val + "(" +
+                         cond->get_metric_value_string() + "), ";
     }
+    cond_eval_msg = cond_eval_msg.substr(0, cond_eval_msg.size() - 2);
+    query_info_ptr->cond_eval_msg = cond_eval_msg;
     return true;
 }
 

From f1d37cb951ee4b83bc9e3ff1d6dca7fb5bf37ac5 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Thu, 28 Nov 2024 10:13:12 +0800
Subject: [PATCH 044/399] [improve](backup) Limit the output size of show
 backup/restore job progress (#44681)

A backup/restore job might involve millions of tablets, and the output
of the show backup/restore command might be huge without limitation.
---
 .../org/apache/doris/alter/RollupJobV2.java   |  4 ++--
 .../org/apache/doris/backup/BackupJob.java    | 22 ++++++++++++++-----
 .../org/apache/doris/backup/RestoreJob.java   | 22 ++++++++++++++-----
 3 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 8eb87050bf6d77..1d09224e6fa38f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -570,7 +570,7 @@ protected void runRunningJob() throws AlterCancelException {
                     continue;
                 }
 
-                long visiableVersion = partition.getVisibleVersion();
+                long visibleVersion = partition.getVisibleVersion();
                 short expectReplicationNum = tbl.getPartitionInfo().getReplicaAllocation(
                         partitionId).getTotalReplicaNum();
                 MaterializedIndex rollupIndex = entry.getValue();
@@ -579,7 +579,7 @@ protected void runRunningJob() throws AlterCancelException {
                     int healthyReplicaNum = 0;
                     for (Replica replica : replicas) {
                         if (!replica.isBad() && replica.getLastFailedVersion() < 0
-                                && replica.checkVersionCatchUp(visiableVersion, false)) {
+                                && replica.checkVersionCatchUp(visibleVersion, false)) {
                             healthyReplicaNum++;
                         }
                     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 621a2b1d9f7d29..d0a957dcd9ae0d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -1059,6 +1059,19 @@ public synchronized Snapshot getSnapshot() {
     }
 
     public synchronized List<String> getInfo() {
+        String unfinishedTaskIdsStr = unfinishedTaskIds.entrySet().stream()
+                .map(e -> "[" + e.getKey() + "=" + e.getValue() + "]")
+                .limit(100)
+                .collect(Collectors.joining(", "));
+        String taskProgressStr = taskProgress.entrySet().stream()
+                .map(e -> "[" + e.getKey() + ": " + e.getValue().first + "/" + e.getValue().second + "]")
+                .limit(100)
+                .collect(Collectors.joining(", "));
+        String taskErrMsgStr = taskErrMsg.entrySet().stream()
+                .map(e -> "[" + e.getKey() + ": " + e.getValue() + "]")
+                .limit(100)
+                .collect(Collectors.joining(", "));
+
         List<String> info = Lists.newArrayList();
         info.add(String.valueOf(jobId));
         info.add(label);
@@ -1069,12 +1082,9 @@ public synchronized List<String> getInfo() {
         info.add(TimeUtils.longToTimeString(snapshotFinishedTime));
         info.add(TimeUtils.longToTimeString(snapshotUploadFinishedTime));
         info.add(TimeUtils.longToTimeString(finishedTime));
-        info.add(Joiner.on(", ").join(unfinishedTaskIds.entrySet()));
-        info.add(Joiner.on(", ").join(taskProgress.entrySet().stream().map(
-                e -> "[" + e.getKey() + ": " + e.getValue().first + "/" + e.getValue().second + "]").collect(
-                        Collectors.toList())));
-        info.add(Joiner.on(", ").join(taskErrMsg.entrySet().stream().map(n -> "[" + n.getKey() + ": " + n.getValue()
-                + "]").collect(Collectors.toList())));
+        info.add(unfinishedTaskIdsStr);
+        info.add(taskProgressStr);
+        info.add(taskErrMsgStr);
         info.add(status.toString());
         info.add(String.valueOf(timeoutMs / 1000));
         return info;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 0b5b0cb03390f3..f1abb0c9e632bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -2208,6 +2208,11 @@ public List<String> getFullInfo() {
     }
 
     public synchronized List<String> getInfo(boolean isBrief) {
+        String unfinishedTaskIdsStr = unfinishedSignatureToId.entrySet().stream()
+                .map(e -> "[" + e.getKey() + "=" + e.getValue() + "]")
+                .limit(100)
+                .collect(Collectors.joining(", "));
+
         List<String> info = Lists.newArrayList();
         info.add(String.valueOf(jobId));
         info.add(label);
@@ -2227,13 +2232,18 @@ public synchronized List<String> getInfo(boolean isBrief) {
         info.add(TimeUtils.longToTimeString(snapshotFinishedTime));
         info.add(TimeUtils.longToTimeString(downloadFinishedTime));
         info.add(TimeUtils.longToTimeString(finishedTime));
-        info.add(Joiner.on(", ").join(unfinishedSignatureToId.entrySet()));
+        info.add(unfinishedTaskIdsStr);
         if (!isBrief) {
-            info.add(Joiner.on(", ").join(taskProgress.entrySet().stream().map(
-                    e -> "[" + e.getKey() + ": " + e.getValue().first + "/" + e.getValue().second + "]").collect(
-                    Collectors.toList())));
-            info.add(Joiner.on(", ").join(taskErrMsg.entrySet().stream().map(n -> "[" + n.getKey() + ": "
-                    + n.getValue() + "]").collect(Collectors.toList())));
+            String taskProgressStr = taskProgress.entrySet().stream()
+                    .map(e -> "[" + e.getKey() + ": " + e.getValue().first + "/" + e.getValue().second + "]")
+                    .limit(100)
+                    .collect(Collectors.joining(", "));
+            String taskErrMsgStr = taskErrMsg.entrySet().stream()
+                    .map(n -> "[" + n.getKey() + ": " + n.getValue() + "]")
+                    .limit(100)
+                    .collect(Collectors.joining(", "));
+            info.add(taskProgressStr);
+            info.add(taskErrMsgStr);
         }
         info.add(status.toString());
         info.add(String.valueOf(timeoutMs / 1000));

From 913cda6a96351ac58e22e628d16100574c49b0b8 Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Thu, 28 Nov 2024 10:22:20 +0800
Subject: [PATCH 045/399] [feat](mtmv)Unified external table interface
 supporting partition refresh and partition pruning (#44673)

### What problem does this PR solve?

- Add `MvccTable` to represent a table that supports querying specified
version data
- Add the `MvccSnapshot` interface to store snapshot information of mvcc
at a certain moment in time
- Add the `MvccSnapshot` parameter to the method of the
`MTMVRelatedTableIf `interface to retrieve data of a specified version
- Partition pruning related methods combined with the `MvccSnapshot`
parameter are used to obtain partition information for a specified
version
- Load the snapshot information of mvccTable at the beginning of the
query plan and store it in StatementContext

### Release note

Unified external table interface supporting partition refresh and
partition pruning
---
 .../java/org/apache/doris/catalog/MTMV.java   |  7 +-
 .../org/apache/doris/catalog/OlapTable.java   | 25 +++++-
 .../doris/datasource/ExternalTable.java       | 18 ++--
 .../datasource/hive/HMSExternalTable.java     | 33 ++++----
 .../doris/datasource/mvcc/MvccSnapshot.java   | 25 ++++++
 .../doris/datasource/mvcc/MvccTable.java      | 33 ++++++++
 .../doris/datasource/mvcc/MvccTableInfo.java  | 84 +++++++++++++++++++
 .../paimon/PaimonExternalTable.java           | 14 ++--
 .../mtmv/MTMVPartitionExprDateTrunc.java      |  2 +-
 .../apache/doris/mtmv/MTMVPartitionInfo.java  |  3 +-
 .../apache/doris/mtmv/MTMVPartitionUtil.java  | 12 +--
 ...MTMVRelatedPartitionDescInitGenerator.java |  4 +-
 ...MVRelatedPartitionDescRollUpGenerator.java |  3 +-
 .../apache/doris/mtmv/MTMVRelatedTableIf.java | 27 +++---
 .../apache/doris/nereids/CascadesContext.java |  8 ++
 .../apache/doris/nereids/NereidsPlanner.java  |  2 +-
 .../doris/nereids/StatementContext.java       | 31 +++++++
 .../exploration/mv/MaterializedViewUtils.java |  4 +-
 .../rules/rewrite/PruneFileScanPartition.java |  6 +-
 .../plans/commands/info/CreateMTMVInfo.java   |  2 +-
 .../info/MTMVPartitionDefinition.java         |  3 +-
 .../trees/plans/logical/LogicalFileScan.java  |  3 +-
 .../doris/mtmv/MTMVPartitionUtilTest.java     |  6 +-
 23 files changed, 278 insertions(+), 77 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTable.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTableInfo.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index 825bdef9f09819..955bfd4279fd5c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -59,7 +59,6 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
-import java.util.OptionalLong;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -364,7 +363,7 @@ public MTMVRefreshSnapshot getRefreshSnapshot() {
      * @return mvPartitionName ==> mvPartitionKeyDesc
      */
     public Map<String, PartitionKeyDesc> generateMvPartitionDescs() throws AnalysisException {
-        Map<String, PartitionItem> mtmvItems = getAndCopyPartitionItems(OptionalLong.empty());
+        Map<String, PartitionItem> mtmvItems = getAndCopyPartitionItems();
         Map<String, PartitionKeyDesc> result = Maps.newHashMap();
         for (Entry<String, PartitionItem> entry : mtmvItems.entrySet()) {
             result.put(entry.getKey(), entry.getValue().toPartitionKeyDesc());
@@ -393,7 +392,7 @@ public Pair<Map<String, Set<String>>, Map<String, String>> calculateDoublyPartit
         Map<String, String> baseToMv = Maps.newHashMap();
         Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs = MTMVPartitionUtil
                 .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties);
-        Map<String, PartitionItem> mvPartitionItems = getAndCopyPartitionItems(OptionalLong.empty());
+        Map<String, PartitionItem> mvPartitionItems = getAndCopyPartitionItems();
         for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet()) {
             Set<String> basePartitionNames = relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
                     Sets.newHashSet());
@@ -426,7 +425,7 @@ public Map<String, Set<String>> calculatePartitionMappings() throws AnalysisExce
         Map<String, Set<String>> res = Maps.newHashMap();
         Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs = MTMVPartitionUtil
                 .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties);
-        Map<String, PartitionItem> mvPartitionItems = getAndCopyPartitionItems(OptionalLong.empty());
+        Map<String, PartitionItem> mvPartitionItems = getAndCopyPartitionItems();
         for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet()) {
             res.put(entry.getKey(),
                     relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Sets.newHashSet()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index ec3bd2acbc57d6..5d57540017f067 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -50,6 +50,7 @@
 import org.apache.doris.common.util.PropertyAnalyzer;
 import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
 import org.apache.doris.mtmv.MTMVRefreshContext;
 import org.apache.doris.mtmv.MTMVRelatedTableIf;
 import org.apache.doris.mtmv.MTMVSnapshotIf;
@@ -110,7 +111,6 @@
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.OptionalLong;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
@@ -1049,6 +1049,10 @@ public PartitionInfo getPartitionInfo() {
     }
 
     @Override
+    public Set<String> getPartitionColumnNames(Optional<MvccSnapshot> snapshot) throws DdlException {
+        return getPartitionColumnNames();
+    }
+
     public Set<String> getPartitionColumnNames() throws DdlException {
         Set<String> partitionColumnNames = Sets.newHashSet();
         if (partitionInfo instanceof SinglePartitionInfo) {
@@ -3251,12 +3255,21 @@ public long getVisibleVersionTime() {
     }
 
     @Override
+    public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
+        return getPartitionType();
+    }
+
     public PartitionType getPartitionType() {
         return partitionInfo.getType();
     }
 
     @Override
-    public Map<String, PartitionItem> getAndCopyPartitionItems(OptionalLong snapshotId) throws AnalysisException {
+    public Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot)
+            throws AnalysisException {
+        return getAndCopyPartitionItems();
+    }
+
+    public Map<String, PartitionItem> getAndCopyPartitionItems() throws AnalysisException {
         if (!tryReadLock(1, TimeUnit.MINUTES)) {
             throw new AnalysisException("get table read lock timeout, database=" + getDBName() + ",table=" + getName());
         }
@@ -3275,13 +3288,17 @@ public Map<String, PartitionItem> getAndCopyPartitionItems(OptionalLong snapshot
     }
 
     @Override
+    public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
+        return getPartitionColumns();
+    }
+
     public List<Column> getPartitionColumns() {
         return getPartitionInfo().getPartitionColumns();
     }
 
     @Override
     public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
-            OptionalLong snapshotId)
+            Optional<MvccSnapshot> snapshot)
             throws AnalysisException {
         Map<String, Long> partitionVersions = context.getBaseVersions().getPartitionVersions();
         long partitionId = getPartitionOrAnalysisException(partitionName).getId();
@@ -3291,7 +3308,7 @@ public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshCont
     }
 
     @Override
-    public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, OptionalLong snapshotId) {
+    public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot) {
         Map<Long, Long> tableVersions = context.getBaseVersions().getTableVersions();
         long visibleVersion = tableVersions.containsKey(id) ? tableVersions.get(id) : getVisibleVersion();
         return new MTMVVersionSnapshot(visibleVersion, id);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 3aee5550acf646..d82959954f2607 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -31,6 +31,7 @@
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.common.util.PropertyAnalyzer;
 import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
 import org.apache.doris.persist.gson.GsonPostProcessable;
 import org.apache.doris.persist.gson.GsonUtils;
@@ -55,7 +56,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.OptionalLong;
 import java.util.Set;
 
 /**
@@ -373,17 +373,17 @@ public TableIndexes getTableIndexes() {
     /**
      * Retrieve all partitions and initialize SelectedPartitions
      *
-     * @param snapshotId if not support mvcc, ignore this
+     * @param snapshot if not support mvcc, ignore this
      * @return
      */
-    public SelectedPartitions initSelectedPartitions(OptionalLong snapshotId) {
+    public SelectedPartitions initSelectedPartitions(Optional<MvccSnapshot> snapshot) {
         if (!supportPartitionPruned()) {
             return SelectedPartitions.NOT_PRUNED;
         }
-        if (CollectionUtils.isEmpty(this.getPartitionColumns(snapshotId))) {
+        if (CollectionUtils.isEmpty(this.getPartitionColumns(snapshot))) {
             return SelectedPartitions.NOT_PRUNED;
         }
-        Map<String, PartitionItem> nameToPartitionItems = getNameToPartitionItems(snapshotId);
+        Map<String, PartitionItem> nameToPartitionItems = getNameToPartitionItems(snapshot);
         return new SelectedPartitions(nameToPartitionItems.size(), nameToPartitionItems, false);
     }
 
@@ -391,10 +391,10 @@ public SelectedPartitions initSelectedPartitions(OptionalLong snapshotId) {
      * get partition map
      * If partition related operations are supported, this method needs to be implemented in the subclass
      *
-     * @param snapshotId if not support mvcc, ignore this
+     * @param snapshot if not support mvcc, ignore this
      * @return partitionName ==> PartitionItem
      */
-    public Map<String, PartitionItem> getNameToPartitionItems(OptionalLong snapshotId) {
+    public Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
         return Collections.emptyMap();
     }
 
@@ -402,10 +402,10 @@ public Map<String, PartitionItem> getNameToPartitionItems(OptionalLong snapshotI
      * get partition column list
      * If partition related operations are supported, this method needs to be implemented in the subclass
      *
-     * @param snapshotId if not support mvcc, ignore this
+     * @param snapshot if not support mvcc, ignore this
      * @return
      */
-    public List<Column> getPartitionColumns(OptionalLong snapshotId) {
+    public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
         return Collections.emptyList();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 1c30fa24cfb51e..6d65f8bcdbccb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -33,6 +33,7 @@
 import org.apache.doris.datasource.SchemaCacheValue;
 import org.apache.doris.datasource.hudi.HudiUtils;
 import org.apache.doris.datasource.iceberg.IcebergUtils;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
 import org.apache.doris.mtmv.MTMVBaseTableIf;
 import org.apache.doris.mtmv.MTMVMaxTimestampSnapshot;
 import org.apache.doris.mtmv.MTMVRefreshContext;
@@ -83,7 +84,6 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
-import java.util.OptionalLong;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -288,7 +288,6 @@ public List<Type> getPartitionColumnTypes() {
                 .orElse(Collections.emptyList());
     }
 
-    @Override
     public List<Column> getPartitionColumns() {
         makeSureInitialized();
         Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
@@ -297,7 +296,7 @@ public List<Column> getPartitionColumns() {
     }
 
     @Override
-    public List<Column> getPartitionColumns(OptionalLong snapshotId) {
+    public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
         return getPartitionColumns();
     }
 
@@ -307,7 +306,7 @@ public boolean supportPartitionPruned() {
     }
 
     @Override
-    public Map<String, PartitionItem> getNameToPartitionItems(OptionalLong snapshotId) {
+    public Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
         return getNameToPartitionItems();
     }
 
@@ -756,34 +755,32 @@ public Set<String> getDistributionColumnNames() {
     }
 
     @Override
+    public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
+        return getPartitionType();
+    }
+
     public PartitionType getPartitionType() {
         return getPartitionColumns().size() > 0 ? PartitionType.LIST : PartitionType.UNPARTITIONED;
     }
 
     @Override
+    public Set<String> getPartitionColumnNames(Optional<MvccSnapshot> snapshot) {
+        return getPartitionColumnNames();
+    }
+
     public Set<String> getPartitionColumnNames() {
         return getPartitionColumns().stream()
                 .map(c -> c.getName().toLowerCase()).collect(Collectors.toSet());
     }
 
     @Override
-    public Map<String, PartitionItem> getAndCopyPartitionItems(OptionalLong snapshotId) {
-        HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
-                .getMetaStoreCache((HMSExternalCatalog) getCatalog());
-        HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues(
-                getDbName(), getName(), getPartitionColumnTypes());
-        Map<String, PartitionItem> res = Maps.newHashMap();
-        Map<Long, PartitionItem> idToPartitionItem = hivePartitionValues.getIdToPartitionItem();
-        BiMap<Long, String> idToName = hivePartitionValues.getPartitionNameToIdMap().inverse();
-        for (Entry<Long, PartitionItem> entry : idToPartitionItem.entrySet()) {
-            res.put(idToName.get(entry.getKey()), entry.getValue());
-        }
-        return res;
+    public Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot) {
+        return getNameToPartitionItems();
     }
 
     @Override
     public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
-            OptionalLong snapshotId) throws AnalysisException {
+            Optional<MvccSnapshot> snapshot) throws AnalysisException {
         HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
                 .getMetaStoreCache((HMSExternalCatalog) getCatalog());
         HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues(
@@ -795,7 +792,7 @@ public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshCont
     }
 
     @Override
-    public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, OptionalLong snapshotId)
+    public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
             throws AnalysisException {
         if (getPartitionType() == PartitionType.UNPARTITIONED) {
             return new MTMVMaxTimestampSnapshot(getName(), getLastDdlTime());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java
new file mode 100644
index 00000000000000..d7826b0a5de19e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.mvcc;
+
+/**
+ * The snapshot information of mvcc is defined by each table,
+ * but it should be ensured that the table information queried through this snapshot remains unchanged
+ */
+public interface MvccSnapshot {
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTable.java
new file mode 100644
index 00000000000000..d69e0f3114df0c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTable.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.mvcc;
+
+import org.apache.doris.catalog.TableIf;
+
+/**
+ * The table that needs to query data based on the version needs to implement this interface.
+ */
+public interface MvccTable extends TableIf {
+    /**
+     * Retrieve the current snapshot information of the table,
+     * and the returned result will be used for the entire process of this query
+     *
+     * @return MvccSnapshot
+     */
+    MvccSnapshot loadSnapshot();
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTableInfo.java
new file mode 100644
index 00000000000000..0d865f837c8c4e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTableInfo.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.mvcc;
+
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.datasource.CatalogIf;
+
+import com.google.common.base.Objects;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class MvccTableInfo {
+    private static final Logger LOG = LogManager.getLogger(MvccTableInfo.class);
+
+    private String tableName;
+    private String dbName;
+    private String ctlName;
+
+    public MvccTableInfo(TableIf table) {
+        java.util.Objects.requireNonNull(table, "table is null");
+        DatabaseIf database = table.getDatabase();
+        java.util.Objects.requireNonNull(database, "database is null");
+        CatalogIf catalog = database.getCatalog();
+        java.util.Objects.requireNonNull(database, "catalog is null");
+        this.tableName = table.getName();
+        this.dbName = database.getFullName();
+        this.ctlName = catalog.getName();
+    }
+
+    public String getTableName() {
+        return tableName;
+    }
+
+    public String getDbName() {
+        return dbName;
+    }
+
+    public String getCtlName() {
+        return ctlName;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        MvccTableInfo that = (MvccTableInfo) o;
+        return Objects.equal(tableName, that.tableName) && Objects.equal(
+                dbName, that.dbName) && Objects.equal(ctlName, that.ctlName);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(tableName, dbName, ctlName);
+    }
+
+    @Override
+    public String toString() {
+        return "MvccTableInfo{"
+                + "tableName='" + tableName + '\''
+                + ", dbName='" + dbName + '\''
+                + ", ctlName='" + ctlName + '\''
+                + '}';
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
index 632a0da0ebd316..7fe3c858448e3f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
@@ -28,6 +28,7 @@
 import org.apache.doris.common.DdlException;
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.SchemaCacheValue;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
 import org.apache.doris.mtmv.MTMVBaseTableIf;
 import org.apache.doris.mtmv.MTMVRefreshContext;
 import org.apache.doris.mtmv.MTMVRelatedTableIf;
@@ -67,7 +68,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.OptionalLong;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -313,29 +313,29 @@ public void beforeMTMVRefresh(MTMV mtmv) throws DdlException {
     }
 
     @Override
-    public Map<String, PartitionItem> getAndCopyPartitionItems(OptionalLong snapshotId) {
+    public Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot) {
         return Maps.newHashMap(getPartitionInfoFromCache().getNameToPartitionItem());
     }
 
     @Override
-    public PartitionType getPartitionType() {
+    public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
         return getPartitionColumnsFromCache().size() > 0 ? PartitionType.LIST : PartitionType.UNPARTITIONED;
     }
 
     @Override
-    public Set<String> getPartitionColumnNames() {
+    public Set<String> getPartitionColumnNames(Optional<MvccSnapshot> snapshot) {
         return getPartitionColumnsFromCache().stream()
                 .map(c -> c.getName().toLowerCase()).collect(Collectors.toSet());
     }
 
     @Override
-    public List<Column> getPartitionColumns() {
+    public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
         return getPartitionColumnsFromCache();
     }
 
     @Override
     public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
-            OptionalLong snapshotId)
+            Optional<MvccSnapshot> snapshot)
             throws AnalysisException {
         PaimonPartition paimonPartition = getPartitionInfoFromCache().getNameToPartition().get(partitionName);
         if (paimonPartition == null) {
@@ -345,7 +345,7 @@ public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshCont
     }
 
     @Override
-    public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, OptionalLong snapshotId)
+    public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
             throws AnalysisException {
         return new MTMVVersionSnapshot(getLatestSnapshotIdFromCache());
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
index ea15c84d1b925d..95a8717e01c4c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
@@ -69,7 +69,7 @@ public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException
                     String.format("timeUnit not support: %s, only support: %s", this.timeUnit, timeUnits));
         }
         MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable();
-        PartitionType partitionType = relatedTable.getPartitionType();
+        PartitionType partitionType = relatedTable.getPartitionType(Optional.empty());
         if (partitionType == PartitionType.RANGE) {
             Type partitionColumnType = MTMVPartitionUtil
                     .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
index b3cd239269abc7..7eae44db0af4cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
@@ -25,6 +25,7 @@
 import com.google.gson.annotations.SerializedName;
 
 import java.util.List;
+import java.util.Optional;
 
 /**
  * MTMVPartitionInfo
@@ -115,7 +116,7 @@ public int getRelatedColPos() throws AnalysisException {
         if (partitionType == MTMVPartitionType.SELF_MANAGE) {
             throw new AnalysisException("partitionType is: " + partitionType);
         }
-        List<Column> partitionColumns = getRelatedTable().getPartitionColumns();
+        List<Column> partitionColumns = getRelatedTable().getPartitionColumns(Optional.empty());
         for (int i = 0; i < partitionColumns.size(); i++) {
             if (partitionColumns.get(i).getName().equalsIgnoreCase(relatedCol)) {
                 return i;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 1bbc51fb004c57..8ba022de415006 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -50,7 +50,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
-import java.util.OptionalLong;
+import java.util.Optional;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -330,7 +330,7 @@ public static boolean isSyncWithPartitions(MTMVRefreshContext context, String mt
         }
         for (String relatedPartitionName : relatedPartitionNames) {
             MTMVSnapshotIf relatedPartitionCurrentSnapshot = relatedTable
-                    .getPartitionSnapshot(relatedPartitionName, context, OptionalLong.empty());
+                    .getPartitionSnapshot(relatedPartitionName, context, Optional.empty());
             if (!mtmv.getRefreshSnapshot()
                     .equalsWithRelatedPartition(mtmvPartitionName, relatedPartitionName,
                             relatedPartitionCurrentSnapshot)) {
@@ -447,7 +447,7 @@ private static boolean isSyncWithBaseTable(MTMVRefreshContext context, String mt
         if (!baseTable.needAutoRefresh()) {
             return true;
         }
-        MTMVSnapshotIf baseTableCurrentSnapshot = baseTable.getTableSnapshot(context, OptionalLong.empty());
+        MTMVSnapshotIf baseTableCurrentSnapshot = baseTable.getTableSnapshot(context, Optional.empty());
         return mtmv.getRefreshSnapshot()
                 .equalsWithBaseTable(mtmvPartitionName, new BaseTableInfo(baseTable), baseTableCurrentSnapshot);
     }
@@ -483,7 +483,7 @@ private static MTMVRefreshPartitionSnapshot generatePartitionSnapshot(MTMVRefres
             MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable();
             for (String relatedPartitionName : relatedPartitionNames) {
                 MTMVSnapshotIf partitionSnapshot = relatedTable
-                        .getPartitionSnapshot(relatedPartitionName, context, OptionalLong.empty());
+                        .getPartitionSnapshot(relatedPartitionName, context, Optional.empty());
                 refreshPartitionSnapshot.getPartitions()
                         .put(relatedPartitionName, partitionSnapshot);
             }
@@ -498,13 +498,13 @@ private static MTMVRefreshPartitionSnapshot generatePartitionSnapshot(MTMVRefres
                 continue;
             }
             refreshPartitionSnapshot.addTableSnapshot(baseTableInfo,
-                    ((MTMVRelatedTableIf) table).getTableSnapshot(context, OptionalLong.empty()));
+                    ((MTMVRelatedTableIf) table).getTableSnapshot(context, Optional.empty()));
         }
         return refreshPartitionSnapshot;
     }
 
     public static Type getPartitionColumnType(MTMVRelatedTableIf relatedTable, String col) throws AnalysisException {
-        List<Column> partitionColumns = relatedTable.getPartitionColumns();
+        List<Column> partitionColumns = relatedTable.getPartitionColumns(Optional.empty());
         for (Column column : partitionColumns) {
             if (column.getName().equals(col)) {
                 return column.getType();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
index ef3100dec4c732..c6b4e331184e2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
@@ -20,7 +20,7 @@
 import org.apache.doris.common.AnalysisException;
 
 import java.util.Map;
-import java.util.OptionalLong;
+import java.util.Optional;
 
 /**
  * get all related partition descs
@@ -30,6 +30,6 @@ public class MTMVRelatedPartitionDescInitGenerator implements MTMVRelatedPartiti
     @Override
     public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String> mvProperties,
             RelatedPartitionDescResult lastResult) throws AnalysisException {
-        lastResult.setItems(mvPartitionInfo.getRelatedTable().getAndCopyPartitionItems(OptionalLong.empty()));
+        lastResult.setItems(mvPartitionInfo.getRelatedTable().getAndCopyPartitionItems(Optional.empty()));
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
index 76e20ef70f5d92..325fab819d9a09 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
@@ -31,6 +31,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -45,7 +46,7 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String> mvPrope
             return;
         }
         MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable();
-        PartitionType partitionType = relatedTable.getPartitionType();
+        PartitionType partitionType = relatedTable.getPartitionType(Optional.empty());
         if (partitionType == PartitionType.RANGE) {
             lastResult.setDescs(rollUpRange(lastResult.getDescs(), mvPartitionInfo));
         } else if (partitionType == PartitionType.LIST) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
index e18784ae253a0f..c4261aa78f10be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
@@ -23,10 +23,11 @@
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
 
 import java.util.List;
 import java.util.Map;
-import java.util.OptionalLong;
+import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -39,32 +40,35 @@ public interface MTMVRelatedTableIf extends TableIf {
      * Note: This method is called every time there is a refresh and transparent rewrite,
      * so if this method is slow, it will significantly reduce query performance
      *
-     * @param snapshotId
+     * @param snapshot
      * @return partitionName->PartitionItem
      */
-    Map<String, PartitionItem> getAndCopyPartitionItems(OptionalLong snapshotId) throws AnalysisException;
+    Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot) throws AnalysisException;
 
     /**
      * getPartitionType LIST/RANGE/UNPARTITIONED
      *
+     * @param snapshot
      * @return
      */
-    PartitionType getPartitionType();
+    PartitionType getPartitionType(Optional<MvccSnapshot> snapshot);
 
     /**
      * getPartitionColumnNames
      *
+     * @param snapshot
      * @return
      * @throws DdlException
      */
-    Set<String> getPartitionColumnNames() throws DdlException;
+    Set<String> getPartitionColumnNames(Optional<MvccSnapshot> snapshot) throws DdlException;
 
     /**
      * getPartitionColumns
      *
+     * @param snapshot
      * @return
      */
-    List<Column> getPartitionColumns();
+    List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot);
 
     /**
      * getPartitionSnapshot
@@ -72,14 +76,14 @@ public interface MTMVRelatedTableIf extends TableIf {
      * If snapshots have already been obtained in bulk in the context,
      * the results should be obtained directly from the context
      *
-     * @param snapshotId
+     * @param snapshot
      * @param partitionName
      * @param context
      * @return partition snapshot at current time
      * @throws AnalysisException
      */
-    MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context, OptionalLong snapshotId)
-            throws AnalysisException;
+    MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
+            Optional<MvccSnapshot> snapshot) throws AnalysisException;
 
     /**
      * getTableSnapshot
@@ -87,12 +91,13 @@ MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext con
      * If snapshots have already been obtained in bulk in the context,
      * the results should be obtained directly from the context
      *
-     * @param snapshotId
+     * @param snapshot
      * @param context
      * @return table snapshot at current time
      * @throws AnalysisException
      */
-    MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, OptionalLong snapshotId) throws AnalysisException;
+    MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
+            throws AnalysisException;
 
     /**
      * Does the current type of table allow timed triggering
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index a5c966370f030d..17ae5883063fb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -70,6 +70,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
+import org.apache.commons.collections.MapUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -462,6 +463,13 @@ private Set<List<String>> getTables(LogicalPlan logicalPlan) {
         return tableNames;
     }
 
+    public Map<List<String>, TableIf> getOrExtractTables(LogicalPlan logicalPlan) {
+        if (MapUtils.isEmpty(tables)) {
+            extractTables(logicalPlan);
+        }
+        return tables;
+    }
+
     private Set<List<String>> extractTableNamesFromHaving(LogicalHaving<?> having) {
         Set<SubqueryExpr> subqueryExprs = having.getPredicate()
                 .collect(SubqueryExpr.class::isInstance);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 8fb0f6e77976da..c7478411a5de11 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -216,7 +216,7 @@ public Plan planWithLock(LogicalPlan plan, PhysicalProperties requireProperties,
             plan = preprocess(plan);
 
             initCascadesContext(plan, requireProperties);
-
+            statementContext.loadSnapshots(cascadesContext.getOrExtractTables(plan));
             try (Lock lock = new Lock(plan, cascadesContext)) {
                 Plan resultPlan = planWithoutLock(plan, explainLevel, showPlanProcess, requireProperties);
                 lockCallback.accept(resultPlan);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 6b6e335b74a676..b172f9dc591bd9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -24,6 +24,9 @@
 import org.apache.doris.common.Id;
 import org.apache.doris.common.IdGenerator;
 import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
+import org.apache.doris.datasource.mvcc.MvccTable;
+import org.apache.doris.datasource.mvcc.MvccTableInfo;
 import org.apache.doris.nereids.hint.Hint;
 import org.apache.doris.nereids.memo.Group;
 import org.apache.doris.nereids.rules.analysis.ColumnAliasGenerator;
@@ -174,6 +177,8 @@ public class StatementContext implements Closeable {
 
     private Backend groupCommitMergeBackend;
 
+    private final Map<MvccTableInfo, MvccSnapshot> snapshots = Maps.newHashMap();
+
     private boolean privChecked;
 
     public StatementContext() {
@@ -510,6 +515,32 @@ public void addPlannerHook(PlannerHook plannerHook) {
         this.plannerHooks.add(plannerHook);
     }
 
+    /**
+     * Load snapshot information of mvcc
+     *
+     * @param tables Tables used in queries
+     */
+    public void loadSnapshots(Map<List<String>, TableIf> tables) {
+        if (tables == null) {
+            return;
+        }
+        for (TableIf tableIf : tables.values()) {
+            if (tableIf instanceof MvccTable) {
+                snapshots.put(new MvccTableInfo(tableIf), ((MvccTable) tableIf).loadSnapshot());
+            }
+        }
+    }
+
+    /**
+     * Obtain snapshot information of mvcc
+     *
+     * @param mvccTable mvccTable
+     * @return MvccSnapshot
+     */
+    public MvccSnapshot getSnapshot(MvccTable mvccTable) {
+        return snapshots.get(new MvccTableInfo(mvccTable));
+    }
+
     private static class CloseableResource implements Closeable {
         public final String resourceName;
         public final String threadName;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index a659c2f9990a3f..484abd11f01e72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -481,13 +481,13 @@ public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerConte
                 return null;
             }
             MTMVRelatedTableIf relatedTable = (MTMVRelatedTableIf) table;
-            PartitionType type = relatedTable.getPartitionType();
+            PartitionType type = relatedTable.getPartitionType(Optional.empty());
             if (PartitionType.UNPARTITIONED.equals(type)) {
                 context.addFailReason(String.format("related base table is not partition table, the table is %s",
                         table.getName()));
                 return null;
             }
-            Set<Column> partitionColumnSet = new HashSet<>(relatedTable.getPartitionColumns());
+            Set<Column> partitionColumnSet = new HashSet<>(relatedTable.getPartitionColumns(Optional.empty()));
             Column mvReferenceColumn = contextPartitionColumn.getColumn().get();
             Expr definExpr = mvReferenceColumn.getDefineExpr();
             if (definExpr instanceof SlotRef) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
index d50219383072df..4bbb0a8aa76270 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
@@ -36,7 +36,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.OptionalLong;
+import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -75,7 +75,7 @@ private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable,
             LogicalFilter<LogicalFileScan> filter, LogicalFileScan scan, CascadesContext ctx) {
         Map<String, PartitionItem> selectedPartitionItems = Maps.newHashMap();
         // todo: real snapshotId
-        if (CollectionUtils.isEmpty(externalTable.getPartitionColumns(OptionalLong.empty()))) {
+        if (CollectionUtils.isEmpty(externalTable.getPartitionColumns(Optional.empty()))) {
             // non partitioned table, return NOT_PRUNED.
             // non partition table will be handled in HiveScanNode.
             return SelectedPartitions.NOT_PRUNED;
@@ -84,7 +84,7 @@ private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable,
                 .stream()
                 .collect(Collectors.toMap(slot -> slot.getName().toLowerCase(), Function.identity()));
         // todo: real snapshotId
-        List<Slot> partitionSlots = externalTable.getPartitionColumns(OptionalLong.empty())
+        List<Slot> partitionSlots = externalTable.getPartitionColumns(Optional.empty())
                 .stream()
                 .map(column -> scanOutput.get(column.getName().toLowerCase()))
                 .collect(Collectors.toList());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index de5e188d5a65bf..8c44b42a5ccfbd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -351,7 +351,7 @@ private PartitionDesc generatePartitionDesc(ConnectContext ctx) {
                     allPartitionDescs.size(), ctx.getSessionVariable().getCreateTablePartitionMaxNum()));
         }
         try {
-            PartitionType type = relatedTable.getPartitionType();
+            PartitionType type = relatedTable.getPartitionType(Optional.empty());
             if (type == PartitionType.RANGE) {
                 return new RangePartitionDesc(Lists.newArrayList(mvPartitionInfo.getPartitionCol()),
                         allPartitionDescs);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
index 427e2368e7ab2b..c4117e8608e29d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
@@ -54,6 +54,7 @@
 import com.google.common.collect.Sets;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -147,7 +148,7 @@ private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectCont
             MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo());
             Set<String> partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
             try {
-                partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames());
+                partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames(Optional.empty()));
             } catch (DdlException e) {
                 throw new AnalysisException(e.getMessage(), e);
             }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index 010c30d915d529..96b8e032d11274 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -36,7 +36,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.OptionalLong;
 
 /**
  * Logical file scan for external catalog.
@@ -64,7 +63,7 @@ public LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifie
                            Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
         // todo: real snapshotId
         this(id, table, qualifier, Optional.empty(), Optional.empty(),
-                table.initSelectedPartitions(OptionalLong.empty()), tableSample, tableSnapshot);
+                table.initSelectedPartitions(Optional.empty()), tableSample, tableSnapshot);
     }
 
     public SelectedPartitions getSelectedPartitions() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
index 96ac59b81216bc..e5d2e21a8db626 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
@@ -35,7 +35,7 @@
 import org.junit.Test;
 
 import java.util.List;
-import java.util.OptionalLong;
+import java.util.Optional;
 import java.util.Set;
 
 public class MTMVPartitionUtilTest {
@@ -113,7 +113,7 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc
                 minTimes = 0;
                 result = true;
 
-                baseOlapTable.getTableSnapshot((MTMVRefreshContext) any, (OptionalLong) any);
+                baseOlapTable.getTableSnapshot((MTMVRefreshContext) any, (Optional) any);
                 minTimes = 0;
                 result = baseSnapshotIf;
 
@@ -133,7 +133,7 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc
                 minTimes = 0;
                 result = true;
 
-                baseOlapTable.getPartitionSnapshot(anyString, (MTMVRefreshContext) any, (OptionalLong) any);
+                baseOlapTable.getPartitionSnapshot(anyString, (MTMVRefreshContext) any, (Optional) any);
                 minTimes = 0;
                 result = baseSnapshotIf;
 

From 016d389e14c5d98b2c689373fa68cb5434bb09bb Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Thu, 28 Nov 2024 10:25:41 +0800
Subject: [PATCH 046/399] [fix](ip-search)fix ip search with inverted index
 (#44580)

this pr is to solve is_ip_address_in_range with invalid param
---
 be/src/vec/functions/function_ip.h                       | 5 +++++
 .../test_ip_cidr_search_with_inverted_index.out          | 4 ----
 .../test_ip_cidr_search_with_inverted_index.groovy       | 9 +++++++--
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/be/src/vec/functions/function_ip.h b/be/src/vec/functions/function_ip.h
index b90d1b2bcf9434..724121ce57c109 100644
--- a/be/src/vec/functions/function_ip.h
+++ b/be/src/vec/functions/function_ip.h
@@ -694,6 +694,11 @@ class FunctionIsIPAddressInRange : public IFunction {
                             reinterpret_cast<char*>(&cidr_range_ipv6_data[1]), cidr._prefix);
             min_ip = cidr_range_ipv6_data[0];
             max_ip = cidr_range_ipv6_data[1];
+        } else {
+            // if here param is invalid for current column to calcute min_ip|max_ip we just return
+            return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>(
+                    "Inverted index evaluate skipped, data type " + arg_type->get_name() +
+                    " can not support this cidr " + arg_column->get_data_at(0).to_string());
         }
         // apply for inverted index
         std::shared_ptr<roaring::Roaring> res_roaring = std::make_shared<roaring::Roaring>();
diff --git a/regression-test/data/inverted_index_p0/test_ip_cidr_search_with_inverted_index.out b/regression-test/data/inverted_index_p0/test_ip_cidr_search_with_inverted_index.out
index f365449b9e5244..2211a11b95a697 100644
--- a/regression-test/data/inverted_index_p0/test_ip_cidr_search_with_inverted_index.out
+++ b/regression-test/data/inverted_index_p0/test_ip_cidr_search_with_inverted_index.out
@@ -50,13 +50,9 @@
 
 -- !sql --
 
--- !sql --
-
 -- !sql --
 2	42.117.228.166	2001:16a0:2:200a::2
 
--- !sql --
-
 -- !sql --
 5	255.255.255.255	\N
 
diff --git a/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy
index 61751f2404f2d2..191c818f351fde 100644
--- a/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy
+++ b/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy
@@ -118,8 +118,13 @@ suite("test_ip_cidr_search_with_inverted_index", "nonConcurrent"){
     def execute_sql = { sqlList ->
         def i = 0
         for (sqlStr in sqlList) {
+            if (i == 0 || i == 4 ) {
+                    // inverted index will skip and do eval expr
+		++i;
+                continue;
+            }
             try {
-                log.info("execute sql: i")
+		log.info("execute sql: ${i} : ${sqlStr}")
                 GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [req_id: i])
                 order_qt_sql """ ${sqlStr} """
             } finally {
@@ -131,4 +136,4 @@ suite("test_ip_cidr_search_with_inverted_index", "nonConcurrent"){
 
     execute_sql.call(create_sql.call())
 
-}
\ No newline at end of file
+}

From 0f90d46ec8b064df54be986c24860fdc2e912671 Mon Sep 17 00:00:00 2001
From: qiye <luen@selectdb.com>
Date: Thu, 28 Nov 2024 10:32:50 +0800
Subject: [PATCH 047/399] [fix](compaction)Fix cluster key column duplicated in
 value columns group (#44610)

Issue Number: close #44609

Related pr: #40372

Problem Summary:

`key_columns` will be used after move.
This will cause skip in construction of `value_columns` useless.
cluster key  column will be duplicated added in value column group.
---
 be/src/olap/merger.cpp                        |   2 +-
 .../test_index_compaction_unique_keys_arr.out |  68 +++++++
 ...st_index_compaction_unique_keys_arr.groovy | 189 ++++++++++--------
 3 files changed, 178 insertions(+), 81 deletions(-)

diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index 694b7d6db84d8e..b207cc4c5ad22b 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -201,7 +201,7 @@ void Merger::vertical_split_columns(const TabletSchema& tablet_schema,
                 << ", delete_sign_idx=" << delete_sign_idx;
     // for duplicate no keys
     if (!key_columns.empty()) {
-        column_groups->emplace_back(std::move(key_columns));
+        column_groups->emplace_back(key_columns);
     }
 
     std::vector<uint32_t> value_columns;
diff --git a/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out b/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out
index 3edad07916c9b5..a401117495f32d 100644
--- a/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out
+++ b/regression-test/data/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.out
@@ -67,3 +67,71 @@
 2	bason	bason hate pear	99
 3	bason	bason hate pear	99
 
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
diff --git a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
index 013d5a558111e7..e07618be6c1370 100644
--- a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
+++ b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
@@ -112,69 +112,22 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index")
         }
     }
 
-    boolean invertedIndexCompactionEnable = false
-    boolean has_update_be_config = false
-    try {
-        String backend_id;
-        backend_id = backendId_to_backendIP.keySet()[0]
-        def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
-        
-        logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
-        assertEquals(code, 0)
-        def configList = parseJson(out.trim())
-        assert configList instanceof List
+    def run_test = { table_name -> 
 
-        for (Object ele in (List) configList) {
-            assert ele instanceof List<String>
-            if (((List<String>) ele)[0] == "inverted_index_compaction_enable") {
-                invertedIndexCompactionEnable = Boolean.parseBoolean(((List<String>) ele)[2])
-                logger.info("inverted_index_compaction_enable: ${((List<String>) ele)[2]}")
-            }
-            if (((List<String>) ele)[0] == "disable_auto_compaction") {
-                disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
-                logger.info("disable_auto_compaction: ${((List<String>) ele)[2]}")
-            }
-        }
-        set_be_config.call("inverted_index_compaction_enable", "true")
-        has_update_be_config = true
-        // check updated config
-        check_config.call("inverted_index_compaction_enable", "true");
+        sql """ INSERT INTO ${table_name} VALUES (1, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${table_name} VALUES (1, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${table_name} VALUES (2, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${table_name} VALUES (2, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${table_name} VALUES (3, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${table_name} VALUES (3, "bason", "bason hate pear", 99); """
 
-        sql """ DROP TABLE IF EXISTS ${tableName}; """
-        sql """
-            CREATE TABLE ${tableName} (
-                `id` int(11) NULL,
-                `name` varchar(255) NULL,
-                `hobbies` text NULL,
-                `score` int(11) NULL,
-                index index_name (name) using inverted,
-                index index_hobbies (hobbies) using inverted properties("parser"="english"),
-                index index_score (score) using inverted
-            ) ENGINE=OLAP
-            UNIQUE KEY(`id`)
-            COMMENT 'OLAP'
-            DISTRIBUTED BY HASH(`id`) BUCKETS 1
-            PROPERTIES ( 
-                "replication_num" = "1",
-                "disable_auto_compaction" = "true",
-                "enable_unique_key_merge_on_write" = "true"
-            );
-        """
-
-        sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """
-        sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """
-        sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple", 100); """
-        sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate pear", 99); """
-        sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple", 100); """
-        sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """
-
-        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where score < 100 order by id, name, hobbies, score """
 
         //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
-        def tablets = sql_return_maparray """ show tablets from ${tableName}; """
+        def tablets = sql_return_maparray """ show tablets from ${table_name}; """
         def dedup_tablets = deduplicate_tablets(tablets)
 
         // In the p0 testing environment, there are no expected operations such as scaling down BE (backend) services
@@ -189,7 +142,7 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index")
         int rowsetCount = get_rowset_count.call(tablets);
         assert (rowsetCount == 7 * replicaNum)
 
-        // trigger full compactions for all tablets in ${tableName}
+        // trigger full compactions for all tablets in ${table_name}
         trigger_full_compaction_on_tablets.call(tablets)
 
         // wait for full compaction done
@@ -203,23 +156,23 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index")
             assert (rowsetCount == 1 * replicaNum)
         }
 
-        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where score < 100 order by id, name, hobbies, score """
 
         // insert more data and trigger full compaction again
-        sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """
-        sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """
-        sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple", 100); """
-        sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate pear", 99); """
-        sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple", 100); """
-        sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """
-
-        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+        sql """ INSERT INTO ${table_name} VALUES (1, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${table_name} VALUES (1, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${table_name} VALUES (2, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${table_name} VALUES (2, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${table_name} VALUES (3, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${table_name} VALUES (3, "bason", "bason hate pear", 99); """
+
+        qt_sql """ select * from ${table_name} order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where score < 100 order by id, name, hobbies, score """
 
         rowsetCount = get_rowset_count.call(tablets);
         if (isCloudMode) {
@@ -228,7 +181,7 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index")
             assert (rowsetCount == 7 * replicaNum)
         }
 
-        // trigger full compactions for all tablets in ${tableName}
+        // trigger full compactions for all tablets in ${table_name}
         trigger_full_compaction_on_tablets.call(tablets)
 
         // wait for full compaction done
@@ -242,10 +195,86 @@ suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index")
             assert (rowsetCount == 1 * replicaNum)
         }
 
-        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
-        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${table_name} where score < 100 order by id, name, hobbies, score """
+    }
+
+    boolean invertedIndexCompactionEnable = false
+    boolean has_update_be_config = false
+    try {
+        String backend_id;
+        backend_id = backendId_to_backendIP.keySet()[0]
+        def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+        
+        logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+        assertEquals(code, 0)
+        def configList = parseJson(out.trim())
+        assert configList instanceof List
+
+        for (Object ele in (List) configList) {
+            assert ele instanceof List<String>
+            if (((List<String>) ele)[0] == "inverted_index_compaction_enable") {
+                invertedIndexCompactionEnable = Boolean.parseBoolean(((List<String>) ele)[2])
+                logger.info("inverted_index_compaction_enable: ${((List<String>) ele)[2]}")
+            }
+            if (((List<String>) ele)[0] == "disable_auto_compaction") {
+                disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
+                logger.info("disable_auto_compaction: ${((List<String>) ele)[2]}")
+            }
+        }
+        set_be_config.call("inverted_index_compaction_enable", "true")
+        has_update_be_config = true
+        // check updated config
+        check_config.call("inverted_index_compaction_enable", "true");
+
+        sql """ DROP TABLE IF EXISTS ${tableName}; """
+        sql """
+            CREATE TABLE ${tableName} (
+                `id` int(11) NULL,
+                `name` varchar(255) NULL,
+                `hobbies` text NULL,
+                `score` int(11) NULL,
+                index index_name (name) using inverted,
+                index index_hobbies (hobbies) using inverted properties("parser"="english"),
+                index index_score (score) using inverted
+            ) ENGINE=OLAP
+            UNIQUE KEY(`id`)
+            COMMENT 'OLAP'
+            DISTRIBUTED BY HASH(`id`) BUCKETS 1
+            PROPERTIES ( 
+                "replication_num" = "1",
+                "disable_auto_compaction" = "true",
+                "enable_unique_key_merge_on_write" = "true"
+            );
+        """
+
+        run_test.call(tableName)
+
+        tableName = "test_index_compaction_unique_keys_arr_cluster_key"
+        sql """ DROP TABLE IF EXISTS ${tableName}; """
+        sql """
+            CREATE TABLE ${tableName} (
+                `id` int(11) NULL,
+                `name` varchar(255) NULL,
+                `hobbies` text NULL,
+                `score` int(11) NULL,
+                index index_name (name) using inverted,
+                index index_hobbies (hobbies) using inverted properties("parser"="english"),
+                index index_score (score) using inverted
+            ) ENGINE=OLAP
+            UNIQUE KEY(`id`)
+            CLUSTER BY (`score`)
+            COMMENT 'OLAP'
+            DISTRIBUTED BY HASH(`id`) BUCKETS 1
+            PROPERTIES ( 
+                "replication_num" = "1",
+                "disable_auto_compaction" = "true",
+                "enable_unique_key_merge_on_write" = "true"
+            );
+        """
+        run_test.call(tableName)
 
     } finally {
         if (has_update_be_config) {

From cf0b4c95af89bd4ffadc53c09aff2e9e78a3e542 Mon Sep 17 00:00:00 2001
From: morrySnow <zhangwenxin@selectdb.com>
Date: Thu, 28 Nov 2024 10:42:55 +0800
Subject: [PATCH 048/399] [fix](planner) FunctionCallExpr clone should be deep
 copy (#44678)

---
 .../java/org/apache/doris/analysis/FunctionCallExpr.java     | 2 +-
 .../main/java/org/apache/doris/analysis/FunctionName.java    | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 53bb2ba95acb25..51e18d750f263f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -432,7 +432,7 @@ public FunctionCallExpr(FunctionCallExpr e, FunctionParams params) {
 
     protected FunctionCallExpr(FunctionCallExpr other) {
         super(other);
-        fnName = other.fnName;
+        fnName = other.fnName != null ? other.fnName.clone() : null;
         orderByElements = other.orderByElements;
         isAnalyticFnCall = other.isAnalyticFnCall;
         // aggOp = other.aggOp;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionName.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionName.java
index b6feddb7d256ed..d0e695f6660a43 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionName.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionName.java
@@ -183,4 +183,9 @@ public static FunctionName read(DataInput in) throws IOException {
     public int hashCode() {
         return 31 * Objects.hashCode(db) + Objects.hashCode(fn);
     }
+
+    @Override
+    public FunctionName clone() {
+        return new FunctionName(db, fn);
+    }
 }

From ee5691b9dd4c7a1ed4ee9364c2261ebd3be52881 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Thu, 28 Nov 2024 11:18:28 +0800
Subject: [PATCH 049/399] [test](nereids)add regression case: disable join
 reorder if there are invalid stats (#43501)

### What problem does this PR solve?
add test case for pr #41790.
this case is blocked by #43279
---
 .../data/nereids_p0/stats/invalid_stats.out   |  31 +++++
 .../stats/invalid_stats/invalid_stats.out     |  31 +++++
 .../stats/{ => col_stats}/column_stats.groovy |   0
 .../nereids_p0/stats/invalid_stats.groovy     | 111 ------------------
 .../stats/invalid_stats/invalid_stats.groovy  |  77 ++++++++++++
 5 files changed, 139 insertions(+), 111 deletions(-)
 create mode 100644 regression-test/data/nereids_p0/stats/invalid_stats.out
 create mode 100644 regression-test/data/nereids_p0/stats/invalid_stats/invalid_stats.out
 rename regression-test/suites/nereids_p0/stats/{ => col_stats}/column_stats.groovy (100%)
 delete mode 100644 regression-test/suites/nereids_p0/stats/invalid_stats.groovy
 create mode 100644 regression-test/suites/nereids_p0/stats/invalid_stats/invalid_stats.groovy

diff --git a/regression-test/data/nereids_p0/stats/invalid_stats.out b/regression-test/data/nereids_p0/stats/invalid_stats.out
new file mode 100644
index 00000000000000..9b1b2e2aa97528
--- /dev/null
+++ b/regression-test/data/nereids_p0/stats/invalid_stats.out
@@ -0,0 +1,31 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !reorder_1 --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[nation]
+----PhysicalOlapScan[region]
+
+-- !ndv_min_max_invalid --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[region]
+----PhysicalOlapScan[nation]
+
+-- !reorder_2 --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[nation]
+----PhysicalOlapScan[region]
+
+-- !order_3 --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[nation]
+----PhysicalOlapScan[region]
+
+-- !ndv_row_invalid --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[region]
+----PhysicalOlapScan[nation]
+
diff --git a/regression-test/data/nereids_p0/stats/invalid_stats/invalid_stats.out b/regression-test/data/nereids_p0/stats/invalid_stats/invalid_stats.out
new file mode 100644
index 00000000000000..9b1b2e2aa97528
--- /dev/null
+++ b/regression-test/data/nereids_p0/stats/invalid_stats/invalid_stats.out
@@ -0,0 +1,31 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !reorder_1 --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[nation]
+----PhysicalOlapScan[region]
+
+-- !ndv_min_max_invalid --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[region]
+----PhysicalOlapScan[nation]
+
+-- !reorder_2 --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[nation]
+----PhysicalOlapScan[region]
+
+-- !order_3 --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[nation]
+----PhysicalOlapScan[region]
+
+-- !ndv_row_invalid --
+PhysicalResultSink
+--hashJoin[INNER_JOIN broadcast] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=()
+----PhysicalOlapScan[region]
+----PhysicalOlapScan[nation]
+
diff --git a/regression-test/suites/nereids_p0/stats/column_stats.groovy b/regression-test/suites/nereids_p0/stats/col_stats/column_stats.groovy
similarity index 100%
rename from regression-test/suites/nereids_p0/stats/column_stats.groovy
rename to regression-test/suites/nereids_p0/stats/col_stats/column_stats.groovy
diff --git a/regression-test/suites/nereids_p0/stats/invalid_stats.groovy b/regression-test/suites/nereids_p0/stats/invalid_stats.groovy
deleted file mode 100644
index 5304cd8c2c1fa9..00000000000000
--- a/regression-test/suites/nereids_p0/stats/invalid_stats.groovy
+++ /dev/null
@@ -1,111 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("invalid_stats") {
-    // multi_sql """
-    //     set global enable_auto_analyze=false;
-    //     SET enable_nereids_planner=true;
-    //     SET enable_fallback_to_original_planner=false;
-    //     set disable_nereids_rules=PRUNE_EMPTY_PARTITION;
-
-        
-    //     drop table if exists region;
-    //     CREATE TABLE region  (
-    //     r_regionkey      int NOT NULL,
-    //     r_name       VARCHAR(25) NOT NULL,
-    //     r_comment    VARCHAR(152)
-    //     )ENGINE=OLAP
-    //     DUPLICATE KEY(`r_regionkey`)
-    //     COMMENT "OLAP"
-    //     DISTRIBUTED BY HASH(`r_regionkey`) BUCKETS 1
-    //     PROPERTIES (
-    //         "replication_num" = "1"
-    //     );
-
-    //     drop table if exists nation;
-    //      CREATE TABLE `nation` (
-    //     `n_nationkey` int(11) NOT NULL,
-    //     `n_name`      varchar(25) NOT NULL,
-    //     `n_regionkey` int(11) NOT NULL,
-    //     `n_comment`   varchar(152) NULL
-    //     ) ENGINE=OLAP
-    //     DUPLICATE KEY(`N_NATIONKEY`)
-    //     COMMENT "OLAP"
-    //     DISTRIBUTED BY HASH(`N_NATIONKEY`) BUCKETS 1
-    //     PROPERTIES (
-    //         "replication_num" = "1"
-    //     );
-    //     alter table nation modify column n_nationkey set stats ('ndv'='25', 'num_nulls'='0', 'min_value'='0', 'max_value'='24', 'row_count'='25');
-
-    //     alter table nation modify column n_regionkey set stats ('ndv'='5', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='25');
-
-    // """
-
-    // explain {
-    //     sql "select * from region"
-    //     notContains("join reorder with unknown column statistics")
-    // }
-
-    // explain {
-    //     sql "select * from region where r_regionkey=1"
-    //     contains("join reorder with unknown column statistics")
-    // }
-
-    // explain {
-    //     sql "select r_regionkey from region group by r_regionkey"
-    //     contains("join reorder with unknown column statistics")
-    // }
-
-    // explain {
-    //     sql "select r_regionkey from region join nation on r_regionkey=n_regionkey"
-    //     contains("join reorder with unknown column statistics")
-    // }
-
-    // sql "alter table region modify column r_regionkey set stats ('ndv'='5', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='5');"
-    
-    // explain {
-    //     sql "select * from region where r_regionkey=1"
-    //     notContains("join reorder with unknown column statistics")
-    // }
-
-    // explain {
-    //     sql "select r_regionkey from region group by r_regionkey"
-    //     notContains("join reorder with unknown column statistics")
-    // }
-
-    // explain {
-    //     sql "select r_regionkey from region join nation on r_regionkey=n_regionkey"
-    //     notContains("join reorder with unknown column statistics")
-    // }
-
-    // explain {
-    //     sql "select r_name from region join nation on r_regionkey=n_regionkey"
-    //     notContains("join reorder with unknown column statistics")
-    // }
-
-    // explain {
-    //     sql """
-    //         select r_name 
-    //         from (select r_name, r_regionkey + 1 x from region) T join nation on T.x=n_regionkey
-    //         """
-    //     notContains("join reorder with unknown column statistics")
-    // }
-}
-// disable jo: alter table region modify column r_regionkey set stats ('ndv'='0', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='0');
-// disable jo: alter table region modify column r_regionkey set stats ('ndv'='11', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='1');
-
-// alter table region modify column r_regionkey set stats ('ndv'='10', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='1');
diff --git a/regression-test/suites/nereids_p0/stats/invalid_stats/invalid_stats.groovy b/regression-test/suites/nereids_p0/stats/invalid_stats/invalid_stats.groovy
new file mode 100644
index 00000000000000..51f1a37f118a42
--- /dev/null
+++ b/regression-test/suites/nereids_p0/stats/invalid_stats/invalid_stats.groovy
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("invalid_stats") {
+    multi_sql """
+        set global enable_auto_analyze=false;
+        SET enable_nereids_planner=true;
+        SET enable_fallback_to_original_planner=false;
+        set disable_nereids_rules=PRUNE_EMPTY_PARTITION;
+        set ignore_shape_nodes=PhysicalProject;
+        
+        drop table if exists region;
+        CREATE TABLE region  (
+        r_regionkey      int NOT NULL,
+        r_name       VARCHAR(25) NOT NULL,
+        r_comment    VARCHAR(152)
+        )ENGINE=OLAP
+        DUPLICATE KEY(`r_regionkey`)
+        COMMENT "OLAP"
+        DISTRIBUTED BY HASH(`r_regionkey`) BUCKETS 1
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+
+        drop table if exists nation;
+         CREATE TABLE `nation` (
+        `n_nationkey` int(11) NOT NULL,
+        `n_name`      varchar(25) NOT NULL,
+        `n_regionkey` int(11) NOT NULL,
+        `n_comment`   varchar(152) NULL
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`N_NATIONKEY`)
+        COMMENT "OLAP"
+        DISTRIBUTED BY HASH(`N_NATIONKEY`) BUCKETS 1
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+        alter table nation modify column n_nationkey set stats ('ndv'='25', 'num_nulls'='0', 'min_value'='0', 'max_value'='24', 'row_count'='25');
+
+        alter table region modify column r_regionkey set stats ('ndv'='5', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='5');
+
+    """
+
+    qt_reorder_1 "explain shape plan select r_regionkey from region join nation on r_regionkey=n_regionkey"
+   
+    sql "alter table region modify column r_regionkey set stats ('ndv'='0', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='0');"
+    
+    // r_regionkey stats invalid: ndv=0, but min or max is not null
+    qt_ndv_min_max_invalid "explain shape plan select r_regionkey from region join nation on r_regionkey=n_regionkey"
+    
+    // inject normal stats and check join order is nation-region
+    sql "alter table region modify column r_regionkey set stats ('ndv'='5', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='5');"
+    
+    qt_reorder_2 "explain shape plan select r_regionkey from region join nation on r_regionkey=n_regionkey"
+
+    // r_regionkey stats invalid: ndv > 10*row
+    sql "alter table region modify column r_regionkey set stats ('ndv'='10', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='1');"
+    qt_order_3 "explain shape plan select r_regionkey from region join nation on r_regionkey=n_regionkey"
+    
+    sql "alter table region modify column r_regionkey set stats ('ndv'='11', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='1');"
+    qt_ndv_row_invalid "explain shape plan select r_regionkey from region join nation on r_regionkey=n_regionkey"
+
+}

From fa3bdbce966512da6986046df8558c1d04e93f61 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Thu, 28 Nov 2024 11:19:32 +0800
Subject: [PATCH 050/399] [opt](nereids) enhance
 PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE (#43856)

### What problem does this PR solve?
PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE has some restrictions

do not support count(*)
do not support join with other join conditions
do not support the project between agg and join that contains non-slot
expressions
this pr removes above restrictions for pattern: agg-project-join
---
 .../PushDownAggThroughJoinOneSide.java        | 123 +++++++++++++-----
 .../PushDownMinMaxSumThroughJoinTest.java     |  16 +--
 .../push_down_count_through_join_one_side.out |  22 ++++
 ...sh_down_count_through_join_one_side.groovy |  95 ++++++++++++++
 4 files changed, 212 insertions(+), 44 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOneSide.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOneSide.java
index f32bf8ea91b355..c5d3d0fb49a0a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOneSide.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOneSide.java
@@ -36,6 +36,7 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.Lists;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -74,8 +75,8 @@ public List<Rule> buildRules() {
                             Set<AggregateFunction> funcs = agg.getAggregateFunctions();
                             return !funcs.isEmpty() && funcs.stream()
                                     .allMatch(f -> (f instanceof Min || f instanceof Max || f instanceof Sum
-                                            || (f instanceof Count && !((Count) f).isCountStar())) && !f.isDistinct()
-                                            && f.child(0) instanceof Slot);
+                                            || f instanceof Count && !f.isDistinct()
+                                            && (f.children().isEmpty() || f.child(0) instanceof Slot)));
                         })
                         .thenApply(ctx -> {
                             Set<Integer> enableNereidsRules = ctx.cascadesContext.getConnectContext()
@@ -88,15 +89,16 @@ public List<Rule> buildRules() {
                         })
                         .toRule(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE),
                 logicalAggregate(logicalProject(innerLogicalJoin()))
-                        .when(agg -> agg.child().isAllSlots())
-                        .when(agg -> agg.child().child().getOtherJoinConjuncts().isEmpty())
-                        .whenNot(agg -> agg.child().children().stream().anyMatch(p -> p instanceof LogicalAggregate))
+                        // .when(agg -> agg.child().isAllSlots())
+                        // .when(agg -> agg.child().child().getOtherJoinConjuncts().isEmpty())
+                        .whenNot(agg -> agg.child()
+                                .child(0).children().stream().anyMatch(p -> p instanceof LogicalAggregate))
                         .when(agg -> {
                             Set<AggregateFunction> funcs = agg.getAggregateFunctions();
                             return !funcs.isEmpty() && funcs.stream()
                                     .allMatch(f -> (f instanceof Min || f instanceof Max || f instanceof Sum
-                                            || (f instanceof Count && (!((Count) f).isCountStar()))) && !f.isDistinct()
-                                            && f.child(0) instanceof Slot);
+                                            || f instanceof Count) && !f.isDistinct()
+                                            && (f.children().isEmpty() || f.child(0) instanceof Slot));
                         })
                         .thenApply(ctx -> {
                             Set<Integer> enableNereidsRules = ctx.cascadesContext.getConnectContext()
@@ -118,23 +120,6 @@ public static LogicalAggregate<Plan> pushMinMaxSumCount(LogicalAggregate<? exten
             LogicalJoin<Plan, Plan> join, List<NamedExpression> projects) {
         List<Slot> leftOutput = join.left().getOutput();
         List<Slot> rightOutput = join.right().getOutput();
-
-        List<AggregateFunction> leftFuncs = new ArrayList<>();
-        List<AggregateFunction> rightFuncs = new ArrayList<>();
-        for (AggregateFunction func : agg.getAggregateFunctions()) {
-            Slot slot = (Slot) func.child(0);
-            if (leftOutput.contains(slot)) {
-                leftFuncs.add(func);
-            } else if (rightOutput.contains(slot)) {
-                rightFuncs.add(func);
-            } else {
-                throw new IllegalStateException("Slot " + slot + " not found in join output");
-            }
-        }
-        if (leftFuncs.isEmpty() && rightFuncs.isEmpty()) {
-            return null;
-        }
-
         Set<Slot> leftGroupBy = new HashSet<>();
         Set<Slot> rightGroupBy = new HashSet<>();
         for (Expression e : agg.getGroupByExpressions()) {
@@ -144,18 +129,71 @@ public static LogicalAggregate<Plan> pushMinMaxSumCount(LogicalAggregate<? exten
             } else if (rightOutput.contains(slot)) {
                 rightGroupBy.add(slot);
             } else {
-                return null;
+                if (projects.isEmpty()) {
+                    // TODO: select ... from ... group by A , B, 1.2; 1.2 is constant
+                    return null;
+                } else {
+                    for (NamedExpression proj : projects) {
+                        if (proj instanceof Alias && proj.toSlot().equals(slot)) {
+                            Set<Slot> inputForAliasSet = proj.getInputSlots();
+                            for (Slot aliasInputSlot : inputForAliasSet) {
+                                if (leftOutput.contains(aliasInputSlot)) {
+                                    leftGroupBy.add(aliasInputSlot);
+                                } else if (rightOutput.contains(aliasInputSlot)) {
+                                    rightGroupBy.add(aliasInputSlot);
+                                } else {
+                                    return null;
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
             }
         }
-        join.getHashJoinConjuncts().forEach(e -> e.getInputSlots().forEach(slot -> {
-            if (leftOutput.contains(slot)) {
-                leftGroupBy.add(slot);
-            } else if (rightOutput.contains(slot)) {
-                rightGroupBy.add(slot);
+
+        List<AggregateFunction> leftFuncs = new ArrayList<>();
+        List<AggregateFunction> rightFuncs = new ArrayList<>();
+        Count countStar = null;
+        Count rewrittenCountStar = null;
+        for (AggregateFunction func : agg.getAggregateFunctions()) {
+            if (func instanceof Count && ((Count) func).isCountStar()) {
+                countStar = (Count) func;
+            } else {
+                Slot slot = (Slot) func.child(0);
+                if (leftOutput.contains(slot)) {
+                    leftFuncs.add(func);
+                } else if (rightOutput.contains(slot)) {
+                    rightFuncs.add(func);
+                } else {
+                    throw new IllegalStateException("Slot " + slot + " not found in join output");
+                }
+            }
+        }
+        // rewrite count(*) to count(A), where A is slot from left/right group by key
+        if (countStar != null) {
+            if (!leftGroupBy.isEmpty()) {
+                rewrittenCountStar = (Count) countStar.withChildren(leftGroupBy.iterator().next());
+                leftFuncs.add(rewrittenCountStar);
+            } else if (!rightGroupBy.isEmpty()) {
+                rewrittenCountStar = (Count) countStar.withChildren(rightGroupBy.iterator().next());
+                rightFuncs.add(rewrittenCountStar);
             } else {
-                throw new IllegalStateException("Slot " + slot + " not found in join output");
+                return null;
+            }
+        }
+        for (Expression condition : join.getHashJoinConjuncts()) {
+            for (Slot joinConditionSlot : condition.getInputSlots()) {
+                if (leftOutput.contains(joinConditionSlot)) {
+                    leftGroupBy.add(joinConditionSlot);
+                } else if (rightOutput.contains(joinConditionSlot)) {
+                    rightGroupBy.add(joinConditionSlot);
+                } else {
+                    // apply failed
+                    return null;
+                }
             }
-        }));
+        }
 
         Plan left = join.left();
         Plan right = join.right();
@@ -196,6 +234,10 @@ public static LogicalAggregate<Plan> pushMinMaxSumCount(LogicalAggregate<? exten
         for (NamedExpression ne : agg.getOutputExpressions()) {
             if (ne instanceof Alias && ((Alias) ne).child() instanceof AggregateFunction) {
                 AggregateFunction func = (AggregateFunction) ((Alias) ne).child();
+                if (func instanceof Count && ((Count) func).isCountStar()) {
+                    // countStar is already rewritten as count(left_slot) or count(right_slot)
+                    func = rewrittenCountStar;
+                }
                 Slot slot = (Slot) func.child(0);
                 if (leftSlotToOutput.containsKey(slot)) {
                     Expression newFunc = replaceAggFunc(func, leftSlotToOutput.get(slot).toSlot());
@@ -210,9 +252,20 @@ public static LogicalAggregate<Plan> pushMinMaxSumCount(LogicalAggregate<? exten
                 newOutputExprs.add(ne);
             }
         }
-
-        // TODO: column prune project
-        return agg.withAggOutputChild(newOutputExprs, newJoin);
+        Plan newAggChild = newJoin;
+        if (agg.child() instanceof LogicalProject) {
+            LogicalProject project = (LogicalProject) agg.child();
+            List<NamedExpression> newProjections = Lists.newArrayList();
+            newProjections.addAll(project.getProjects());
+            Set<NamedExpression> leftDifference = new HashSet<NamedExpression>(left.getOutput());
+            leftDifference.removeAll(project.getProjects());
+            newProjections.addAll(leftDifference);
+            Set<NamedExpression> rightDifference = new HashSet<NamedExpression>(right.getOutput());
+            rightDifference.removeAll(project.getProjects());
+            newProjections.addAll(rightDifference);
+            newAggChild = ((LogicalProject) agg.child()).withProjectsAndChild(newProjections, newJoin);
+        }
+        return agg.withAggOutputChild(newOutputExprs, newAggChild);
     }
 
     private static Expression replaceAggFunc(AggregateFunction func, Slot inputSlot) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxSumThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxSumThroughJoinTest.java
index 58ab7fbe9e925f..cffe91045d0ab2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxSumThroughJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxSumThroughJoinTest.java
@@ -323,11 +323,11 @@ void testSingleCountStar() {
                 .applyTopDown(new PushDownAggThroughJoinOneSide())
                 .printlnTree()
                 .matches(
-                        logicalAggregate(
-                                logicalJoin(
-                                        logicalOlapScan(),
+                        logicalJoin(
+                                logicalAggregate(
                                         logicalOlapScan()
-                                )
+                                ),
+                                logicalOlapScan()
                         )
                 );
     }
@@ -346,11 +346,9 @@ void testBothSideCountAndCountStar() {
         PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
                 .applyTopDown(new PushDownAggThroughJoinOneSide())
                 .matches(
-                        logicalAggregate(
-                                logicalJoin(
-                                        logicalOlapScan(),
-                                        logicalOlapScan()
-                                )
+                        logicalJoin(
+                                logicalAggregate(logicalOlapScan()),
+                                logicalAggregate(logicalOlapScan())
                         )
                 );
     }
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.out
index da69919becd7f2..8267eb3e38ff91 100644
--- a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.out
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.out
@@ -1034,3 +1034,25 @@ Used:
 UnUsed: use_push_down_agg_through_join_one_side
 SyntaxError:
 
+-- !shape --
+PhysicalResultSink
+--PhysicalTopN[MERGE_SORT]
+----PhysicalTopN[LOCAL_SORT]
+------hashAgg[GLOBAL]
+--------hashAgg[LOCAL]
+----------hashJoin[INNER_JOIN] hashCondition=((dwd_tracking_sensor_init_tmp_ymd.dt = dw_user_b2c_tracking_info_tmp_ymd.dt) and (dwd_tracking_sensor_init_tmp_ymd.guid = dw_user_b2c_tracking_info_tmp_ymd.guid)) otherCondition=((dwd_tracking_sensor_init_tmp_ymd.dt >= substring(first_visit_time, 1, 10)))
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((dwd_tracking_sensor_init_tmp_ymd.dt = '2024-08-19') and (dwd_tracking_sensor_init_tmp_ymd.tracking_type = 'click'))
+------------------PhysicalOlapScan[dwd_tracking_sensor_init_tmp_ymd]
+------------filter((dw_user_b2c_tracking_info_tmp_ymd.dt = '2024-08-19'))
+--------------PhysicalOlapScan[dw_user_b2c_tracking_info_tmp_ymd]
+
+Hint log:
+Used: use_PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE
+UnUsed:
+SyntaxError:
+
+-- !agg_pushed --
+2	是	2024-08-19
+
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy
index 02e06710296333..e551fa04c9110a 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy
@@ -426,4 +426,99 @@ suite("push_down_count_through_join_one_side") {
     qt_with_hint_groupby_pushdown_nested_queries """
         explain shape plan select /*+ USE_CBO_RULE(push_down_agg_through_join_one_side) */  count(*) from (select * from count_t_one_side where score > 20) t1 join (select * from count_t_one_side where id < 100) t2 on t1.id = t2.id group by t1.name;
     """
+
+    sql """
+    drop table if exists dw_user_b2c_tracking_info_tmp_ymd;
+    create table dw_user_b2c_tracking_info_tmp_ymd (
+        guid int,
+        dt varchar,
+        first_visit_time varchar
+    )Engine=Olap
+    DUPLICATE KEY(guid)
+    distributed by hash(dt) buckets 3
+    properties('replication_num' = '1');
+
+    insert into dw_user_b2c_tracking_info_tmp_ymd values (1, '2024-08-19', '2024-08-19');
+
+    drop table if exists dwd_tracking_sensor_init_tmp_ymd;
+    create table dwd_tracking_sensor_init_tmp_ymd (
+        guid int,
+        dt varchar,
+        tracking_type varchar
+    )Engine=Olap
+    DUPLICATE KEY(guid)
+    distributed by hash(dt) buckets 3
+    properties('replication_num' = '1');
+
+    insert into dwd_tracking_sensor_init_tmp_ymd values(1, '2024-08-19', 'click'), (1, '2024-08-19', 'click');
+    """
+    sql """ 
+    set ENABLE_NEREIDS_RULES = "PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE";
+    set disable_join_reorder=true;
+    """
+
+    qt_shape """
+    explain shape plan
+    SELECT /*+use_cbo_rule(PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE)*/
+        Count(*)                            AS accee593,
+       CASE
+         WHEN dwd_tracking_sensor_init_tmp_ymd.dt =
+              Substring(dw_user_b2c_tracking_info_tmp_ymd.first_visit_time, 1,
+              10) THEN
+         '是'
+         WHEN dwd_tracking_sensor_init_tmp_ymd.dt >
+              Substring(dw_user_b2c_tracking_info_tmp_ymd.first_visit_time, 1,
+              10) THEN
+         '否'
+         ELSE '-1'
+       end                                 AS a1302fb2,
+       dwd_tracking_sensor_init_tmp_ymd.dt AS ad466123
+    FROM   dwd_tracking_sensor_init_tmp_ymd
+        LEFT JOIN dw_user_b2c_tracking_info_tmp_ymd
+                ON dwd_tracking_sensor_init_tmp_ymd.guid =
+                    dw_user_b2c_tracking_info_tmp_ymd.guid
+                    AND dwd_tracking_sensor_init_tmp_ymd.dt =
+                        dw_user_b2c_tracking_info_tmp_ymd.dt
+    WHERE  dwd_tracking_sensor_init_tmp_ymd.dt = '2024-08-19'
+        AND dw_user_b2c_tracking_info_tmp_ymd.dt = '2024-08-19'
+        AND dwd_tracking_sensor_init_tmp_ymd.dt >=
+            Substring(dw_user_b2c_tracking_info_tmp_ymd.first_visit_time, 1, 10)
+        AND dwd_tracking_sensor_init_tmp_ymd.tracking_type = 'click'
+    GROUP  BY 2,
+            3
+    ORDER  BY 3 ASC
+    LIMIT  10000; 
+    """
+
+    qt_agg_pushed  """
+    SELECT /*+use_cbo_rule(PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE)*/
+        Count(*)                            AS accee593,
+       CASE
+         WHEN dwd_tracking_sensor_init_tmp_ymd.dt =
+              Substring(dw_user_b2c_tracking_info_tmp_ymd.first_visit_time, 1,
+              10) THEN
+         '是'
+         WHEN dwd_tracking_sensor_init_tmp_ymd.dt >
+              Substring(dw_user_b2c_tracking_info_tmp_ymd.first_visit_time, 1,
+              10) THEN
+         '否'
+         ELSE '-1'
+       end                                 AS a1302fb2,
+       dwd_tracking_sensor_init_tmp_ymd.dt AS ad466123
+    FROM   dwd_tracking_sensor_init_tmp_ymd
+        LEFT JOIN dw_user_b2c_tracking_info_tmp_ymd
+                ON dwd_tracking_sensor_init_tmp_ymd.guid =
+                    dw_user_b2c_tracking_info_tmp_ymd.guid
+                    AND dwd_tracking_sensor_init_tmp_ymd.dt =
+                        dw_user_b2c_tracking_info_tmp_ymd.dt
+    WHERE  dwd_tracking_sensor_init_tmp_ymd.dt = '2024-08-19'
+        AND dw_user_b2c_tracking_info_tmp_ymd.dt = '2024-08-19'
+        AND dwd_tracking_sensor_init_tmp_ymd.dt >=
+            Substring(dw_user_b2c_tracking_info_tmp_ymd.first_visit_time, 1, 10)
+        AND dwd_tracking_sensor_init_tmp_ymd.tracking_type = 'click'
+    GROUP  BY 2,
+            3
+    ORDER  BY 3 ASC
+    LIMIT  10000; 
+    """
 }

From b7d985d8ff85504b20e6fd59597b143797cee020 Mon Sep 17 00:00:00 2001
From: LiBinfeng <1204975323@qq.com>
Date: Thu, 28 Nov 2024 11:36:11 +0800
Subject: [PATCH 051/399] [feat](Nereids) support cancel commands (#44422)

Support
cancelLoadCommand
cancelExportCommand
cancelWarmUpJobCommand
---
 .../org/apache/doris/nereids/DorisParser.g4   |  10 +-
 .../doris/analysis/CancelExportStmt.java      |   9 +
 .../apache/doris/analysis/CancelLoadStmt.java |   9 +
 .../apache/doris/job/manager/JobManager.java  |  89 ++++++++++
 .../java/org/apache/doris/load/ExportMgr.java |  74 ++++++++
 .../apache/doris/load/loadv2/LoadManager.java |  94 +++++++++-
 .../nereids/parser/LogicalPlanBuilder.java    |  38 ++++
 .../doris/nereids/trees/plans/PlanType.java   |   3 +
 .../trees/plans/commands/CancelCommand.java   | 162 ++++++++++++++++++
 .../plans/commands/CancelExportCommand.java   |  89 ++++++++++
 .../plans/commands/CancelJobTaskCommand.java  |   2 +-
 .../plans/commands/CancelLoadCommand.java     |  94 ++++++++++
 .../commands/CancelWarmUpJobCommand.java      | 116 +++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  15 ++
 14 files changed, 792 insertions(+), 12 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelExportCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelLoadCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelWarmUpJobCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index cee972a63d210b..279329089a4ade 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -59,6 +59,7 @@ statementBase
     | supportedRefreshStatement         #supportedRefreshStatementAlias
     | supportedShowStatement            #supportedShowStatementAlias
     | supportedLoadStatement            #supportedLoadStatementAlias
+    | supportedCancelStatement          #supportedCancelStatementAlias
     | supportedRecoverStatement         #supportedRecoverStatementAlias
     | supportedAdminStatement           #supportedAdminStatementAlias
     | unsupportedStatement              #unsupported
@@ -455,10 +456,14 @@ unsupportedCleanStatement
     | CLEAN ALL QUERY STATS                                                         #cleanAllQueryStats
     ;
 
-unsupportedCancelStatement
+supportedCancelStatement
     : CANCEL LOAD ((FROM | IN) database=identifier)? wildWhere?                     #cancelLoad
     | CANCEL EXPORT ((FROM | IN) database=identifier)? wildWhere?                   #cancelExport
-    | CANCEL ALTER TABLE (ROLLUP | (MATERIALIZED VIEW) | COLUMN)
+    | CANCEL WARM UP JOB wildWhere?                                                 #cancelWarmUpJob
+    ;
+
+unsupportedCancelStatement
+    : CANCEL ALTER TABLE (ROLLUP | (MATERIALIZED VIEW) | COLUMN)
         FROM tableName=multipartIdentifier (LEFT_PAREN jobIds+=INTEGER_VALUE
             (COMMA jobIds+=INTEGER_VALUE)* RIGHT_PAREN)?                            #cancelAlterTable
     | CANCEL BUILD INDEX ON tableName=multipartIdentifier
@@ -468,7 +473,6 @@ unsupportedCancelStatement
         (COMMA hostPorts+=STRING_LITERAL)*                                          #cancelDecommisionBackend
     | CANCEL BACKUP ((FROM | IN) database=identifier)?                              #cancelBackup
     | CANCEL RESTORE ((FROM | IN) database=identifier)?                             #cancelRestore
-    | CANCEL WARM UP JOB wildWhere?                                                 #cancelWarmUp
     ;
 
 supportedAdminStatement
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelExportStmt.java
index 63448cf35a027a..fa6d7640781e17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelExportStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelExportStmt.java
@@ -58,6 +58,15 @@ public CancelExportStmt(String dbName, Expr whereClause) {
         this.whereClause = whereClause;
     }
 
+    public CancelExportStmt(String dbName, Expr whereClause, String label, CompoundPredicate.Operator operator,
+                            String state) {
+        this.dbName = dbName;
+        this.whereClause = whereClause;
+        this.label = label;
+        this.operator = operator;
+        this.state = state;
+    }
+
     private void checkColumn(Expr expr, boolean like) throws AnalysisException {
         String inputCol = ((SlotRef) expr.getChild(0)).getColumnName();
         if (!SUPPORT_COLUMNS.contains(inputCol.toLowerCase())) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java
index 1371e308675708..bbffb2649738fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java
@@ -59,6 +59,15 @@ public CancelLoadStmt(String dbName, Expr whereClause) {
         this.SUPPORT_COLUMNS.add("state");
     }
 
+    public CancelLoadStmt(String dbName, Expr whereClause, String label, CompoundPredicate.Operator operator,
+                          String state) {
+        this.dbName = dbName;
+        this.whereClause = whereClause;
+        this.label = label;
+        this.operator = operator;
+        this.state = state;
+    }
+
     private void checkColumn(Expr expr, boolean like) throws AnalysisException {
         String inputCol = ((SlotRef) expr.getChild(0)).getColumnName();
         if (!SUPPORT_COLUMNS.contains(inputCol)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
index 47a3a0c5c19d5c..db2ca6eb81d156 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
@@ -41,6 +41,8 @@
 import org.apache.doris.job.scheduler.JobScheduler;
 import org.apache.doris.load.loadv2.JobState;
 import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.expressions.And;
+import org.apache.doris.nereids.trees.expressions.BinaryOperator;
 import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.collect.Lists;
@@ -522,4 +524,91 @@ private static void addNeedCancelLoadJob(String label, String state,
     //            job.updateLoadingStatus(beId, loadId, fragmentId, scannedRows, scannedBytes, isDone);
     //        }
     //    }
+
+    /**
+     * used for nereids planner
+     */
+    public void cancelLoadJob(String dbName, String label, String state, BinaryOperator operator)
+            throws JobException, AnalysisException, DdlException {
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
+        // List of load jobs waiting to be cancelled
+        List<InsertJob> unfinishedLoadJob;
+        readLock();
+        try {
+            List<InsertJob> loadJobs = Env.getCurrentEnv().getLabelProcessor().getJobs(db);
+            List<InsertJob> matchLoadJobs = Lists.newArrayList();
+            addNeedCancelLoadJob(label, state, operator, loadJobs, matchLoadJobs);
+            if (matchLoadJobs.isEmpty()) {
+                throw new JobException("Load job does not exist");
+            }
+            // check state here
+            unfinishedLoadJob =
+                matchLoadJobs.stream().filter(InsertJob::isRunning)
+                    .collect(Collectors.toList());
+            if (unfinishedLoadJob.isEmpty()) {
+                throw new JobException("There is no uncompleted job");
+            }
+        } finally {
+            readUnlock();
+        }
+        // check auth
+        if (unfinishedLoadJob.size() > 1 || unfinishedLoadJob.get(0).getTableNames().isEmpty()) {
+            if (Env.getCurrentEnv().getAccessManager()
+                    .checkDbPriv(ConnectContext.get(), InternalCatalog.INTERNAL_CATALOG_NAME, dbName,
+                    PrivPredicate.LOAD)) {
+                ErrorReport.reportAnalysisException(ErrorCode.ERR_DBACCESS_DENIED_ERROR, "LOAD",
+                        ConnectContext.get().getQualifiedUser(),
+                        ConnectContext.get().getRemoteIP(), dbName);
+            }
+        } else {
+            for (String tableName : unfinishedLoadJob.get(0).getTableNames()) {
+                if (Env.getCurrentEnv().getAccessManager()
+                        .checkTblPriv(ConnectContext.get(), InternalCatalog.INTERNAL_CATALOG_NAME, dbName,
+                        tableName,
+                        PrivPredicate.LOAD)) {
+                    ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD",
+                            ConnectContext.get().getQualifiedUser(),
+                            ConnectContext.get().getRemoteIP(), dbName + ":" + tableName);
+                }
+            }
+        }
+        for (InsertJob loadJob : unfinishedLoadJob) {
+            try {
+                alterJobStatus(loadJob.getJobId(), JobStatus.STOPPED);
+            } catch (JobException e) {
+                log.warn("Fail to cancel job, its label: {}", loadJob.getLabelName());
+            }
+        }
+    }
+
+    private static void addNeedCancelLoadJob(String label, String state,
+                                             BinaryOperator operator, List<InsertJob> loadJobs,
+                                             List<InsertJob> matchLoadJobs)
+            throws AnalysisException {
+        PatternMatcher matcher = PatternMatcherWrapper.createMysqlPattern(label,
+                CaseSensibility.LABEL.getCaseSensibility());
+        matchLoadJobs.addAll(
+                loadJobs.stream()
+                .filter(job -> !job.isCancelled())
+                .filter(job -> {
+                    if (operator != null) {
+                        // compound
+                        boolean labelFilter =
+                                label.contains("%") ? matcher.match(job.getLabelName())
+                                : job.getLabelName().equalsIgnoreCase(label);
+                        boolean stateFilter = job.getJobStatus().name().equalsIgnoreCase(state);
+                        return operator instanceof And ? labelFilter && stateFilter :
+                            labelFilter || stateFilter;
+                    }
+                    if (StringUtils.isNotEmpty(label)) {
+                        return label.contains("%") ? matcher.match(job.getLabelName())
+                            : job.getLabelName().equalsIgnoreCase(label);
+                    }
+                    if (StringUtils.isNotEmpty(state)) {
+                        return job.getJobStatus().name().equalsIgnoreCase(state);
+                    }
+                    return false;
+                }).collect(Collectors.toList())
+        );
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index 49ebbfe7dcddb0..398cc8f4ac2518 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -38,6 +38,8 @@
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.datasource.InternalCatalog;
 import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.expressions.BinaryOperator;
+import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.scheduler.exception.JobException;
 
@@ -160,6 +162,78 @@ public void cancelExportJob(CancelExportStmt stmt) throws DdlException, Analysis
         }
     }
 
+    private List<ExportJob> getWaitingCancelJobs(String label, String state, BinaryOperator operator)
+            throws AnalysisException {
+        Predicate<ExportJob> jobFilter = buildCancelJobFilter(label, state, operator);
+        readLock();
+        try {
+            return getJobs().stream().filter(jobFilter).collect(Collectors.toList());
+        } finally {
+            readUnlock();
+        }
+    }
+
+    @VisibleForTesting
+    public static Predicate<ExportJob> buildCancelJobFilter(String label, String state, BinaryOperator operator)
+            throws AnalysisException {
+        PatternMatcher matcher = PatternMatcherWrapper.createMysqlPattern(label,
+                CaseSensibility.LABEL.getCaseSensibility());
+
+        return job -> {
+            boolean labelFilter = true;
+            boolean stateFilter = true;
+            if (StringUtils.isNotEmpty(label)) {
+                labelFilter = label.contains("%") ? matcher.match(job.getLabel()) :
+                    job.getLabel().equalsIgnoreCase(label);
+            }
+            if (StringUtils.isNotEmpty(state)) {
+                stateFilter = job.getState().name().equalsIgnoreCase(state);
+            }
+
+            if (operator != null && operator instanceof Or) {
+                return labelFilter || stateFilter;
+            }
+
+            return labelFilter && stateFilter;
+        };
+    }
+
+    /**
+     * used for Nereids planner
+     */
+    public void cancelExportJob(String label, String state, BinaryOperator operator, String dbName)
+            throws DdlException, AnalysisException {
+        // List of export jobs waiting to be cancelled
+        List<ExportJob> matchExportJobs = getWaitingCancelJobs(label, state, operator);
+        if (matchExportJobs.isEmpty()) {
+            throw new DdlException("Export job(s) do not exist");
+        }
+        matchExportJobs = matchExportJobs.stream()
+            .filter(job -> !job.isFinalState()).collect(Collectors.toList());
+        if (matchExportJobs.isEmpty()) {
+            throw new DdlException("All export job(s) are at final state (CANCELLED/FINISHED)");
+        }
+
+        // check auth
+        checkCancelExportJobAuth(InternalCatalog.INTERNAL_CATALOG_NAME, dbName, matchExportJobs);
+        // Must add lock to protect export job.
+        // Because job may be cancelled when generating task executors,
+        // the cancel process may clear the task executor list at same time,
+        // which will cause ConcurrentModificationException
+        writeLock();
+        try {
+            for (ExportJob exportJob : matchExportJobs) {
+                // exportJob.cancel(ExportFailMsg.CancelType.USER_CANCEL, "user cancel");
+                exportJob.updateExportJobState(ExportJobState.CANCELLED, 0L, null,
+                        ExportFailMsg.CancelType.USER_CANCEL, "user cancel");
+            }
+        } catch (JobException e) {
+            throw new AnalysisException(e.getMessage());
+        } finally {
+            writeUnlock();
+        }
+    }
+
     public void checkCancelExportJobAuth(String ctlName, String dbName, List<ExportJob> jobs) throws AnalysisException {
         if (jobs.size() > 1) {
             if (!Env.getCurrentEnv().getAccessManager()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
index 07c459d61cf056..5c39a27e24f828 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
@@ -46,6 +46,8 @@
 import org.apache.doris.load.FailMsg.CancelType;
 import org.apache.doris.load.Load;
 import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.expressions.And;
+import org.apache.doris.nereids.trees.expressions.BinaryOperator;
 import org.apache.doris.persist.CleanLabelOperationLog;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.OriginStatement;
@@ -246,23 +248,22 @@ public void recordFinishedLoadJob(String label, long transactionId, String dbNam
      * Match need cancel loadJob by stmt.
      **/
     @VisibleForTesting
-    public static void addNeedCancelLoadJob(CancelLoadStmt stmt, List<LoadJob> loadJobs, List<LoadJob> matchLoadJobs)
+    public static void addNeedCancelLoadJob(String label, String state, BinaryOperator operator,
+                                            List<LoadJob> loadJobs, List<LoadJob> matchLoadJobs)
             throws AnalysisException {
-        String label = stmt.getLabel();
-        String state = stmt.getState();
         PatternMatcher matcher = PatternMatcherWrapper.createMysqlPattern(label,
                 CaseSensibility.LABEL.getCaseSensibility());
         matchLoadJobs.addAll(
                 loadJobs.stream()
                         .filter(job -> job.getState() != JobState.CANCELLED)
                         .filter(job -> {
-                            if (stmt.getOperator() != null) {
+                            if (operator != null) {
                                 // compound
                                 boolean labelFilter =
                                         label.contains("%") ? matcher.match(job.getLabel())
                                                 : job.getLabel().equalsIgnoreCase(label);
                                 boolean stateFilter = job.getState().name().equalsIgnoreCase(state);
-                                return Operator.AND.equals(stmt.getOperator()) ? labelFilter && stateFilter :
+                                return operator instanceof And ? labelFilter && stateFilter :
                                         labelFilter || stateFilter;
                             }
                             if (StringUtils.isNotEmpty(label)) {
@@ -280,8 +281,9 @@ public static void addNeedCancelLoadJob(CancelLoadStmt stmt, List<LoadJob> loadJ
     /**
      * Cancel load job by stmt.
      **/
-    public void cancelLoadJob(CancelLoadStmt stmt) throws DdlException, AnalysisException {
-        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(stmt.getDbName());
+    public void cancelLoadJob(String dbName, String label, String state, BinaryOperator operator)
+            throws DdlException, AnalysisException {
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
         // List of load jobs waiting to be cancelled
         List<LoadJob> unfinishedLoadJob;
         readLock();
@@ -291,7 +293,7 @@ public void cancelLoadJob(CancelLoadStmt stmt) throws DdlException, AnalysisExce
                 throw new DdlException("Load job does not exist");
             }
             List<LoadJob> matchLoadJobs = Lists.newArrayList();
-            addNeedCancelLoadJob(stmt,
+            addNeedCancelLoadJob(label, state, operator,
                     labelToLoadJobs.values().stream().flatMap(Collection::stream).collect(Collectors.toList()),
                     matchLoadJobs);
             if (matchLoadJobs.isEmpty()) {
@@ -318,6 +320,82 @@ public void cancelLoadJob(CancelLoadStmt stmt) throws DdlException, AnalysisExce
         }
     }
 
+    /**
+     * Match need cancel loadJob by stmt.
+     **/
+    @VisibleForTesting
+    public static void addNeedCancelLoadJob(CancelLoadStmt stmt, List<LoadJob> loadJobs, List<LoadJob> matchLoadJobs)
+            throws AnalysisException {
+        String label = stmt.getLabel();
+        String state = stmt.getState();
+        PatternMatcher matcher = PatternMatcherWrapper.createMysqlPattern(label,
+                CaseSensibility.LABEL.getCaseSensibility());
+        matchLoadJobs.addAll(
+                loadJobs.stream()
+                .filter(job -> job.getState() != JobState.CANCELLED)
+                .filter(job -> {
+                    if (stmt.getOperator() != null) {
+                        // compound
+                        boolean labelFilter =
+                                label.contains("%") ? matcher.match(job.getLabel())
+                                : job.getLabel().equalsIgnoreCase(label);
+                        boolean stateFilter = job.getState().name().equalsIgnoreCase(state);
+                        return Operator.AND.equals(stmt.getOperator()) ? labelFilter && stateFilter :
+                            labelFilter || stateFilter;
+                    }
+                    if (StringUtils.isNotEmpty(label)) {
+                        return label.contains("%") ? matcher.match(job.getLabel())
+                            : job.getLabel().equalsIgnoreCase(label);
+                    }
+                    if (StringUtils.isNotEmpty(state)) {
+                        return job.getState().name().equalsIgnoreCase(state);
+                    }
+                    return false;
+                }).collect(Collectors.toList())
+        );
+    }
+
+    /**
+     * Cancel load job by stmt.
+     **/
+    public void cancelLoadJob(CancelLoadStmt stmt) throws DdlException, AnalysisException {
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(stmt.getDbName());
+        // List of load jobs waiting to be cancelled
+        List<LoadJob> unfinishedLoadJob;
+        readLock();
+        try {
+            Map<String, List<LoadJob>> labelToLoadJobs = dbIdToLabelToLoadJobs.get(db.getId());
+            if (labelToLoadJobs == null) {
+                throw new DdlException("Load job does not exist");
+            }
+            List<LoadJob> matchLoadJobs = Lists.newArrayList();
+            addNeedCancelLoadJob(stmt,
+                    labelToLoadJobs.values().stream().flatMap(Collection::stream).collect(Collectors.toList()),
+                    matchLoadJobs);
+            if (matchLoadJobs.isEmpty()) {
+                throw new DdlException("Load job does not exist");
+            }
+            // check state here
+            unfinishedLoadJob =
+                matchLoadJobs.stream().filter(entity -> !entity.isTxnDone()).collect(Collectors.toList());
+            if (unfinishedLoadJob.isEmpty()) {
+                throw new DdlException("There is no uncompleted job");
+            }
+        } finally {
+            readUnlock();
+        }
+        for (LoadJob loadJob : unfinishedLoadJob) {
+            try {
+                loadJob.cancelJob(new FailMsg(FailMsg.CancelType.USER_CANCEL, "user cancel"));
+            } catch (DdlException e) {
+                throw new DdlException(
+                    "Cancel load job [" + loadJob.getId() + "] fail, " + "label=[" + loadJob.getLabel()
+                        +
+                        "] failed msg=" + e.getMessage());
+            }
+        }
+    }
+
     /**
      * Replay end load job.
      **/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 1a49c7ba65fa56..abd6afb84bc7f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -457,8 +457,11 @@
 import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadPolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CallCommand;
+import org.apache.doris.nereids.trees.plans.commands.CancelExportCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelJobTaskCommand;
+import org.apache.doris.nereids.trees.plans.commands.CancelLoadCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
+import org.apache.doris.nereids.trees.plans.commands.CancelWarmUpJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.Constraint;
@@ -1086,6 +1089,41 @@ public ShowCreateMTMVCommand visitShowCreateMTMV(ShowCreateMTMVContext ctx) {
         return new ShowCreateMTMVCommand(new ShowCreateMTMVInfo(new TableNameInfo(nameParts)));
     }
 
+    @Override
+    public CancelExportCommand visitCancelExport(DorisParser.CancelExportContext ctx) {
+        String databaseName = null;
+        if (ctx.database != null) {
+            databaseName = stripQuotes(ctx.database.getText());
+        }
+        Expression wildWhere = null;
+        if (ctx.wildWhere() != null) {
+            wildWhere = getWildWhere(ctx.wildWhere());
+        }
+        return new CancelExportCommand(databaseName, wildWhere);
+    }
+
+    @Override
+    public CancelLoadCommand visitCancelLoad(DorisParser.CancelLoadContext ctx) {
+        String databaseName = null;
+        if (ctx.database != null) {
+            databaseName = stripQuotes(ctx.database.getText());
+        }
+        Expression wildWhere = null;
+        if (ctx.wildWhere() != null) {
+            wildWhere = getWildWhere(ctx.wildWhere());
+        }
+        return new CancelLoadCommand(databaseName, wildWhere);
+    }
+
+    @Override
+    public CancelWarmUpJobCommand visitCancelWarmUpJob(DorisParser.CancelWarmUpJobContext ctx) {
+        Expression wildWhere = null;
+        if (ctx.wildWhere() != null) {
+            wildWhere = getWildWhere(ctx.wildWhere());
+        }
+        return new CancelWarmUpJobCommand(wildWhere);
+    }
+
     @Override
     public CancelMTMVTaskCommand visitCancelMTMVTask(CancelMTMVTaskContext ctx) {
         List<String> nameParts = visitMultipartIdentifier(ctx.mvName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 20d46d612d02ac..09fa71f5614e87 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -158,6 +158,9 @@ public enum PlanType {
     PAUSE_MTMV_COMMAND,
     RESUME_MTMV_COMMAND,
     SHOW_CREATE_MTMV_COMMAND,
+    CANCEL_EXPORT_COMMAND,
+    CANCEL_LOAD_COMMAND,
+    CANCEL_WARM_UP_JOB_COMMAND,
     CANCEL_MTMV_TASK_COMMAND,
     CALL_COMMAND,
     CREATE_PROCEDURE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelCommand.java
new file mode 100644
index 00000000000000..88ecf3cba1e6d8
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelCommand.java
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.glue.translator.ExpressionTranslator;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.BinaryOperator;
+import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Like;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+/**
+ * cancel load command
+ */
+public abstract class CancelCommand extends Command implements ForwardWithSync {
+    public CancelCommand(PlanType type) {
+        super(type);
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return null;
+    }
+
+    /**
+     * translate to legacy expr, which do not need complex expression and table columns
+     */
+    public Expr translateToLegacyExpr(ConnectContext ctx, Expression expression) {
+        LogicalEmptyRelation plan = new LogicalEmptyRelation(
+                ConnectContext.get().getStatementContext().getNextRelationId(),
+                new ArrayList<>());
+        CascadesContext cascadesContext = CascadesContext.initContext(ctx.getStatementContext(), plan,
+                PhysicalProperties.ANY);
+        PlanTranslatorContext planTranslatorContext = new PlanTranslatorContext(cascadesContext);
+        ExpressionToExpr translator = new ExpressionToExpr();
+        return expression.accept(translator, planTranslatorContext);
+    }
+
+    private static class ExpressionToExpr extends ExpressionTranslator {
+        @Override
+        public Expr visitUnboundSlot(UnboundSlot unboundSlot, PlanTranslatorContext context) {
+            String inputCol = unboundSlot.getName();
+            return new SlotRef(null, inputCol);
+        }
+    }
+
+    /**
+     * check where filter for cancel load/export commands
+     * @param expression where clause
+     * @param supportedColumns only these kind of columns is supported
+     * @throws AnalysisException analyze exceptions
+     */
+    public void checkWhereFilter(Expression expression, Map<String, String> supportedColumns) throws AnalysisException {
+        if (null == expression) {
+            throw new AnalysisException("Where clause can't be null");
+        } else if (expression instanceof Like) {
+            likeCheck(expression, supportedColumns);
+        } else if (expression instanceof BinaryOperator) {
+            binaryCheck(expression, supportedColumns);
+        } else if (expression instanceof CompoundPredicate) {
+            compoundCheck(expression, supportedColumns);
+        } else {
+            throw new AnalysisException("Only support like/binary/compound predicate");
+        }
+    }
+
+    private void checkColumn(Expression expr, boolean like, Map<String, String> supportedColumns)
+            throws AnalysisException {
+        if (!(expr.child(0) instanceof UnboundSlot)) {
+            throw new AnalysisException("Current only support label and state, invalid column: "
+                + expr.child(0).toSql());
+        }
+        String inputCol = ((UnboundSlot) expr.child(0)).getName();
+        if (!supportedColumns.keySet().contains(inputCol.toLowerCase())) {
+            throw new AnalysisException("Current only support label and state, invalid column: " + inputCol);
+        }
+        if (!(expr.child(1) instanceof StringLikeLiteral)) {
+            throw new AnalysisException("Value must be a string");
+        }
+
+        String inputValue = ((StringLikeLiteral) expr.child(1)).getStringValue();
+        if (Strings.isNullOrEmpty(inputValue)) {
+            throw new AnalysisException("Value can't be null");
+        }
+
+        if (inputCol.equalsIgnoreCase("label")) {
+            supportedColumns.put("label", inputValue);
+        }
+
+        if (inputCol.equalsIgnoreCase("state")) {
+            if (like) {
+                throw new AnalysisException("Only label can use like");
+            }
+            supportedColumns.put("state", inputValue);
+        }
+    }
+
+    private void likeCheck(Expression expr, Map<String, String> supportedColumns) throws AnalysisException {
+        checkColumn(expr, true, supportedColumns);
+    }
+
+    private void binaryCheck(Expression expr, Map<String, String> supportedColumns) throws AnalysisException {
+        checkColumn(expr, false, supportedColumns);
+    }
+
+    private void compoundCheck(Expression expr, Map<String, String> supportedColumns) throws AnalysisException {
+        // current only support label and state
+        if (expr instanceof Not) {
+            throw new AnalysisException("Current not support NOT operator");
+        }
+        for (int i = 0; i < 2; i++) {
+            Expression child = expr.child(i);
+            if (child instanceof CompoundPredicate) {
+                throw new AnalysisException("Current not support nested clause");
+            } else if (child instanceof Like) {
+                likeCheck(child, supportedColumns);
+            } else if (child instanceof BinaryOperator) {
+                binaryCheck(child, supportedColumns);
+            } else {
+                throw new AnalysisException("Only support like/binary predicate");
+            }
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelExportCommand.java
new file mode 100644
index 00000000000000..f7defefb422476
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelExportCommand.java
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.UserException;
+import org.apache.doris.load.ExportJobState;
+import org.apache.doris.nereids.trees.expressions.BinaryOperator;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * cancel export command
+ */
+public class CancelExportCommand extends CancelCommand implements ForwardWithSync {
+    private Map<String, String> supportedColumns = new HashMap<>();
+
+    private String dbName;
+
+    private String label;
+
+    private String state;
+
+    private Expression whereClause;
+
+    public CancelExportCommand(String dbName, Expression whereClause) {
+        super(PlanType.CANCEL_EXPORT_COMMAND);
+        this.dbName = dbName;
+        this.whereClause = whereClause;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        ctx.getEnv().getExportMgr().cancelExportJob(label, state, (BinaryOperator) whereClause, dbName);
+    }
+
+    private void validate(ConnectContext ctx) throws UserException {
+        if (Strings.isNullOrEmpty(dbName)) {
+            dbName = ctx.getDatabase();
+            if (Strings.isNullOrEmpty(dbName)) {
+                throw new AnalysisException("No database selected");
+            }
+        }
+
+        supportedColumns.put("label", "");
+        supportedColumns.put("state", "");
+        checkWhereFilter(whereClause, supportedColumns);
+        if (!Strings.isNullOrEmpty(supportedColumns.get("label"))) {
+            label = supportedColumns.get("label");
+        }
+        if (!Strings.isNullOrEmpty(supportedColumns.get("state"))) {
+            state = supportedColumns.get("state");
+            ExportJobState jobState = ExportJobState.valueOf(state);
+            if (jobState != ExportJobState.PENDING
+                    && jobState != ExportJobState.EXPORTING) {
+                throw new AnalysisException("Only support PENDING/EXPORTING, invalid state: " + state);
+            }
+        }
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCancelExportCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelJobTaskCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelJobTaskCommand.java
index a9ea241e3b63ec..e02c731911a18e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelJobTaskCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelJobTaskCommand.java
@@ -36,7 +36,7 @@
 /**
  * base class for all drop commands
  */
-public class CancelJobTaskCommand extends Command implements ForwardWithSync {
+public class CancelJobTaskCommand extends CancelCommand implements ForwardWithSync {
     private static final String jobNameKey = "jobName";
 
     private static final String taskIdKey = "taskId";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelLoadCommand.java
new file mode 100644
index 00000000000000..4d3068ed08f559
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelLoadCommand.java
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.UserException;
+import org.apache.doris.job.exception.JobException;
+import org.apache.doris.load.ExportJobState;
+import org.apache.doris.nereids.trees.expressions.BinaryOperator;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * cancel load command
+ */
+public class CancelLoadCommand extends CancelCommand implements ForwardWithSync {
+    private Map<String, String> supportedColumns = new HashMap<>();
+
+    private String dbName;
+
+    private String label;
+
+    private String state;
+
+    private Expression whereClause;
+
+    public CancelLoadCommand(String dbName, Expression whereClause) {
+        super(PlanType.CANCEL_EXPORT_COMMAND);
+        this.dbName = dbName;
+        this.whereClause = whereClause;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        try {
+            ctx.getEnv().getJobManager().cancelLoadJob(dbName, label, state, (BinaryOperator) whereClause);
+        } catch (JobException e) {
+            ctx.getEnv().getLoadManager().cancelLoadJob(dbName, label, state, (BinaryOperator) whereClause);
+        }
+    }
+
+    private void validate(ConnectContext ctx) throws UserException {
+        if (Strings.isNullOrEmpty(dbName)) {
+            dbName = ctx.getDatabase();
+            if (Strings.isNullOrEmpty(dbName)) {
+                throw new AnalysisException("No database selected");
+            }
+        }
+
+        supportedColumns.put("label", "");
+        supportedColumns.put("state", "");
+        checkWhereFilter(whereClause, supportedColumns);
+        if (!Strings.isNullOrEmpty(supportedColumns.get("label"))) {
+            label = supportedColumns.get("label");
+        }
+        if (!Strings.isNullOrEmpty(supportedColumns.get("state"))) {
+            state = supportedColumns.get("state");
+            ExportJobState jobState = ExportJobState.valueOf(state);
+            if (jobState != ExportJobState.PENDING
+                    && jobState != ExportJobState.EXPORTING) {
+                throw new AnalysisException("Only support PENDING/EXPORTING, invalid state: " + state);
+            }
+        }
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCancelLoadCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelWarmUpJobCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelWarmUpJobCommand.java
new file mode 100644
index 00000000000000..b5d26b4c4213ff
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelWarmUpJobCommand.java
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.CancelCloudWarmUpStmt;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.cloud.catalog.CloudEnv;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.glue.translator.ExpressionTranslator;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import java.util.ArrayList;
+
+/**
+ * cancel warm up job command
+ */
+public class CancelWarmUpJobCommand extends Command implements ForwardWithSync {
+    private Expression whereClause;
+    private long jobId;
+    private Expr legacyWhereClause;
+
+    public CancelWarmUpJobCommand(Expression whereClause) {
+        super(PlanType.CANCEL_WARM_UP_JOB_COMMAND);
+        this.whereClause = whereClause;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        if (Config.isCloudMode()) {
+            CancelCloudWarmUpStmt stmt = new CancelCloudWarmUpStmt(legacyWhereClause);
+            ((CloudEnv) ctx.getEnv()).cancelCloudWarmUp(stmt);
+        }
+    }
+
+    public long getJobId() {
+        return jobId;
+    }
+
+    /**
+     * validate cloud warm up job
+     * @param ctx connect context
+     * @throws AnalysisException check whether this sql is legal
+     */
+    public void validate(ConnectContext ctx) throws AnalysisException {
+        if (!Config.isCloudMode()) {
+            throw new AnalysisException("The sql is illegal in disk mode ");
+        }
+        if (whereClause == null) {
+            throw new AnalysisException("Missing job id");
+        }
+        boolean valid = true;
+        CHECK: {
+            if (!(whereClause instanceof EqualTo)) {
+                valid = false;
+                break CHECK;
+            }
+
+            // left child
+            if (!(whereClause.child(0) instanceof UnboundSlot)) {
+                valid = false;
+                break CHECK;
+            }
+            String leftKey = ((UnboundSlot) whereClause.child(0)).getName();
+            if (leftKey.equalsIgnoreCase("id") && (whereClause.child(1) instanceof IntegerLiteral)) {
+                jobId = ((IntegerLiteral) whereClause.child(1)).getLongValue();
+            } else {
+                valid = false;
+            }
+        }
+
+        if (!valid) {
+            throw new AnalysisException("Where clause should looks like one of them: id = 123");
+        }
+
+        LogicalEmptyRelation plan = new LogicalEmptyRelation(
+                ConnectContext.get().getStatementContext().getNextRelationId(),
+                new ArrayList<>());
+        CascadesContext cascadesContext = CascadesContext.initContext(ctx.getStatementContext(), plan,
+                PhysicalProperties.ANY);
+        PlanTranslatorContext planTranslatorContext = new PlanTranslatorContext(cascadesContext);
+        legacyWhereClause = ExpressionTranslator.translate(whereClause, planTranslatorContext);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCancelWarmUpJobCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 0af9ce4dc9c4cc..ef1688c3398b82 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -25,8 +25,11 @@
 import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadPolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CallCommand;
+import org.apache.doris.nereids.trees.plans.commands.CancelExportCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelJobTaskCommand;
+import org.apache.doris.nereids.trees.plans.commands.CancelLoadCommand;
 import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
+import org.apache.doris.nereids.trees.plans.commands.CancelWarmUpJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
@@ -223,6 +226,18 @@ default R visitShowCreateMTMVCommand(ShowCreateMTMVCommand showCreateMTMVCommand
         return visitCommand(showCreateMTMVCommand, context);
     }
 
+    default R visitCancelLoadCommand(CancelLoadCommand cancelLoadCommand, C context) {
+        return visitCommand(cancelLoadCommand, context);
+    }
+
+    default R visitCancelExportCommand(CancelExportCommand cancelExportCommand, C context) {
+        return visitCommand(cancelExportCommand, context);
+    }
+
+    default R visitCancelWarmUpJobCommand(CancelWarmUpJobCommand cancelWarmUpJobCommand, C context) {
+        return visitCommand(cancelWarmUpJobCommand, context);
+    }
+
     default R visitCancelMTMVTaskCommand(CancelMTMVTaskCommand cancelMTMVTaskCommand, C context) {
         return visitCommand(cancelMTMVTaskCommand, context);
     }

From c4288a0c14914e02077cd94ecdb3e8efb59d4706 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Thu, 28 Nov 2024 12:01:14 +0800
Subject: [PATCH 052/399] [feat](binlog) Support drop rollup binlog (#44677)

ref #43435

Related PR: https://github.com/selectdb/ccr-syncer/pull/269
---
 .../doris/alter/MaterializedViewHandler.java  | 16 +++++------
 .../apache/doris/binlog/BinlogManager.java    | 17 ++++++++++++
 .../org/apache/doris/binlog/DBBinlog.java     |  9 +++++++
 .../doris/datasource/InternalCatalog.java     |  4 +--
 .../apache/doris/persist/BatchDropInfo.java   | 12 +++++++--
 .../org/apache/doris/persist/DropInfo.java    | 20 ++++++++++++--
 .../org/apache/doris/persist/EditLog.java     | 27 ++++++++++++++-----
 .../doris/persist/DropAndRecoverInfoTest.java | 10 +++----
 gensrc/thrift/FrontendService.thrift          |  4 +--
 9 files changed, 90 insertions(+), 29 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index d02e91a379f560..a6f1cae9987678 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -944,14 +944,12 @@ public void processBatchDropRollup(List<AlterClause> dropRollupClauses, Database
             }
 
             // drop data in memory
-            Set<Long> indexIdSet = new HashSet<>();
-            Set<String> rollupNameSet = new HashSet<>();
+            Map<Long, String> rollupNameMap = new HashMap<>();
             for (AlterClause alterClause : dropRollupClauses) {
                 DropRollupClause dropRollupClause = (DropRollupClause) alterClause;
                 String rollupIndexName = dropRollupClause.getRollupName();
                 long rollupIndexId = dropMaterializedView(rollupIndexName, olapTable);
-                indexIdSet.add(rollupIndexId);
-                rollupNameSet.add(rollupIndexName);
+                rollupNameMap.put(rollupIndexId, rollupIndexName);
             }
 
             // batch log drop rollup operation
@@ -959,10 +957,10 @@ public void processBatchDropRollup(List<AlterClause> dropRollupClauses, Database
             long dbId = db.getId();
             long tableId = olapTable.getId();
             String tableName = olapTable.getName();
-            editLog.logBatchDropRollup(new BatchDropInfo(dbId, tableId, tableName, indexIdSet));
-            deleteIndexList = indexIdSet.stream().collect(Collectors.toList());
+            editLog.logBatchDropRollup(new BatchDropInfo(dbId, tableId, tableName, rollupNameMap));
+            deleteIndexList = rollupNameMap.keySet().stream().collect(Collectors.toList());
             LOG.info("finished drop rollup index[{}] in table[{}]",
-                    String.join("", rollupNameSet), olapTable.getName());
+                    String.join("", rollupNameMap.values()), olapTable.getName());
         } finally {
             olapTable.writeUnlock();
         }
@@ -982,8 +980,8 @@ public void processDropMaterializedView(DropMaterializedViewStmt dropMaterialize
             long mvIndexId = dropMaterializedView(mvName, olapTable);
             // Step3: log drop mv operation
             EditLog editLog = Env.getCurrentEnv().getEditLog();
-            editLog.logDropRollup(
-                    new DropInfo(db.getId(), olapTable.getId(), olapTable.getName(), mvIndexId, false, false, 0));
+            editLog.logDropRollup(new DropInfo(db.getId(), olapTable.getId(), olapTable.getName(),
+                    mvIndexId, mvName, false, false, 0));
             deleteIndexList.add(mvIndexId);
             LOG.info("finished drop materialized view [{}] in table [{}]", mvName, olapTable.getName());
         } catch (MetaNotFoundException e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
index 3a033c981038c2..67bb99a8bcdc18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
@@ -30,6 +30,7 @@
 import org.apache.doris.persist.BarrierLog;
 import org.apache.doris.persist.BatchModifyPartitionsInfo;
 import org.apache.doris.persist.BinlogGcInfo;
+import org.apache.doris.persist.DropInfo;
 import org.apache.doris.persist.DropPartitionInfo;
 import org.apache.doris.persist.ModifyCommentOperationLog;
 import org.apache.doris.persist.ModifyTablePropertyOperationLog;
@@ -429,6 +430,22 @@ public void addIndexChangeJob(IndexChangeJob indexChangeJob, long commitSeq) {
         addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false, indexChangeJob);
     }
 
+    public void addDropRollup(DropInfo info, long commitSeq) {
+        if (StringUtils.isEmpty(info.getIndexName())) {
+            LOG.warn("skip drop rollup binlog, because indexName is empty. info: {}", info);
+            return;
+        }
+
+        long dbId = info.getDbId();
+        List<Long> tableIds = Lists.newArrayList();
+        tableIds.add(info.getTableId());
+        long timestamp = -1;
+        TBinlogType type = TBinlogType.DROP_ROLLUP;
+        String data = info.toJson();
+
+        addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false, info);
+    }
+
     // get binlog by dbId, return first binlog.version > version
     public Pair<TStatus, TBinlog> getBinlog(long dbId, long tableId, long prevCommitSeq) {
         TStatus status = new TStatus(TStatusCode.OK);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
index c96e994be91c3a..b78ed389a0fe86 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
@@ -23,6 +23,7 @@
 import org.apache.doris.common.Pair;
 import org.apache.doris.common.proc.BaseProcResult;
 import org.apache.doris.persist.BarrierLog;
+import org.apache.doris.persist.DropInfo;
 import org.apache.doris.persist.DropPartitionInfo;
 import org.apache.doris.persist.ReplaceTableOperationLog;
 import org.apache.doris.thrift.TBinlog;
@@ -649,6 +650,9 @@ private void recordDroppedResources(TBinlogType binlogType, long commitSeq, Stri
                 case REPLACE_TABLE:
                     raw = ReplaceTableOperationLog.fromJson(data);
                     break;
+                case DROP_ROLLUP:
+                    raw = DropInfo.fromJson(data);
+                    break;
                 case BARRIER:
                     raw = BarrierLog.fromJson(data);
                     break;
@@ -693,6 +697,11 @@ private void recordDroppedResources(TBinlogType binlogType, long commitSeq, Obje
             if (!record.isSwapTable()) {
                 droppedTables.add(Pair.of(record.getOrigTblId(), commitSeq));
             }
+        } else if (binlogType == TBinlogType.DROP_ROLLUP && raw instanceof DropInfo) {
+            long indexId = ((DropInfo) raw).getIndexId();
+            if (indexId > 0) {
+                droppedIndexes.add(Pair.of(indexId, commitSeq));
+            }
         } else if (binlogType == TBinlogType.BARRIER && raw instanceof BarrierLog) {
             BarrierLog log = (BarrierLog) raw;
             // keep compatible with doris 2.0/2.1
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 18f2e333e9119b..bdc6626f63df81 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1015,7 +1015,7 @@ private void dropTableInternal(Database db, Table table, boolean isView, boolean
 
         Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getCurrentCatalog().getId(),
                 db.getId(), table.getId());
-        DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, -1L, isView, forceDrop, recycleTime);
+        DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, isView, forceDrop, recycleTime);
         Env.getCurrentEnv().getEditLog().logDropTable(info);
         Env.getCurrentEnv().getMtmvService().dropTable(table);
     }
@@ -3259,7 +3259,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
             try {
                 dropTable(db, tableId, true, false, 0L);
                 if (hadLogEditCreateTable) {
-                    DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, false, true, 0L);
+                    DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), false, true, 0L);
                     Env.getCurrentEnv().getEditLog().logDropTable(info);
                 }
             } catch (Exception ex) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/BatchDropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/BatchDropInfo.java
index fdfc44e27bbebc..260ad316d3cc24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/BatchDropInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/BatchDropInfo.java
@@ -26,6 +26,7 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -43,12 +44,15 @@ public class BatchDropInfo implements Writable {
     private String tableName; // not used in equals and hashCode
     @SerializedName(value = "indexIdSet")
     private Set<Long> indexIdSet;
+    @SerializedName(value = "indexNameMap")
+    private Map<Long, String> indexNameMap; // not used in equals and hashCode
 
-    public BatchDropInfo(long dbId, long tableId, String tableName, Set<Long> indexIdSet) {
+    public BatchDropInfo(long dbId, long tableId, String tableName, Map<Long, String> indexNameMap) {
         this.dbId = dbId;
         this.tableId = tableId;
         this.tableName = tableName;
-        this.indexIdSet = indexIdSet;
+        this.indexIdSet = indexNameMap.keySet();
+        this.indexNameMap = indexNameMap;
     }
 
     @Override
@@ -82,6 +86,10 @@ public Set<Long> getIndexIdSet() {
         return indexIdSet;
     }
 
+    public Map<Long, String> getIndexNameMap() {
+        return indexNameMap;
+    }
+
     public long getDbId() {
         return dbId;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
index 461f3ddd67d5a7..69994caf23d5dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
@@ -38,6 +38,8 @@ public class DropInfo implements Writable {
     private String tableName; // not used in equals and hashCode
     @SerializedName(value = "indexId")
     private long indexId;
+    @SerializedName(value = "indexName")
+    private String indexName; // not used in equals and hashCode
     @SerializedName(value = "isView")
     private boolean isView = false;
     @SerializedName(value = "forceDrop")
@@ -48,12 +50,18 @@ public class DropInfo implements Writable {
     public DropInfo() {
     }
 
-    public DropInfo(long dbId, long tableId, String tableName, long indexId, boolean isView, boolean forceDrop,
-                    long recycleTime) {
+    public DropInfo(long dbId, long tableId, String tableName, boolean isView, boolean forceDrop,
+            long recycleTime) {
+        this(dbId, tableId, tableName, -1, "", isView, forceDrop, recycleTime);
+    }
+
+    public DropInfo(long dbId, long tableId, String tableName, long indexId, String indexName, boolean isView,
+            boolean forceDrop, long recycleTime) {
         this.dbId = dbId;
         this.tableId = tableId;
         this.tableName = tableName;
         this.indexId = indexId;
+        this.indexName = indexName;
         this.isView = isView;
         this.forceDrop = forceDrop;
         this.recycleTime = recycleTime;
@@ -75,6 +83,10 @@ public long getIndexId() {
         return this.indexId;
     }
 
+    public String getIndexName() {
+        return this.indexName;
+    }
+
     public boolean isView() {
         return this.isView;
     }
@@ -133,4 +145,8 @@ public boolean equals(Object obj) {
     public String toJson() {
         return GsonUtils.GSON.toJson(this);
     }
+
+    public static DropInfo fromJson(String json) {
+        return GsonUtils.GSON.fromJson(json, DropInfo.class);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index 1e70eb634b2073..7d1f2127eecaaf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -105,6 +105,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 /**
  * EditLog maintains a log of the memory modifications.
@@ -341,15 +342,18 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) {
                 case OperationType.OP_DROP_ROLLUP: {
                     DropInfo info = (DropInfo) journal.getData();
                     env.getMaterializedViewHandler().replayDropRollup(info, env);
+                    env.getBinlogManager().addDropRollup(info, logId);
                     break;
                 }
                 case OperationType.OP_BATCH_DROP_ROLLUP: {
                     BatchDropInfo batchDropInfo = (BatchDropInfo) journal.getData();
-                    for (long indexId : batchDropInfo.getIndexIdSet()) {
-                        env.getMaterializedViewHandler().replayDropRollup(
-                                new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(),
-                                        batchDropInfo.getTableName(), indexId, false, false, 0),
-                                env);
+                    for (Map.Entry<Long, String> entry : batchDropInfo.getIndexNameMap().entrySet()) {
+                        long indexId = entry.getKey();
+                        String indexName = entry.getValue();
+                        DropInfo info = new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(),
+                                        batchDropInfo.getTableName(), indexId, indexName, false, false, 0);
+                        env.getMaterializedViewHandler().replayDropRollup(info, env);
+                        env.getBinlogManager().addDropRollup(info, logId);
                     }
                     break;
                 }
@@ -1463,11 +1467,20 @@ public void logRecoverTable(RecoverInfo info) {
     }
 
     public void logDropRollup(DropInfo info) {
-        logEdit(OperationType.OP_DROP_ROLLUP, info);
+        long logId = logEdit(OperationType.OP_DROP_ROLLUP, info);
+        Env.getCurrentEnv().getBinlogManager().addDropRollup(info, logId);
     }
 
     public void logBatchDropRollup(BatchDropInfo batchDropInfo) {
-        logEdit(OperationType.OP_BATCH_DROP_ROLLUP, batchDropInfo);
+        long logId = logEdit(OperationType.OP_BATCH_DROP_ROLLUP, batchDropInfo);
+        for (Map.Entry<Long, String> entry : batchDropInfo.getIndexNameMap().entrySet()) {
+            DropInfo info = new DropInfo(batchDropInfo.getDbId(),
+                    batchDropInfo.getTableId(),
+                    batchDropInfo.getTableName(),
+                    entry.getKey(), entry.getValue(),
+                    false, true, 0);
+            Env.getCurrentEnv().getBinlogManager().addDropRollup(info, logId);
+        }
     }
 
     public void logFinishConsistencyCheck(ConsistencyCheckInfo info) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
index 88aa22ded22e5e..8c74fba275352d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
@@ -44,7 +44,7 @@ public void testDropInfoSerialization() throws Exception {
         DropInfo info1 = new DropInfo();
         info1.write(dos);
 
-        DropInfo info2 = new DropInfo(1, 2, "t2", -1, false, true, 0);
+        DropInfo info2 = new DropInfo(1, 2, "t2", -1, "", false, true, 0);
         info2.write(dos);
 
         dos.flush();
@@ -65,10 +65,10 @@ public void testDropInfoSerialization() throws Exception {
 
         Assert.assertEquals(rInfo2, rInfo2);
         Assert.assertNotEquals(rInfo2, this);
-        Assert.assertNotEquals(info2, new DropInfo(0, 2, "t2", -1L, false, true, 0));
-        Assert.assertNotEquals(info2, new DropInfo(1, 0, "t0", -1L, false, true, 0));
-        Assert.assertNotEquals(info2, new DropInfo(1, 2, "t2", -1L, false, false, 0));
-        Assert.assertEquals(info2, new DropInfo(1, 2, "t2", -1L, false, true, 0));
+        Assert.assertNotEquals(info2, new DropInfo(0, 2, "t2", -1L, "", false, true, 0));
+        Assert.assertNotEquals(info2, new DropInfo(1, 0, "t0", -1L, "", false, true, 0));
+        Assert.assertNotEquals(info2, new DropInfo(1, 2, "t2", -1L, "", false, false, 0));
+        Assert.assertEquals(info2, new DropInfo(1, 2, "t2", -1L, "", false, true, 0));
 
         // 3. delete files
         dis.close();
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index fc7f98e16e3f74..e2af8937425d0c 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1196,6 +1196,7 @@ enum TBinlogType {
   INDEX_CHANGE_JOB = 20,
   RENAME_ROLLUP = 21,
   RENAME_PARTITION = 22,
+  DROP_ROLLUP = 23,
 
   // Keep some IDs for allocation so that when new binlog types are added in the
   // future, the changes can be picked back to the old versions without breaking
@@ -1212,8 +1213,7 @@ enum TBinlogType {
   //    MODIFY_XXX = 17,
   //    MIN_UNKNOWN = 18,
   //    UNKNOWN_3 = 19,
-  MIN_UNKNOWN = 23,
-  UNKNOWN_8 = 24,
+  MIN_UNKNOWN = 24,
   UNKNOWN_9 = 25,
   UNKNOWN_10 = 26,
   UNKNOWN_11 = 27,

From 6ba91dc39c763018f9030b877b435aa9ea704c28 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Thu, 28 Nov 2024 13:38:10 +0800
Subject: [PATCH 053/399] [fix](runtime filter) Avoid to use  ignored runtime
 filter (#44708)

### What problem does this PR solve?

Filter size should be set by a global size instead of a local size. This
behaviour is introduced by #43885 .
---
 be/src/exprs/runtime_filter.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 6eed49e8567e1c..efe96ae3395432 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -1114,9 +1114,6 @@ Status IRuntimeFilter::send_filter_size(RuntimeState* state, uint64_t local_filt
         std::lock_guard l(*local_merge_filters->lock);
         local_merge_filters->merge_size_times--;
         local_merge_filters->local_merged_size += local_filter_size;
-        if (_has_local_target) {
-            set_synced_size(local_filter_size);
-        }
         if (local_merge_filters->merge_size_times) {
             return Status::OK();
         } else {
@@ -1539,9 +1536,10 @@ void IRuntimeFilter::update_runtime_filter_type_to_profile(uint64_t local_merge_
 std::string IRuntimeFilter::debug_string() const {
     return fmt::format(
             "RuntimeFilter: (id = {}, type = {}, is_broadcast: {}, "
-            "build_bf_cardinality: {}, error_msg: {}",
+            "build_bf_cardinality: {}, ignored: {},  error_msg: {}",
             _filter_id, to_string(_runtime_filter_type), _is_broadcast_join,
-            _wrapper->get_build_bf_cardinality(), _wrapper->_context->err_msg);
+            _wrapper->get_build_bf_cardinality(), _wrapper->is_ignored(),
+            _wrapper->_context->err_msg);
 }
 
 Status IRuntimeFilter::merge_from(const RuntimePredicateWrapper* wrapper) {

From 47a74146bb57a657190d377a137a02c1d894f210 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Thu, 28 Nov 2024 11:31:31 +0530
Subject: [PATCH 054/399] [Enhancement] Support move truncated old data to
 recycle bin (#43107)

---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 fe/fe-core/src/main/cup/sql_parser.cup        |   4 +-
 .../doris/analysis/TruncateTableStmt.java     |  14 +-
 .../org/apache/doris/catalog/OlapTable.java   | 154 +++++++++++-------
 .../doris/catalog/RecyclePartitionParam.java  |  31 ++++
 .../doris/datasource/InternalCatalog.java     |  54 +++---
 .../doris/persist/TruncateTableInfo.java      |  10 +-
 .../doris/catalog/TruncateTableTest.java      |   2 +-
 .../test_insert_overwrite_multiple.out        |  15 ++
 .../test_truncate_multiple.out                |  16 ++
 .../test_truncate_recover.out                 |  22 +++
 .../test_truncate_recover_list.out            |  18 ++
 .../test_truncate_recover_no_partition.out    |  13 ++
 .../test_truncate_recover_restart_fe.out      |  27 +++
 .../data/ddl_p0/test_truncate_table_force.out |  12 ++
 .../test_insert_overwrite_multiple.groovy     |  68 ++++++++
 .../test_insert_overwrite_recover.groovy      |   2 +-
 ...sert_overwrite_recover_no_partition.groovy |   4 +-
 .../test_truncate_multiple.groovy             |  75 +++++++++
 .../test_truncate_recover.groovy              |  76 +++++++++
 .../test_truncate_recover_list.groovy         |  56 +++++++
 .../test_truncate_recover_no_partition.groovy |  76 +++++++++
 .../test_truncate_recover_restart_fe.groovy   |  90 ++++++++++
 .../suites/ddl_p0/test_truncate_table.groovy  |   5 +-
 .../ddl_p0/test_truncate_table_force.groovy   |  79 +++++++++
 25 files changed, 833 insertions(+), 92 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java
 create mode 100644 regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_multiple.out
 create mode 100644 regression-test/data/catalog_recycle_bin_p0/test_truncate_multiple.out
 create mode 100644 regression-test/data/catalog_recycle_bin_p0/test_truncate_recover.out
 create mode 100644 regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_list.out
 create mode 100644 regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_no_partition.out
 create mode 100644 regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_restart_fe.out
 create mode 100644 regression-test/data/ddl_p0/test_truncate_table_force.out
 create mode 100644 regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_multiple.groovy
 create mode 100644 regression-test/suites/catalog_recycle_bin_p0/test_truncate_multiple.groovy
 create mode 100644 regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover.groovy
 create mode 100644 regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_list.groovy
 create mode 100644 regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_no_partition.groovy
 create mode 100644 regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_restart_fe.groovy
 create mode 100644 regression-test/suites/ddl_p0/test_truncate_table_force.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 279329089a4ade..08f9058bbb14d6 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -869,7 +869,7 @@ unsupportedUseStatement
     ;
 
 unsupportedDmlStatement
-    : TRUNCATE TABLE multipartIdentifier specifiedPartition?                        #truncateTable
+    : TRUNCATE TABLE multipartIdentifier specifiedPartition?  FORCE?                 #truncateTable
     | COPY INTO name=multipartIdentifier columns=identifierList? FROM
         (stageAndPattern | (LEFT_PAREN SELECT selectColumnClause
             FROM stageAndPattern whereClause? RIGHT_PAREN))
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 1f4083d6ea0edc..03f1f02f158f84 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -7976,9 +7976,9 @@ admin_stmt ::=
     ;
 
 truncate_stmt ::=
-    KW_TRUNCATE KW_TABLE base_table_ref:tblRef
+    KW_TRUNCATE KW_TABLE base_table_ref:tblRef opt_force:force
     {:
-        RESULT = new TruncateTableStmt(tblRef);
+        RESULT = new TruncateTableStmt(tblRef, force);
     :}
     ;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
index 9543ff1853b742..f751934316d722 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
@@ -30,15 +30,21 @@
 public class TruncateTableStmt extends DdlStmt implements NotFallbackInParser {
 
     private TableRef tblRef;
+    private boolean forceDrop;
 
-    public TruncateTableStmt(TableRef tblRef) {
+    public TruncateTableStmt(TableRef tblRef, boolean forceDrop) {
         this.tblRef = tblRef;
+        this.forceDrop = forceDrop;
     }
 
     public TableRef getTblRef() {
         return tblRef;
     }
 
+    public boolean isForceDrop() {
+        return forceDrop;
+    }
+
     @Override
     public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
         super.analyze(analyzer);
@@ -75,6 +81,9 @@ public String toSql() {
         if (tblRef.getPartitionNames() != null) {
             sb.append(tblRef.getPartitionNames().toSql());
         }
+        if (isForceDrop()) {
+            sb.append(" FORCE");
+        }
         return sb.toString();
     }
 
@@ -83,6 +92,9 @@ public String toSqlWithoutTable() {
         if (tblRef.getPartitionNames() != null) {
             sb.append(tblRef.getPartitionNames().toSql());
         }
+        if (isForceDrop()) {
+            sb.append(" FORCE");
+        }
         return sb.toString();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 5d57540017f067..8925e483c29df7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1131,63 +1131,9 @@ private Partition dropPartition(long dbId, String partitionName, boolean isForce
         if (partition != null) {
             idToPartition.remove(partition.getId());
             nameToPartition.remove(partitionName);
-
-            if (!isForceDrop) {
-                // recycle partition
-                if (partitionInfo.getType() == PartitionType.RANGE) {
-                    Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
-                            partitionInfo.getItem(partition.getId()).getItems(),
-                            new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
-                            partitionInfo.getDataProperty(partition.getId()),
-                            partitionInfo.getReplicaAllocation(partition.getId()),
-                            partitionInfo.getIsInMemory(partition.getId()),
-                            partitionInfo.getIsMutable(partition.getId()));
-
-                } else if (partitionInfo.getType() == PartitionType.LIST) {
-                    // construct a dummy range
-                    List<Column> dummyColumns = new ArrayList<>();
-                    dummyColumns.add(new Column("dummy", PrimitiveType.INT));
-                    PartitionKey dummyKey = null;
-                    try {
-                        dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
-                    } catch (AnalysisException e) {
-                        LOG.warn("should not happen", e);
-                    }
-                    Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);
-
-                    Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
-                            dummyRange,
-                            partitionInfo.getItem(partition.getId()),
-                            partitionInfo.getDataProperty(partition.getId()),
-                            partitionInfo.getReplicaAllocation(partition.getId()),
-                            partitionInfo.getIsInMemory(partition.getId()),
-                            partitionInfo.getIsMutable(partition.getId()));
-                } else {
-                    // unpartition
-                    // construct a dummy range and dummy list.
-                    List<Column> dummyColumns = new ArrayList<>();
-                    dummyColumns.add(new Column("dummy", PrimitiveType.INT));
-                    PartitionKey dummyKey = null;
-                    try {
-                        dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
-                    } catch (AnalysisException e) {
-                        LOG.warn("should not happen", e);
-                    }
-                    Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);
-                    Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
-                            dummyRange,
-                            new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
-                            partitionInfo.getDataProperty(partition.getId()),
-                            partitionInfo.getReplicaAllocation(partition.getId()),
-                            partitionInfo.getIsInMemory(partition.getId()),
-                            partitionInfo.getIsMutable(partition.getId()));
-                }
-            } else if (!reserveTablets) {
-                Env.getCurrentEnv().onErasePartition(partition);
-            }
-
-            // drop partition info
-            partitionInfo.dropPartition(partition.getId());
+            RecyclePartitionParam recyclePartitionParam = new RecyclePartitionParam();
+            fillInfo(partition, recyclePartitionParam);
+            dropPartitionCommon(dbId, isForceDrop, recyclePartitionParam, partition, reserveTablets);
         }
         return partition;
     }
@@ -1200,6 +1146,81 @@ public Partition dropPartition(long dbId, String partitionName, boolean isForceD
         return dropPartition(dbId, partitionName, isForceDrop, !isForceDrop);
     }
 
+    private void dropPartitionCommon(long dbId, boolean isForceDrop,
+                                        RecyclePartitionParam recyclePartitionParam,
+                                        Partition partition,
+                                        boolean reserveTablets) {
+        if (!isForceDrop) {
+            // recycle partition
+            if (partitionInfo.getType() == PartitionType.RANGE) {
+                Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
+                        recyclePartitionParam.partitionItem.getItems(),
+                        new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
+                        recyclePartitionParam.dataProperty,
+                        recyclePartitionParam.replicaAlloc,
+                        recyclePartitionParam.isInMemory,
+                        recyclePartitionParam.isMutable);
+
+            } else if (partitionInfo.getType() == PartitionType.LIST) {
+                // construct a dummy range
+                List<Column> dummyColumns = new ArrayList<>();
+                dummyColumns.add(new Column("dummy", PrimitiveType.INT));
+                PartitionKey dummyKey = null;
+                try {
+                    dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
+                } catch (AnalysisException e) {
+                    LOG.warn("should not happen", e);
+                }
+                Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);
+
+                Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
+                        dummyRange,
+                        recyclePartitionParam.partitionItem,
+                        recyclePartitionParam.dataProperty,
+                        recyclePartitionParam.replicaAlloc,
+                        recyclePartitionParam.isInMemory,
+                        recyclePartitionParam.isMutable);
+            } else {
+                // unpartition
+                // construct a dummy range and dummy list.
+                List<Column> dummyColumns = new ArrayList<>();
+                dummyColumns.add(new Column("dummy", PrimitiveType.INT));
+                PartitionKey dummyKey = null;
+                try {
+                    dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
+                } catch (AnalysisException e) {
+                    LOG.warn("should not happen", e);
+                }
+                Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);
+                Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
+                        dummyRange,
+                        new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
+                        recyclePartitionParam.dataProperty,
+                        recyclePartitionParam.replicaAlloc,
+                        recyclePartitionParam.isInMemory,
+                        recyclePartitionParam.isMutable);
+            }
+        } else if (!reserveTablets) {
+            Env.getCurrentEnv().onErasePartition(partition);
+        }
+
+        // drop partition info
+        partitionInfo.dropPartition(partition.getId());
+    }
+
+    public Partition dropPartitionForTruncate(long dbId, boolean isForceDrop,
+                                            RecyclePartitionParam recyclePartitionParam) {
+        // 1. If "isForceDrop" is false, the partition will be added to the Catalog Recyle bin, and all tablets of this
+        //    partition will not be deleted.
+        // 2. If "ifForceDrop" is true, the partition will be dropped immediately
+        Partition partition = recyclePartitionParam.partition;
+        if (partition != null) {
+            idToPartition.remove(partition.getId());
+            dropPartitionCommon(dbId, isForceDrop, recyclePartitionParam, partition, false);
+        }
+        return partition;
+    }
+
     /*
      * A table may contain both formal and temporary partitions.
      * There are several methods to get the partition of a table.
@@ -2031,13 +2052,24 @@ public static OlapTable read(DataInput in) throws IOException {
         return GsonUtils.GSON.fromJson(Text.readString(in), OlapTable.class);
     }
 
+
+    public void fillInfo(Partition partition, RecyclePartitionParam recyclePartitionParam) {
+        recyclePartitionParam.dataProperty = partitionInfo.getDataProperty(partition.getId());
+        recyclePartitionParam.replicaAlloc = partitionInfo.getReplicaAllocation(partition.getId());
+        recyclePartitionParam.isInMemory = partitionInfo.getIsInMemory(partition.getId());
+        recyclePartitionParam.isMutable = partitionInfo.getIsMutable(partition.getId());
+        recyclePartitionParam.partitionItem = partitionInfo.getItem(partition.getId());
+        recyclePartitionParam.partition = partition;
+    }
+
     /*
      * this method is currently used for truncating table(partitions).
      * the new partition has new id, so we need to change all 'id-related' members
      *
      * return the old partition.
      */
-    public Partition replacePartition(Partition newPartition) {
+    public Partition replacePartition(Partition newPartition,
+                                        RecyclePartitionParam recyclePartitionParam) {
         Partition oldPartition = nameToPartition.remove(newPartition.getName());
         idToPartition.remove(oldPartition.getId());
 
@@ -2048,6 +2080,12 @@ public Partition replacePartition(Partition newPartition) {
         ReplicaAllocation replicaAlloc = partitionInfo.getReplicaAllocation(oldPartition.getId());
         boolean isInMemory = partitionInfo.getIsInMemory(oldPartition.getId());
         boolean isMutable = partitionInfo.getIsMutable(oldPartition.getId());
+        recyclePartitionParam.dataProperty = dataProperty;
+        recyclePartitionParam.replicaAlloc = replicaAlloc;
+        recyclePartitionParam.isInMemory = isInMemory;
+        recyclePartitionParam.isMutable = isMutable;
+        recyclePartitionParam.partitionItem = partitionInfo.getItem(oldPartition.getId());
+        recyclePartitionParam.partition = oldPartition;
 
         if (partitionInfo.getType() == PartitionType.RANGE
                 || partitionInfo.getType() == PartitionType.LIST) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java
new file mode 100644
index 00000000000000..51f38638938db5
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+public class RecyclePartitionParam {
+    public Partition partition;
+    public PartitionItem partitionItem;
+    public DataProperty dataProperty;
+    public ReplicaAllocation replicaAlloc;
+    public boolean isInMemory;
+    public boolean isMutable = true;
+
+    public RecyclePartitionParam() {
+        // do nothing.
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index bdc6626f63df81..1430d4a32b1789 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -103,6 +103,7 @@
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.RandomDistributionInfo;
 import org.apache.doris.catalog.RangePartitionItem;
+import org.apache.doris.catalog.RecyclePartitionParam;
 import org.apache.doris.catalog.Replica;
 import org.apache.doris.catalog.Replica.ReplicaState;
 import org.apache.doris.catalog.ReplicaAllocation;
@@ -3545,7 +3546,7 @@ private void checkAutoIncColumns(List<Column> columns, KeysType type) throws Ddl
     public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException {
         TableRef tblRef = truncateTableStmt.getTblRef();
         TableName dbTbl = tblRef.getName();
-
+        boolean isForceDrop = truncateTableStmt.isForceDrop();
         // check, and save some info which need to be checked again later
         Map<String, Long> origPartitions = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
         Map<Long, DistributionInfo> partitionsDistributionInfo = Maps.newHashMap();
@@ -3574,7 +3575,9 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti
                     }
                     // If need absolutely correct, should check running txn here.
                     // But if the txn is in prepare state, cann't known which partitions had load data.
-                    if (!partition.hasData()) {
+                    if ((isForceDrop) && (!partition.hasData())) {
+                        // if not force drop, then need to add partition to
+                        // recycle bin, so behavior for recover would be clear
                         continue;
                     }
                     origPartitions.put(partName, partition.getId());
@@ -3585,7 +3588,9 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti
                 for (Partition partition : olapTable.getPartitions()) {
                     // If need absolutely correct, should check running txn here.
                     // But if the txn is in prepare state, cann't known which partitions had load data.
-                    if (!partition.hasData()) {
+                    if ((isForceDrop) && (!partition.hasData())) {
+                        // if not force drop, then need to add partition to
+                        // recycle bin, so behavior for recover would be clear
                         continue;
                     }
                     origPartitions.put(partition.getName(), partition.getId());
@@ -3732,13 +3737,16 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti
                 throw new DdlException("Table[" + copiedTbl.getName() + "]'s meta has been changed. try again.");
             }
 
-            // replace
-            oldPartitions = truncateTableInternal(olapTable, newPartitions, truncateEntireTable);
+            //replace
+            Map<Long, RecyclePartitionParam> recyclePartitionParamMap  =  new HashMap<>();
+            oldPartitions = truncateTableInternal(olapTable, newPartitions,
+                                    truncateEntireTable, recyclePartitionParamMap, isForceDrop);
 
             // write edit log
             TruncateTableInfo info =
                     new TruncateTableInfo(db.getId(), db.getFullName(), olapTable.getId(), olapTable.getName(),
-                            newPartitions, truncateEntireTable, truncateTableStmt.toSqlWithoutTable(), oldPartitions);
+                            newPartitions, truncateEntireTable,
+                            truncateTableStmt.toSqlWithoutTable(), oldPartitions, isForceDrop);
             Env.getCurrentEnv().getEditLog().logTruncateTable(info);
         } catch (DdlException e) {
             failedCleanCallback.run();
@@ -3749,8 +3757,6 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti
             }
         }
 
-        erasePartitionDropBackendReplicas(oldPartitions);
-
         PartitionNames partitionNames = truncateEntireTable ? null
                 : new PartitionNames(false, tblRef.getPartitionNames().getPartitionNames());
         Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable, partitionNames);
@@ -3759,47 +3765,49 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti
     }
 
     private List<Partition> truncateTableInternal(OlapTable olapTable, List<Partition> newPartitions,
-            boolean isEntireTable) {
+            boolean isEntireTable, Map<Long, RecyclePartitionParam> recyclePartitionParamMap, boolean isforceDrop) {
         // use new partitions to replace the old ones.
         List<Partition> oldPartitions = Lists.newArrayList();
-        Set<Long> oldTabletIds = Sets.newHashSet();
         for (Partition newPartition : newPartitions) {
-            Partition oldPartition = olapTable.replacePartition(newPartition);
+            RecyclePartitionParam recyclePartitionParam = new RecyclePartitionParam();
+            Partition oldPartition = olapTable.replacePartition(newPartition, recyclePartitionParam);
             oldPartitions.add(oldPartition);
-            // save old tablets to be removed
-            for (MaterializedIndex index : oldPartition.getMaterializedIndices(IndexExtState.ALL)) {
-                index.getTablets().forEach(t -> {
-                    oldTabletIds.add(t.getId());
-                });
-            }
+            recyclePartitionParamMap.put(oldPartition.getId(), recyclePartitionParam);
         }
 
         if (isEntireTable) {
             Set<Long> oldPartitionsIds = oldPartitions.stream().map(Partition::getId).collect(Collectors.toSet());
             for (Partition partition : olapTable.getAllTempPartitions()) {
                 if (!oldPartitionsIds.contains(partition.getId())) {
+                    RecyclePartitionParam recyclePartitionParam = new RecyclePartitionParam();
+                    olapTable.fillInfo(partition, recyclePartitionParam);
                     oldPartitions.add(partition);
+                    recyclePartitionParamMap.put(partition.getId(), recyclePartitionParam);
+                    // clear temp partition from memory.
+                    // tablet may be moved to recycle bin or deleted inside
+                    // dropPartitionForTruncate function.
+                    olapTable.dropTempPartition(partition.getName(), false);
                 }
             }
-            // drop all temp partitions
-            olapTable.dropAllTempPartitions();
         }
 
-        // remove the tablets in old partitions
-        for (Long tabletId : oldTabletIds) {
-            Env.getCurrentInvertedIndex().deleteTablet(tabletId);
+        for (Map.Entry<Long, RecyclePartitionParam> pair : recyclePartitionParamMap.entrySet()) {
+            olapTable.dropPartitionForTruncate(olapTable.getDatabase().getId(), isforceDrop, pair.getValue());
         }
 
         return oldPartitions;
     }
 
     public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundException {
+        boolean isForceDrop = info.getForce();
         List<Partition> oldPartitions = Lists.newArrayList();
         Database db = (Database) getDbOrMetaException(info.getDbId());
         OlapTable olapTable = (OlapTable) db.getTableOrMetaException(info.getTblId(), TableType.OLAP);
         olapTable.writeLock();
         try {
-            truncateTableInternal(olapTable, info.getPartitions(), info.isEntireTable());
+            Map<Long, RecyclePartitionParam> recyclePartitionParamMap =  new HashMap<>();
+            truncateTableInternal(olapTable, info.getPartitions(), info.isEntireTable(),
+                                    recyclePartitionParamMap, isForceDrop);
 
             // add tablet to inverted index
             TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
index b252b2a38233f7..ffb59d2afdffd9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java
@@ -49,13 +49,15 @@ public class TruncateTableInfo implements Writable {
     private String rawSql = "";
     @SerializedName(value = "op")
     private Map<Long, String> oldPartitions = new HashMap<>();
+    @SerializedName(value = "force")
+    private boolean force = true; // older version it was forced always.
 
     public TruncateTableInfo() {
 
     }
 
     public TruncateTableInfo(long dbId, String db, long tblId, String table, List<Partition> partitions,
-            boolean isEntireTable, String rawSql, List<Partition> oldPartitions) {
+            boolean isEntireTable, String rawSql, List<Partition> oldPartitions, boolean force) {
         this.dbId = dbId;
         this.db = db;
         this.tblId = tblId;
@@ -66,6 +68,7 @@ public TruncateTableInfo(long dbId, String db, long tblId, String table, List<Pa
         for (Partition partition : oldPartitions) {
             this.oldPartitions.put(partition.getId(), partition.getName());
         }
+        this.force = force;
     }
 
     public long getDbId() {
@@ -96,6 +99,11 @@ public boolean isEntireTable() {
         return isEntireTable;
     }
 
+    public boolean getForce() {
+        return force;
+    }
+
+
     public String getRawSql() {
         return rawSql;
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
index 8c742787bcb2a1..346245de4ec09b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
@@ -111,7 +111,7 @@ public void testTruncateWithCaseInsensitivePartitionName() throws Exception {
         Env.getCurrentEnv().truncateTable(truncateTableStmt);
         Assert.assertNotEquals(p20211008Id, tbl.getPartition("P20211008").getId());
         // 2. truncate P20211007
-        truncateStr = "TRUNCATE TABLE test.case_sensitive_table PARTITION P20211007; \n";
+        truncateStr = "TRUNCATE TABLE test.case_sensitive_table PARTITION P20211007 force; \n";
         truncateTableStmt = (TruncateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
         Env.getCurrentEnv().truncateTable(truncateTableStmt);
         Assert.assertEquals(3, tbl.getPartitionInfo().idToDataProperty.size());
diff --git a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_multiple.out b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_multiple.out
new file mode 100644
index 00000000000000..64fd2a2dabd665
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_multiple.out
@@ -0,0 +1,15 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_check_1 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check__value_should_be_3 --
+3	a	2024-01-02
+
+-- !select_check__value_should_be_2 --
+2	a	2024-01-02
+
+-- !select_check__value_should_be_3 --
+3	a	2024-01-02
+
diff --git a/regression-test/data/catalog_recycle_bin_p0/test_truncate_multiple.out b/regression-test/data/catalog_recycle_bin_p0/test_truncate_multiple.out
new file mode 100644
index 00000000000000..0d267a16bbfe27
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/test_truncate_multiple.out
@@ -0,0 +1,16 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_check_1 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_2 --
+
+-- !select_check_3 --
+100	a	2022-01-02
+
+-- !select_check_4 --
+
+-- !select_check_5 --
+100	a	2022-01-02
+
diff --git a/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover.out b/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover.out
new file mode 100644
index 00000000000000..41caf633ac5699
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_check_1 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_2 --
+
+-- !select_check_3 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_4 --
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_5 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
diff --git a/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_list.out b/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_list.out
new file mode 100644
index 00000000000000..8bf63b404004b8
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_list.out
@@ -0,0 +1,18 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_check_1 --
+a	b
+c	d
+
+-- !select_check_2 --
+
+-- !select_check_3 --
+a	b
+c	d
+
+-- !select_check_4 --
+c	d
+
+-- !select_check_5 --
+a	b
+c	d
+
diff --git a/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_no_partition.out b/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_no_partition.out
new file mode 100644
index 00000000000000..f2e00edddb9cb1
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_no_partition.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_check_1 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_2 --
+
+-- !select_check_3 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
diff --git a/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_restart_fe.out b/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_restart_fe.out
new file mode 100644
index 00000000000000..a9eb9c09033083
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/test_truncate_recover_restart_fe.out
@@ -0,0 +1,27 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_check_1 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_2 --
+
+-- !select_check_3 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_4 --
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_5 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_6 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
diff --git a/regression-test/data/ddl_p0/test_truncate_table_force.out b/regression-test/data/ddl_p0/test_truncate_table_force.out
new file mode 100644
index 00000000000000..95ed72f34af5c9
--- /dev/null
+++ b/regression-test/data/ddl_p0/test_truncate_table_force.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_1 --
+2020-01-01	1.00	a	1
+2020-03-10	1.00	a	1
+
+-- !select_2 --
+
+-- !select_3 --
+2020-02-10	1.00	a	1
+
+-- !select_4 --
+
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_multiple.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_multiple.groovy
new file mode 100644
index 00000000000000..94bc3f96ec4433
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_multiple.groovy
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_insert_overwrite_multiple") {
+    def table = "test_insert_overwrite_multiple"
+
+    // create table and insert data
+    sql """ drop table if exists ${table}"""
+    sql """
+    create table ${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+
+    sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+    sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+    sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+    sql """ SYNC;"""
+
+    qt_select_check_1 """ select * from  ${table} order by id,name,da; """
+
+    sql """ insert overwrite  table ${table} values(3, 'a', '2024-01-02'); """
+
+    qt_select_check__value_should_be_3 """ select * from  ${table} order by id,name,da; """
+
+    sql """ insert overwrite  table ${table} values(2, 'a', '2024-01-02'); """
+
+    qt_select_check__value_should_be_2 """ select * from  ${table} order by id,name,da; """
+
+    sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
+
+    sql """ recover partition p3  from ${table}; """
+    sql """ recover partition p4  from ${table}; """
+    sql """ recover partition p5  from ${table}; """    
+
+    qt_select_check__value_should_be_3 """ select * from  ${table} order by id,name,da; """
+
+}
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover.groovy
index 71faaf849ba44a..2668f1b2a715b3 100644
--- a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover.groovy
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover.groovy
@@ -19,7 +19,7 @@ suite("test_insert_overwrite_recover") {
     def table = "test_insert_overwrite_recover"
 
     // create table and insert data
-    sql """ drop table if exists ${table} force"""
+    sql """ drop table if exists ${table}"""
     sql """
     create table ${table} (
         `id` int(11),
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
index f3123d71b8dbff..af5526a22b209a 100644
--- a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
@@ -17,7 +17,7 @@
 
 suite("test_insert_overwrite_recover_no_partition") {
     def table = "test_insert_overwrite_recover_no_partition"
-
+    def table_bk = "test_insert_overwrite_recover_no_partition_backup"
     // create table and insert data for range.
     sql """ drop table if exists ${table} force"""
     sql """
@@ -52,7 +52,7 @@ suite("test_insert_overwrite_recover_no_partition") {
     sql """ recover partition ${table} as p2  from ${table}; """
 
     // create a table to copy the data only for partition p2.
-    table_bk = "test_insert_overwrite_recover_no_partition_backup"
+
     sql """ drop table if exists ${table_bk} force"""
     sql """
     create table ${table_bk} (
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_truncate_multiple.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_multiple.groovy
new file mode 100644
index 00000000000000..e6c424f7a55a8d
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_multiple.groovy
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_truncate_multiple") {
+    def table = "test_truncate_multiple"
+
+    // create table and insert data
+    sql """ drop table if exists ${table}"""
+    sql """
+    create table ${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+
+    sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+    sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+    sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+    sql """ SYNC;"""
+
+    qt_select_check_1 """ select * from  ${table} order by id,name,da; """
+
+    sql """ truncate  table ${table}; """
+
+    qt_select_check_2 """ select * from  ${table} order by id,name,da; """
+
+    // add only one record, so that some paritions can be empty
+    sql """ insert into ${table} values(100, 'a', '2022-01-02'); """
+    // check empty partition drop
+    sql """ SYNC;"""
+    qt_select_check_3 """ select * from  ${table} order by id,name,da; """    
+    sql """ truncate  table ${table}; """
+
+    qt_select_check_4 """ select * from  ${table} order by id,name,da; """    
+    // after truncate , there would be new partition created,
+    // drop forcefully so that this partitions not kept in recycle bin.
+    sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
+
+    // recover the data from the recycel bin , data present before truncate.
+    sql """ recover partition p3  from ${table}; """
+    sql """ recover partition p4  from ${table}; """
+    sql """ recover partition p5  from ${table}; """    
+
+    // data should match whatever before truncate.
+    qt_select_check_5 """ select * from  ${table} order by id,name,da; """
+}
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover.groovy
new file mode 100644
index 00000000000000..fd603e1e8494a5
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover.groovy
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_truncate_recover") {
+    def table = "test_truncate_recover"
+
+    // create table and insert data
+    sql """ drop table if exists ${table}"""
+    sql """
+    create table ${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+
+    sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+    sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+    sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+    sql """ SYNC;"""
+
+    qt_select_check_1 """ select * from  ${table} order by id,name,da; """
+
+    sql """ truncate  table ${table}; """
+
+    qt_select_check_2 """ select * from  ${table} order by id,name,da; """
+    
+    // after truncate , there would be new partition created,
+    // drop forcefully so that this partitions not kept in recycle bin.
+    sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
+
+    // recover the data from the recycel bin , data present before truncate.
+    sql """ recover partition p3  from ${table}; """
+    sql """ recover partition p4  from ${table}; """
+    sql """ recover partition p5  from ${table}; """    
+
+    // data should match whatever before truncate.
+    qt_select_check_3 """ select * from  ${table} order by id,name,da; """
+    
+    // now lets test partition.
+    sql """ truncate  table ${table} PARTITION(p3); """
+    qt_select_check_4 """ select * from  ${table} order by id,name,da; """ 
+    // drop forcefully the partition which is created in the truncate partition process.
+    sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
+    // recover data from the partition before truncate partition process.
+    sql """ recover partition p3  from ${table}; """
+    qt_select_check_5 """ select * from  ${table} order by id,name,da; """
+}
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_list.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_list.groovy
new file mode 100644
index 00000000000000..08013aa70b9748
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_list.groovy
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_truncate_recover_list") {
+    def table = "test_truncate_recover_list"
+
+    // create table and insert data
+    sql """ drop table if exists ${table}"""
+    sql """
+        CREATE TABLE  ${table} (k1 CHAR(5) NOT NULL, v1 CHAR(5) REPLACE NOT NULL )
+        AGGREGATE KEY(k1)
+        PARTITION BY LIST(k1) ( PARTITION p1 VALUES IN ("a","b"), PARTITION p2 VALUES IN ("c","d") )
+        DISTRIBUTED BY HASH(k1) BUCKETS 5 PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+        """
+    sql "insert into ${table} values('a', 'b')"
+    sql "insert into ${table} values('c', 'd')"
+
+    sql """ SYNC;"""
+
+    qt_select_check_1 """ select * from  ${table} order by k1,v1; """
+
+    sql """ truncate  table ${table}; """
+
+    qt_select_check_2 """ select * from  ${table} order by k1,v1; """
+    // after truncate , there would be new partition created,
+    // drop forcefully so that this partitions not kept in recycle bin.    
+    sql """ ALTER TABLE ${table} DROP PARTITION p1 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p2 force; """
+
+    sql """ recover partition p1  from ${table}; """
+    sql """ recover partition p2  from ${table}; """
+
+    qt_select_check_3 """ select * from  ${table} order by k1,v1; """
+    
+    sql """ truncate  table ${table} PARTITION(p1); """
+    qt_select_check_4 """ select * from  ${table} order by k1,v1; """
+    // drop forcefully the partition which is created in the truncate partition process.
+    sql """ ALTER TABLE ${table} DROP PARTITION p1 force; """
+    sql """ recover partition p1  from ${table}; """
+    qt_select_check_5 """ select * from  ${table} order by k1,v1; """   
+
+}
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_no_partition.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_no_partition.groovy
new file mode 100644
index 00000000000000..5f49211fc774a1
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_no_partition.groovy
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_truncate_recover_no_partition") {
+    def table = "test_truncate_recover_no_partition"
+    def table_bk = "test_truncate_no_partition_backup"
+    // create table and insert data for range.
+    sql """ drop table if exists ${table}"""
+    sql """
+    create table ${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+
+    sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+    sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+    sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+    sql """ SYNC;"""
+
+    qt_select_check_1 """ select * from  ${table} order by id,name,da; """
+
+    sql """ truncate  table ${table}; """
+
+    
+    qt_select_check_2 """ select * from  ${table} order by id,name,da; """
+    
+    // now unpartition data is kept inside the recycle bin.
+    // we need to recover it as another partition in the table.
+    sql """ recover partition ${table} as p2  from ${table}; """
+
+    // create a table to copy the data only for partition p2.
+    sql """ drop table if exists ${table_bk} force"""
+    sql """
+    create table ${table_bk} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+    sql """ insert into ${table_bk} select * from ${table} partition p2; """
+
+    sql """ alter table ${table} replace with table ${table_bk}; """
+
+    // data from the select should be same as data before truncate
+    qt_select_check_3 """ select * from  ${table} order by id,name,da; """
+}
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_restart_fe.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_restart_fe.groovy
new file mode 100644
index 00000000000000..f604f7bd4ef38d
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_truncate_recover_restart_fe.groovy
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import org.apache.doris.regression.suite.ClusterOptions
+import org.apache.doris.regression.util.NodeType
+
+suite("test_truncate_recover_restart_fe", 'docker') {
+    def options = new ClusterOptions()
+    options.setFeNum(1)
+    options.enableDebugPoints()
+    options.feConfigs.add('publish_wait_time_second=-1')
+    options.feConfigs.add('commit_timeout_second=10')
+    options.feConfigs.add('sys_log_verbose_modules=org.apache.doris')
+    options.beConfigs.add('enable_java_support=false')
+    docker(options) {
+        def table = "test_truncate_recover_restart_fe"
+        // create table and insert data
+        sql """ drop table if exists ${table}"""
+        sql """
+        create table ${table} (
+            `id` int(11),
+            `name` varchar(128),
+            `da` date
+        )
+        engine=olap
+        duplicate key(id)
+        partition by range(da)(
+            PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+            PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+            PARTITION p5 VALUES LESS THAN ('2025-01-01')
+        )
+        distributed by hash(id) buckets 2
+        properties(
+            "replication_num"="1",
+            "light_schema_change"="true"
+        );
+        """
+
+        sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+        sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+        sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+        sql """ SYNC;"""
+
+        qt_select_check_1 """ select * from  ${table} order by id,name,da; """
+
+        sql """ truncate  table ${table}; """
+        cluster.restartFrontends()
+        sleep(30000)
+        context.reconnectFe()
+        qt_select_check_2 """ select * from  ${table} order by id,name,da; """
+        
+        // after truncate , there would be new partition created,
+        // drop forcefully so that this partitions not kept in recycle bin.
+        sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
+        sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
+        sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
+
+        sql """ recover partition p3  from ${table}; """
+        sql """ recover partition p4  from ${table}; """
+        sql """ recover partition p5  from ${table}; """    
+
+        qt_select_check_3 """ select * from  ${table} order by id,name,da; """
+        
+        sql """ truncate  table ${table} PARTITION(p3); """
+
+        qt_select_check_4 """ select * from  ${table} order by id,name,da; """ 
+        // drop forcefully the partition which is created in the truncate partition process.
+        sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
+        sql """ recover partition p3  from ${table}; """
+        qt_select_check_5 """ select * from  ${table} order by id,name,da; """    
+
+        cluster.restartFrontends()
+        sleep(30000)
+        context.reconnectFe()        
+        qt_select_check_6 """ select * from  ${table} order by id,name,da; """    
+    }
+}
diff --git a/regression-test/suites/ddl_p0/test_truncate_table.groovy b/regression-test/suites/ddl_p0/test_truncate_table.groovy
index 7a79d04c70e2b7..c4f93e586113f1 100644
--- a/regression-test/suites/ddl_p0/test_truncate_table.groovy
+++ b/regression-test/suites/ddl_p0/test_truncate_table.groovy
@@ -53,11 +53,12 @@ suite("test_truncate_table") {
     sql "insert into ${testTable} values ('2020-03-10', 1.0, 'a', 1)"
     order_qt_select_1 "SELECT * FROM ${testTable}"
 
+    // if truncate without force, empty partions also kept in recycle bin.
     sql """truncate table ${testTable};"""
     def partitionIds2 = getPartitionIds()
     assertEquals(["p1", "p2", "p3"].toSet(), partitionIds2.keySet())
     assertNotEquals(partitionIds1.get("p1"), partitionIds2.get("p1"))
-    assertEquals(partitionIds1.get("p2"), partitionIds2.get("p2"))
+    assertNotEquals(partitionIds1.get("p2"), partitionIds2.get("p2"))
     assertNotEquals(partitionIds1.get("p3"), partitionIds2.get("p3"))
     order_qt_select_2 "SELECT * FROM ${testTable}"
 
@@ -68,7 +69,7 @@ suite("test_truncate_table") {
 
     def partitionIds3 = getPartitionIds()
     assertEquals(["p1", "p2", "p3"].toSet(), partitionIds3.keySet())
-    assertEquals(partitionIds2.get("p1"), partitionIds3.get("p1"))
+    assertNotEquals(partitionIds2.get("p1"), partitionIds3.get("p1"))
     assertNotEquals(partitionIds2.get("p2"), partitionIds3.get("p2"))
     assertEquals(partitionIds2.get("p3"), partitionIds3.get("p3"))
 
diff --git a/regression-test/suites/ddl_p0/test_truncate_table_force.groovy b/regression-test/suites/ddl_p0/test_truncate_table_force.groovy
new file mode 100644
index 00000000000000..f14134b9823d59
--- /dev/null
+++ b/regression-test/suites/ddl_p0/test_truncate_table_force.groovy
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_truncate_table_force") {
+    def testTable = "test_truncate_table_force"
+
+    sql "DROP TABLE IF EXISTS ${testTable}"
+
+    sql """
+        CREATE TABLE ${testTable}
+		(
+			k1 DATE,
+			k2 DECIMAL(10, 2) DEFAULT "10.5",
+			k3 CHAR(10) COMMENT "string column",
+			k4 INT NOT NULL DEFAULT "1" COMMENT "int column"
+		)
+		PARTITION BY RANGE(k1)
+		(
+			PARTITION p1 VALUES LESS THAN ("2020-02-01"),
+			PARTITION p2 VALUES LESS THAN ("2020-03-01"),
+			PARTITION p3 VALUES LESS THAN ("2020-04-01")
+		)
+		DISTRIBUTED BY HASH(k2) BUCKETS 32
+		PROPERTIES (
+			"replication_num" = "1"
+		);
+		"""
+
+    def getPartitionIds = { ->
+        def result = sql_return_maparray("show partitions from ${testTable}")
+        return result.collectEntries { [it.PartitionName, it.PartitionId as long] }
+    }
+
+    def partitionIds1 = getPartitionIds()
+    assertEquals(["p1", "p2", "p3"].toSet(), partitionIds1.keySet())
+
+    sql "insert into ${testTable} values ('2020-01-01', 1.0, 'a', 1)"
+    sql "insert into ${testTable} values ('2020-03-10', 1.0, 'a', 1)"
+    order_qt_select_1 "SELECT * FROM ${testTable}"
+
+    sql """truncate table ${testTable} force;"""
+    // if we use force, the empty partitions skiped, 
+    // so that partition ID doesnt change.
+    def partitionIds2 = getPartitionIds()
+    assertEquals(["p1", "p2", "p3"].toSet(), partitionIds2.keySet())
+    assertNotEquals(partitionIds1.get("p1"), partitionIds2.get("p1"))
+    assertEquals(partitionIds1.get("p2"), partitionIds2.get("p2"))
+    assertNotEquals(partitionIds1.get("p3"), partitionIds2.get("p3"))
+    order_qt_select_2 "SELECT * FROM ${testTable}"
+
+    sql "insert into ${testTable} values ('2020-02-10', 1.0, 'a', 1)"
+    order_qt_select_3 "SELECT * FROM ${testTable}"
+    sql """truncate table ${testTable} partitions (p1, p2) force;"""
+    order_qt_select_4 "SELECT * FROM ${testTable}"
+
+    def partitionIds3 = getPartitionIds()
+    assertEquals(["p1", "p2", "p3"].toSet(), partitionIds3.keySet())
+    assertEquals(partitionIds2.get("p1"), partitionIds3.get("p1"))
+    assertNotEquals(partitionIds2.get("p2"), partitionIds3.get("p2"))
+    assertEquals(partitionIds2.get("p3"), partitionIds3.get("p3"))
+
+    sql "DROP TABLE IF EXISTS ${testTable}"
+}
+

From 325a67c736ee202ac03d231899135819df2d7743 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Thu, 28 Nov 2024 11:36:21 +0530
Subject: [PATCH 055/399] [chore](test) Add info about buildKit required with
 docker-compose (#44095)

got error when trying to create docker cluster using private image.
adding more info so that can help others.
---
 docker/runtime/doris-compose/Readme.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/runtime/doris-compose/Readme.md b/docker/runtime/doris-compose/Readme.md
index c4c4dc0990f0fc..fea13ab1821882 100644
--- a/docker/runtime/doris-compose/Readme.md
+++ b/docker/runtime/doris-compose/Readme.md
@@ -32,6 +32,8 @@ docker run hello-world
 
 if have problem with permission denied, then [add-docker-permission](https://docs.docker.com/engine/install/linux-postinstall/).
 
+Make sure BuildKit configured in the machine. if not follow [docker-with-BuildKit](https://docs.docker.com/build/buildkit/).
+
 ##### 2. The doris image should contains
 
 ```

From 96079bcc127f83af91b53ab0b4f8f70306d0a5a2 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Thu, 28 Nov 2024 11:37:08 +0530
Subject: [PATCH 056/399] [Enhancement](recyclebin) drop table keep temp
 partitions in recycle-bin (#44097)

---
 .../doris/datasource/InternalCatalog.java     |  4 --
 .../test_temp_partitions_table.out            | 28 +++++++++
 .../test_temp_partitions_table.groovy         | 63 +++++++++++++++++++
 3 files changed, 91 insertions(+), 4 deletions(-)
 create mode 100644 regression-test/data/catalog_recycle_bin_p0/test_temp_partitions_table.out
 create mode 100644 regression-test/suites/catalog_recycle_bin_p0/test_temp_partitions_table.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 1430d4a32b1789..64a759b9747044 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1037,10 +1037,6 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop,
             long recycleTime) {
         if (table.getType() == TableType.ELASTICSEARCH) {
             esRepository.deRegisterTable(table.getId());
-        } else if (table.isManagedTable()) {
-            // drop all temp partitions of this table, so that there is no temp partitions in recycle bin,
-            // which make things easier.
-            ((OlapTable) table).dropAllTempPartitions();
         }
         if (table.getType() == TableType.MATERIALIZED_VIEW) {
             Env.getCurrentEnv().getMtmvService().deregisterMTMV((MTMV) table);
diff --git a/regression-test/data/catalog_recycle_bin_p0/test_temp_partitions_table.out b/regression-test/data/catalog_recycle_bin_p0/test_temp_partitions_table.out
new file mode 100644
index 00000000000000..c1715c55c89247
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/test_temp_partitions_table.out
@@ -0,0 +1,28 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql_1 --
+1	1	1
+
+-- !sql_2 --
+7	1	3
+
+-- !sql_3 --
+16	1234	t
+
+-- !sql_recover1 --
+1	1	1
+
+-- !sql_recover2 --
+7	1	3
+
+-- !sql_recover3 --
+16	1234	t
+
+-- !sql_newrecover1 --
+1	1	1
+
+-- !sql_newrecover2 --
+7	1	3
+
+-- !sql_newrecover3 --
+16	1234	t
+
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_temp_partitions_table.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_temp_partitions_table.groovy
new file mode 100644
index 00000000000000..e1e68846c52bf6
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_temp_partitions_table.groovy
@@ -0,0 +1,63 @@
+  
+  // Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_temp_partitions_table") {
+    def table = "test_temp_partitions_table"
+    def table2 = "test_temp_partitions_table_2"
+   
+   sql """
+        CREATE TABLE ${table}  (
+            id BIGINT,
+            val BIGINT,
+            str VARCHAR(114)
+        ) DUPLICATE KEY(`id`)
+        PARTITION BY RANGE(`id`)
+        (
+            PARTITION `p1` VALUES LESS THAN ('5'),
+            PARTITION `p2` VALUES LESS THAN ('10')
+        )
+        DISTRIBUTED BY HASH(`id`) BUCKETS 3
+        PROPERTIES (
+        "replication_num"="1"
+        );
+    """
+
+    sql """ALTER TABLE ${table}   ADD TEMPORARY PARTITION tp1 VALUES [("15"), ("20"));"""
+
+    sql "INSERT INTO ${table}   VALUES(1, 1,'1')"
+    sql "INSERT INTO ${table}   VALUES(7, 1,'3')"
+    sql "INSERT INTO ${table}   TEMPORARY PARTITION(tp1) values(16,1234, 't');"
+    sql "SYNC"
+    qt_sql_1 "SELECT * FROM ${table}   PARTITION p1"
+    qt_sql_2 "SELECT * FROM ${table}   PARTITION p2"
+    qt_sql_3 """select * from ${table}   temporary partition(tp1);"""
+    sql "DROP table ${table} "
+    sql "recover table ${table} "
+
+    qt_sql_recover1 "SELECT * FROM ${table}   PARTITION p1"
+    qt_sql_recover2 "SELECT * FROM ${table}   PARTITION p2"
+    qt_sql_recover3 """select * from ${table}   temporary partition(tp1);"""
+
+    sql "DROP table ${table} "
+    sql "recover table ${table}  as ${table2}"    
+
+    qt_sql_newrecover1 "SELECT * FROM ${table2}   PARTITION p1"
+    qt_sql_newrecover2 "SELECT * FROM ${table2}   PARTITION p2"
+    qt_sql_newrecover3 """select * from ${table2}   temporary partition(tp1);"""
+
+}
\ No newline at end of file

From 5c36328b1008dd306e6caa80a67e7071ab8427c4 Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Thu, 28 Nov 2024 14:15:39 +0800
Subject: [PATCH 057/399] [fix](case) fix some cluster key case (#44674)

---
 .../org/apache/doris/regression/Config.groovy | 29 ++++++-
 .../doris/regression/suite/Suite.groovy       |  4 +
 .../insert_p0/transaction/txn_insert.groovy   |  2 +-
 .../suites/nereids_function_p0/load.groovy    |  5 +-
 .../test_col_data_type_boundary.groovy        | 79 +++++++++++++++----
 .../point_query_p0/test_point_query_ck.groovy |  5 +-
 ...l_update_insert_light_schema_change.groovy |  3 +
 ...partial_update_insert_schema_change.groovy |  3 +
 .../test_partial_update_schema_change.groovy  |  3 +
 ...tial_update_schema_change_row_store.groovy |  3 +
 .../test_partial_update_seq_col.groovy        |  3 +
 .../test_partial_update_seq_col_delete.groovy |  3 +
 .../test_partial_update_seq_type.groovy       |  3 +
 ...test_partial_update_seq_type_delete.groovy |  3 +
 .../test_partial_update_strict_mode.groovy    |  3 +
 .../test_partial_update_upsert.groovy         |  3 +
 ..._partial_update_with_inverted_index.groovy |  3 +
 ...test_partial_update_with_row_column.groovy |  4 +
 .../test_update_on_current_timestamp.groovy   |  4 +
 .../test_create_table.groovy                  |  4 +
 .../test_schema_change_ck.groovy              |  6 +-
 21 files changed, 152 insertions(+), 23 deletions(-)

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 7b0d24482a29a1..5aa6d7ce7e7218 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -524,7 +524,7 @@ class Config {
             // docker suite no need external cluster.
             // so can ignore error here.
         }
-
+        config.excludeUnsupportedCase()
         return config
     }
 
@@ -1008,6 +1008,33 @@ class Config {
         }
     }
 
+    boolean isClusterKeyEnabled() {
+        try {
+            def result = JdbcUtils.executeToMapArray(getRootConnection(), "SHOW FRONTEND CONFIG LIKE 'random_add_cluster_keys_for_mow'")
+            log.info("show random_add_cluster_keys_for_mow config: ${result}".toString())
+            return result[0].Value.toString().equalsIgnoreCase("true")
+        } catch (Throwable t) {
+            log.warn("Fetch server config 'random_add_cluster_keys_for_mow' failed, jdbcUrl: ${jdbcUrl}".toString(), t)
+            return false
+        }
+    }
+
+    void excludeUnsupportedCase() {
+        boolean isCKEnabled = isClusterKeyEnabled()
+        log.info("random_add_cluster_keys_for_mow in fe.conf: ${isCKEnabled}".toString())
+        if (isCKEnabled) {
+            excludeDirectorySet.add("unique_with_mow_p0/partial_update")
+            excludeDirectorySet.add("unique_with_mow_p0/flexible")
+            excludeDirectorySet.add("doc")
+            List<String> excludeCases = ["test_table_properties", "test_default_hll", "test_default_pi", "test_full_compaction", "test_full_compaction_by_table_id", "test_create_table", "txn_insert", "test_update_mow", "test_new_update", "test_update_unique", "test_partial_update_generated_column", "nereids_partial_update_native_insert_stmt", "partial_update", "nereids_update_on_current_timestamp", "update_on_current_timestamp", "test_default_bitmap_empty", "nereids_delete_mow_partial_update", "delete_mow_partial_update", "partial_update_seq_col", "nereids_partial_update_native_insert_stmt_complex", "regression_test_variant_delete_and_update", "test_unique_table_auto_inc_partial_update_correct_stream_load", "test_unique_table_auto_inc", "test_unique_table_auto_inc_partial_update_correct_insert", "test_update_schema_change"]
+            for (def excludeCase in excludeCases) {
+                excludeSuiteWildcard.add(excludeCase)
+            }
+            log.info("excludeDirectorySet: ${excludeDirectorySet}".toString())
+            log.info("excludeSuiteWildcard: ${excludeSuiteWildcard}".toString())
+        }
+    }
+
     Connection getConnection() {
         return DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword)
     }
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index bd16342268f1e3..2a5ba5acc414b7 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1641,6 +1641,10 @@ class Suite implements GroovyInterceptable {
         }
     }
 
+    boolean isClusterKeyEnabled() {
+        return getFeConfig("random_add_cluster_keys_for_mow").equals("true")
+    }
+
     boolean enableStoragevault() {
         if (Strings.isNullOrEmpty(context.config.metaServiceHttpAddress)
                 || Strings.isNullOrEmpty(context.config.instanceId)
diff --git a/regression-test/suites/insert_p0/transaction/txn_insert.groovy b/regression-test/suites/insert_p0/transaction/txn_insert.groovy
index a4868ca6b11ffe..b54fc66e714fbd 100644
--- a/regression-test/suites/insert_p0/transaction/txn_insert.groovy
+++ b/regression-test/suites/insert_p0/transaction/txn_insert.groovy
@@ -717,7 +717,7 @@ suite("txn_insert") {
         }
 
         // 18. column update(mow table)
-        if (use_nereids_planner) {
+        if (use_nereids_planner && !isClusterKeyEnabled()) {
             def unique_table = "txn_insert_cu"
             for (def i in 0..3) {
                 sql """ drop table if exists ${unique_table}_${i} """
diff --git a/regression-test/suites/nereids_function_p0/load.groovy b/regression-test/suites/nereids_function_p0/load.groovy
index d4aeea530412cb..2777774449e3e4 100644
--- a/regression-test/suites/nereids_function_p0/load.groovy
+++ b/regression-test/suites/nereids_function_p0/load.groovy
@@ -155,6 +155,7 @@ suite("load") {
         }
     }
 
+    if (!isClusterKeyEnabled()) {
     // test fn_test_ip_nullable_rowstore table with update action
     sql "update fn_test_ip_nullable_rowstore set ip4 = '' where id = 1;"
     sql_res = sql "select * from fn_test_ip_nullable_rowstore where id = 1;"
@@ -169,6 +170,7 @@ suite("load") {
     sql "update fn_test_ip_nullable_rowstore set ip6 = '::1' where id = 1;"
     sql_res = sql "select * from fn_test_ip_nullable_rowstore where id = 1;"
     assertEquals(sql_res[0].toString(), '[1, 127.0.0.1, ::1, "127.0.0.1", "::1"]')
+    }
 
     streamLoad {
         table "fn_test_ip_not_nullable"
@@ -218,6 +220,7 @@ suite("load") {
         }
     }
 
+    if (!isClusterKeyEnabled()) {
     // test fn_test_ip_not_nullable_rowstore table with update action
     // not null will throw exception if we has data in table
     test {
@@ -237,7 +240,7 @@ suite("load") {
     sql "update fn_test_ip_not_nullable_rowstore set ip6 = '::2' where id = 1;"
     sql_res1 = sql "select * from fn_test_ip_not_nullable_rowstore where id = 1;"
     assertEquals(sql_res1[0].toString(), '[1, 192.10.10.1, ::2, "127.0.0.1", "::1"]')
-
+    }
 
     // make some normal ipv4/ipv6 data for sql function , which is increased one by one
     // 29-50 A 类地址 ; 51-68 B 类地址 ; 69-87 C 类地址 ; 88-100 D 类地址
diff --git a/regression-test/suites/partition_p0/auto_partition/test_col_data_type_boundary.groovy b/regression-test/suites/partition_p0/auto_partition/test_col_data_type_boundary.groovy
index 47fe118860e71a..65098e0fd23f20 100644
--- a/regression-test/suites/partition_p0/auto_partition/test_col_data_type_boundary.groovy
+++ b/regression-test/suites/partition_p0/auto_partition/test_col_data_type_boundary.groovy
@@ -68,7 +68,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_tinyint modify column c_tinyint largeint key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
     sql """drop table if exists table_smallint"""
@@ -97,7 +101,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_smallint modify column c_smallint largeint key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
     sql """drop table if exists table_int"""
@@ -126,7 +134,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_int modify column c_int largeint key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
     sql """drop table if exists table_bigint"""
@@ -155,7 +167,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_bigint modify column c_bigint largeint key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
     sql """drop table if exists table_largeint"""
@@ -225,7 +241,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_date_range modify column c_date datetime key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
 
@@ -249,7 +269,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_date_list modify column c_date datetime key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
     sql """drop table if exists table_datetime_range"""
@@ -274,7 +298,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_datetime_range modify column c_datetime date key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
     sql """drop table if exists table_datetime_list"""
@@ -299,7 +327,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_datetime_list modify column c_datetime date key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
 
@@ -323,7 +355,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_char modify column c_char varchar key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
 
@@ -347,7 +383,11 @@ suite("test_col_data_type_boundary") {
         sql """alter table table_varchar modify column c_varchar date key;"""
     } catch (Exception e) {
         log.info(e.getMessage())
-        assertTrue(e.getMessage().contains("Can not modify partition column"))
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            assertTrue(e.getMessage().contains("Can not modify partition column"))
+        }
     }
 
 
@@ -371,10 +411,19 @@ suite("test_col_data_type_boundary") {
     assertEquals(select_rows[0][0], 3)
     assertEquals(partitions_res1.size(), 2)
 
-    sql """alter table table_datetime_range modify column c_int largeint;"""
-    partitions_res1 = sql """show partitions from table_datetime_range order by PartitionId;"""
-    select_rows = sql """select count() from table_datetime_range;"""
-    assertEquals(select_rows[0][0], 3)
-    assertEquals(partitions_res1.size(), 2)
+    try {
+        sql """alter table table_datetime_range modify column c_int largeint;"""
+        partitions_res1 = sql """show partitions from table_datetime_range order by PartitionId;"""
+        select_rows = sql """select count() from table_datetime_range;"""
+        assertEquals(select_rows[0][0], 3)
+        assertEquals(partitions_res1.size(), 2)
+    } catch (Exception e) {
+        log.info(e.getMessage())
+        if (isClusterKeyEnabled()) {
+            assertTrue(e.getMessage().contains("Can not modify "))
+        } else {
+            throw e
+        }
+    }
 
 }
diff --git a/regression-test/suites/point_query_p0/test_point_query_ck.groovy b/regression-test/suites/point_query_p0/test_point_query_ck.groovy
index 09cf644cb9a48a..33c31dc0970929 100644
--- a/regression-test/suites/point_query_p0/test_point_query_ck.groovy
+++ b/regression-test/suites/point_query_p0/test_point_query_ck.groovy
@@ -202,7 +202,7 @@ suite("test_point_query_ck") {
                 qe_point_select stmt
                 qe_point_select stmt
                 // invalidate cache
-                sql "sync"
+                // sql "sync"
                 nprep_sql """ INSERT INTO ${tableName} VALUES(1235, 120939.11130, "a    ddd", "xxxxxx", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", 0, 1929111.1111,[119291.19291], ["111", "222", "333"], 2) """
                 qe_point_select stmt
                 qe_point_select stmt
@@ -218,9 +218,10 @@ suite("test_point_query_ck") {
                 qe_point_select stmt
                 qe_point_select stmt
 
-                sql """
+                nprep_sql """
                   ALTER table ${tableName} ADD COLUMN new_column1 INT default "0";
                 """
+                sql "select 1"
                 qe_point_select stmt
             }
             // disable useServerPrepStmts
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_light_schema_change.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
index 33f71ab307661b..cc6faa03dc0d79 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
@@ -19,6 +19,9 @@ import java.util.concurrent.TimeUnit
 import org.awaitility.Awaitility
 
 suite("test_partial_update_insert_light_schema_change", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_schema_change.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_schema_change.groovy
index 07740b9a09de20..62140ac58bf1cf 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_schema_change.groovy
@@ -19,6 +19,9 @@ import java.util.concurrent.TimeUnit
 import org.awaitility.Awaitility
 
 suite("test_partial_update_insert_schema_change", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     // ===== light schema change =====
     // test add value column
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_schema_change.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_schema_change.groovy
index c40e98f6badcbc..d2e27699524635 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_schema_change.groovy
@@ -19,6 +19,9 @@ import java.util.concurrent.TimeUnit
 import org.awaitility.Awaitility
 
 suite("test_partial_update_schema_change", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     /* ============================================== light schema change cases: ============================================== */
 
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_schema_change_row_store.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_schema_change_row_store.groovy
index 72698530f0a7b8..cffb682488a2d3 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_schema_change_row_store.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_schema_change_row_store.groovy
@@ -19,6 +19,9 @@ import java.util.concurrent.TimeUnit
 import org.awaitility.Awaitility
 
 suite("test_partial_update_row_store_schema_change", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     /* ============================================== light schema change cases: ============================================== */
 
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col.groovy
index 111037ae18f6b8..884882b7c93934 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col.groovy
@@ -17,6 +17,9 @@
 // under the License.
 
 suite("test_primary_key_partial_update_seq_col", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.groovy
index dca646948b4951..b37cdc68525d77 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.groovy
@@ -17,6 +17,9 @@
 // under the License.
 
 suite("test_primary_key_partial_update_seq_col_delete", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type.groovy
index d7d55725df6a62..3075faa022240f 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type.groovy
@@ -17,6 +17,9 @@
 // under the License.
 
 suite("test_primary_key_partial_update_seq_type", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.groovy
index 6ad60e0cd75f31..142feebef559c1 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.groovy
@@ -17,6 +17,9 @@
 // under the License.
 
 suite("test_primary_key_partial_update_seq_type_delete", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy
index fe47a5ebc89450..313fc97d06ba55 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy
@@ -17,6 +17,9 @@
 // under the License.
 
 suite("test_partial_update_strict_mode", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy
index 5e4a02f5407904..c8203cb0f9da53 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy
@@ -17,6 +17,9 @@
 // under the License.
 
 suite("test_partial_update_upsert", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_inverted_index.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_inverted_index.groovy
index f72160bb594ee9..3395757d876516 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_inverted_index.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_inverted_index.groovy
@@ -17,6 +17,9 @@
 // under the License.
 
 suite("test_partial_update_with_inverted_index", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_row_column.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_row_column.groovy
index 4b4de014b6772d..ea80d898ff18e8 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_row_column.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_row_column.groovy
@@ -17,6 +17,10 @@
 // under the License.
 
 suite("test_primary_key_partial_update_with_row_column", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
+
     def tableName = "test_primary_key_partial_update_with_row_column"
 
     // create table
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_update_on_current_timestamp.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_update_on_current_timestamp.groovy
index 85f4f721273587..794097ffa14f4a 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_update_on_current_timestamp.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_update_on_current_timestamp.groovy
@@ -17,6 +17,10 @@
 // under the License.
 
 suite("test_mow_update_on_current_timestamp", "p0") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
+
     sql 'set experimental_enable_nereids_planner=true'
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set enable_nereids_dml=true'
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_create_table.groovy b/regression-test/suites/unique_with_mow_c_p0/test_create_table.groovy
index a7810c2a6c3b12..c8cb94904c328e 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_create_table.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_create_table.groovy
@@ -16,6 +16,10 @@
 // under the License.
 
 suite("test_create_table") {
+    if (isClusterKeyEnabled()) {
+        return
+    }
+
     def tableName = "cluster_key_test_create_table"
     sql """ DROP TABLE IF EXISTS ${tableName} """
     onFinish {
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_ck.groovy b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_ck.groovy
index 09849d9fce22db..fb59aabf05dcd1 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_ck.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_ck.groovy
@@ -242,8 +242,8 @@ suite("test_schema_change_ck") {
         order_qt_select_restore_roll2 """select k2, k1, c4, c3 from ${tableName};"""
 
         // restore
-        logger.info(""" RESTORE SNAPSHOT ${context.dbName}.${backup} FROM `${repoName}` ON (`${tableName}`) PROPERTIES ("backup_timestamp" = "${snapshot}","replication_num" = "1" ) """)
-        sql """ RESTORE SNAPSHOT ${context.dbName}.${backup} FROM `${repoName}` ON (`${tableName}`) PROPERTIES ("backup_timestamp" = "${snapshot}","replication_num" = "1" ) """
+        logger.info(""" RESTORE SNAPSHOT ${context.dbName}.${backup} FROM `${repoName}` ON (`${tableName}`) PROPERTIES ("backup_timestamp" = "${snapshot}","reserve_replica" = "true" ) """)
+        sql """ RESTORE SNAPSHOT ${context.dbName}.${backup} FROM `${repoName}` ON (`${tableName}`) PROPERTIES ("backup_timestamp" = "${snapshot}","reserve_replica" = "true" ) """
         syncer.waitAllRestoreFinish(context.dbName)
         result = sql """ show tablets from ${tableName}; """
         logger.info("tablets 1: ${result}")
@@ -255,7 +255,7 @@ suite("test_schema_change_ck") {
 
         // restore
         sql """ drop table ${tableName}; """
-        sql """ RESTORE SNAPSHOT ${context.dbName}.${backup} FROM `${repoName}` ON (`${tableName}`) PROPERTIES ("backup_timestamp" = "${snapshot}","replication_num" = "1" ) """
+        sql """ RESTORE SNAPSHOT ${context.dbName}.${backup} FROM `${repoName}` ON (`${tableName}`) PROPERTIES ("backup_timestamp" = "${snapshot}","reserve_replica" = "true" ) """
         syncer.waitAllRestoreFinish(context.dbName)
         result = sql """ show tablets from ${tableName}; """
         logger.info("tablets 2: ${result}")

From 42f9c217f935ac621aa0d28256b0ece622655c02 Mon Sep 17 00:00:00 2001
From: shee <13843187+qzsee@users.noreply.github.com>
Date: Thu, 28 Nov 2024 14:45:07 +0800
Subject: [PATCH 058/399] [Metric](FE) add slow query metric (#41800)

---
 .../java/org/apache/doris/metric/MetricCalculator.java    | 8 ++++++++
 .../src/main/java/org/apache/doris/metric/MetricRepo.java | 6 ++++++
 .../src/main/java/org/apache/doris/qe/AuditLogHelper.java | 1 +
 3 files changed, 15 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricCalculator.java
index f4beed94b0c636..06d206377d7811 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricCalculator.java
@@ -33,6 +33,7 @@ public class MetricCalculator extends TimerTask {
     private long lastQueryCounter = -1;
     private long lastRequestCounter = -1;
     private long lastQueryErrCounter = -1;
+    private long lastQuerySlowCounter = -1;
 
     private Map<String, Long> clusterLastRequestCounter = new HashMap<>();
     private Map<String, Long> clusterLastQueryCounter = new HashMap<>();
@@ -50,6 +51,7 @@ private void update() {
             lastQueryCounter = MetricRepo.COUNTER_QUERY_ALL.getValue();
             lastRequestCounter = MetricRepo.COUNTER_REQUEST_ALL.getValue();
             lastQueryErrCounter = MetricRepo.COUNTER_QUERY_ERR.getValue();
+            lastQuerySlowCounter = MetricRepo.COUNTER_QUERY_SLOW.getValue();
             initCloudMetrics();
             return;
         }
@@ -74,6 +76,12 @@ private void update() {
         MetricRepo.GAUGE_QUERY_ERR_RATE.setValue(errRate < 0 ? 0.0 : errRate);
         lastQueryErrCounter = currentErrCounter;
 
+        // slow rate
+        long currentSlowCounter = MetricRepo.COUNTER_QUERY_SLOW.getValue();
+        double slowRate = (double) (currentSlowCounter - lastQuerySlowCounter) / interval;
+        MetricRepo.GAUGE_QUERY_SLOW_RATE.setValue(slowRate < 0 ? 0.0 : slowRate);
+        lastQuerySlowCounter = currentSlowCounter;
+
         updateCloudMetrics(interval);
         lastTs = currentTs;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index 86741758250e84..ca731eacf4b0bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -74,6 +74,7 @@ public final class MetricRepo {
     public static LongCounterMetric COUNTER_REQUEST_ALL;
     public static LongCounterMetric COUNTER_QUERY_ALL;
     public static LongCounterMetric COUNTER_QUERY_ERR;
+    public static LongCounterMetric COUNTER_QUERY_SLOW;
     public static LongCounterMetric COUNTER_QUERY_TABLE;
     public static LongCounterMetric COUNTER_QUERY_OLAP_TABLE;
     public static LongCounterMetric COUNTER_QUERY_HIVE_TABLE;
@@ -142,6 +143,7 @@ public final class MetricRepo {
     public static GaugeMetricImpl<Double> GAUGE_QUERY_PER_SECOND;
     public static GaugeMetricImpl<Double> GAUGE_REQUEST_PER_SECOND;
     public static GaugeMetricImpl<Double> GAUGE_QUERY_ERR_RATE;
+    public static GaugeMetricImpl<Double> GAUGE_QUERY_SLOW_RATE;
     public static GaugeMetricImpl<Long> GAUGE_MAX_TABLET_COMPACTION_SCORE;
 
     private static Map<Pair<EtlJobType, JobState>, Long> loadJobNum = Maps.newHashMap();
@@ -292,6 +294,8 @@ public Long getValue() {
         DORIS_METRIC_REGISTER.addMetrics(GAUGE_REQUEST_PER_SECOND);
         GAUGE_QUERY_ERR_RATE = new GaugeMetricImpl<>("query_err_rate", MetricUnit.NOUNIT, "query error rate", 0.0);
         DORIS_METRIC_REGISTER.addMetrics(GAUGE_QUERY_ERR_RATE);
+        GAUGE_QUERY_SLOW_RATE = new GaugeMetricImpl<>("query_slow_rate", MetricUnit.NOUNIT, "query slow rate", 0.0);
+        DORIS_METRIC_REGISTER.addMetrics(GAUGE_QUERY_SLOW_RATE);
         GAUGE_MAX_TABLET_COMPACTION_SCORE = new GaugeMetricImpl<>("max_tablet_compaction_score", MetricUnit.NOUNIT,
                 "max tablet compaction score of all backends", 0L);
         DORIS_METRIC_REGISTER.addMetrics(GAUGE_MAX_TABLET_COMPACTION_SCORE);
@@ -303,6 +307,8 @@ public Long getValue() {
         DORIS_METRIC_REGISTER.addMetrics(COUNTER_QUERY_ALL);
         COUNTER_QUERY_ERR = new LongCounterMetric("query_err", MetricUnit.REQUESTS, "total error query");
         DORIS_METRIC_REGISTER.addMetrics(COUNTER_QUERY_ERR);
+        COUNTER_QUERY_SLOW = new LongCounterMetric("query_slow", MetricUnit.REQUESTS, "total slow query");
+        DORIS_METRIC_REGISTER.addMetrics(COUNTER_QUERY_SLOW);
         COUNTER_QUERY_TABLE = new LongCounterMetric("query_table", MetricUnit.REQUESTS, "total query from table");
         DORIS_METRIC_REGISTER.addMetrics(COUNTER_QUERY_TABLE);
         COUNTER_QUERY_OLAP_TABLE = new LongCounterMetric("query_olap_table", MetricUnit.REQUESTS,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
index 12958d53390c89..903b72898c9512 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
@@ -249,6 +249,7 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme
                 if (elapseMs > Config.qe_slow_log_ms) {
                     String sqlDigest = DigestUtils.md5Hex(((Queriable) parsedStmt).toDigest());
                     auditEventBuilder.setSqlDigest(sqlDigest);
+                    MetricRepo.COUNTER_QUERY_SLOW.increase(1L);
                 }
             }
             auditEventBuilder.setIsQuery(true)

From 10c74f3e829b6005cb3254e663940031ad26fe93 Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Thu, 28 Nov 2024 12:18:46 +0530
Subject: [PATCH 059/399] [Enhancement] (nereids)implement
 showCreateViewCommand in nereids (#43145)

Issue Number: close #42737

implement showCreateViewCommand in nereids
---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |   9 ++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../plans/commands/ShowCreateViewCommand.java | 126 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../nereids_p0/show/test_show_create_view.out |   7 +
 .../show/test_show_create_view.groovy         |  76 +++++++++++
 7 files changed, 225 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateViewCommand.java
 create mode 100644 regression-test/data/nereids_p0/show/test_show_create_view.out
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_create_view.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 08f9058bbb14d6..6444f533ac0458 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -236,6 +236,7 @@ supportedShowStatement
     | SHOW STORAGE? ENGINES                                                         #showStorageEngines
     | SHOW CREATE CATALOG name=identifier                                           #showCreateCatalog
     | SHOW SQL_BLOCK_RULE (FOR ruleName=identifier)?                                #showSqlBlockRule
+    | SHOW CREATE VIEW name=multipartIdentifier                                     #showCreateView
     | SHOW CREATE MATERIALIZED VIEW mvName=identifier
         ON tableName=multipartIdentifier                                            #showCreateMaterializedView   
     | SHOW BACKENDS                                                                 #showBackends
@@ -292,7 +293,6 @@ unsupportedShowStatement
     | SHOW FULL? VIEWS ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showViews
     | SHOW FULL? PROCESSLIST                                                        #showProcessList
     | SHOW (GLOBAL | SESSION | LOCAL)? STATUS wildWhere?                            #showStatus
-    | SHOW CREATE VIEW name=multipartIdentifier                                     #showCreateView
     | SHOW CREATE MATERIALIZED VIEW name=multipartIdentifier                        #showMaterializedView
     | SHOW CREATE (DATABASE | SCHEMA) name=multipartIdentifier                      #showCreateDatabase
     | SHOW CREATE (GLOBAL | SESSION | LOCAL)? FUNCTION functionIdentifier
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index abd6afb84bc7f2..f2119ddd61d3bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -230,6 +230,7 @@
 import org.apache.doris.nereids.DorisParser.ShowCreateMaterializedViewContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateProcedureContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateTableContext;
+import org.apache.doris.nereids.DorisParser.ShowCreateViewContext;
 import org.apache.doris.nereids.DorisParser.ShowDeleteContext;
 import org.apache.doris.nereids.DorisParser.ShowDynamicPartitionContext;
 import org.apache.doris.nereids.DorisParser.ShowEventsContext;
@@ -514,6 +515,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMaterializedViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowEventsCommand;
@@ -4563,6 +4565,12 @@ public LogicalPlan visitShowCreateTable(ShowCreateTableContext ctx) {
         return new ShowCreateTableCommand(new TableNameInfo(nameParts), ctx.BRIEF() != null);
     }
 
+    @Override
+    public LogicalPlan visitShowCreateView(ShowCreateViewContext ctx) {
+        List<String> nameParts = visitMultipartIdentifier(ctx.name);
+        return new ShowCreateViewCommand(new TableNameInfo(nameParts));
+    }
+
     @Override
     public LogicalPlan visitShowCreateMaterializedView(ShowCreateMaterializedViewContext ctx) {
         List<String> nameParts = visitMultipartIdentifier(ctx.tableName);
@@ -4642,6 +4650,7 @@ public RecoverPartitionCommand visitRecoverPartition(RecoverPartitionContext ctx
     }
 
     @Override
+
     public LogicalPlan visitShowBroker(ShowBrokerContext ctx) {
         return new ShowBrokerCommand();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 09fa71f5614e87..dced9a0a579827 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -200,6 +200,7 @@ public enum PlanType {
     SHOW_CREATE_CATALOG_COMMAND,
     SHOW_CREATE_MATERIALIZED_VIEW_COMMAND,
     SHOW_CREATE_TABLE_COMMAND,
+    SHOW_CREATE_VIEW_COMMAND,
     SHOW_DELETE_COMMAND,
     SHOW_DYNAMIC_PARTITION_COMMAND,
     SHOW_EVENTS_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateViewCommand.java
new file mode 100644
index 00000000000000..96fb047c542f85
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateViewCommand.java
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.View;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * Represents the command for SHOW CREATE VIEW.
+ */
+public class ShowCreateViewCommand extends ShowCommand {
+    private static final ShowResultSetMetaData VIEW_META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("View", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Create View", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("character_set_client", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("collation_connection", ScalarType.createVarchar(30)))
+                    .build();
+
+    private final TableNameInfo tblNameInfo;
+
+    public ShowCreateViewCommand(TableNameInfo tableNameInfo) {
+        super(PlanType.SHOW_CREATE_VIEW_COMMAND);
+        this.tblNameInfo = tableNameInfo;
+    }
+
+    private void validate(ConnectContext ctx) throws AnalysisException {
+        tblNameInfo.analyze(ctx);
+
+        TableIf tableIf = Env.getCurrentEnv().getCatalogMgr()
+                .getCatalogOrAnalysisException(tblNameInfo.getCtl())
+                .getDbOrAnalysisException(tblNameInfo.getDb()).getTableOrAnalysisException(tblNameInfo.getTbl());
+
+        if (tableIf instanceof MTMV) {
+            ErrorReport.reportAnalysisException("not support async materialized view, "
+                    + "please use `show create materialized view`");
+        }
+
+        PrivPredicate wanted;
+        if (tableIf instanceof View) {
+            wanted = PrivPredicate.SHOW_VIEW;
+        } else {
+            wanted = PrivPredicate.SHOW;
+        }
+
+        if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(),
+                tblNameInfo.getCtl(), tblNameInfo.getDb(), tblNameInfo.getTbl(), wanted)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW CREATE TABLE",
+                                                ConnectContext.get().getQualifiedUser(),
+                                                ConnectContext.get().getRemoteIP(),
+                                                tblNameInfo.getDb() + ": " + tblNameInfo.getTbl());
+        }
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowCreateViewCommand(this, context);
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        // Fetch the catalog, database, and view metadata
+        DatabaseIf db = ctx.getEnv().getCatalogMgr().getCatalogOrAnalysisException(tblNameInfo.getCtl())
+                        .getDbOrMetaException(tblNameInfo.getDb());
+        TableIf view = db.getTableOrMetaException(tblNameInfo.getTbl());
+
+        if (!(view instanceof View)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_OBJECT, tblNameInfo.getDb(), tblNameInfo.getTbl(),
+                        "VIEW", "Use 'SHOW CREATE TABLE '" + tblNameInfo.getTbl());
+        }
+
+        List<List<String>> rows = Lists.newArrayList();
+        // Lock the view to ensure consistent metadata access
+        view.readLock();
+        try {
+            List<String> createViewStmt = Lists.newArrayList();
+            ctx.getEnv().getDdlStmt(null, null, view, createViewStmt, null, null, false, true,
+                        false, -1L, false, false);
+
+            if (!createViewStmt.isEmpty()) {
+                rows.add(Lists.newArrayList(view.getName(), createViewStmt.get(0), "utf8mb4", "utf8mb4_0900_bin"));
+            }
+        } finally {
+            view.readUnlock();
+        }
+
+        // Set the result set and send it using the executor
+        return new ShowResultSet(VIEW_META_DATA, rows);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index ef1688c3398b82..b4a56f71d4166e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -79,6 +79,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMaterializedViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowEventsCommand;
@@ -396,6 +397,10 @@ default R visitShowCreateMaterializedViewCommand(ShowCreateMaterializedViewComma
         return visitCommand(showCreateMtlzViewCommand, context);
     }
 
+    default R visitShowCreateViewCommand(ShowCreateViewCommand showCreateViewCommand, C context) {
+        return visitCommand(showCreateViewCommand, context);
+    }
+
     default R visitAlterRoleCommand(AlterRoleCommand alterRoleCommand, C context) {
         return visitCommand(alterRoleCommand, context);
     }
diff --git a/regression-test/data/nereids_p0/show/test_show_create_view.out b/regression-test/data/nereids_p0/show/test_show_create_view.out
new file mode 100644
index 00000000000000..8ac669337b482c
--- /dev/null
+++ b/regression-test/data/nereids_p0/show/test_show_create_view.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !cmd --
+view_show_create_view	CREATE VIEW `view_show_create_view` AS SELECT `internal`.`regression_test_nereids_p0_show`.`table_for_view_test`.`id`, `internal`.`regression_test_nereids_p0_show`.`table_for_view_test`.`name` FROM `internal`.`regression_test_nereids_p0_show`.`table_for_view_test`;	utf8mb4	utf8mb4_0900_bin
+
+-- !cmd --
+view_show_create_view_2	CREATE VIEW `view_show_create_view_2` AS SELECT `internal`.`regression_test_nereids_p0_show`.`table_for_view_test_2`.`key_field`, `internal`.`regression_test_nereids_p0_show`.`table_for_view_test_2`.`value` FROM `internal`.`regression_test_nereids_p0_show`.`table_for_view_test_2`;	utf8mb4	utf8mb4_0900_bin
+
diff --git a/regression-test/suites/nereids_p0/show/test_show_create_view.groovy b/regression-test/suites/nereids_p0/show/test_show_create_view.groovy
new file mode 100644
index 00000000000000..65ed4758a9ae66
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_create_view.groovy
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_create_view", "query,arrow_flight_sql") {
+    String view_name = "view_show_create_view";
+    String table_name = "table_for_view_test";
+    try {
+        // Create a table for testing
+        sql """
+            CREATE TABLE IF NOT EXISTS ${table_name} (
+                id INT COMMENT "Primary key",
+                name STRING COMMENT "Name field"
+            )
+            DISTRIBUTED BY HASH(id) BUCKETS 5
+            PROPERTIES ("replication_num" = "1");
+        """
+
+        // Create a view based on the created table
+        sql """
+            CREATE VIEW IF NOT EXISTS ${view_name} AS
+            SELECT id, name FROM ${table_name}
+        """
+        
+        // Execute the SHOW CREATE VIEW command
+        checkNereidsExecute("""show create view `${view_name}`;""")
+        qt_cmd("""show create view `${view_name}`;""")
+    } finally {
+        // Drop the view and table after testing
+        try_sql("DROP VIEW IF EXISTS `${view_name}`")
+        try_sql("DROP TABLE IF EXISTS `${table_name}`")
+    }
+
+    // Additional case: Create another view based on a different table
+    String view_name_2 = "view_show_create_view_2";
+    String table_name_2 = "table_for_view_test_2";
+    try {
+        // Create another table for testing
+        sql """
+            CREATE TABLE IF NOT EXISTS ${table_name_2} (
+                `key_field` INT COMMENT "Key field",
+                `value` STRING COMMENT "Value field"
+            )
+            DISTRIBUTED BY HASH(key_field) BUCKETS 3
+            PROPERTIES ("replication_num" = "1");
+        """
+
+        // Create a view based on the new table
+        sql """
+            CREATE VIEW IF NOT EXISTS ${view_name_2} AS
+            SELECT key_field, value FROM ${table_name_2}
+        """
+        
+        // Execute the SHOW CREATE VIEW command for the new view
+        checkNereidsExecute("""show create view `${view_name_2}`;""")
+        qt_cmd("""show create view `${view_name_2}`;""")
+
+    } finally {
+        // Drop the view and table after testing
+        try_sql("DROP VIEW IF EXISTS `${view_name_2}`")
+        try_sql("DROP TABLE IF EXISTS `${table_name_2}`")
+    }
+}

From b761e4ec34be7a0bba2fb805e604e7c8283032cf Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Thu, 28 Nov 2024 12:19:48 +0530
Subject: [PATCH 060/399] [Enhancement] (nereids)implement
 showDiagnoseTabletCommand in nereids (#43159)

Issue Number: close #42836

implement showDiagnoseTabletCommand in nereids
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  8 ++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/ShowDiagnoseTabletCommand.java   | 84 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../show/test_show_diagnose_tablet.groovy     | 39 +++++++++
 6 files changed, 138 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDiagnoseTabletCommand.java
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_diagnose_tablet.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 6444f533ac0458..1b1e218a958146 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -242,6 +242,7 @@ supportedShowStatement
     | SHOW BACKENDS                                                                 #showBackends
     | SHOW REPLICA DISTRIBUTION FROM baseTableRef                                   #showReplicaDistribution
     | SHOW FULL? TRIGGERS ((FROM | IN) database=multipartIdentifier)? wildWhere?    #showTriggers    
+    | SHOW TABLET DIAGNOSIS tabletId=INTEGER_VALUE                                  #showDiagnoseTablet
     | SHOW FRONTENDS name=identifier?                                               #showFrontends 
     | SHOW TABLE tableId=INTEGER_VALUE                                              #showTableId
     | SHOW WHITELIST                                                                #showWhitelist
@@ -355,7 +356,6 @@ unsupportedShowStatement
     | SHOW CONVERT_LSC ((FROM | IN) database=multipartIdentifier)?                  #showConvertLsc
     | SHOW REPLICA STATUS FROM baseTableRef wildWhere?                              #showReplicaStatus
     | SHOW TABLET STORAGE FORMAT VERBOSE?                                           #showTabletStorageFormat
-    | SHOW TABLET DIAGNOSIS tabletId=INTEGER_VALUE                                  #showDiagnoseTablet
     | SHOW COPY ((FROM | IN) database=multipartIdentifier)?
         whereClause? sortClause? limitClause?                                       #showCopy
     | SHOW WARM UP JOB wildWhere?                                                   #showWarmUpJob
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index f2119ddd61d3bd..2db26dc15011f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -232,6 +232,7 @@
 import org.apache.doris.nereids.DorisParser.ShowCreateTableContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateViewContext;
 import org.apache.doris.nereids.DorisParser.ShowDeleteContext;
+import org.apache.doris.nereids.DorisParser.ShowDiagnoseTabletContext;
 import org.apache.doris.nereids.DorisParser.ShowDynamicPartitionContext;
 import org.apache.doris.nereids.DorisParser.ShowEventsContext;
 import org.apache.doris.nereids.DorisParser.ShowFrontendsContext;
@@ -517,6 +518,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowEventsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowFrontendsCommand;
@@ -4559,6 +4561,12 @@ public LogicalPlan visitShowStorageEngines(ShowStorageEnginesContext ctx) {
         return new ShowStorageEnginesCommand();
     }
 
+    @Override
+    public LogicalPlan visitShowDiagnoseTablet(ShowDiagnoseTabletContext ctx) {
+        long tabletId = Long.parseLong(ctx.INTEGER_VALUE().getText());
+        return new ShowDiagnoseTabletCommand(tabletId);
+    }
+
     @Override
     public LogicalPlan visitShowCreateTable(ShowCreateTableContext ctx) {
         List<String> nameParts = visitMultipartIdentifier(ctx.name);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index dced9a0a579827..8f474ea395f889 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -202,6 +202,7 @@ public enum PlanType {
     SHOW_CREATE_TABLE_COMMAND,
     SHOW_CREATE_VIEW_COMMAND,
     SHOW_DELETE_COMMAND,
+    SHOW_DIAGNOSE_TABLET_COMMAND,
     SHOW_DYNAMIC_PARTITION_COMMAND,
     SHOW_EVENTS_COMMAND,
     SHOW_FRONTENDS_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDiagnoseTabletCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDiagnoseTabletCommand.java
new file mode 100644
index 00000000000000..6dac6ca214ede5
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDiagnoseTabletCommand.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.RedirectStatus;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.system.Diagnoser;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * Represents the command for SHOW TABLET DIAGNOSIS
+ */
+public class ShowDiagnoseTabletCommand extends ShowCommand {
+    public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
+            .add("Item").add("Info").add("Suggestion")
+            .build();
+
+    private final long tabletId;
+
+    public ShowDiagnoseTabletCommand(long tabletId) {
+        super(PlanType.SHOW_DIAGNOSE_TABLET_COMMAND);
+        this.tabletId = tabletId;
+    }
+
+    public ShowResultSetMetaData getMetaData() {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        for (String title : TITLE_NAMES) {
+            builder.addColumn(new Column(title, ScalarType.createVarchar(1024)));
+        }
+        return builder.build();
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        List<List<String>> resultRowSet = Diagnoser.diagnoseTablet(tabletId);
+        // Set the result set and send it using the executor
+        return new ShowResultSet(getMetaData(), resultRowSet);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowDiagnoseTabletCommand(this, context);
+    }
+
+    @Override
+    public RedirectStatus toRedirectStatus() {
+        return RedirectStatus.FORWARD_NO_SYNC;
+    }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index b4a56f71d4166e..c00c0ef90ca018 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -81,6 +81,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowEventsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowFrontendsCommand;
@@ -433,6 +434,10 @@ default R visitRecoverDatabaseCommand(RecoverDatabaseCommand recoverDatabaseComm
         return visitCommand(recoverDatabaseCommand, context);
     }
 
+    default R visitShowDiagnoseTabletCommand(ShowDiagnoseTabletCommand showDiagnoseTabletCommand, C context) {
+        return visitCommand(showDiagnoseTabletCommand, context);
+    }
+
     default R visitRecoverTableCommand(RecoverTableCommand recoverTableCommand, C context) {
         return visitCommand(recoverTableCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/show/test_show_diagnose_tablet.groovy b/regression-test/suites/nereids_p0/show/test_show_diagnose_tablet.groovy
new file mode 100644
index 00000000000000..029fac7428e0bb
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_diagnose_tablet.groovy
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_diagnose_tablet_nereids", "query,diagnose") {
+    String tableName = "test_table_for_diagnosis";
+    String tabletId = "";
+    try {
+        // Create a new table to test the SHOW TABLET DIAGNOSIS command
+        sql "CREATE TABLE IF NOT EXISTS ${tableName} (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES('replication_num'='1');"
+
+        // Extract tablet ID from the created table
+        def showTabletsResult = sql "SHOW TABLETS FROM ${tableName}"
+        assert showTabletsResult.size() > 0
+        tabletId = showTabletsResult[0][0]  // Assuming the first tablet ID is used
+
+        // Execute the SHOW TABLET DIAGNOSIS command and verify the output
+        checkNereidsExecute("SHOW TABLET DIAGNOSIS ${tabletId}")
+    } catch (Exception e) {
+        log.error("Failed to execute SHOW TABLET DIAGNOSIS command", e)
+        throw e
+    } finally {
+        try_sql("DROP TABLE IF EXISTS ${tableName}")
+    }
+}
+

From a4d9aaa75cf1c059d36bbddc456effffb50396f0 Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Thu, 28 Nov 2024 14:56:39 +0800
Subject: [PATCH 061/399] [opt](nereids) Use a unified approach to deal with
 monotonic function in partition prune (#42934)

This pr use a unified approach to deal with monotonic function in
partition prune. When adding a monotonic function in partition prune, we
don't need to add visitXXX in OneRangePartitionEvaluator, only need to
implement the Monotonic interface.
---
 .../rules/OneRangePartitionEvaluator.java     | 76 +++++++------------
 .../expressions/functions/Monotonic.java      |  3 +-
 .../functions/scalar/ConvertTz.java           |  3 +-
 .../expressions/functions/scalar/Date.java    |  3 +-
 .../functions/scalar/DateTrunc.java           |  3 +-
 5 files changed, 31 insertions(+), 57 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index 1fb8954ab16547..7e91d5502f7fad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -43,9 +43,8 @@
 import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.Date;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
 import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.literal.MaxLiteral;
@@ -91,6 +90,7 @@ public class OneRangePartitionEvaluator<K>
     private final List<Literal> lowers;
     private final List<Literal> uppers;
     private final List<List<Expression>> inputs;
+    // whether the Expression in partition range may be null.
     private final Map<Expression, Boolean> partitionSlotContainsNull;
     private final Map<Slot, PartitionSlotType> slotToType;
 
@@ -456,7 +456,12 @@ private EvaluateRangeResult evaluateChildrenThenThis(Expression expr, EvaluateRa
 
         for (int i = 0; i < children.size(); i++) {
             Expression child = children.get(i);
-            EvaluateRangeResult childResult = child.accept(this, context);
+            EvaluateRangeResult childResult;
+            if (child instanceof Monotonic) {
+                childResult = visitMonotonic(child, context);
+            } else {
+                childResult = child.accept(this, context);
+            }
             if (!childResult.result.equals(child)) {
                 hasNewChildren = true;
             }
@@ -615,45 +620,6 @@ private List<Literal> toMultiNereidsLiterals(PartitionKey partitionKey) {
         return literals;
     }
 
-    @Override
-    public EvaluateRangeResult visitDateTrunc(DateTrunc dateTrunc, EvaluateRangeInput context) {
-        EvaluateRangeResult result = super.visitDateTrunc(dateTrunc, context);
-        if (!(result.result instanceof DateTrunc)) {
-            return result;
-        }
-        Expression dateTruncChild = dateTrunc.child(0);
-        if (partitionSlotContainsNull.containsKey(dateTruncChild)) {
-            partitionSlotContainsNull.put(dateTrunc, true);
-        }
-        return computeMonotonicFunctionRange(result, context.rangeMap);
-    }
-
-    @Override
-    public EvaluateRangeResult visitDate(Date date, EvaluateRangeInput context) {
-        EvaluateRangeResult result = super.visitDate(date, context);
-        if (!(result.result instanceof Date)) {
-            return result;
-        }
-        Expression dateChild = date.child(0);
-        if (partitionSlotContainsNull.containsKey(dateChild)) {
-            partitionSlotContainsNull.put(date, true);
-        }
-        return computeMonotonicFunctionRange(result, context.rangeMap);
-    }
-
-    @Override
-    public EvaluateRangeResult visitConvertTz(ConvertTz convertTz, EvaluateRangeInput context) {
-        EvaluateRangeResult result = super.visitConvertTz(convertTz, context);
-        if (!(result.result instanceof ConvertTz)) {
-            return result;
-        }
-        Expression converTzChild = convertTz.child(0);
-        if (partitionSlotContainsNull.containsKey(converTzChild)) {
-            partitionSlotContainsNull.put(convertTz, true);
-        }
-        return computeMonotonicFunctionRange(result, context.rangeMap);
-    }
-
     private boolean isPartitionSlot(Slot slot) {
         return slotToType.containsKey(slot);
     }
@@ -820,15 +786,27 @@ private List<Map<Slot, PartitionSlotInput>> commonComputeOnePartitionInputs() {
         return onePartitionInputs;
     }
 
-    private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result,
-            Map<Expression, ColumnRange> rangeMap) {
+    public EvaluateRangeResult visitMonotonic(Expression monotonic, EvaluateRangeInput context) {
+        EvaluateRangeResult rangeResult = evaluateChildrenThenThis(monotonic, context);
+        if (!rangeResult.result.getClass().equals(monotonic.getClass())) {
+            return rangeResult;
+        }
+        return computeMonotonicFunctionRange(rangeResult, context);
+    }
+
+    private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result, EvaluateRangeInput context) {
         Monotonic func = (Monotonic) result.result;
-        if (rangeMap.containsKey(func)) {
+        if (context.rangeMap.containsKey(func)) {
             return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func,
-                    rangeMap.get(func)), result.childrenResult);
+                    context.rangeMap.get(func)), result.childrenResult);
         }
         int childIndex = func.getMonotonicFunctionChildIndex();
         Expression funcChild = func.child(childIndex);
+        boolean isNullable = partitionSlotContainsNull.getOrDefault(funcChild, true);
+        Expression withNullable = func.withConstantArgs(isNullable ? new Nullable(funcChild)
+                : new NonNullable(funcChild));
+        partitionSlotContainsNull.put((Expression) func, withNullable.nullable());
+
         if (!result.childrenResult.get(0).columnRanges.containsKey(funcChild)) {
             return result;
         }
@@ -854,7 +832,7 @@ private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult re
         ColumnRange newRange = ColumnRange.all();
         if (lowerValue instanceof Literal && upperValue instanceof Literal && lowerValue.equals(upperValue)) {
             newRange = ColumnRange.singleton((Literal) lowerValue);
-            rangeMap.put((Expression) func, newRange);
+            context.rangeMap.put((Expression) func, newRange);
             newRanges.put((Expression) func, newRange);
             return new EvaluateRangeResult(lowerValue, newRanges, result.childrenResult);
         } else {
@@ -864,7 +842,7 @@ private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult re
             if (upperValue instanceof Literal) {
                 newRange = newRange.withUpperBound((Literal) upperValue);
             }
-            rangeMap.put((Expression) func, newRange);
+            context.rangeMap.put((Expression) func, newRange);
             newRanges.put((Expression) func, newRange);
             return new EvaluateRangeResult((Expression) func, newRanges, result.childrenResult);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Monotonic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Monotonic.java
index 2fdde0e7415613..bcaa040cb2a650 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Monotonic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Monotonic.java
@@ -18,7 +18,6 @@
 package org.apache.doris.nereids.trees.expressions.functions;
 
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
 
 /** monotonicity of expressions */
 public interface Monotonic extends ExpressionTrait {
@@ -32,5 +31,5 @@ public interface Monotonic extends ExpressionTrait {
     // return the function with the arguments replaced by literal
     // e.g. date_trunc(dt, 'day'), dt in range ['2020-01-01 10:00:00', '2020-01-03 10:00:00']
     // return date_trunc('2020-01-01 10:00:00', 'day')
-    Expression withConstantArgs(Literal literal);
+    Expression withConstantArgs(Expression literal);
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java
index faee04cacd82b8..e0d717cd04f66a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java
@@ -24,7 +24,6 @@
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
 import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression;
@@ -99,7 +98,7 @@ public int getMonotonicFunctionChildIndex() {
     }
 
     @Override
-    public Expression withConstantArgs(Literal literal) {
+    public Expression withConstantArgs(Expression literal) {
         return new ConvertTz(literal, child(1), child(2));
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Date.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Date.java
index 3f7ae84975031a..80c6178cd3ffb8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Date.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Date.java
@@ -23,7 +23,6 @@
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
 import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.DateTimeType;
@@ -84,7 +83,7 @@ public int getMonotonicFunctionChildIndex() {
     }
 
     @Override
-    public Expression withConstantArgs(Literal literal) {
+    public Expression withConstantArgs(Expression literal) {
         return new Date(literal);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java
index cbd2da5627b375..2a4d838c4fd5ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java
@@ -23,7 +23,6 @@
 import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -105,7 +104,7 @@ public int getMonotonicFunctionChildIndex() {
     }
 
     @Override
-    public Expression withConstantArgs(Literal literal) {
+    public Expression withConstantArgs(Expression literal) {
         return new DateTrunc(literal, child(1));
     }
 }

From 12192739e052ad6c03aeab7126bf2e7cfb35fa90 Mon Sep 17 00:00:00 2001
From: yiguolei <guolei@selectdb.com>
Date: Thu, 28 Nov 2024 15:59:52 +0800
Subject: [PATCH 062/399] [chore](compile) fix some compile warning in BE
 (#44454)

---
 .../serde/data_type_date64_serde.cpp          |  8 ++++----
 .../array/function_array_contains_all.cpp     | 17 ++++++++--------
 .../functions/array/function_array_element.h  |  8 ++++++--
 .../array/function_array_intersect.cpp        |  3 ++-
 .../vec/functions/array/function_array_join.h |  2 ++
 .../vec/functions/array/function_array_map.h  | 18 +++++++++--------
 .../functions/array/function_array_mapped.h   |  3 ++-
 .../vec/functions/array/function_array_nary.h |  2 ++
 .../functions/array/function_array_range.cpp  |  7 ++++---
 .../array/function_array_shuffle.cpp          |  3 ++-
 .../functions/array/function_array_sortby.cpp |  2 +-
 .../functions/array/function_array_split.cpp  |  2 +-
 .../functions/array/function_array_union.cpp  |  1 +
 .../functions/array/function_array_utils.h    |  4 +++-
 .../array/function_array_with_constant.cpp    |  6 ++++--
 be/src/vec/functions/date_format_type.h       | 15 +++++++-------
 be/src/vec/functions/date_time_transforms.h   | 20 ++++++++++++++-----
 .../functions/function_binary_arithmetic.h    |  1 -
 be/src/vec/functions/function_bit_count.cpp   |  6 ++++--
 be/src/vec/functions/function_bitmap.cpp      | 15 ++++++++------
 be/src/vec/functions/function_case.h          | 10 ++++++----
 be/src/vec/functions/function_cast.h          |  1 -
 be/src/vec/runtime/vdatetime_value.cpp        | 10 +++++-----
 be/src/vec/runtime/vdatetime_value.h          | 14 ++++++-------
 24 files changed, 107 insertions(+), 71 deletions(-)

diff --git a/be/src/vec/data_types/serde/data_type_date64_serde.cpp b/be/src/vec/data_types/serde/data_type_date64_serde.cpp
index 6cb5b31ae387a2..8e102ec6e3a729 100644
--- a/be/src/vec/data_types/serde/data_type_date64_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_date64_serde.cpp
@@ -167,12 +167,12 @@ void DataTypeDate64SerDe::write_column_to_arrow(const IColumn& column, const Nul
     for (size_t i = start; i < end; ++i) {
         char buf[64];
         const VecDateTimeValue* time_val = (const VecDateTimeValue*)(&col_data[i]);
-        int len = time_val->to_buffer(buf);
+        size_t len = time_val->to_buffer(buf);
         if (null_map && (*null_map)[i]) {
             checkArrowStatus(string_builder.AppendNull(), column.get_name(),
                              array_builder->type()->name());
         } else {
-            checkArrowStatus(string_builder.Append(buf, len), column.get_name(),
+            checkArrowStatus(string_builder.Append(buf, cast_set<int32_t>(len)), column.get_name(),
                              array_builder->type()->name());
         }
     }
@@ -305,8 +305,8 @@ Status DataTypeDate64SerDe::write_column_to_orc(const std::string& timezone, con
             continue;
         }
 
-        int len = binary_cast<Int64, VecDateTimeValue>(col_data[row_id])
-                          .to_buffer(const_cast<char*>(bufferRef.data) + offset);
+        size_t len = binary_cast<Int64, VecDateTimeValue>(col_data[row_id])
+                             .to_buffer(const_cast<char*>(bufferRef.data) + offset);
 
         REALLOC_MEMORY_FOR_ORC_WRITER()
 
diff --git a/be/src/vec/functions/array/function_array_contains_all.cpp b/be/src/vec/functions/array/function_array_contains_all.cpp
index 298289278bfa5f..67279babd5f49b 100644
--- a/be/src/vec/functions/array/function_array_contains_all.cpp
+++ b/be/src/vec/functions/array/function_array_contains_all.cpp
@@ -29,6 +29,7 @@
 #include "vec/functions/simple_function_factory.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 class FunctionArrayContainsAll : public IFunction {
 public:
@@ -189,12 +190,12 @@ class FunctionArrayContainsAll : public IFunction {
         for (ssize_t row = 0; row < input_rows_count; ++row) {
             auto left_index = index_check_const(row, left_is_const);
             auto right_index = index_check_const(row, right_is_const);
-            ssize_t left_start = (*left_data.offsets_ptr)[left_index - 1];
-            ssize_t left_end = (*left_data.offsets_ptr)[left_index];
-            ssize_t left_size = left_end - left_start;
-            ssize_t right_start = (*right_data.offsets_ptr)[right_index - 1];
-            ssize_t right_end = (*right_data.offsets_ptr)[right_index];
-            ssize_t right_size = right_end - right_start;
+            size_t left_start = (*left_data.offsets_ptr)[left_index - 1];
+            size_t left_end = (*left_data.offsets_ptr)[left_index];
+            size_t left_size = left_end - left_start;
+            size_t right_start = (*right_data.offsets_ptr)[right_index - 1];
+            size_t right_end = (*right_data.offsets_ptr)[right_index];
+            size_t right_size = right_end - right_start;
             // case: [1,2,3] : []
             if (right_size == 0) {
                 dst_data[row] = 1;
@@ -216,7 +217,7 @@ class FunctionArrayContainsAll : public IFunction {
                     is_equal_value = false;
                     break;
                 }
-                int left_nested_loop_pos = left_pos;
+                size_t left_nested_loop_pos = left_pos;
                 right_pos = right_start;
                 while (right_pos < right_end) {
                     bool left_nested_data_is_null =
@@ -261,5 +262,5 @@ void register_function_array_contains_all(SimpleFunctionFactory& factory) {
     factory.register_function<FunctionArrayContainsAll>();
     factory.register_alias("array_contains_all", "hasSubstr");
 }
-
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/array/function_array_element.h b/be/src/vec/functions/array/function_array_element.h
index 7a4807608c3ef9..e4b57348c883ae 100644
--- a/be/src/vec/functions/array/function_array_element.h
+++ b/be/src/vec/functions/array/function_array_element.h
@@ -58,6 +58,7 @@ class FunctionContext;
 } // namespace doris
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 class FunctionArrayElement : public IFunction {
 public:
@@ -147,14 +148,16 @@ class FunctionArrayElement : public IFunction {
             size_t end = offsets[i];
             for (size_t j = begin; j < end; j++) {
                 if (nested_ptr->compare_at(j, i, *right_column, -1) == 0) {
-                    matched_indices->insert_value(j - begin + 1);
+                    matched_indices->insert_value(
+                            cast_set<MapIndiceDataType::FieldType, size_t, false>(j - begin + 1));
                     matched = true;
                     break;
                 }
             }
 
             if (!matched) {
-                matched_indices->insert_value(end - begin + 1); // make indices for null
+                matched_indices->insert_value(cast_set<MapIndiceDataType::FieldType, size_t, false>(
+                        end - begin + 1)); // make indices for null
             }
         }
 
@@ -415,4 +418,5 @@ class FunctionArrayElement : public IFunction {
     }
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/array/function_array_intersect.cpp b/be/src/vec/functions/array/function_array_intersect.cpp
index e68ddf30542e6a..369b85631f38a7 100644
--- a/be/src/vec/functions/array/function_array_intersect.cpp
+++ b/be/src/vec/functions/array/function_array_intersect.cpp
@@ -24,6 +24,7 @@
 
 namespace doris::vectorized {
 
+#include "common/compile_check_begin.h"
 struct NameArrayIntersect {
     static constexpr auto name = "array_intersect";
 };
@@ -41,7 +42,7 @@ struct IntersectAction {
     bool apply_null() { return result_null_flag; }
 
     // Handle Non-Null element.
-    void apply(Map& map, const int arg_idx, const int row_idx,
+    void apply(Map& map, const size_t arg_idx, const size_t row_idx,
                const ColumnArrayExecutionData& param) {
         current_null_flag = false;
         size_t start_off = (*param.offsets_ptr)[row_idx - 1];
diff --git a/be/src/vec/functions/array/function_array_join.h b/be/src/vec/functions/array/function_array_join.h
index f7ffd4dfca3449..957b2288fb746a 100644
--- a/be/src/vec/functions/array/function_array_join.h
+++ b/be/src/vec/functions/array/function_array_join.h
@@ -25,6 +25,7 @@
 
 namespace doris::vectorized {
 
+#include "common/compile_check_begin.h"
 struct NameArrayJoin {
     static constexpr auto name = "array_join";
 };
@@ -266,4 +267,5 @@ struct ArrayJoinImpl {
     }
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/array/function_array_map.h b/be/src/vec/functions/array/function_array_map.h
index bcb71a4a79f264..fd4a2fc59f3548 100644
--- a/be/src/vec/functions/array/function_array_map.h
+++ b/be/src/vec/functions/array/function_array_map.h
@@ -26,6 +26,7 @@
 #include "vec/functions/function_helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 enum class MapOperation { INTERSECT, UNION };
 
@@ -64,9 +65,9 @@ struct OpenMapImpl {
 
     // this method calculate rows to get a rest dst data
     void apply(ColumnArrayMutableData& dst, const ColumnArrayExecutionDatas params,
-               std::vector<bool>& col_const, int start_row, int end_row) {
+               std::vector<bool>& col_const, size_t start_row, size_t end_row) {
         size_t dst_off = 0;
-        for (int row = start_row; row < end_row; ++row) {
+        for (size_t row = start_row; row < end_row; ++row) {
             reset();
             for (int i = 0; i < params.size(); ++i) {
                 action.apply(map, i, index_check_const(row, col_const[i]), params[i]);
@@ -110,11 +111,11 @@ struct OpenMapImpl<operation, ColumnString> {
     }
 
     void apply(ColumnArrayMutableData& dst, const ColumnArrayExecutionDatas params,
-               std::vector<bool>& col_const, int start_row, int end_row) {
+               std::vector<bool>& col_const, size_t start_row, size_t end_row) {
         size_t dst_off = 0;
-        for (int row = start_row; row < end_row; ++row) {
+        for (size_t row = start_row; row < end_row; ++row) {
             reset();
-            for (int i = 0; i < params.size(); ++i) {
+            for (size_t i = 0; i < params.size(); ++i) {
                 action.apply(map, i, index_check_const(row, col_const[i]), params[i]);
             }
             // nullmap
@@ -162,7 +163,7 @@ struct ArrayMapImpl {
     }
 
     static Status execute(ColumnPtr& res_ptr, ColumnArrayExecutionDatas datas,
-                          std::vector<bool>& col_const, int start_row, int end_row) {
+                          std::vector<bool>& col_const, size_t start_row, size_t end_row) {
         ColumnArrayMutableData dst =
                 create_mutable_data(datas[0].nested_col, datas[0].nested_nullmap_data);
         if (_execute_internal<ALL_COLUMNS_SIMPLE>(dst, datas, col_const, start_row, end_row)) {
@@ -175,7 +176,7 @@ struct ArrayMapImpl {
 private:
     template <typename ColumnType>
     static bool _execute_internal(ColumnArrayMutableData& dst, ColumnArrayExecutionDatas datas,
-                                  std::vector<bool>& col_const, int start_row, int end_row) {
+                                  std::vector<bool>& col_const, size_t start_row, size_t end_row) {
         for (auto data : datas) {
             if (!check_column<ColumnType>(*data.nested_col)) {
                 return false;
@@ -192,10 +193,11 @@ struct ArrayMapImpl {
     template <typename T, typename... Ts>
         requires(sizeof...(Ts) > 0)
     static bool _execute_internal(ColumnArrayMutableData& dst, ColumnArrayExecutionDatas datas,
-                                  std::vector<bool>& col_const, int start_row, int end_row) {
+                                  std::vector<bool>& col_const, size_t start_row, size_t end_row) {
         return _execute_internal<T>(dst, datas, col_const, start_row, end_row) ||
                _execute_internal<Ts...>(dst, datas, col_const, start_row, end_row);
     }
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/array/function_array_mapped.h b/be/src/vec/functions/array/function_array_mapped.h
index 93f7d068e4b01d..5c84baf9dfadb8 100644
--- a/be/src/vec/functions/array/function_array_mapped.h
+++ b/be/src/vec/functions/array/function_array_mapped.h
@@ -26,7 +26,7 @@
 
 namespace doris {
 namespace vectorized {
-
+#include "common/compile_check_begin.h"
 /** Higher-order functions for arrays.
   * These functions optionally apply a map (transform) to array (or multiple arrays of identical size) by lambda function,
   *  and return some result based on that transformation.
@@ -72,5 +72,6 @@ class FunctionArrayMapped : public IFunction {
     }
 };
 
+#include "common/compile_check_end.h"
 } // namespace vectorized
 } // namespace doris
diff --git a/be/src/vec/functions/array/function_array_nary.h b/be/src/vec/functions/array/function_array_nary.h
index 923b64473a8d1d..e9b2009f68484c 100644
--- a/be/src/vec/functions/array/function_array_nary.h
+++ b/be/src/vec/functions/array/function_array_nary.h
@@ -25,6 +25,7 @@
 #include "vec/functions/function_helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 // Functions with more than two arrays of the same element type.
 template <typename Impl, typename Name>
@@ -78,4 +79,5 @@ class FunctionArrayNary : public IFunction {
     }
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/array/function_array_range.cpp b/be/src/vec/functions/array/function_array_range.cpp
index b6b814a26e339a..8a3de3754503ae 100644
--- a/be/src/vec/functions/array/function_array_range.cpp
+++ b/be/src/vec/functions/array/function_array_range.cpp
@@ -52,6 +52,7 @@ class FunctionContext;
 } // namespace doris
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename Impl>
 class FunctionArrayRange : public IFunction {
@@ -174,7 +175,7 @@ struct RangeImplUtil {
                          PaddedPODArray<SourceDataType>& nested_column,
                          PaddedPODArray<UInt8>& dest_nested_null_map,
                          ColumnArray::Offsets64& dest_offsets) {
-        int rows = start.size();
+        size_t rows = start.size();
         for (auto row = 0; row < rows; ++row) {
             auto idx = start[row];
             auto end_row = end[row];
@@ -193,7 +194,7 @@ struct RangeImplUtil {
                         return Status::InvalidArgument("Array size exceeds the limit {}",
                                                        max_array_size_as_field);
                     }
-                    int offset = dest_offsets.back();
+                    size_t offset = dest_offsets.back();
                     while (idx < end[row]) {
                         nested_column.push_back(idx);
                         dest_nested_null_map.push_back(0);
@@ -213,7 +214,7 @@ struct RangeImplUtil {
                     dest_offsets.push_back(dest_offsets.back());
                     continue;
                 } else {
-                    int offset = dest_offsets.back();
+                    size_t offset = dest_offsets.back();
                     using UNIT = std::conditional_t<std::is_same_v<TimeUnitOrVoid, void>,
                                                     std::integral_constant<TimeUnit, TimeUnit::DAY>,
                                                     TimeUnitOrVoid>;
diff --git a/be/src/vec/functions/array/function_array_shuffle.cpp b/be/src/vec/functions/array/function_array_shuffle.cpp
index 648b06318df3dc..fbf5bcfcdf1686 100644
--- a/be/src/vec/functions/array/function_array_shuffle.cpp
+++ b/be/src/vec/functions/array/function_array_shuffle.cpp
@@ -45,6 +45,7 @@ class FunctionContext;
 } // namespace doris
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 class FunctionArrayShuffle : public IFunction {
 public:
@@ -71,7 +72,7 @@ class FunctionArrayShuffle : public IFunction {
                 block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
         const auto& src_column_array = assert_cast<const ColumnArray&>(*src_column);
 
-        uint32_t seed = time(nullptr);
+        size_t seed = time(nullptr);
         if (arguments.size() == 2) {
             ColumnPtr seed_column =
                     block.get_by_position(arguments[1]).column->convert_to_full_column_if_const();
diff --git a/be/src/vec/functions/array/function_array_sortby.cpp b/be/src/vec/functions/array/function_array_sortby.cpp
index 499a5e37ab9380..899bb40fba1423 100644
--- a/be/src/vec/functions/array/function_array_sortby.cpp
+++ b/be/src/vec/functions/array/function_array_sortby.cpp
@@ -46,7 +46,7 @@ class FunctionContext;
 } // namespace doris
 
 namespace doris::vectorized {
-
+#include "common/compile_check_begin.h"
 class FunctionArraySortBy : public IFunction {
 public:
     static constexpr auto name = "array_sortby";
diff --git a/be/src/vec/functions/array/function_array_split.cpp b/be/src/vec/functions/array/function_array_split.cpp
index 7f7a847abc71f4..5e58a9f189a657 100644
--- a/be/src/vec/functions/array/function_array_split.cpp
+++ b/be/src/vec/functions/array/function_array_split.cpp
@@ -46,7 +46,7 @@ class FunctionContext;
 } // namespace doris
 
 namespace doris::vectorized {
-
+#include "common/compile_check_begin.h"
 template <bool reverse>
 class FunctionArraySplit : public IFunction {
 public:
diff --git a/be/src/vec/functions/array/function_array_union.cpp b/be/src/vec/functions/array/function_array_union.cpp
index 75f7ec211b00c1..8d40682193584d 100644
--- a/be/src/vec/functions/array/function_array_union.cpp
+++ b/be/src/vec/functions/array/function_array_union.cpp
@@ -28,6 +28,7 @@
 
 namespace doris::vectorized {
 
+#include "common/compile_check_begin.h"
 struct NameArrayUnion {
     static constexpr auto name = "array_union";
 };
diff --git a/be/src/vec/functions/array/function_array_utils.h b/be/src/vec/functions/array/function_array_utils.h
index 36bf811b770899..dc522ed813760a 100644
--- a/be/src/vec/functions/array/function_array_utils.h
+++ b/be/src/vec/functions/array/function_array_utils.h
@@ -29,6 +29,7 @@ class IColumn;
 } // namespace doris
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 struct ColumnArrayMutableData {
 public:
@@ -70,7 +71,7 @@ struct ColumnArrayExecutionData {
             dst.offsets_ptr->push_back((*offsets_ptr)[row]);
             size_t off = (*offsets_ptr)[row - 1];
             size_t len = (*offsets_ptr)[row] - off;
-            for (int start = off; start < off + len; ++start) {
+            for (size_t start = off; start < off + len; ++start) {
                 if (nested_nullmap_data && nested_nullmap_data[start]) {
                     dst.nested_col->insert_default();
                     dst.nested_nullmap_data->push_back(1);
@@ -94,4 +95,5 @@ void slice_array(ColumnArrayMutableData& dst, ColumnArrayExecutionData& src,
                  const IColumn& offset_column, const IColumn* length_column);
 
 using ColumnArrayExecutionDatas = std::vector<ColumnArrayExecutionData>;
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/array/function_array_with_constant.cpp b/be/src/vec/functions/array/function_array_with_constant.cpp
index ba385535ebb3a6..7b9f3b43271f05 100644
--- a/be/src/vec/functions/array/function_array_with_constant.cpp
+++ b/be/src/vec/functions/array/function_array_with_constant.cpp
@@ -48,6 +48,7 @@ class Block;
 } // namespace doris
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 /* array_with_constant(num, T) / array_repeat(T, num)  - return array of constants with length num.
  * array_with_constant(2, 'xxx') = ['xxx', 'xxx']
@@ -89,7 +90,8 @@ class FunctionArrayWithConstant : public IFunction {
         ColumnArray::Offset64 offset = 0;
         std::vector<uint32_t> array_sizes;
         array_sizes.reserve(input_rows_count);
-        for (size_t i = 0; i < input_rows_count; ++i) {
+        // The array size will never gt int max value.
+        for (int i = 0; i < input_rows_count; ++i) {
             auto array_size = num->get_int(i);
             if (UNLIKELY(array_size < 0) || UNLIKELY(array_size > max_array_size_as_field)) {
                 return Status::InvalidArgument("Array size should in range(0, {}) in function: {}",
@@ -132,5 +134,5 @@ void register_function_array_with_constant(SimpleFunctionFactory& factory) {
     factory.register_function<FunctionArrayWithConstant<NameArrayWithConstant>>();
     factory.register_function<FunctionArrayWithConstant<NameArrayRepeat>>();
 }
-
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/date_format_type.h b/be/src/vec/functions/date_format_type.h
index 071ecf44853e1d..d7a0dcea2b028f 100644
--- a/be/src/vec/functions/date_format_type.h
+++ b/be/src/vec/functions/date_format_type.h
@@ -22,6 +22,7 @@
 #include "vec/common/string_ref.h"
 
 namespace doris::vectorized::time_format_type {
+#include "common/compile_check_begin.h"
 // Used to optimize commonly used date formats.
 
 inline StringRef rewrite_specific_format(const char* raw_str, size_t str_size) {
@@ -41,18 +42,18 @@ inline StringRef rewrite_specific_format(const char* raw_str, size_t str_size) {
 template <typename T>
 void put_year(T y, char* buf, int& i) {
     int t = y / 100;
-    buf[i++] = t / 10 + '0';
-    buf[i++] = t % 10 + '0';
+    buf[i++] = cast_set<char, int, false>(t / 10 + '0');
+    buf[i++] = cast_set<char, int, false>(t % 10 + '0');
 
     t = y % 100;
-    buf[i++] = t / 10 + '0';
-    buf[i++] = t % 10 + '0';
+    buf[i++] = cast_set<char, int, false>(t / 10 + '0');
+    buf[i++] = cast_set<char, int, false>(t % 10 + '0');
 }
 
 template <typename T>
 void put_other(T m, char* buf, int& i) {
-    buf[i++] = m / 10 + '0';
-    buf[i++] = m % 10 + '0';
+    buf[i++] = cast_set<char, int, false>(m / 10 + '0');
+    buf[i++] = cast_set<char, int, false>(m % 10 + '0');
 }
 
 // NoneImpl indicates that no specific optimization has been applied, and the general logic is used for processing.
@@ -152,5 +153,5 @@ inline FormatImplVariant string_to_impl(const std::string& format) {
         return NoneImpl {};
     }
 }
-
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized::time_format_type
diff --git a/be/src/vec/functions/date_time_transforms.h b/be/src/vec/functions/date_time_transforms.h
index d2c30d9d706332..c23fe147400412 100644
--- a/be/src/vec/functions/date_time_transforms.h
+++ b/be/src/vec/functions/date_time_transforms.h
@@ -38,6 +38,7 @@
 #include "vec/utils/util.hpp"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 #define TIME_FUNCTION_IMPL(CLASS, UNIT, FUNCTION)                                \
     template <typename ArgType>                                                  \
@@ -313,7 +314,8 @@ struct TransformerToStringOneArgument {
             const auto& date_time_value =
                     reinterpret_cast<const typename DateTraits<typename Transform::OpArgType>::T&>(
                             t);
-            res_offsets[i] = Transform::execute(date_time_value, res_data, offset);
+            res_offsets[i] =
+                    cast_set<UInt32>(Transform::execute(date_time_value, res_data, offset));
             null_map[i] = !date_time_value.is_valid_date();
         }
     }
@@ -331,7 +333,8 @@ struct TransformerToStringOneArgument {
             const auto& date_time_value =
                     reinterpret_cast<const typename DateTraits<typename Transform::OpArgType>::T&>(
                             t);
-            res_offsets[i] = Transform::execute(date_time_value, res_data, offset);
+            res_offsets[i] =
+                    cast_set<UInt32>(Transform::execute(date_time_value, res_data, offset));
             DCHECK(date_time_value.is_valid_date());
         }
     }
@@ -360,7 +363,7 @@ struct TransformerToStringTwoArgument {
             } else {
                 std::tie(new_offset, is_null) = Transform::execute(t, format, res_data, offset);
             }
-            res_offsets[i] = new_offset;
+            res_offsets[i] = cast_set<UInt32>(new_offset);
             null_map[i] = is_null;
         }
     }
@@ -375,7 +378,11 @@ struct Transformer {
         null_map.resize(size);
 
         for (size_t i = 0; i < size; ++i) {
-            vec_to[i] = Transform::execute(vec_from[i]);
+            // The transform result maybe an int, int32, but the result maybe short
+            // for example, year function. It is only a short.
+            auto res = Transform::execute(vec_from[i]);
+            using RESULT_TYPE = std::decay_t<decltype(res)>;
+            vec_to[i] = cast_set<ToType, RESULT_TYPE, false>(res);
             null_map[i] = !((typename DateTraits<typename Transform::OpArgType>::T&)(vec_from[i]))
                                    .is_valid_date();
         }
@@ -386,7 +393,9 @@ struct Transformer {
         vec_to.resize(size);
 
         for (size_t i = 0; i < size; ++i) {
-            vec_to[i] = Transform::execute(vec_from[i]);
+            auto res = Transform::execute(vec_from[i]);
+            using RESULT_TYPE = std::decay_t<decltype(res)>;
+            vec_to[i] = cast_set<ToType, RESULT_TYPE, false>(res);
             DCHECK(((typename DateTraits<typename Transform::OpArgType>::T&)(vec_from[i]))
                            .is_valid_date());
         }
@@ -465,4 +474,5 @@ struct DateTimeTransformImpl {
     }
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_binary_arithmetic.h b/be/src/vec/functions/function_binary_arithmetic.h
index 31e1d50f45b03c..4c0b8e7a0890dc 100644
--- a/be/src/vec/functions/function_binary_arithmetic.h
+++ b/be/src/vec/functions/function_binary_arithmetic.h
@@ -42,7 +42,6 @@
 #include "vec/utils/template_helpers.hpp"
 
 namespace doris::vectorized {
-
 // Arithmetic operations: +, -, *, |, &, ^, ~
 // need implement apply(a, b)
 
diff --git a/be/src/vec/functions/function_bit_count.cpp b/be/src/vec/functions/function_bit_count.cpp
index 4c3e45d81c8a8c..d4d9306a7c58bb 100644
--- a/be/src/vec/functions/function_bit_count.cpp
+++ b/be/src/vec/functions/function_bit_count.cpp
@@ -28,7 +28,7 @@
 #include "vec/functions/simple_function_factory.h"
 
 namespace doris::vectorized {
-
+#include "common/compile_check_begin.h"
 struct NameBitCount {
     static constexpr auto name = "bit_count";
 };
@@ -43,7 +43,9 @@ struct BitCountImpl {
         if constexpr (std::is_same_v<T, Int128> || std::is_same_v<T, Int64> ||
                       std::is_same_v<T, Int32> || std::is_same_v<T, Int16> ||
                       std::is_same_v<T, Int8>) {
-            return std::popcount(static_cast<std::make_unsigned_t<T>>(a));
+            // ResultType already check the length
+            return cast_set<ResultType, int, false>(
+                    std::popcount(static_cast<std::make_unsigned_t<T>>(a)));
         } else {
             throw Exception(ErrorCode::INVALID_ARGUMENT,
                             "bit_count only support using INTEGER as operator");
diff --git a/be/src/vec/functions/function_bitmap.cpp b/be/src/vec/functions/function_bitmap.cpp
index 64059472efcee9..d5f2075dffb6d2 100644
--- a/be/src/vec/functions/function_bitmap.cpp
+++ b/be/src/vec/functions/function_bitmap.cpp
@@ -76,6 +76,7 @@ class FunctionContext;
 } // namespace doris
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 struct BitmapEmpty {
     static constexpr auto name = "bitmap_empty";
@@ -112,7 +113,7 @@ struct ToBitmap {
                     continue;
                 } else {
                     const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
-                    size_t str_size = offsets[i] - offsets[i - 1];
+                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
                     StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
                     uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
                             raw_str, str_size, &parse_result);
@@ -168,7 +169,8 @@ struct ToBitmapWithCheck {
                     continue;
                 } else {
                     const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
-                    size_t str_size = offsets[i] - offsets[i - 1];
+                    // The string lenght is less than 2G, so that cast the str size to int, not use size_t
+                    int str_size = cast_set<int>(offsets[i] - offsets[i - 1]);
                     StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
                     uint64_t int_value = StringParser::string_to_unsigned_int<uint64_t>(
                             raw_str, str_size, &parse_result);
@@ -265,11 +267,11 @@ struct BitmapFromBase64 {
             return Status::OK();
         }
         std::string decode_buff;
-        int last_decode_buff_len = 0;
-        int curr_decode_buff_len = 0;
+        size_t last_decode_buff_len = 0;
+        size_t curr_decode_buff_len = 0;
         for (size_t i = 0; i < input_rows_count; ++i) {
             const char* src_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
-            int64_t src_size = offsets[i] - offsets[i - 1];
+            size_t src_size = offsets[i] - offsets[i - 1];
             if (0 != src_size % 4) {
                 // return Status::InvalidArgument(
                 //         fmt::format("invalid base64: {}", std::string(src_str, src_size)));
@@ -984,6 +986,7 @@ struct BitmapToBase64 {
     using Chars = ColumnString::Chars;
     using Offsets = ColumnString::Offsets;
 
+    // ColumnString not support 64bit, only 32bit, so that the max size is 4G
     static Status vector(const std::vector<BitmapValue>& data, Chars& chars, Offsets& offsets) {
         size_t size = data.size();
         offsets.resize(size);
@@ -1015,7 +1018,7 @@ struct BitmapToBase64 {
             DCHECK(outlen > 0);
 
             encoded_offset += (int)(4.0 * ceil((double)cur_ser_size / 3.0));
-            offsets[i] = encoded_offset;
+            offsets[i] = cast_set<UInt32>(encoded_offset);
         }
         return Status::OK();
     }
diff --git a/be/src/vec/functions/function_case.h b/be/src/vec/functions/function_case.h
index fdfecc211caed8..af44ea0d9b1ace 100644
--- a/be/src/vec/functions/function_case.h
+++ b/be/src/vec/functions/function_case.h
@@ -46,6 +46,7 @@
 #include "vec/utils/template_helpers.hpp"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <bool has_case, bool has_else>
 struct FunctionCaseName;
@@ -89,7 +90,7 @@ struct CaseWhenColumnHolder {
                         : std::nullopt);
 
         int begin = 0 + has_case;
-        int end = arguments.size() - has_else;
+        int end = cast_set<int>(arguments.size() - has_else);
         pair_count = (end - begin) / 2 + 1; // when/then at [1: pair_count)
 
         for (int i = begin; i < end; i += 2) {
@@ -131,7 +132,7 @@ class FunctionCase : public IFunction {
 
     DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
         int loop_start = has_case ? 2 : 1;
-        int loop_end = has_else ? arguments.size() - 1 : arguments.size();
+        int loop_end = cast_set<int>(has_else ? arguments.size() - 1 : arguments.size());
 
         bool is_nullable = false;
         if (!has_else || arguments[loop_end].get()->is_nullable()) {
@@ -156,7 +157,7 @@ class FunctionCase : public IFunction {
     Status execute_short_circuit(const DataTypePtr& data_type, Block& block, uint32_t result,
                                  CaseWhenColumnHolder column_holder) const {
         auto case_column_ptr = column_holder.when_ptrs[0].value_or(nullptr);
-        int rows_count = column_holder.rows_count;
+        size_t rows_count = column_holder.rows_count;
 
         // `then` data index corresponding to each row of results, 0 represents `else`.
         auto then_idx_uptr = std::unique_ptr<int[]>(new int[rows_count]);
@@ -203,7 +204,7 @@ class FunctionCase : public IFunction {
                                                                            column_holder);
         }
 
-        int rows_count = column_holder.rows_count;
+        size_t rows_count = column_holder.rows_count;
 
         // `then` data index corresponding to each row of results, 0 represents `else`.
         auto then_idx_uptr = std::unique_ptr<uint8_t[]>(new uint8_t[rows_count]);
@@ -426,4 +427,5 @@ class FunctionCase : public IFunction {
     }
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
index 7027a4e9c622bc..3ad456d60d201f 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -2401,5 +2401,4 @@ class FunctionBuilderCast : public FunctionBuilderImpl {
     bool use_default_implementation_for_nulls() const override { return false; }
     bool use_default_implementation_for_low_cardinality_columns() const override { return false; }
 };
-
 } // namespace doris::vectorized
diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp
index f7e72efa0899ea..86c50f0936f30d 100644
--- a/be/src/vec/runtime/vdatetime_value.cpp
+++ b/be/src/vec/runtime/vdatetime_value.cpp
@@ -591,7 +591,7 @@ static char* append_with_prefix(const char* str, int str_len, char prefix, int t
     return to + str_len;
 }
 
-int VecDateTimeValue::compute_format_len(const char* format, int len) {
+int VecDateTimeValue::compute_format_len(const char* format, size_t len) {
     int size = 0;
     const char* ptr = format;
     const char* end = format + len;
@@ -684,8 +684,8 @@ char* write_four_digits_to_string(int number, char* dst) {
     return dst + 4;
 }
 
-bool VecDateTimeValue::to_format_string_conservative(const char* format, int len, char* to,
-                                                     int max_valid_length) const {
+bool VecDateTimeValue::to_format_string_conservative(const char* format, size_t len, char* to,
+                                                     size_t max_valid_length) const {
     if (check_range(_year, _month, _day, _hour, _minute, _second, _type)) {
         return false;
     }
@@ -3450,8 +3450,8 @@ void DateV2Value<T>::set_microsecond(uint64_t microsecond) {
 }
 
 template <typename T>
-bool DateV2Value<T>::to_format_string_conservative(const char* format, int len, char* to,
-                                                   int max_valid_length) const {
+bool DateV2Value<T>::to_format_string_conservative(const char* format, size_t len, char* to,
+                                                   size_t max_valid_length) const {
     if (is_invalid(year(), month(), day(), hour(), minute(), second(), microsecond())) {
         return false;
     }
diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h
index 2cf7636347851a..dc7ba70b0b04ac 100644
--- a/be/src/vec/runtime/vdatetime_value.h
+++ b/be/src/vec/runtime/vdatetime_value.h
@@ -398,11 +398,11 @@ class VecDateTimeValue { // Now this type is a temp solution with little changes
     // for performance of checking, may return false when just APPROACH BUT NOT REACH max_valid_length.
     // so need a little big buffer and its length as max_valid_length to make sure store valid data.
     // to make sure of this. make the buffer size = <data_need_length> + SAFE_FORMAT_STRING_MARGIN. and pass this size as max_valid_length
-    bool to_format_string_conservative(const char* format, int len, char* to,
-                                       int max_valid_length) const;
+    bool to_format_string_conservative(const char* format, size_t len, char* to,
+                                       size_t max_valid_length) const;
 
     // compute the length of data format pattern
-    static int compute_format_len(const char* format, int len);
+    static int compute_format_len(const char* format, size_t len);
 
     // Return true if range or date is invalid
     static bool check_range(uint32_t year, uint32_t month, uint32_t day, uint32_t hour,
@@ -830,11 +830,11 @@ class DateV2Value {
     // for performance of checking, may return false when just APPROACH BUT NOT REACH max_valid_length.
     // so need a little big buffer and its length as max_valid_length to make sure store valid data.
     // to make sure of this. make the buffer size = <data_need_length> + SAFE_FORMAT_STRING_MARGIN. and pass this size as max_valid_length
-    bool to_format_string_conservative(const char* format, int len, char* to,
-                                       int max_valid_length) const;
+    bool to_format_string_conservative(const char* format, size_t len, char* to,
+                                       size_t max_valid_length) const;
 
-    bool from_date_format_str(const char* format, int format_len, const char* value,
-                              int64_t value_len) {
+    bool from_date_format_str(const char* format, size_t format_len, const char* value,
+                              size_t value_len) {
         return from_date_format_str(format, format_len, value, value_len, nullptr);
     }
 

From d77bfa09d85d9759965c109cc50ee89e8fbde499 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Thu, 28 Nov 2024 16:33:16 +0800
Subject: [PATCH 063/399] =?UTF-8?q?[Improvement](shuffle)=20Use=20a=20knob?=
 =?UTF-8?q?=20to=20decide=20whether=20a=20serial=20exchange=E2=80=A6=20(#4?=
 =?UTF-8?q?4676)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… should be used

This improvement was completed in #43199 and reverted by #44075 due to performance fallback. After fixing it, this improvement is re-submited.

A new knob to control a exchange node should be serial or not.
For example, a partitioned hash join should be executed like below:
```
┌────────────────────────────┐                  ┌────────────────────────────┐
│                            │                  │                            │
│Exchange(HASH PARTITIONED N)│                  │Exchange(HASH PARTITIONED N)│
│                            │                  │                            │
└────────────────────────────┴─────────┬────────┴────────────────────────────┘
                                       │
                                       │
                                       │
                                       │
                                       │
                                       │
                                ┌──────▼──────┐
                                │             │
                                │ HASH  JOIN  │
                                │             │
                                └─────────────┘
```

After turning on this knob, the real plan should be:
```
  ┌──────────────────────────────┐                        ┌──────────────────────────────┐
  │                              │                        │                              │
  │ Exchange (HASH PARTITIONED 1)│                        │ Exchange (HASH PARTITIONED 1)│
  │                              │                        │                              │
  └────────────┬─────────────────┘                        └────────────┬─────────────────┘
               │                                                       │
               │                                                       │
               │                                                       │
               │                                                       │
               │                                                       │
┌──────────────▼─────────────────────┐                  ┌──────────────▼─────────────────────┐
│                                    │                  │                                    │
│ Local  Exchange(HASH PARTITIONED N)│                  │ Local  Exchange(HASH PARTITIONED N)│
│              1 -> N                │                  │              1 -> N                │
└────────────────────────────────────┴─────────┬────────┴────────────────────────────────────┴
                                               │
                                               │
                                               │
                                               │
                                               │
                                               │
                                        ┌──────▼──────┐
                                        │             │
                                        │ HASH  JOIN  │
                                        │             │
                                        └─────────────┘
```

For large cluster, X (mappers) * Y (reducers) rpc channels can be reduced to X (mappers) * Z (BEs).
---
 .../org/apache/doris/planner/ExchangeNode.java     | 13 ++++++++++++-
 .../org/apache/doris/planner/PlanFragment.java     | 14 ++++++--------
 .../java/org/apache/doris/planner/PlanNode.java    |  7 +++++++
 .../java/org/apache/doris/planner/ScanNode.java    |  5 +++++
 .../main/java/org/apache/doris/qe/Coordinator.java | 10 ++--------
 .../java/org/apache/doris/qe/SessionVariable.java  | 11 +++++++++++
 6 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
index 1ca1db56bfc7c6..cb6628b01c556b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
@@ -25,6 +25,7 @@
 import org.apache.doris.analysis.TupleDescriptor;
 import org.apache.doris.analysis.TupleId;
 import org.apache.doris.common.UserException;
+import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.statistics.StatsRecursiveDerive;
 import org.apache.doris.thrift.TExchangeNode;
@@ -169,6 +170,10 @@ public void setMergeInfo(SortInfo info) {
 
     @Override
     protected void toThrift(TPlanNode msg) {
+        // If this fragment has another scan node, this exchange node is serial or not should be decided by the scan
+        // node.
+        msg.setIsSerialOperator((isSerialOperator() || fragment.hasSerialScanNode())
+                && fragment.useSerialSource(ConnectContext.get()));
         msg.node_type = TPlanNodeType.EXCHANGE_NODE;
         msg.exchange_node = new TExchangeNode();
         for (TupleId tid : tupleIds) {
@@ -228,11 +233,17 @@ public void setRightChildOfBroadcastHashJoin(boolean value) {
      */
     @Override
     public boolean isSerialOperator() {
-        return partitionType == TPartitionType.UNPARTITIONED && mergeInfo != null;
+        return (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isUseSerialExchange()
+                || partitionType == TPartitionType.UNPARTITIONED) && mergeInfo != null;
     }
 
     @Override
     public boolean hasSerialChildren() {
         return isSerialOperator();
     }
+
+    @Override
+    public boolean hasSerialScanChildren() {
+        return false;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
index fef3de9b696e9e..ab5307c07e904a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
@@ -515,15 +515,13 @@ public boolean useSerialSource(ConnectContext context) {
                 && !hasNullAwareLeftAntiJoin()
                 // If planRoot is not a serial operator and has serial children, we can use serial source and improve
                 // parallelism of non-serial operators.
-                && sink instanceof DataStreamSink && !planRoot.isSerialOperator()
-                && planRoot.hasSerialChildren();
+                // For bucket shuffle / colocate join fragment, always use serial source if the bucket scan nodes are
+                // serial.
+                && (hasSerialScanNode() || (sink instanceof DataStreamSink && !planRoot.isSerialOperator()
+                && planRoot.hasSerialChildren()));
     }
 
-    public int getNumBackends() {
-        return numBackends;
-    }
-
-    public void setNumBackends(int numBackends) {
-        this.numBackends = numBackends;
+    public boolean hasSerialScanNode() {
+        return planRoot.hasSerialScanChildren();
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 14bd34e93e1f43..73768435154b76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -1388,4 +1388,11 @@ public boolean hasSerialChildren() {
         }
         return children.stream().allMatch(PlanNode::hasSerialChildren);
     }
+
+    public boolean hasSerialScanChildren() {
+        if (children.isEmpty()) {
+            return false;
+        }
+        return children.stream().anyMatch(PlanNode::hasSerialScanChildren);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index a2583868346704..b4033a0535ef3d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -861,4 +861,9 @@ public boolean isSerialOperator() {
                 < ConnectContext.get().getSessionVariable().getParallelExecInstanceNum() * numScanBackends()
                 || (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isForceToLocalShuffle());
     }
+
+    @Override
+    public boolean hasSerialScanChildren() {
+        return isSerialOperator();
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 3a6f6e4f84069d..262b5836689ab0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -1886,17 +1886,11 @@ protected void computeFragmentHosts() throws Exception {
                             return scanNode.getId().asInt() == planNodeId;
                         }).findFirst();
 
-                        /**
-                         * Ignore storage data distribution iff:
-                         * 1. `parallelExecInstanceNum * numBackends` is larger than scan ranges.
-                         * 2. Use Nereids planner.
-                         */
                         boolean sharedScan = true;
                         int expectedInstanceNum = Math.min(parallelExecInstanceNum,
                                 leftMostNode.getNumInstances());
-                        boolean ignoreStorageDataDistribution = node.isPresent()
-                                && fragment.useSerialSource(context);
-                        if (node.isPresent() && ignoreStorageDataDistribution) {
+                        boolean ignoreStorageDataDistribution = fragment.useSerialSource(context);
+                        if (ignoreStorageDataDistribution) {
                             expectedInstanceNum = Math.max(expectedInstanceNum, 1);
                             // if have limit and no conjuncts, only need 1 instance to save cpu and
                             // mem resource
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 71b746c7907262..115614a41873a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -268,6 +268,8 @@ public class SessionVariable implements Serializable, Writable {
 
     public static final String IGNORE_STORAGE_DATA_DISTRIBUTION = "ignore_storage_data_distribution";
 
+    public static final String USE_SERIAL_EXCHANGE = "use_serial_exchange";
+
     public static final String ENABLE_PARALLEL_SCAN = "enable_parallel_scan";
 
     // Limit the max count of scanners to prevent generate too many scanners.
@@ -1112,6 +1114,10 @@ public enum IgnoreSplitType {
             varType = VariableAnnotation.EXPERIMENTAL, needForward = true)
     private boolean ignoreStorageDataDistribution = true;
 
+    @VariableMgr.VarAttr(name = USE_SERIAL_EXCHANGE, fuzzy = true,
+            varType = VariableAnnotation.EXPERIMENTAL, needForward = true)
+    private boolean useSerialExchange = false;
+
     @VariableMgr.VarAttr(
             name = ENABLE_LOCAL_SHUFFLE, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL,
             description = {"是否在pipelineX引擎上开启local shuffle优化",
@@ -2353,6 +2359,7 @@ public void initFuzzyModeVariables() {
         this.parallelPrepareThreshold = random.nextInt(32) + 1;
         this.enableCommonExprPushdown = random.nextBoolean();
         this.enableLocalExchange = random.nextBoolean();
+        this.useSerialExchange = random.nextBoolean();
         // This will cause be dead loop, disable it first
         // this.disableJoinReorder = random.nextBoolean();
         this.enableCommonExpPushDownForInvertedIndex = random.nextBoolean();
@@ -4563,6 +4570,10 @@ public boolean isEnableCooldownReplicaAffinity() {
         return enableCooldownReplicaAffinity;
     }
 
+    public boolean isUseSerialExchange() {
+        return useSerialExchange && getEnableLocalExchange();
+    }
+
     public void setDisableInvertedIndexV1ForVaraint(boolean disableInvertedIndexV1ForVaraint) {
         this.disableInvertedIndexV1ForVaraint = disableInvertedIndexV1ForVaraint;
     }

From 1ae91b89936e35d4ea329169cbf0dae3be75668a Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Thu, 28 Nov 2024 17:38:40 +0800
Subject: [PATCH 064/399] [fix](nereids) fix months_add/ months_sub/
 years_add/years_sub compute wrong result because
 SimplifyArithmeticComparisonRule (#44725)

fix months_add/ months_sub/ years_add/years_sub compute wrong result
because SimplifyArithmeticComparisonRule
introduced by #25180
---
 .../rules/SimplifyArithmeticComparisonRule.java       | 11 +++--------
 .../rules/expression/SimplifyArithmeticRuleTest.java  |  8 ++++----
 .../datetime_functions/test_date_function.groovy      | 11 +++++++++++
 .../datetime_functions/test_date_function.groovy      | 11 +++++++++++
 4 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java
index 6d18bc7b3807a6..863bfd18f458fd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java
@@ -32,14 +32,10 @@
 import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursSub;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.MinutesAdd;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.MinutesSub;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.MonthsAdd;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.MonthsSub;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.SecondsAdd;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.SecondsSub;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.WeeksAdd;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.WeeksSub;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub;
 import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.util.TypeCoercionUtils;
@@ -66,10 +62,9 @@ public class SimplifyArithmeticComparisonRule implements ExpressionPatternRuleFa
             .put(Add.class, Subtract.class)
             .put(Subtract.class, Add.class)
             .put(Divide.class, Multiply.class)
-            .put(YearsSub.class, YearsAdd.class)
-            .put(YearsAdd.class, YearsSub.class)
-            .put(MonthsSub.class, MonthsAdd.class)
-            .put(MonthsAdd.class, MonthsSub.class)
+            // ATTN: YearsAdd, MonthsAdd can not reverse
+            //       for example, months_add(date '2024-01-31', 1) = date '2024-02-29' can not reverse to
+            //       date '2024-01-31' = months_sub(date '2024-02-29', 1)
             .put(WeeksSub.class, WeeksAdd.class)
             .put(WeeksAdd.class, WeeksSub.class)
             .put(DaysSub.class, DaysAdd.class)
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java
index 55ce93da63d5aa..f23aefe5267c1a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java
@@ -151,10 +151,10 @@ void testSimplifyDateTimeComparison() {
                     FoldConstantRule.INSTANCE
                 )
         ));
-        assertRewriteAfterTypeCoercion("years_add(IA, 1) > '2021-01-01 00:00:00'", "(cast(IA as DATETIMEV2(0)) > '2020-01-01 00:00:00')");
-        assertRewriteAfterTypeCoercion("years_sub(IA, 1) > '2021-01-01 00:00:00'", "(cast(IA as DATETIMEV2(0)) > '2022-01-01 00:00:00')");
-        assertRewriteAfterTypeCoercion("months_add(IA, 1) > '2021-01-01 00:00:00'", "(cast(IA as DATETIMEV2(0)) > '2020-12-01 00:00:00')");
-        assertRewriteAfterTypeCoercion("months_sub(IA, 1) > '2021-01-01 00:00:00'", "(cast(IA as DATETIMEV2(0)) > '2021-02-01 00:00:00')");
+        assertRewriteAfterTypeCoercion("years_add(IA, 1) > '2021-01-01 00:00:00'", "(years_add(cast(IA as DATETIMEV2(0)), 1) > '2021-01-01 00:00:00')");
+        assertRewriteAfterTypeCoercion("years_sub(IA, 1) > '2021-01-01 00:00:00'", "(years_sub(cast(IA as DATETIMEV2(0)), 1) > '2021-01-01 00:00:00')");
+        assertRewriteAfterTypeCoercion("months_add(IA, 1) > '2021-01-01 00:00:00'", "(months_add(cast(IA as DATETIMEV2(0)), 1) > '2021-01-01 00:00:00')");
+        assertRewriteAfterTypeCoercion("months_sub(IA, 1) > '2021-01-01 00:00:00'", "(months_sub(cast(IA as DATETIMEV2(0)), 1) > '2021-01-01 00:00:00')");
         assertRewriteAfterTypeCoercion("weeks_add(IA, 1) > '2021-01-01 00:00:00'", "(cast(IA as DATETIMEV2(0)) > '2020-12-25 00:00:00')");
         assertRewriteAfterTypeCoercion("weeks_sub(IA, 1) > '2021-01-01 00:00:00'", "(cast(IA as DATETIMEV2(0)) > '2021-01-08 00:00:00')");
         assertRewriteAfterTypeCoercion("days_add(IA, 1) > '2021-01-01 00:00:00'", "(cast(IA as DATETIMEV2(0)) > '2020-12-31 00:00:00')");
diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
index 5bde4423253744..bdd92166f7e686 100644
--- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
@@ -720,4 +720,15 @@ suite("test_date_function") {
 
 
     qt_sql_time_value """ select  cast(4562632 as time),  hour(cast(4562632 as time)) ,  minute(cast(4562632 as time)) , second(cast(4562632 as time)); """
+
+    def test_simplify = {
+        test {
+            sql "select months_add(dt, 1) = date '2024-02-29' from (select date '2024-01-31' as dt)a"
+            result([[true]])
+        }
+        test {
+            sql "select years_add(dt, 1) = date '2025-02-28' from (select date '2024-02-29' as dt)a"
+            result([[true]])
+        }
+    }()
 }
diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
index 511f2bc35a67a7..8bd43dae685afc 100644
--- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
@@ -818,4 +818,15 @@ suite("test_date_function") {
     qt_sql_varchar1 """ select dt, fmt, unix_timestamp(dt, fmt) as k1 from date_varchar order by k1,dt,fmt; """
     qt_sql_varchar1 """ select dt, unix_timestamp(dt, "%Y-%m-%d") as k1 from date_varchar order by k1,dt,fmt; """
     qt_sql_varchar1 """ select fmt, unix_timestamp("1990-12-12", fmt) as k1 from date_varchar order by k1,dt,fmt; """
+
+    def test_simplify = {
+        test {
+            sql "select months_add(dt, 1) = date '2024-02-29' from (select date '2024-01-31' as dt)a"
+            result([[true]])
+        }
+        test {
+            sql "select years_add(dt, 1) = date '2025-02-28' from (select date '2024-02-29' as dt)a"
+            result([[true]])
+        }
+    }()
 }

From 51ad321b84cf211efd63415a1052e2f72a3415c5 Mon Sep 17 00:00:00 2001
From: zclllhhjj <zhaochangle@selectdb.com>
Date: Thu, 28 Nov 2024 18:24:16 +0800
Subject: [PATCH 065/399] [Enhancement](function) Print exact column name when
 execute non_nullable failed (#44679)

### What problem does this PR solve?
Before it print the column type. now more precisely print the column
name
---
 be/src/vec/functions/function_nullables.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/vec/functions/function_nullables.cpp b/be/src/vec/functions/function_nullables.cpp
index 88bb2b38fab65c..91bce24f48fc8b 100644
--- a/be/src/vec/functions/function_nullables.cpp
+++ b/be/src/vec/functions/function_nullables.cpp
@@ -96,7 +96,7 @@ class FunctionNonNullable : public IFunction {
             if (col_null->has_null()) [[unlikely]] {
                 return Status::InvalidArgument(
                         "There's NULL value in column {} which is illegal for non_nullable",
-                        data.column->get_name());
+                        data.name);
             }
             const ColumnPtr& nest_col = col_null->get_nested_column_ptr();
             block.replace_by_position(result, nest_col->clone_resized(nest_col->size()));

From 9a3622409966af5d5ab220ffc881df4ed0d64cf2 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Thu, 28 Nov 2024 22:12:25 +0800
Subject: [PATCH 066/399] [Chore](log)  set some variable's fuzzy to true and
 display inited for bf func check (#44752)

### What problem does this PR solve?
 set some variable's fuzzy to true and display inited for bf func check
---
 be/src/exprs/bloom_filter_func.h                   | 14 +++++++-------
 .../java/org/apache/doris/qe/SessionVariable.java  |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h
index 54ad7502810484..674a429581643b 100644
--- a/be/src/exprs/bloom_filter_func.h
+++ b/be/src/exprs/bloom_filter_func.h
@@ -157,19 +157,18 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
             return Status::InternalError("bloomfilter_func is nullptr");
         }
         if (bloomfilter_func->_bloom_filter == nullptr) {
-            return Status::InternalError("bloomfilter_func->_bloom_filter is nullptr");
+            return Status::InternalError(
+                    "bloomfilter_func->_bloom_filter is nullptr, bloomfilter_func->inited: {}",
+                    bloomfilter_func->_inited);
         }
         // If `_inited` is false, there is no memory allocated in bloom filter and this is the first
         // call for `merge` function. So we just reuse this bloom filter, and we don't need to
         // allocate memory again.
         if (!_inited) {
-            auto* other_func = static_cast<BloomFilterFuncBase*>(bloomfilter_func);
             if (_bloom_filter != nullptr) {
-                return Status::InternalError("_bloom_filter must is nullptr");
+                return Status::InternalError("_bloom_filter must is nullptr, inited: {}", _inited);
             }
-            _bloom_filter = bloomfilter_func->_bloom_filter;
-            _bloom_filter_alloced = other_func->_bloom_filter_alloced;
-            _inited = true;
+            light_copy(bloomfilter_func);
             return Status::OK();
         }
         auto* other_func = static_cast<BloomFilterFuncBase*>(bloomfilter_func);
@@ -207,7 +206,8 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
 
     bool contain_null() const {
         if (!_bloom_filter) {
-            throw Exception(ErrorCode::INTERNAL_ERROR, "_bloom_filter is nullptr");
+            throw Exception(ErrorCode::INTERNAL_ERROR, "_bloom_filter is nullptr, inited: {}",
+                            _inited);
         }
         return _bloom_filter->contain_null();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 115614a41873a6..f977c3ccb05f17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1202,10 +1202,10 @@ public enum IgnoreSplitType {
     @VariableMgr.VarAttr(name = RUNTIME_FILTER_TYPE, fuzzy = true, needForward = true)
     private int runtimeFilterType = 12;
 
-    @VariableMgr.VarAttr(name = RUNTIME_FILTER_MAX_IN_NUM, needForward = true)
+    @VariableMgr.VarAttr(name = RUNTIME_FILTER_MAX_IN_NUM, needForward = true, fuzzy = true)
     private int runtimeFilterMaxInNum = 1024;
 
-    @VariableMgr.VarAttr(name = ENABLE_SYNC_RUNTIME_FILTER_SIZE, needForward = true)
+    @VariableMgr.VarAttr(name = ENABLE_SYNC_RUNTIME_FILTER_SIZE, needForward = true, fuzzy = true)
     private boolean enableSyncRuntimeFilterSize = true;
 
     @VariableMgr.VarAttr(name = ENABLE_PARALLEL_RESULT_SINK, needForward = true, fuzzy = true)
@@ -1456,7 +1456,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
     @VariableMgr.VarAttr(name = BROADCAST_HASHTABLE_MEM_LIMIT_PERCENTAGE, needForward = true)
     private double broadcastHashtableMemLimitPercentage = 0.2;
 
-    @VariableMgr.VarAttr(name = ENABLE_RUNTIME_FILTER_PRUNE, needForward = true)
+    @VariableMgr.VarAttr(name = ENABLE_RUNTIME_FILTER_PRUNE, needForward = true, fuzzy = true)
     public boolean enableRuntimeFilterPrune = true;
 
     /**

From 358ed8caac231f88f0ce40d484ac5c2fdc626f8b Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Thu, 28 Nov 2024 22:13:27 +0800
Subject: [PATCH 067/399] [feature](hive)support hive catalog read json table. 
 (#43469)

### What problem does this PR solve?

Problem Summary:
Support reading json format hive table like:
```mysql
mysql> show create table basic_json_table;
CREATE TABLE `basic_json_table`(
  `id` int,
  `name` string,
  `age` tinyint,
  `salary` float,
  `is_active` boolean,
  `join_date` date,
  `last_login` timestamp,
  `height` double,
  `profile` binary,
  `rating` decimal(10,2))
ROW FORMAT SERDE
  'org.apache.hive.hcatalog.data.JsonSerDe'
```

Behavior changed:
To implement this feature, this pr modifies `new_json_reader`.
Previously, `new_json_reader` could only insert data into columnString.
In order to support inserting data into columns of other types,
`DataTypeSerDe` is introduced to insert data into columns. To maintain
compatibility with previous versions, changes to this pr are triggered
only when reading hive json tables.

Limitation of Use:
1. Currently, only query is supported, and writing is not supported.
2. Currently, only the `ROW FORMAT SERDE
'org.apache.hive.hcatalog.data.JsonSerDe';` scenario is supported. For
some properties specified in `with serdeproperties`, Doris does not take
effect.
3. Since Hive does not allow columns with the same name but different
case when creating a table in Json format (including inside a Struct),
we convert the field names in the Json data to lowercase when reading
the Json data file, and then match according to the lowercase field
names. For field names that are duplicated after being converted to
lowercase in the data, the value of the last field is used (consistent
with Hive behavior).
example:
```
create table json_table(
    column int
)ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';

a.json:
{"column":1,"COLumn",2,"COLUMN":3}
{"column":10,"COLumn",20}
{"column":100}
in Hive : load a.json to table json_table

in Doris query:
---
3
20
100
---
```

Todo(in next pr):
Merge `serde` and `json_reader` ,because they have logical conflicts.

### Release note
Hive catalog support read json format table.
---
 .../data_types/serde/data_type_array_serde.h  |   2 +
 .../data_types/serde/data_type_map_serde.h    |   4 +
 .../serde/data_type_nullable_serde.h          |   2 +
 be/src/vec/data_types/serde/data_type_serde.h |  12 +-
 .../data_types/serde/data_type_struct_serde.h |   2 +
 .../vec/exec/format/json/new_json_reader.cpp  | 559 +++++++++++++-----
 be/src/vec/exec/format/json/new_json_reader.h |  32 +-
 be/src/vec/exec/scan/vfile_scanner.cpp        |   4 +-
 .../create_preinstalled_scripts/run69.hql     |  35 ++
 .../create_preinstalled_scripts/run70.hql     |  73 +++
 .../create_preinstalled_scripts/run71.hql     |  13 +
 .../json_all_complex_types/dt=dt1/000000_0    |   3 +
 .../json_all_complex_types/dt=dt2/000000_0    |   1 +
 .../json_all_complex_types/dt=dt3/000000_0    |   2 +
 .../json/json_load_data_table/1               |  13 +
 .../json/json_nested_complex_table/1          |   2 +
 .../json/json_nested_complex_table/2          |   1 +
 .../json/json_nested_complex_table/modify_2   |   2 +
 .../hive/HiveMetaStoreClientHelper.java       |   3 +
 .../datasource/hive/source/HiveScanNode.java  |  41 +-
 .../hive/hive_json_basic_test.out             | 115 ++++
 .../hive/hive_json_basic_test.groovy          |  71 +++
 22 files changed, 832 insertions(+), 160 deletions(-)
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run69.hql
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run70.hql
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run71.hql
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt1/000000_0
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt2/000000_0
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt3/000000_0
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_load_data_table/1
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/1
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/2
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/modify_2
 create mode 100644 regression-test/data/external_table_p0/hive/hive_json_basic_test.out
 create mode 100644 regression-test/suites/external_table_p0/hive/hive_json_basic_test.groovy

diff --git a/be/src/vec/data_types/serde/data_type_array_serde.h b/be/src/vec/data_types/serde/data_type_array_serde.h
index 5b15f48f502b81..cdd2115576030e 100644
--- a/be/src/vec/data_types/serde/data_type_array_serde.h
+++ b/be/src/vec/data_types/serde/data_type_array_serde.h
@@ -101,6 +101,8 @@ class DataTypeArraySerDe : public DataTypeSerDe {
         nested_serde->set_return_object_as_string(value);
     }
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return {nested_serde}; }
+
 private:
     template <bool is_binary_format>
     Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer<is_binary_format>& result,
diff --git a/be/src/vec/data_types/serde/data_type_map_serde.h b/be/src/vec/data_types/serde/data_type_map_serde.h
index 599533eb4d0951..51c11300d44c01 100644
--- a/be/src/vec/data_types/serde/data_type_map_serde.h
+++ b/be/src/vec/data_types/serde/data_type_map_serde.h
@@ -96,6 +96,10 @@ class DataTypeMapSerDe : public DataTypeSerDe {
         value_serde->set_return_object_as_string(value);
     }
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const override {
+        return {key_serde, value_serde};
+    }
+
 private:
     template <bool is_binary_format>
     Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer<is_binary_format>& result,
diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.h b/be/src/vec/data_types/serde/data_type_nullable_serde.h
index 6051c7f722d62a..159db890540dbc 100644
--- a/be/src/vec/data_types/serde/data_type_nullable_serde.h
+++ b/be/src/vec/data_types/serde/data_type_nullable_serde.h
@@ -99,6 +99,8 @@ class DataTypeNullableSerDe : public DataTypeSerDe {
                                   int64_t row_num) const override;
     Status read_one_cell_from_json(IColumn& column, const rapidjson::Value& result) const override;
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return {nested_serde}; }
+
 private:
     template <bool is_binary_format>
     Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer<is_binary_format>& result,
diff --git a/be/src/vec/data_types/serde/data_type_serde.h b/be/src/vec/data_types/serde/data_type_serde.h
index f0e9eb27961439..bec0fabfcca9c3 100644
--- a/be/src/vec/data_types/serde/data_type_serde.h
+++ b/be/src/vec/data_types/serde/data_type_serde.h
@@ -100,6 +100,10 @@ class IColumn;
 class Arena;
 class IDataType;
 
+class DataTypeSerDe;
+using DataTypeSerDeSPtr = std::shared_ptr<DataTypeSerDe>;
+using DataTypeSerDeSPtrs = std::vector<DataTypeSerDeSPtr>;
+
 // Deserialize means read from different file format or memory format,
 // for example read from arrow, read from parquet.
 // Serialize means write the column cell or the total column into another
@@ -337,6 +341,11 @@ class DataTypeSerDe {
                                           Arena& mem_pool, int64_t row_num) const;
     virtual Status read_one_cell_from_json(IColumn& column, const rapidjson::Value& result) const;
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const {
+        throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
+                               "Method get_nested_serdes is not supported for this serde");
+    }
+
 protected:
     bool _return_object_as_string = false;
     // This parameter indicates what level the serde belongs to and is mainly used for complex types
@@ -379,9 +388,6 @@ inline void checkArrowStatus(const arrow::Status& status, const std::string& col
     }
 }
 
-using DataTypeSerDeSPtr = std::shared_ptr<DataTypeSerDe>;
-using DataTypeSerDeSPtrs = std::vector<DataTypeSerDeSPtr>;
-
 DataTypeSerDeSPtrs create_data_type_serdes(
         const std::vector<std::shared_ptr<const IDataType>>& types);
 DataTypeSerDeSPtrs create_data_type_serdes(const std::vector<SlotDescriptor*>& slots);
diff --git a/be/src/vec/data_types/serde/data_type_struct_serde.h b/be/src/vec/data_types/serde/data_type_struct_serde.h
index 2ec871838d7ed7..fc77e26b71f882 100644
--- a/be/src/vec/data_types/serde/data_type_struct_serde.h
+++ b/be/src/vec/data_types/serde/data_type_struct_serde.h
@@ -172,6 +172,8 @@ class DataTypeStructSerDe : public DataTypeSerDe {
         }
     }
 
+    virtual DataTypeSerDeSPtrs get_nested_serdes() const override { return elem_serdes_ptrs; }
+
 private:
     std::optional<size_t> try_get_position_by_name(const String& name) const;
 
diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp
index d726de3d99817e..d79e86520741cd 100644
--- a/be/src/vec/exec/format/json/new_json_reader.cpp
+++ b/be/src/vec/exec/format/json/new_json_reader.cpp
@@ -54,8 +54,11 @@
 #include "util/slice.h"
 #include "util/uid_util.h"
 #include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
 #include "vec/columns/column_nullable.h"
 #include "vec/columns/column_string.h"
+#include "vec/columns/column_struct.h"
 #include "vec/common/assert_cast.h"
 #include "vec/common/typeid_cast.h"
 #include "vec/core/block.h"
@@ -164,10 +167,18 @@ void NewJsonReader::_init_file_description() {
 }
 
 Status NewJsonReader::init_reader(
-        const std::unordered_map<std::string, VExprContextSPtr>& col_default_value_ctx) {
+        const std::unordered_map<std::string, VExprContextSPtr>& col_default_value_ctx,
+        bool is_load) {
+    _is_load = is_load;
+
     // generate _col_default_value_map
     RETURN_IF_ERROR(_get_column_default_value(_file_slot_descs, col_default_value_ctx));
 
+    //use serde insert data to column.
+    for (auto* slot_desc : _file_slot_descs) {
+        _serdes.emplace_back(slot_desc->get_data_type_ptr()->get_serde());
+    }
+
     // create decompressor.
     // _decompressor may be nullptr if this is not a compressed file
     RETURN_IF_ERROR(Decompressor::create_decompressor(_file_compress_type, &_decompressor));
@@ -390,6 +401,9 @@ Status NewJsonReader::_get_range_params() {
     if (_params.file_attributes.__isset.fuzzy_parse) {
         _fuzzy_parse = _params.file_attributes.fuzzy_parse;
     }
+    if (_range.table_format_params.table_format_type == "hive") {
+        _is_hive_table = true;
+    }
     return Status::OK();
 }
 
@@ -477,8 +491,8 @@ Status NewJsonReader::_vhandle_simple_json(RuntimeState* /*state*/, Block& block
         bool valid = false;
         if (_next_row >= _total_rows) { // parse json and generic document
             Status st = _parse_json(is_empty_row, eof);
-            if (st.is<DATA_QUALITY_ERROR>()) {
-                continue; // continue to read next
+            if (_is_load && st.is<DATA_QUALITY_ERROR>()) {
+                continue; // continue to read next (for load, after this , already append error to file.)
             }
             RETURN_IF_ERROR(st);
             if (*is_empty_row) {
@@ -761,7 +775,20 @@ Status NewJsonReader::_set_column_value(rapidjson::Value& objectValue, Block& bl
         _append_empty_skip_bitmap_value(block, cur_row_count);
     }
 
-    for (auto* slot_desc : slot_descs) {
+    if (_is_hive_table) {
+        //don't like _fuzzy_parse,each line read in must modify name_map once.
+
+        for (int i = 0; i < objectValue.MemberCount(); ++i) {
+            auto it = objectValue.MemberBegin() + i;
+            std::string field_name(it->name.GetString(), it->name.GetStringLength());
+            std::transform(field_name.begin(), field_name.end(), field_name.begin(), ::tolower);
+
+            //Use the last value with the same name.
+            _name_map.emplace(field_name, i);
+        }
+    }
+    for (size_t slot_idx = 0; slot_idx < slot_descs.size(); ++slot_idx) {
+        auto* slot_desc = slot_descs[slot_idx];
         if (!slot_desc->is_materialized()) {
             continue;
         }
@@ -773,7 +800,7 @@ Status NewJsonReader::_set_column_value(rapidjson::Value& objectValue, Block& bl
         auto* column_ptr = block.get_by_position(dest_index).column->assume_mutable().get();
         rapidjson::Value::ConstMemberIterator it = objectValue.MemberEnd();
 
-        if (_fuzzy_parse) {
+        if (_fuzzy_parse || _is_hive_table) {
             auto idx_it = _name_map.find(slot_desc->col_name());
             if (idx_it != _name_map.end() && idx_it->second < objectValue.MemberCount()) {
                 it = objectValue.MemberBegin() + idx_it->second;
@@ -785,7 +812,8 @@ Status NewJsonReader::_set_column_value(rapidjson::Value& objectValue, Block& bl
 
         if (it != objectValue.MemberEnd()) {
             const rapidjson::Value& value = it->value;
-            RETURN_IF_ERROR(_write_data_to_column(&value, slot_desc, column_ptr, valid));
+            RETURN_IF_ERROR(_write_data_to_column(&value, slot_desc->type(), column_ptr,
+                                                  slot_desc->col_name(), _serdes[slot_idx], valid));
             if (!(*valid)) {
                 return Status::OK();
             }
@@ -814,14 +842,15 @@ Status NewJsonReader::_set_column_value(rapidjson::Value& objectValue, Block& bl
                 column_ptr->insert_default();
             } else {
                 // not found, filling with default value
-                RETURN_IF_ERROR(_fill_missing_column(slot_desc, column_ptr, valid));
+                RETURN_IF_ERROR(
+                        _fill_missing_column(slot_desc, _serdes[slot_idx], column_ptr, valid));
                 if (!(*valid)) {
                     return Status::OK();
                 }
             }
         }
     }
-    if (!has_valid_value) {
+    if (!has_valid_value && _is_load) {
         // there is no valid value in json line but has filled with default value before
         // so remove this line in block
         string col_names;
@@ -845,79 +874,188 @@ Status NewJsonReader::_set_column_value(rapidjson::Value& objectValue, Block& bl
 }
 
 Status NewJsonReader::_write_data_to_column(rapidjson::Value::ConstValueIterator value,
-                                            SlotDescriptor* slot_desc, IColumn* column_ptr,
+                                            const TypeDescriptor& type_desc,
+                                            vectorized::IColumn* column_ptr,
+                                            const std::string& column_name, DataTypeSerDeSPtr serde,
                                             bool* valid) {
-    const char* str_value = nullptr;
-    char tmp_buf[128] = {0};
-    int32_t wbytes = 0;
-    std::string json_str;
-
     ColumnNullable* nullable_column = nullptr;
-    if (slot_desc->is_nullable()) {
+    vectorized::IColumn* data_column_ptr = column_ptr;
+    DataTypeSerDeSPtr data_serde = serde;
+
+    bool value_is_null = (value == nullptr) || (value->GetType() == rapidjson::Type::kNullType);
+
+    if (column_ptr->is_nullable()) {
         nullable_column = reinterpret_cast<ColumnNullable*>(column_ptr);
-        // kNullType will put 1 into the Null map, so there is no need to push 0 for kNullType.
-        if (value->GetType() != rapidjson::Type::kNullType) {
+        data_column_ptr = nullable_column->get_nested_column().get_ptr();
+        data_serde = serde->get_nested_serdes()[0];
+
+        if (value_is_null) {
+            nullable_column->insert_default();
+            *valid = true;
+            return Status::OK();
+        } else {
             nullable_column->get_null_map_data().push_back(0);
+        }
+
+    } else if (value_is_null) [[unlikely]] {
+        if (_is_load) {
+            RETURN_IF_ERROR(_append_error_msg(
+                    *value, "Json value is null, but the column `{}` is not nullable.", column_name,
+                    valid));
+            return Status::OK();
+
         } else {
-            nullable_column->insert_default();
+            return Status::DataQualityError(
+                    "Json value is null, but the column `{}` is not nullable.", column_name);
         }
-        column_ptr = &nullable_column->get_nested_column();
     }
 
-    switch (value->GetType()) {
-    case rapidjson::Type::kStringType:
-        str_value = value->GetString();
-        wbytes = value->GetStringLength();
-        break;
-    case rapidjson::Type::kNumberType:
-        if (value->IsUint()) {
-            wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%u", value->GetUint());
-        } else if (value->IsInt()) {
-            wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%d", value->GetInt());
-        } else if (value->IsUint64()) {
-            wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRIu64, value->GetUint64());
-        } else if (value->IsInt64()) {
-            wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRId64, value->GetInt64());
-        } else if (value->IsFloat() || value->IsDouble()) {
-            auto* end = fmt::format_to(tmp_buf, "{}", value->GetDouble());
-            wbytes = end - tmp_buf;
+    if (_is_load || !type_desc.is_complex_type()) {
+        if (value->IsString()) {
+            Slice slice {value->GetString(), value->GetStringLength()};
+            RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice,
+                                                                       _serde_options));
+
         } else {
-            return Status::InternalError<false>("It should not here.");
+            // We can `switch (value->GetType()) case: kNumberType`.
+            // Note that `if (value->IsInt())`, but column is FloatColumn.
+            // Or for any type, use `NewJsonReader::_print_json_value(*value)`.
+
+            const char* str_value = nullptr;
+            char tmp_buf[128] = {0};
+            size_t wbytes = 0;
+            std::string json_str;
+
+            switch (value->GetType()) {
+            case rapidjson::Type::kStringType:
+                str_value = value->GetString();
+                wbytes = value->GetStringLength();
+                break;
+            case rapidjson::Type::kNumberType:
+                if (value->IsUint()) {
+                    wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%u", value->GetUint());
+                } else if (value->IsInt()) {
+                    wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%d", value->GetInt());
+                } else if (value->IsUint64()) {
+                    wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRIu64, value->GetUint64());
+                } else if (value->IsInt64()) {
+                    wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRId64, value->GetInt64());
+                } else if (value->IsFloat() || value->IsDouble()) {
+                    auto* end = fmt::format_to(tmp_buf, "{}", value->GetDouble());
+                    wbytes = end - tmp_buf;
+                } else {
+                    return Status::InternalError<false>("It should not here.");
+                }
+                str_value = tmp_buf;
+                break;
+            case rapidjson::Type::kFalseType:
+                wbytes = 1;
+                str_value = (char*)"0";
+                break;
+            case rapidjson::Type::kTrueType:
+                wbytes = 1;
+                str_value = (char*)"1";
+                break;
+            default:
+                // for other type, we convert it to string to save
+                json_str = NewJsonReader::_print_json_value(*value);
+                wbytes = json_str.size();
+                str_value = json_str.c_str();
+                break;
+            }
+            Slice slice {str_value, wbytes};
+            RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice,
+                                                                       _serde_options));
         }
-        str_value = tmp_buf;
-        break;
-    case rapidjson::Type::kFalseType:
-        wbytes = 1;
-        str_value = (char*)"0";
-        break;
-    case rapidjson::Type::kTrueType:
-        wbytes = 1;
-        str_value = (char*)"1";
-        break;
-    case rapidjson::Type::kNullType:
-        if (!slot_desc->is_nullable()) {
-            RETURN_IF_ERROR(_append_error_msg(
-                    *value, "Json value is null, but the column `{}` is not nullable.",
-                    slot_desc->col_name(), valid));
-            return Status::OK();
+    } else if (type_desc.type == TYPE_STRUCT) {
+        if (!value->IsObject()) [[unlikely]] {
+            return Status::DataQualityError(
+                    "Json value isn't object, but the column `{}` is struct.", column_name);
         }
 
-        // return immediately to prevent from repeatedly insert_data
-        *valid = true;
-        return Status::OK();
-    default:
-        // for other type like array or object. we convert it to string to save
-        json_str = NewJsonReader::_print_json_value(*value);
-        wbytes = json_str.size();
-        str_value = json_str.c_str();
-        break;
-    }
+        auto sub_col_size = type_desc.children.size();
+        const auto& struct_value = value->GetObject();
 
-    // TODO: if the vexpr can support another 'slot_desc type' than 'TYPE_VARCHAR',
-    // we need use a function to support these types to insert data in columns.
-    DCHECK(slot_desc->type().type == TYPE_VARCHAR || slot_desc->type().type == TYPE_STRING)
-            << slot_desc->type().type << ", query id: " << print_id(_state->query_id());
-    assert_cast<ColumnString*>(column_ptr)->insert_data(str_value, wbytes);
+        auto sub_serdes = data_serde->get_nested_serdes();
+        auto struct_column_ptr = assert_cast<ColumnStruct*>(data_column_ptr);
+
+        std::map<std::string, size_t> sub_col_name_to_idx;
+        for (size_t sub_col_idx = 0; sub_col_idx < sub_col_size; sub_col_idx++) {
+            sub_col_name_to_idx.emplace(type_desc.field_names[sub_col_idx], sub_col_idx);
+        }
+
+        std::vector<rapidjson::Value::ConstValueIterator> sub_values(sub_col_size, nullptr);
+        for (const auto& sub : struct_value) {
+            if (!sub.name.IsString()) [[unlikely]] {
+                return Status::DataQualityError(
+                        "Json file struct column `{}` subfield name isn't a String", column_name);
+            }
+
+            auto sub_key_char = sub.name.GetString();
+            auto sub_key_length = sub.name.GetStringLength();
+
+            std::string sub_key(sub_key_char, sub_key_length);
+            std::transform(sub_key.begin(), sub_key.end(), sub_key.begin(), ::tolower);
+
+            if (sub_col_name_to_idx.find(sub_key) == sub_col_name_to_idx.end()) [[unlikely]] {
+                continue;
+            }
+            size_t sub_column_idx = sub_col_name_to_idx[sub_key];
+            sub_values[sub_column_idx] = &sub.value;
+        }
+
+        for (size_t sub_col_idx = 0; sub_col_idx < sub_col_size; sub_col_idx++) {
+            auto sub_value = sub_values[sub_col_idx];
+
+            const auto& sub_col_type = type_desc.children[sub_col_idx];
+
+            RETURN_IF_ERROR(_write_data_to_column(
+                    sub_value, sub_col_type, struct_column_ptr->get_column(sub_col_idx).get_ptr(),
+                    column_name + "." + type_desc.field_names[sub_col_idx], sub_serdes[sub_col_idx],
+                    valid));
+        }
+    } else if (type_desc.type == TYPE_MAP) {
+        if (!value->IsObject()) [[unlikely]] {
+            return Status::DataQualityError("Json value isn't object, but the column `{}` is map.",
+                                            column_name);
+        }
+        const auto& object_value = value->GetObject();
+        auto sub_serdes = data_serde->get_nested_serdes();
+        auto map_column_ptr = assert_cast<ColumnMap*>(data_column_ptr);
+
+        for (const auto& member_value : object_value) {
+            RETURN_IF_ERROR(_write_data_to_column(
+                    &member_value.name, type_desc.children[0],
+                    map_column_ptr->get_keys_ptr()->assume_mutable()->get_ptr(),
+                    column_name + ".key", sub_serdes[0], valid));
+
+            RETURN_IF_ERROR(_write_data_to_column(
+                    &member_value.value, type_desc.children[1],
+                    map_column_ptr->get_values_ptr()->assume_mutable()->get_ptr(),
+                    column_name + ".value", sub_serdes[1], valid));
+        }
+
+        auto& offsets = map_column_ptr->get_offsets();
+        offsets.emplace_back(offsets.back() + object_value.MemberCount());
+    } else if (type_desc.type == TYPE_ARRAY) {
+        if (!value->IsArray()) [[unlikely]] {
+            return Status::DataQualityError("Json value isn't array, but the column `{}` is array.",
+                                            column_name);
+        }
+        const auto& array_value = value->GetArray();
+        auto sub_serdes = data_serde->get_nested_serdes();
+        auto array_column_ptr = assert_cast<ColumnArray*>(data_column_ptr);
+
+        for (const auto& sub_value : array_value) {
+            RETURN_IF_ERROR(_write_data_to_column(&sub_value, type_desc.children[0],
+                                                  array_column_ptr->get_data().get_ptr(),
+                                                  column_name + ".element", sub_serdes[0], valid));
+        }
+        auto& offsets = array_column_ptr->get_offsets();
+        offsets.emplace_back(offsets.back() + array_value.Size());
+    } else {
+        return Status::InternalError("Not support load to complex column.");
+    }
 
     *valid = true;
     return Status::OK();
@@ -949,20 +1087,21 @@ Status NewJsonReader::_write_columns_by_jsonpath(rapidjson::Value& objectValue,
                 // if json_values' size > 1, it means we just match an array, not a wrapped one, so no need to unwrap.
                 json_values = &((*json_values)[0]);
             }
-            RETURN_IF_ERROR(_write_data_to_column(json_values, slot_descs[i], column_ptr, valid));
+            RETURN_IF_ERROR(_write_data_to_column(json_values, slot_descs[i]->type(), column_ptr,
+                                                  slot_descs[i]->col_name(), _serdes[i], valid));
             if (!(*valid)) {
                 return Status::OK();
             }
             has_valid_value = true;
         } else {
             // not found, filling with default value
-            RETURN_IF_ERROR(_fill_missing_column(slot_desc, column_ptr, valid));
+            RETURN_IF_ERROR(_fill_missing_column(slot_desc, _serdes[i], column_ptr, valid));
             if (!(*valid)) {
                 return Status::OK();
             }
         }
     }
-    if (!has_valid_value) {
+    if (!has_valid_value && _is_load) {
         // there is no valid value in json line but has filled with default value before
         // so remove this line in block
         for (int i = 0; i < block.columns(); ++i) {
@@ -1112,7 +1251,7 @@ Status NewJsonReader::_simdjson_handle_simple_json(RuntimeState* /*state*/, Bloc
 
         // step2: get json value by json doc
         Status st = _get_json_value(&size, eof, &error, is_empty_row);
-        if (st.is<DATA_QUALITY_ERROR>()) {
+        if (_is_load && st.is<DATA_QUALITY_ERROR>()) {
             return Status::OK();
         }
         RETURN_IF_ERROR(st);
@@ -1386,28 +1525,42 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
     for (auto field : *value) {
         std::string_view key = field.unescaped_key();
         StringRef name_ref(key.data(), key.size());
+        std::string key_string;
+        if (_is_hive_table) {
+            key_string = name_ref.to_string();
+            std::transform(key_string.begin(), key_string.end(), key_string.begin(), ::tolower);
+            name_ref = StringRef(key_string);
+        }
         const size_t column_index = _column_index(name_ref, key_index++);
         if (UNLIKELY(ssize_t(column_index) < 0)) {
             // This key is not exist in slot desc, just ignore
             continue;
         }
-        if (_seen_columns[column_index]) {
-            continue;
-        }
         if (column_index == skip_bitmap_col_idx) {
             continue;
         }
+        if (_seen_columns[column_index]) {
+            if (_is_hive_table) {
+                //Since value can only be traversed once,
+                // we can only insert the original value first, then delete it, and then reinsert the new value
+                block.get_by_position(column_index).column->assume_mutable()->pop_back(1);
+            } else {
+                continue;
+            }
+        }
         simdjson::ondemand::value val = field.value();
         auto* column_ptr = block.get_by_position(column_index).column->assume_mutable().get();
-        RETURN_IF_ERROR(
-                _simdjson_write_data_to_column(val, slot_descs[column_index], column_ptr, valid));
+        RETURN_IF_ERROR(_simdjson_write_data_to_column(
+                val, slot_descs[column_index]->type(), column_ptr,
+                slot_descs[column_index]->col_name(), _serdes[column_index], valid));
         if (!(*valid)) {
             return Status::OK();
         }
         _seen_columns[column_index] = true;
         has_valid_value = true;
     }
-    if (!has_valid_value) {
+
+    if (!has_valid_value && _is_load) {
         string col_names;
         for (auto* slot_desc : slot_descs) {
             col_names.append(slot_desc->col_name() + ", ");
@@ -1472,7 +1625,7 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
                 _process_skip_bitmap_mark(slot_desc, column_ptr, block, cur_row_count, valid);
                 column_ptr->insert_default();
             } else {
-                RETURN_IF_ERROR(_fill_missing_column(slot_desc, column_ptr, valid));
+                RETURN_IF_ERROR(_fill_missing_column(slot_desc, _serdes[i], column_ptr, valid));
                 if (!(*valid)) {
                     return Status::OK();
                 }
@@ -1482,12 +1635,6 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
         DCHECK(column_ptr->size() == cur_row_count + 1);
     }
 
-#ifndef NDEBUG
-    // Check all columns rows matched
-    for (size_t i = 0; i < block.columns(); ++i) {
-        DCHECK_EQ(block.get_by_position(i).column->size(), cur_row_count + 1);
-    }
-#endif
     // There is at least one valid value here
     DCHECK(nullcount < block.columns());
     *valid = true;
@@ -1495,54 +1642,180 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
 }
 
 Status NewJsonReader::_simdjson_write_data_to_column(simdjson::ondemand::value& value,
-                                                     SlotDescriptor* slot_desc, IColumn* column,
-                                                     bool* valid) {
-    // write
+                                                     const TypeDescriptor& type_desc,
+                                                     vectorized::IColumn* column_ptr,
+                                                     const std::string& column_name,
+                                                     DataTypeSerDeSPtr serde, bool* valid) {
     ColumnNullable* nullable_column = nullptr;
-    IColumn* column_ptr = nullptr;
-    if (slot_desc->is_nullable()) {
-        nullable_column = assert_cast<ColumnNullable*>(column);
-        column_ptr = &nullable_column->get_nested_column();
-    }
-    // TODO: if the vexpr can support another 'slot_desc type' than 'TYPE_VARCHAR',
-    // we need use a function to support these types to insert data in columns.
-    auto* column_string = assert_cast<ColumnString*>(column_ptr);
-    switch (value.type()) {
-    case simdjson::ondemand::json_type::null: {
-        if (column->is_nullable()) {
-            // insert_default already push 1 to null_map
-            nullable_column->insert_default();
+    vectorized::IColumn* data_column_ptr = column_ptr;
+    DataTypeSerDeSPtr data_serde = serde;
+
+    if (column_ptr->is_nullable()) {
+        nullable_column = reinterpret_cast<ColumnNullable*>(column_ptr);
+
+        data_column_ptr = nullable_column->get_nested_column().get_ptr();
+        data_serde = serde->get_nested_serdes()[0];
+
+        // kNullType will put 1 into the Null map, so there is no need to push 0 for kNullType.
+        if (value.type() != simdjson::ondemand::json_type::null) {
+            nullable_column->get_null_map_data().push_back(0);
         } else {
+            nullable_column->insert_default();
+            *valid = true;
+            return Status::OK();
+        }
+    } else if (value.type() == simdjson::ondemand::json_type::null) [[unlikely]] {
+        if (_is_load) {
             RETURN_IF_ERROR(_append_error_msg(
                     nullptr, "Json value is null, but the column `{}` is not nullable.",
-                    slot_desc->col_name(), valid));
+                    column_name, valid));
             return Status::OK();
-        }
-        break;
-    }
-    case simdjson::ondemand::json_type::boolean: {
-        nullable_column->get_null_map_data().push_back(0);
-        if (value.get_bool()) {
-            column_string->insert_data("1", 1);
         } else {
-            column_string->insert_data("0", 1);
+            return Status::DataQualityError(
+                    "Json value is null, but the column `{}` is not nullable.", column_name);
         }
-        break;
     }
-    default: {
+
+    if (_is_load || !type_desc.is_complex_type()) {
         if (value.type() == simdjson::ondemand::json_type::string) {
-            auto* unescape_buffer =
-                    reinterpret_cast<uint8_t*>(_simdjson_ondemand_unscape_padding_buffer.data());
-            std::string_view unescaped_value =
-                    _ondemand_json_parser->unescape(value.get_raw_json_string(), unescape_buffer);
-            nullable_column->get_null_map_data().push_back(0);
-            column_string->insert_data(unescaped_value.data(), unescaped_value.length());
-            break;
+            std::string_view value_string = value.get_string();
+            Slice slice {value_string.data(), value_string.size()};
+            RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice,
+                                                                       _serde_options));
+
+        } else {
+            // Maybe we can `switch (value->GetType()) case: kNumberType`.
+            // Note that `if (value->IsInt())`, but column is FloatColumn.
+            std::string_view json_str = simdjson::to_json_string(value);
+            Slice slice {json_str.data(), json_str.size()};
+            RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice,
+                                                                       _serde_options));
+        }
+    } else if (type_desc.type == TYPE_STRUCT) {
+        if (value.type() != simdjson::ondemand::json_type::object) [[unlikely]] {
+            return Status::DataQualityError(
+                    "Json value isn't object, but the column `{}` is struct.", column_name);
+        }
+
+        auto sub_col_size = type_desc.children.size();
+        simdjson::ondemand::object struct_value = value.get_object();
+        auto sub_serdes = data_serde->get_nested_serdes();
+        auto struct_column_ptr = assert_cast<ColumnStruct*>(data_column_ptr);
+
+        std::map<std::string, size_t> sub_col_name_to_idx;
+        for (size_t sub_col_idx = 0; sub_col_idx < sub_col_size; sub_col_idx++) {
+            sub_col_name_to_idx.emplace(type_desc.field_names[sub_col_idx], sub_col_idx);
+        }
+        vector<bool> has_value(sub_col_size, false);
+        for (simdjson::ondemand::field sub : struct_value) {
+            std::string_view sub_key_view = sub.unescaped_key();
+            std::string sub_key(sub_key_view.data(), sub_key_view.length());
+            std::transform(sub_key.begin(), sub_key.end(), sub_key.begin(), ::tolower);
+
+            if (sub_col_name_to_idx.find(sub_key) == sub_col_name_to_idx.end()) [[unlikely]] {
+                continue;
+            }
+            size_t sub_column_idx = sub_col_name_to_idx[sub_key];
+            auto sub_column_ptr = struct_column_ptr->get_column(sub_column_idx).get_ptr();
+
+            if (has_value[sub_column_idx]) [[unlikely]] {
+                // Since struct_value can only be traversed once, we can only insert
+                // the original value first, then delete it, and then reinsert the new value.
+                sub_column_ptr->pop_back(1);
+            }
+            has_value[sub_column_idx] = true;
+
+            const auto& sub_col_type = type_desc.children[sub_column_idx];
+            RETURN_IF_ERROR(_simdjson_write_data_to_column(
+                    sub.value(), sub_col_type, sub_column_ptr, column_name + "." + sub_key,
+                    sub_serdes[sub_column_idx], valid));
         }
-        auto value_str = simdjson::to_json_string(value).value();
-        nullable_column->get_null_map_data().push_back(0);
-        column_string->insert_data(value_str.data(), value_str.length());
-    }
+
+        //fill missing subcolumn
+        for (size_t sub_col_idx = 0; sub_col_idx < sub_col_size; sub_col_idx++) {
+            if (has_value[sub_col_idx] == true) {
+                continue;
+            }
+
+            auto sub_column_ptr = struct_column_ptr->get_column(sub_col_idx).get_ptr();
+            if (sub_column_ptr->is_nullable()) {
+                sub_column_ptr->insert_default();
+                continue;
+            } else [[unlikely]] {
+                return Status::DataQualityError(
+                        "Json file structColumn miss field {} and this column isn't nullable.",
+                        column_name + "." + type_desc.field_names[sub_col_idx]);
+            }
+        }
+    } else if (type_desc.type == TYPE_MAP) {
+        if (value.type() != simdjson::ondemand::json_type::object) [[unlikely]] {
+            return Status::DataQualityError("Json value isn't object, but the column `{}` is map.",
+                                            column_name);
+        }
+        simdjson::ondemand::object object_value = value.get_object();
+
+        auto sub_serdes = data_serde->get_nested_serdes();
+        auto map_column_ptr = assert_cast<ColumnMap*>(data_column_ptr);
+
+        size_t field_count = 0;
+        for (simdjson::ondemand::field member_value : object_value) {
+            auto f = [](std::string_view key_view, const TypeDescriptor& type_desc,
+                        vectorized::IColumn* column_ptr, DataTypeSerDeSPtr serde,
+                        vectorized::DataTypeSerDe::FormatOptions serde_options, bool* valid) {
+                auto data_column_ptr = column_ptr;
+                auto data_serde = serde;
+                if (column_ptr->is_nullable()) {
+                    auto nullable_column = static_cast<ColumnNullable*>(column_ptr);
+
+                    nullable_column->get_null_map_data().push_back(0);
+                    data_column_ptr = nullable_column->get_nested_column().get_ptr();
+                    data_serde = serde->get_nested_serdes()[0];
+                }
+                Slice slice(key_view.data(), key_view.length());
+
+                RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice,
+                                                                           serde_options));
+                return Status::OK();
+            };
+
+            RETURN_IF_ERROR(f(member_value.unescaped_key(), type_desc.children[0],
+                              map_column_ptr->get_keys_ptr()->assume_mutable()->get_ptr(),
+                              sub_serdes[0], _serde_options, valid));
+
+            simdjson::ondemand::value field_value = member_value.value();
+            RETURN_IF_ERROR(_simdjson_write_data_to_column(
+                    field_value, type_desc.children[1],
+                    map_column_ptr->get_values_ptr()->assume_mutable()->get_ptr(),
+                    column_name + ".value", sub_serdes[1], valid));
+            field_count++;
+        }
+
+        auto& offsets = map_column_ptr->get_offsets();
+        offsets.emplace_back(offsets.back() + field_count);
+
+    } else if (type_desc.type == TYPE_ARRAY) {
+        if (value.type() != simdjson::ondemand::json_type::array) [[unlikely]] {
+            return Status::DataQualityError("Json value isn't array, but the column `{}` is array.",
+                                            column_name);
+        }
+
+        simdjson::ondemand::array array_value = value.get_array();
+
+        auto sub_serdes = data_serde->get_nested_serdes();
+        auto array_column_ptr = assert_cast<ColumnArray*>(data_column_ptr);
+
+        int field_count = 0;
+        for (simdjson::ondemand::value sub_value : array_value) {
+            RETURN_IF_ERROR(_simdjson_write_data_to_column(
+                    sub_value, type_desc.children[0], array_column_ptr->get_data().get_ptr(),
+                    column_name + ".element", sub_serdes[0], valid));
+            field_count++;
+        }
+        auto& offsets = array_column_ptr->get_offsets();
+        offsets.emplace_back(offsets.back() + field_count);
+
+    } else {
+        return Status::InternalError("Not support load to complex column.");
     }
     *valid = true;
     return Status::OK();
@@ -1750,13 +2023,14 @@ Status NewJsonReader::_simdjson_write_columns_by_jsonpath(
             has_valid_value = true;
         } else if (i >= _parsed_jsonpaths.size() || st.is<NOT_FOUND>()) {
             // not match in jsondata, filling with default value
-            RETURN_IF_ERROR(_fill_missing_column(slot_desc, column_ptr, valid));
+            RETURN_IF_ERROR(_fill_missing_column(slot_desc, _serdes[i], column_ptr, valid));
             if (!(*valid)) {
                 return Status::OK();
             }
         } else {
-            RETURN_IF_ERROR(
-                    _simdjson_write_data_to_column(json_value, slot_desc, column_ptr, valid));
+            RETURN_IF_ERROR(_simdjson_write_data_to_column(json_value, slot_desc->type(),
+                                                           column_ptr, slot_desc->col_name(),
+                                                           _serdes[i], valid));
             if (!(*valid)) {
                 return Status::OK();
             }
@@ -1814,25 +2088,30 @@ Status NewJsonReader::_get_column_default_value(
     return Status::OK();
 }
 
-Status NewJsonReader::_fill_missing_column(SlotDescriptor* slot_desc, IColumn* column_ptr,
-                                           bool* valid) {
-    if (slot_desc->is_nullable()) {
-        auto* nullable_column = reinterpret_cast<ColumnNullable*>(column_ptr);
-        column_ptr = &nullable_column->get_nested_column();
-        auto col_value = _col_default_value_map.find(slot_desc->col_name());
-        if (col_value == _col_default_value_map.end()) {
+Status NewJsonReader::_fill_missing_column(SlotDescriptor* slot_desc, DataTypeSerDeSPtr serde,
+                                           IColumn* column_ptr, bool* valid) {
+    auto col_value = _col_default_value_map.find(slot_desc->col_name());
+    if (col_value == _col_default_value_map.end()) {
+        if (slot_desc->is_nullable()) {
+            auto* nullable_column = static_cast<ColumnNullable*>(column_ptr);
             nullable_column->insert_default();
         } else {
-            const std::string& v_str = col_value->second;
-            nullable_column->get_null_map_data().push_back(0);
-            assert_cast<ColumnString*>(column_ptr)->insert_data(v_str.c_str(), v_str.size());
+            if (_is_load) {
+                RETURN_IF_ERROR(_append_error_msg(
+                        nullptr, "The column `{}` is not nullable, but it's not found in jsondata.",
+                        slot_desc->col_name(), valid));
+            } else {
+                return Status::DataQualityError(
+                        "The column `{}` is not nullable, but it's not found in jsondata.",
+                        slot_desc->col_name());
+            }
         }
     } else {
-        RETURN_IF_ERROR(_append_error_msg(
-                nullptr, "The column `{}` is not nullable, but it's not found in jsondata.",
-                slot_desc->col_name(), valid));
+        const std::string& v_str = col_value->second;
+        Slice column_default_value {v_str};
+        RETURN_IF_ERROR(serde->deserialize_one_cell_from_json(*column_ptr, column_default_value,
+                                                              _serde_options));
     }
-
     *valid = true;
     return Status::OK();
 }
diff --git a/be/src/vec/exec/format/json/new_json_reader.h b/be/src/vec/exec/format/json/new_json_reader.h
index 985bf999676970..482a1ced747cef 100644
--- a/be/src/vec/exec/format/json/new_json_reader.h
+++ b/be/src/vec/exec/format/json/new_json_reader.h
@@ -87,7 +87,8 @@ class NewJsonReader : public GenericReader {
     ~NewJsonReader() override = default;
 
     Status init_reader(const std::unordered_map<std::string, vectorized::VExprContextSPtr>&
-                               col_default_value_ctx);
+                               col_default_value_ctx,
+                       bool is_load);
     Status get_next_block(Block* block, size_t* read_rows, bool* eof) override;
     Status get_columns(std::unordered_map<std::string, TypeDescriptor>* name_to_type,
                        std::unordered_set<std::string>* missing_cols) override;
@@ -128,7 +129,8 @@ class NewJsonReader : public GenericReader {
                              const std::vector<SlotDescriptor*>& slot_descs, bool* valid);
 
     Status _write_data_to_column(rapidjson::Value::ConstValueIterator value,
-                                 SlotDescriptor* slot_desc, vectorized::IColumn* column_ptr,
+                                 const TypeDescriptor& type_desc, vectorized::IColumn* column_ptr,
+                                 const std::string& column_name, DataTypeSerDeSPtr serde,
                                  bool* valid);
 
     Status _write_columns_by_jsonpath(rapidjson::Value& objectValue,
@@ -177,8 +179,10 @@ class NewJsonReader : public GenericReader {
                                       const std::vector<SlotDescriptor*>& slot_descs, bool* valid);
 
     Status _simdjson_write_data_to_column(simdjson::ondemand::value& value,
-                                          SlotDescriptor* slot_desc,
-                                          vectorized::IColumn* column_ptr, bool* valid);
+                                          const TypeDescriptor& type_desc,
+                                          vectorized::IColumn* column_ptr,
+                                          const std::string& column_name, DataTypeSerDeSPtr serde,
+                                          bool* valid);
 
     Status _simdjson_write_columns_by_jsonpath(simdjson::ondemand::object* value,
                                                const std::vector<SlotDescriptor*>& slot_descs,
@@ -196,8 +200,8 @@ class NewJsonReader : public GenericReader {
             const std::unordered_map<std::string, vectorized::VExprContextSPtr>&
                     col_default_value_ctx);
 
-    Status _fill_missing_column(SlotDescriptor* slot_desc, vectorized::IColumn* column_ptr,
-                                bool* valid);
+    Status _fill_missing_column(SlotDescriptor* slot_desc, DataTypeSerDeSPtr serde,
+                                vectorized::IColumn* column_ptr, bool* valid);
 
     // fe will add skip_bitmap_col to _file_slot_descs iff the target olap table has skip_bitmap_col
     // and the current load is a flexible partial update
@@ -292,6 +296,22 @@ class NewJsonReader : public GenericReader {
     std::unordered_map<std::string, std::string> _col_default_value_map;
 
     int32_t skip_bitmap_col_idx {-1};
+
+    bool _is_load = true;
+    //Used to indicate whether it is a stream load. When loading, only data will be inserted into columnString.
+    //If an illegal value is encountered during the load process, `_append_error_msg` should be called
+    //instead of directly returning `Status::DataQualityError`
+
+    bool _is_hive_table = false;
+    // In hive : create table xxx ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
+    // Hive will not allow you to create columns with the same name but different case, including field names inside
+    // structs, and will automatically convert uppercase names in create sql to lowercase.However, when Hive loads data
+    // to table, the column names in the data may be uppercase,and there may be multiple columns with
+    // the same name but different capitalization.We refer to the behavior of hive, convert all column names
+    // in the data to lowercase,and use the last one as the insertion value
+
+    DataTypeSerDeSPtrs _serdes;
+    vectorized::DataTypeSerDe::FormatOptions _serde_options;
 };
 
 } // namespace vectorized
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp
index 982ac6072dc018..3053adebbb521e 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -965,8 +965,8 @@ Status VFileScanner::_get_next_reader() {
             _cur_reader =
                     NewJsonReader::create_unique(_state, _profile, &_counter, *_params, range,
                                                  _file_slot_descs, &_scanner_eof, _io_ctx.get());
-            init_status =
-                    ((NewJsonReader*)(_cur_reader.get()))->init_reader(_col_default_value_ctx);
+            init_status = ((NewJsonReader*)(_cur_reader.get()))
+                                  ->init_reader(_col_default_value_ctx, _is_load);
             break;
         }
         case TFileFormatType::FORMAT_AVRO: {
diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run69.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run69.hql
new file mode 100644
index 00000000000000..adf0f7d56b27d9
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run69.hql
@@ -0,0 +1,35 @@
+use `default`;
+
+
+CREATE TABLE json_nested_complex_table (
+    user_ID STRING,
+    user_PROFILE STRUCT<
+        name: STRING,
+        AGE: INT,
+        preferences: MAP<
+            STRING,
+            STRUCT<
+                preference_ID: INT,
+                preference_VALUES: ARRAY<STRING>
+            >
+        >
+    >,
+    activity_LOG ARRAY<
+        STRUCT<
+            activity_DATE: STRING,
+            activities: MAP<
+                STRING,
+                STRUCT<
+                    `DETAILS`: STRING,
+                    metrics: MAP<STRING, float>
+                >
+            >
+        >
+    >
+) ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
+
+LOCATION
+  '/user/doris/preinstalled_data/json/json_nested_complex_table';
+
+
+msck repair table json_nested_complex_table;
diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run70.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run70.hql
new file mode 100644
index 00000000000000..73df8cba557bcb
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run70.hql
@@ -0,0 +1,73 @@
+use `default`;
+
+
+CREATE TABLE json_all_complex_types (
+  `id` int,
+  `boolean_col` boolean, 
+  `tinyint_col` tinyint, 
+  `smallint_col` smallint, 
+  `int_col` int, 
+  `bigint_col` bigint, 
+  `float_col` float, 
+  `double_col` double, 
+  `decimal_col1` decimal(9,0), 
+  `decimal_col2` decimal(8,4), 
+  `decimal_col3` decimal(18,6), 
+  `decimal_col4` decimal(38,12), 
+  `string_col` string, 
+  `binary_col` binary, 
+  `date_col` date, 
+  `timestamp_col1` timestamp, 
+  `timestamp_col2` timestamp, 
+  `timestamp_col3` timestamp, 
+  `char_col1` char(50), 
+  `char_col2` char(100), 
+  `char_col3` char(255), 
+  `varchar_col1` varchar(50), 
+  `varchar_col2` varchar(100), 
+  `varchar_col3` varchar(255), 
+  `t_map_string` map<string,string>, 
+  `t_map_varchar` map<varchar(65535),varchar(65535)>, 
+  `t_map_char` map<char(10),char(10)>, 
+  `t_map_int` map<int,int>, 
+  `t_map_bigint` map<bigint,bigint>, 
+  `t_map_float` map<float,float>, 
+  `t_map_double` map<double,double>, 
+  `t_map_boolean` map<boolean,boolean>, 
+  `t_map_decimal_precision_2` map<decimal(2,1),decimal(2,1)>, 
+  `t_map_decimal_precision_4` map<decimal(4,2),decimal(4,2)>, 
+  `t_map_decimal_precision_8` map<decimal(8,4),decimal(8,4)>, 
+  `t_map_decimal_precision_17` map<decimal(17,8),decimal(17,8)>, 
+  `t_map_decimal_precision_18` map<decimal(18,8),decimal(18,8)>, 
+  `t_map_decimal_precision_38` map<decimal(38,16),decimal(38,16)>, 
+  `t_array_string` array<string>, 
+  `t_array_int` array<int>, 
+  `t_array_bigint` array<bigint>, 
+  `t_array_float` array<float>, 
+  `t_array_double` array<double>, 
+  `t_array_boolean` array<boolean>, 
+  `t_array_varchar` array<varchar(65535)>, 
+  `t_array_char` array<char(10)>, 
+  `t_array_decimal_precision_2` array<decimal(2,1)>, 
+  `t_array_decimal_precision_4` array<decimal(4,2)>, 
+  `t_array_decimal_precision_8` array<decimal(8,4)>, 
+  `t_array_decimal_precision_17` array<decimal(17,8)>, 
+  `t_array_decimal_precision_18` array<decimal(18,8)>, 
+  `t_array_decimal_precision_38` array<decimal(38,16)>, 
+  `t_struct_bigint` struct<s_bigint:bigint>, 
+  `t_complex` map<string,array<struct<s_int:int>>>, 
+  `t_struct_nested` struct<struct_field:array<string>>, 
+  `t_struct_null` struct<struct_field_null:string,struct_field_null2:string>, 
+  `t_struct_non_nulls_after_nulls` struct<struct_non_nulls_after_nulls1:int,struct_non_nulls_after_nulls2:string>, 
+  `t_nested_struct_non_nulls_after_nulls` struct<struct_field1:int,struct_field2:string,strict_field3:struct<nested_struct_field1:int,nested_struct_field2:string>>, 
+  `t_map_null_value` map<string,string>, 
+  `t_array_string_starting_with_nulls` array<string>, 
+  `t_array_string_with_nulls_in_between` array<string>, 
+  `t_array_string_ending_with_nulls` array<string>, 
+  `t_array_string_all_nulls` array<string>
+    ) PARTITIONED BY (`dt` string)
+ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' 
+LOCATION
+  '/user/doris/preinstalled_data/json/json_all_complex_types';
+
+msck repair table json_all_complex_types;
diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run71.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run71.hql
new file mode 100644
index 00000000000000..ec99e72d2f5780
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run71.hql
@@ -0,0 +1,13 @@
+use `default`;
+
+
+CREATE TABLE json_load_data_table (
+    `id` int,
+    `col1` int,
+    `col2` struct< col2a:int, col2b:string>,
+    `col3` map<int,string>
+) ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
+LOCATION
+  '/user/doris/preinstalled_data/json/json_load_data_table';
+
+msck repair table json_load_data_table;
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt1/000000_0 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt1/000000_0
new file mode 100644
index 00000000000000..5fe37cbc6f098e
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt1/000000_0
@@ -0,0 +1,3 @@
+{"id":1,"boolean_col":true,"tinyint_col":127,"smallint_col":32767,"int_col":2147483647,"bigint_col":9223372036854775807,"float_col":123.45,"double_col":123456.789,"decimal_col1":123456789,"decimal_col2":1234.5678,"decimal_col3":123456.789012,"decimal_col4":123456789.012345678901,"string_col":"string_value","binary_col":"binary_value","date_col":"2024-03-20","timestamp_col1":"2024-03-20 12:00:00","timestamp_col2":"2024-03-20 12:00:00.123456789","timestamp_col3":"2024-03-20 12:00:00.123456789","char_col1":"char_value1                                       ","char_col2":"char_value2                                                                                         ","char_col3":"char_value3                                                                                                                                                                                                                                                    ","varchar_col1":"varchar_value1","varchar_col2":"varchar_value2","varchar_col3":"varchar_value3","t_map_string":{"key1":"value1"},"t_map_varchar":{"key1":"value1"},"t_map_char":{"a         ":"b         "},"t_map_int":{"1":10},"t_map_bigint":{"1":100000000000},"t_map_float":{"1.1":10.1},"t_map_double":{"1.1":10.1},"t_map_boolean":{"true":false},"t_map_decimal_precision_2":{"1.1":1.1},"t_map_decimal_precision_4":{"1.23":1.23},"t_map_decimal_precision_8":{"1.2345":1.2345},"t_map_decimal_precision_17":{"1.23456789":1.23456789},"t_map_decimal_precision_18":{"1.23456789":1.23456789},"t_map_decimal_precision_38":{"1.2345678901234568":1.2345678901234568},"t_array_string":["string1","string2"],"t_array_int":[1,2,3],"t_array_bigint":[100000000000,200000000000],"t_array_float":[1.1,2.2],"t_array_double":[1.123456789,2.123456789],"t_array_boolean":[true,false],"t_array_varchar":["varchar1","varchar2"],"t_array_char":["char1     ","char2     "],"t_array_decimal_precision_2":[1.1,2.2],"t_array_decimal_precision_4":[1.23,2.34],"t_array_decimal_precision_8":[1.2345,2.3456],"t_array_decimal_precision_17":[1.23456789,2.34567891],"t_array_decimal_precision_18":[1.23456789,2.34567891],"t_array_decimal_precision_38":[1.2345678901234568,2.3456789012345679],"t_struct_bigint":{"s_bigint":1234567890},"t_complex":{"key":[{"s_int":123}]},"t_struct_nested":{"struct_field":["value1","value2"]},"t_struct_null":{"struct_field_null":null,"struct_field_null2":null},"t_struct_non_nulls_after_nulls":{"struct_non_nulls_after_nulls1":123,"struct_non_nulls_after_nulls2":"value"},"t_nested_struct_non_nulls_after_nulls":{"struct_field1":123,"struct_field2":"value","strict_field3":{"nested_struct_field1":123,"nested_struct_field2":"nested_value"}},"t_map_null_value":{"null_key":null},"t_array_string_starting_with_nulls":[null,"value1","value2"],"t_array_string_with_nulls_in_between":["value1",null,"value2"],"t_array_string_ending_with_nulls":["value1","value2",null],"t_array_string_all_nulls":[null,null,null]}
+{"id":2,"boolean_col":false,"tinyint_col":58,"smallint_col":12345,"int_col":2147483000,"bigint_col":null,"float_col":789.56,"double_col":654321.123,"decimal_col1":987654321,"decimal_col2":5678.1234,"decimal_col3":987654.321098,"decimal_col4":987654321.098765432109,"string_col":"changed_string","binary_col":"new_binary_value","date_col":"2025-05-25","timestamp_col1":"2025-05-25 15:30:00","timestamp_col2":"2025-05-25 15:30:00.654321987","timestamp_col3":"2025-05-25 15:30:00.654321987","char_col1":"char_new_value1                                   ","char_col2":"char_new_value2                                                                                     ","char_col3":"char_new_value3                                                                                                                                                                                                                                                ","varchar_col1":"varchar_new_value1","varchar_col2":"varchar_new_value2","varchar_col3":"varchar_new_value3","t_map_string":{"key2":"value2"},"t_map_varchar":{"key2":"value2"},"t_map_char":{"x         ":"y         "},"t_map_int":{"2":20},"t_map_bigint":{"2":200000000000},"t_map_float":{"2.2":20.2},"t_map_double":{"2.2":20.2},"t_map_boolean":{"false":true},"t_map_decimal_precision_2":{"2.2":2.2},"t_map_decimal_precision_4":{"2.34":2.34},"t_map_decimal_precision_8":{"2.3456":2.3456},"t_map_decimal_precision_17":{"2.34567891":2.34567891},"t_map_decimal_precision_18":{"2.34567891":2.34567891},"t_map_decimal_precision_38":{"2.3456789012345679":2.3456789012345679},"t_array_string":["string3","string4"],"t_array_int":[4,5,6],"t_array_bigint":[300000000000,400000000000],"t_array_float":[2.2,3.3],"t_array_double":[2.123456789,3.123456789],"t_array_boolean":[false,true],"t_array_varchar":["varchar3","varchar4"],"t_array_char":["char3     ","char4     "],"t_array_decimal_precision_2":[2.2,3.3],"t_array_decimal_precision_4":[2.34,3.45],"t_array_decimal_precision_8":[2.3456,3.4567],"t_array_decimal_precision_17":[2.34567891,3.45678901],"t_array_decimal_precision_18":[2.34567891,3.45678901],"t_array_decimal_precision_38":[2.3456789012345679,3.4567890123456789],"t_struct_bigint":{"s_bigint":9876543210},"t_complex":{"key2":[{"s_int":456}]},"t_struct_nested":{"struct_field":["new_value1","new_value2"]},"t_struct_null":{"struct_field_null":null,"struct_field_null2":null},"t_struct_non_nulls_after_nulls":{"struct_non_nulls_after_nulls1":456,"struct_non_nulls_after_nulls2":"new_value"},"t_nested_struct_non_nulls_after_nulls":{"struct_field1":456,"struct_field2":"new_value","strict_field3":{"nested_struct_field1":456,"nested_struct_field2":"nested_value2"}},"t_map_null_value":{"null_key":null},"t_array_string_starting_with_nulls":[null,"new_value1","new_value2"],"t_array_string_with_nulls_in_between":["new_value1",null,"new_value2"],"t_array_string_ending_with_nulls":["new_value1","new_value2",null],"t_array_string_all_nulls":[null,null,null]}
+{"id":3,"boolean_col":false,"tinyint_col":-128,"smallint_col":-32768,"int_col":-2147483648,"bigint_col":-9223372036854775808,"float_col":-3.4028235E38,"double_col":-1.7976931348623157E308,"decimal_col1":-999999999,"decimal_col2":-9999.9999,"decimal_col3":-999999999.999999,"decimal_col4":null,"string_col":"min_string_value","binary_col":"xxxx","date_col":"2001-01-01","timestamp_col1":"2001-01-01 00:00:00","timestamp_col2":"2001-01-01 00:00:00","timestamp_col3":"2001-01-01 00:00:00","char_col1":"char_min_value1                                   ","char_col2":"char_min_value2                                                                                     ","char_col3":"char_min_value3                                                                                                                                                                                                                                                ","varchar_col1":"varchar_min_value1","varchar_col2":"varchar_min_value2","varchar_col3":"varchar_min_value3","t_map_string":{"min_key":"min_value"},"t_map_varchar":{"min_key":"min_value"},"t_map_char":{"a         ":"z         "},"t_map_int":{"-1":-100},"t_map_bigint":{"-1":-100000000000},"t_map_float":{"-1.1":-10.1},"t_map_double":{"-1.1":-10.1},"t_map_boolean":{"false":true},"t_map_decimal_precision_2":{"-1.1":-1.1},"t_map_decimal_precision_4":{"-1.23":-1.23},"t_map_decimal_precision_8":{"-1.2345":-1.2345},"t_map_decimal_precision_17":{"-1.23456789":-1.23456789},"t_map_decimal_precision_18":{"-1.23456789":-1.23456789},"t_map_decimal_precision_38":{"-1.2345678901234568":-1.2345678901234568},"t_array_string":["min_string1","min_string2"],"t_array_int":[-10,-5,-3],"t_array_bigint":[-100000000000,-200000000000],"t_array_float":[-1.1,-2.2],"t_array_double":[-1.123456789,-2.123456789],"t_array_boolean":[false,true],"t_array_varchar":["min_varchar1","min_varchar2"],"t_array_char":["min_char1 ","min_char2 "],"t_array_decimal_precision_2":[-1.1,-2.2],"t_array_decimal_precision_4":[-1.23,-2.34],"t_array_decimal_precision_8":[-1.2345,-2.3456],"t_array_decimal_precision_17":[-1.23456789,-2.34567891],"t_array_decimal_precision_18":[-1.23456789,-2.34567891],"t_array_decimal_precision_38":[-1.2345678901234568,-2.3456789012345679],"t_struct_bigint":{"s_bigint":-1234567890},"t_complex":{"min_key":[{"s_int":-123}]},"t_struct_nested":{"struct_field":["min_value1","min_value2"]},"t_struct_null":{"struct_field_null":null,"struct_field_null2":null},"t_struct_non_nulls_after_nulls":{"struct_non_nulls_after_nulls1":-123,"struct_non_nulls_after_nulls2":"min_value"},"t_nested_struct_non_nulls_after_nulls":{"struct_field1":-123,"struct_field2":"min_value","strict_field3":{"nested_struct_field1":-123,"nested_struct_field2":"nested_value"}},"t_map_null_value":{"null_key":null},"t_array_string_starting_with_nulls":[null,"min_value1","min_value2"],"t_array_string_with_nulls_in_between":["min_value1",null,"min_value2"],"t_array_string_ending_with_nulls":["min_value1","min_value2",null],"t_array_string_all_nulls":[null,null,null]}
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt2/000000_0 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt2/000000_0
new file mode 100644
index 00000000000000..0a823bee693d76
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt2/000000_0
@@ -0,0 +1 @@
+{"id":4,"boolean_col":null,"tinyint_col":null,"smallint_col":null,"int_col":null,"bigint_col":null,"float_col":123.45,"double_col":null,"decimal_col1":null,"decimal_col2":null,"decimal_col3":null,"decimal_col4":null,"string_col":null,"binary_col":null,"date_col":null,"timestamp_col1":null,"timestamp_col2":null,"timestamp_col3":null,"char_col1":null,"char_col2":null,"char_col3":null,"varchar_col1":null,"varchar_col2":null,"varchar_col3":null,"t_map_string":null,"t_map_varchar":null,"t_map_char":null,"t_map_int":{"1":10},"t_map_bigint":null,"t_map_float":null,"t_map_double":null,"t_map_boolean":null,"t_map_decimal_precision_2":null,"t_map_decimal_precision_4":null,"t_map_decimal_precision_8":null,"t_map_decimal_precision_17":null,"t_map_decimal_precision_18":null,"t_map_decimal_precision_38":null,"t_array_string":null,"t_array_int":null,"t_array_bigint":null,"t_array_float":null,"t_array_double":null,"t_array_boolean":null,"t_array_varchar":null,"t_array_char":null,"t_array_decimal_precision_2":null,"t_array_decimal_precision_4":null,"t_array_decimal_precision_8":[1.2345,2.3456],"t_array_decimal_precision_17":null,"t_array_decimal_precision_18":null,"t_array_decimal_precision_38":null,"t_struct_bigint":null,"t_complex":null,"t_struct_nested":null,"t_struct_null":null,"t_struct_non_nulls_after_nulls":null,"t_nested_struct_non_nulls_after_nulls":null,"t_map_null_value":null,"t_array_string_starting_with_nulls":[null,"value1","value2"],"t_array_string_with_nulls_in_between":null,"t_array_string_ending_with_nulls":null,"t_array_string_all_nulls":null}
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt3/000000_0 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt3/000000_0
new file mode 100644
index 00000000000000..a5e46399fdd553
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_all_complex_types/dt=dt3/000000_0
@@ -0,0 +1,2 @@
+{"id":5,"boolean_col":null,"tinyint_col":null,"smallint_col":null,"int_col":null,"bigint_col":null,"float_col":null,"double_col":null,"decimal_col1":null,"decimal_col2":null,"decimal_col3":null,"decimal_col4":null,"string_col":null,"binary_col":null,"date_col":null,"timestamp_col1":null,"timestamp_col2":null,"timestamp_col3":null,"char_col1":null,"char_col2":null,"char_col3":null,"varchar_col1":null,"varchar_col2":null,"varchar_col3":null,"t_map_string":null,"t_map_varchar":null,"t_map_char":null,"t_map_int":null,"t_map_bigint":null,"t_map_float":null,"t_map_double":null,"t_map_boolean":null,"t_map_decimal_precision_2":null,"t_map_decimal_precision_4":null,"t_map_decimal_precision_8":null,"t_map_decimal_precision_17":null,"t_map_decimal_precision_18":null,"t_map_decimal_precision_38":null,"t_array_string":null,"t_array_int":null,"t_array_bigint":null,"t_array_float":null,"t_array_double":null,"t_array_boolean":null,"t_array_varchar":null,"t_array_char":null,"t_array_decimal_precision_2":null,"t_array_decimal_precision_4":null,"t_array_decimal_precision_8":null,"t_array_decimal_precision_17":null,"t_array_decimal_precision_18":null,"t_array_decimal_precision_38":null,"t_struct_bigint":null,"t_complex":null,"t_struct_nested":null,"t_struct_null":null,"t_struct_non_nulls_after_nulls":null,"t_nested_struct_non_nulls_after_nulls":null,"t_map_null_value":null,"t_array_string_starting_with_nulls":null,"t_array_string_with_nulls_in_between":null,"t_array_string_ending_with_nulls":null,"t_array_string_all_nulls":null}
+{"id":6,"boolean_col":null,"tinyint_col":null,"smallint_col":null,"int_col":null,"bigint_col":null,"float_col":null,"double_col":null,"decimal_col1":null,"decimal_col2":null,"decimal_col3":null,"decimal_col4":null,"string_col":null,"binary_col":null,"date_col":null,"timestamp_col1":null,"timestamp_col2":null,"timestamp_col3":null,"char_col1":null,"char_col2":null,"char_col3":null,"varchar_col1":null,"varchar_col2":null,"varchar_col3":null,"t_map_string":null,"t_map_varchar":null,"t_map_char":null,"t_map_int":null,"t_map_bigint":null,"t_map_float":null,"t_map_double":null,"t_map_boolean":null,"t_map_decimal_precision_2":null,"t_map_decimal_precision_4":null,"t_map_decimal_precision_8":null,"t_map_decimal_precision_17":null,"t_map_decimal_precision_18":null,"t_map_decimal_precision_38":null,"t_array_string":null,"t_array_int":null,"t_array_bigint":null,"t_array_float":null,"t_array_double":null,"t_array_boolean":null,"t_array_varchar":null,"t_array_char":null,"t_array_decimal_precision_2":null,"t_array_decimal_precision_4":null,"t_array_decimal_precision_8":null,"t_array_decimal_precision_17":null,"t_array_decimal_precision_18":null,"t_array_decimal_precision_38":null,"t_struct_bigint":null,"t_complex":null,"t_struct_nested":null,"t_struct_null":null,"t_struct_non_nulls_after_nulls":null,"t_nested_struct_non_nulls_after_nulls":null,"t_map_null_value":null,"t_array_string_starting_with_nulls":null,"t_array_string_with_nulls_in_between":null,"t_array_string_ending_with_nulls":null,"t_array_string_all_nulls":null}
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_load_data_table/1 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_load_data_table/1
new file mode 100644
index 00000000000000..70d1265f98d826
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_load_data_table/1
@@ -0,0 +1,13 @@
+{"id":1,"col1":10,"col2":{"col2a":10,"col2b":"string1"},"col3":{"1":"string10"}}
+{"id":2,"col1":10,"col1":20,"col2":{"col2b":"string2","col2a":0,"Col2A":20},"col3":{"2":"string2"}}
+{"id":3,"col1":10,"col1":20,"COL1":30,"COL2":{"col2a":30,"col2b":"string3"}}
+{"id":4,"COL1":40,"col2":{"col2a":10,"col2b":"string4","new_col":"new_val","col2a":40},"col3":{"4":"string4"}}
+{"id":5}
+{"id":6,"col1":60,"col2":{"COL2a":60,"col2b":600},"col3":{"6":600}}
+{"id":7,"col1":70,"col3":{"7":"string7"},"col2":{"col2b":"string7","col2a":70}}
+
+
+
+
+{}
+{"a":5}
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/1 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/1
new file mode 100644
index 00000000000000..11342c441bce00
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/1
@@ -0,0 +1,2 @@
+{"user_id":"user1","user_profile":{"name":"Alice","age":28,"preferences":{"sports":{"preference_id":101,"preference_values":["soccer","tennis"]},"music":{"preference_id":102,"preference_values":["rock","classical"]}}},"activity_log":[{"activity_date":"2024-08-01","activities":{"workout":{"details":"Morning run","metrics":{"duration":30.5,"calories":200.0}},"reading":{"details":"Read book on Hive","metrics":{"pages":50.0,"time":2.0}}}},{"activity_date":"2024-08-02","activities":{"travel":{"details":"Flight to NY","metrics":{"distance":500.0,"time":3.0}},"meeting":{"details":"Project meeting","metrics":{"duration":1.5,"participants":5.0}}}}]}
+{"user_id":"user2","user_profile":{"name":"Bob","age":32,"preferences":{"books":{"preference_id":201,"preference_values":["fiction","non-fiction"]},"travel":{"preference_id":202,"preference_values":["beaches","mountains"]}}},"activity_log":[{"activity_date":"2024-08-01","activities":{"hiking":{"details":"Mountain trail","metrics":{"distance":10.0,"elevation":500.0}},"photography":{"details":"Wildlife photoshoot","metrics":{"photos_taken":100.0,"time":4.0}}}},{"activity_date":"2024-08-02","activities":{"workshop":{"details":"Photography workshop","metrics":{"duration":3.0,"participants":15.0}},"shopping":{"details":"Bought camera gear","metrics":{"items":5.0,"cost":1500.0}}}}]}
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/2 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/2
new file mode 100644
index 00000000000000..e1b0befc7bca31
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/2
@@ -0,0 +1 @@
+{"user_id":"user3","user_profile":{"name":"Carol","age":24,"preferences":{"food":{"preference_id":301,"preference_values":["vegan","desserts"]},"movies":{"preference_id":302,"preference_values":["action","comedy"]}}},"activity_log":[{"activity_date":"2024-08-01","activities":{"cooking":{"details":"Made vegan meal","metrics":{"time_spent":1.5,"calories":500.0}},"movie":{"details":"Watched action movie","metrics":{"duration":2.0,"rating":8.5}}}},{"activity_date":"2024-08-02","activities":{"gym":{"details":"Strength training","metrics":{"duration":1.0,"calories":300.0}},"shopping":{"details":"Bought groceries","metrics":{"items":10.0,"cost":100.0}}}}]}
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/modify_2 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/modify_2
new file mode 100644
index 00000000000000..08f1586f3aa91c
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json/json_nested_complex_table/modify_2
@@ -0,0 +1,2 @@
+{"user_ID":"user4","user_PROFILE":{"name":"Carol","age":24,"preferences":{"food":{"preference_ID":301,"preference_VALUES":["vegan","desserts"]},"movies":{"preference_ID":302,"preference_VALUES":["action","comedy"]}}},"activity_LOG":[{"activity_DATE":"2024-08-01","activities":{"cooking":{"DETAILS":"Made vegan meal","metrics":{"time_spent":1.5,"calories":500.0}},"movie":{"DETAILS":"Watched action movie","metrics":{"duration":2.0,"rating":8.5}}}},{"activity_DATE":"2024-08-02","activities":{"gym":{"DETAILS":"Strength training","metrics":{"duration":1.0,"calories":300.0}},"shopping":{"DETAILS":"Bought groceries","metrics":{"items":10.0,"cost":100.0}}}}]}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index 97032467cec765..0f839d238b2b1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -94,6 +94,9 @@ public class HiveMetaStoreClientHelper {
 
     private static final Pattern digitPattern = Pattern.compile("(\\d+)");
 
+    public static final String HIVE_JSON_SERDE = "org.apache.hive.hcatalog.data.JsonSerDe";
+    public static final String LEGACY_HIVE_JSON_SERDE = "org.apache.hadoop.hive.serde2.JsonSerDe";
+
     public enum HiveFileFormat {
         TEXT_FILE(0, "text"),
         PARQUET(1, "parquet"),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index e710bdb935d7bc..3a2a4d3eb5c6ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -364,14 +364,21 @@ public TableIf getTargetTable() {
     @Override
     public TFileFormatType getFileFormatType() throws UserException {
         TFileFormatType type = null;
-        String inputFormatName = hmsTable.getRemoteTable().getSd().getInputFormat();
+        Table table = hmsTable.getRemoteTable();
+        String inputFormatName = table.getSd().getInputFormat();
         String hiveFormat = HiveMetaStoreClientHelper.HiveFileFormat.getFormat(inputFormatName);
         if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.PARQUET.getDesc())) {
             type = TFileFormatType.FORMAT_PARQUET;
         } else if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.ORC.getDesc())) {
             type = TFileFormatType.FORMAT_ORC;
         } else if (hiveFormat.equals(HiveMetaStoreClientHelper.HiveFileFormat.TEXT_FILE.getDesc())) {
-            type = TFileFormatType.FORMAT_CSV_PLAIN;
+            String serDeLib = table.getSd().getSerdeInfo().getSerializationLib();
+            if (serDeLib.equals(HiveMetaStoreClientHelper.HIVE_JSON_SERDE)
+                    || serDeLib.equals(HiveMetaStoreClientHelper.LEGACY_HIVE_JSON_SERDE)) {
+                type = TFileFormatType.FORMAT_JSON;
+            } else {
+                type = TFileFormatType.FORMAT_CSV_PLAIN;
+            }
         }
         return type;
     }
@@ -383,11 +390,12 @@ protected Map<String, String> getLocationProperties() throws UserException  {
 
     @Override
     protected TFileAttributes getFileAttributes() throws UserException {
-        TFileTextScanRangeParams textParams = new TFileTextScanRangeParams();
+        TFileAttributes fileAttributes = new TFileAttributes();
         Table table = hmsTable.getRemoteTable();
         // TODO: separate hive text table and OpenCsv table
         String serDeLib = table.getSd().getSerdeInfo().getSerializationLib();
         if (serDeLib.equals("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")) {
+            TFileTextScanRangeParams textParams = new TFileTextScanRangeParams();
             // set properties of LazySimpleSerDe
             // 1. set column separator
             textParams.setColumnSeparator(HiveProperties.getFieldDelimiter(table));
@@ -401,7 +409,10 @@ protected TFileAttributes getFileAttributes() throws UserException {
             HiveProperties.getEscapeDelimiter(table).ifPresent(d -> textParams.setEscape(d.getBytes()[0]));
             // 6. set null format
             textParams.setNullFormat(HiveProperties.getNullFormat(table));
+            fileAttributes.setTextParams(textParams);
+            fileAttributes.setHeaderType("");
         } else if (serDeLib.equals("org.apache.hadoop.hive.serde2.OpenCSVSerde")) {
+            TFileTextScanRangeParams textParams = new TFileTextScanRangeParams();
             // set set properties of OpenCSVSerde
             // 1. set column separator
             textParams.setColumnSeparator(HiveProperties.getSeparatorChar(table));
@@ -411,17 +422,29 @@ protected TFileAttributes getFileAttributes() throws UserException {
             textParams.setEnclose(HiveProperties.getQuoteChar(table).getBytes()[0]);
             // 4. set escape char
             textParams.setEscape(HiveProperties.getEscapeChar(table).getBytes()[0]);
+            fileAttributes.setTextParams(textParams);
+            fileAttributes.setHeaderType("");
+            if (textParams.isSetEnclose()) {
+                fileAttributes.setTrimDoubleQuotes(true);
+            }
+        } else if (serDeLib.equals("org.apache.hive.hcatalog.data.JsonSerDe")) {
+            TFileTextScanRangeParams textParams = new TFileTextScanRangeParams();
+            textParams.setColumnSeparator("\t");
+            textParams.setLineDelimiter("\n");
+            fileAttributes.setTextParams(textParams);
+
+            fileAttributes.setJsonpaths("");
+            fileAttributes.setJsonRoot("");
+            fileAttributes.setNumAsString(true);
+            fileAttributes.setFuzzyParse(false);
+            fileAttributes.setReadJsonByLine(true);
+            fileAttributes.setStripOuterArray(false);
+            fileAttributes.setHeaderType("");
         } else {
             throw new UserException(
                     "unsupported hive table serde: " + serDeLib);
         }
 
-        TFileAttributes fileAttributes = new TFileAttributes();
-        fileAttributes.setTextParams(textParams);
-        fileAttributes.setHeaderType("");
-        if (textParams.isSet(TFileTextScanRangeParams._Fields.ENCLOSE)) {
-            fileAttributes.setTrimDoubleQuotes(true);
-        }
         return fileAttributes;
     }
 
diff --git a/regression-test/data/external_table_p0/hive/hive_json_basic_test.out b/regression-test/data/external_table_p0/hive/hive_json_basic_test.out
new file mode 100644
index 00000000000000..9023f5d72b1ac3
--- /dev/null
+++ b/regression-test/data/external_table_p0/hive/hive_json_basic_test.out
@@ -0,0 +1,115 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q1 --
+1	true	127	32767	2147483647	9223372036854775807	123.45	123456.789	123456789	1234.5678	123456.789012	123456789.012345678901	string_value	binary_value	2024-03-20	2024-03-20T12:00	2024-03-20T12:00:00.123457	2024-03-20T12:00:00.123457	char_value1                                       	char_value2                                                                                         	char_value3                                                                                                                                                                                                                                                    	varchar_value1	varchar_value2	varchar_value3	{"key1":"value1"}	{"key1":"value1"}	{"a         ":"b         "}	{1:10}	{1:100000000000}	{1.1:10.1}	{1.1:10.1}	{1:0}	{1.1:1.1}	{1.23:1.23}	{1.2345:1.2345}	{1.23456789:1.23456789}	{1.23456789:1.23456789}	{1.2345678901234568:1.2345678901234568}	["string1", "string2"]	[1, 2, 3]	[100000000000, 200000000000]	[1.1, 2.2]	[1.123456789, 2.123456789]	[1, 0]	["varchar1", "varchar2"]	["char1     ", "char2     "]	[1.1, 2.2]	[1.23, 2.34]	[1.2345, 2.3456]	[1.23456789, 2.34567891]	[1.23456789, 2.34567891]	[1.2345678901234568, 2.3456789012345679]	{"s_bigint":1234567890}	{"key":[{"s_int":123}]}	{"struct_field":["value1", "value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"}	{"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}}	{"null_key":null}	[null, "value1", "value2"]	["value1", null, "value2"]	["value1", "value2", null]	[null, null, null]	dt1
+2	false	58	12345	2147483000	\N	789.56	654321.123	987654321	5678.1234	987654.321098	987654321.098765432109	changed_string	new_binary_value	2025-05-25	2025-05-25T15:30	2025-05-25T15:30:00.654322	2025-05-25T15:30:00.654322	char_new_value1                                   	char_new_value2                                                                                     	char_new_value3                                                                                                                                                                                                                                                	varchar_new_value1	varchar_new_value2	varchar_new_value3	{"key2":"value2"}	{"key2":"value2"}	{"x         ":"y         "}	{2:20}	{2:200000000000}	{2.2:20.2}	{2.2:20.2}	{0:1}	{2.2:2.2}	{2.34:2.34}	{2.3456:2.3456}	{2.34567891:2.34567891}	{2.34567891:2.34567891}	{2.3456789012345679:2.3456789012345679}	["string3", "string4"]	[4, 5, 6]	[300000000000, 400000000000]	[2.2, 3.3]	[2.123456789, 3.123456789]	[0, 1]	["varchar3", "varchar4"]	["char3     ", "char4     "]	[2.2, 3.3]	[2.34, 3.45]	[2.3456, 3.4567]	[2.34567891, 3.45678901]	[2.34567891, 3.45678901]	[2.3456789012345679, 3.4567890123456789]	{"s_bigint":9876543210}	{"key2":[{"s_int":456}]}	{"struct_field":["new_value1", "new_value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":456, "struct_non_nulls_after_nulls2":"new_value"}	{"struct_field1":456, "struct_field2":"new_value", "strict_field3":{"nested_struct_field1":456, "nested_struct_field2":"nested_value2"}}	{"null_key":null}	[null, "new_value1", "new_value2"]	["new_value1", null, "new_value2"]	["new_value1", "new_value2", null]	[null, null, null]	dt1
+3	false	-128	-32768	-2147483648	-9223372036854775808	-3.4028235e+38	-1.7976931348623157E308	-999999999	-9999.9999	-999999999.999999	\N	min_string_value	xxxx	2001-01-01	2001-01-01T00:00	2001-01-01T00:00	2001-01-01T00:00	char_min_value1                                   	char_min_value2                                                                                     	char_min_value3                                                                                                                                                                                                                                                	varchar_min_value1	varchar_min_value2	varchar_min_value3	{"min_key":"min_value"}	{"min_key":"min_value"}	{"a         ":"z         "}	{-1:-100}	{-1:-100000000000}	{-1.1:-10.1}	{-1.1:-10.1}	{0:1}	{-1.1:-1.1}	{-1.23:-1.23}	{-1.2345:-1.2345}	{-1.23456789:-1.23456789}	{-1.23456789:-1.23456789}	{-1.2345678901234568:-1.2345678901234568}	["min_string1", "min_string2"]	[-10, -5, -3]	[-100000000000, -200000000000]	[-1.1, -2.2]	[-1.123456789, -2.123456789]	[0, 1]	["min_varchar1", "min_varchar2"]	["min_char1 ", "min_char2 "]	[-1.1, -2.2]	[-1.23, -2.34]	[-1.2345, -2.3456]	[-1.23456789, -2.34567891]	[-1.23456789, -2.34567891]	[-1.2345678901234568, -2.3456789012345679]	{"s_bigint":-1234567890}	{"min_key":[{"s_int":-123}]}	{"struct_field":["min_value1", "min_value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"min_value"}	{"struct_field1":-123, "struct_field2":"min_value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}}	{"null_key":null}	[null, "min_value1", "min_value2"]	["min_value1", null, "min_value2"]	["min_value1", "min_value2", null]	[null, null, null]	dt1
+4	\N	\N	\N	\N	\N	123.45	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	{1:10}	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	[1.2345, 2.3456]	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	[null, "value1", "value2"]	\N	\N	\N	dt2
+5	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	dt3
+6	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	dt3
+
+-- !q2 --
+3	false	-128	-32768	-2147483648	-9223372036854775808	-3.4028235e+38	-1.7976931348623157E308	-999999999	-9999.9999	-999999999.999999	\N	min_string_value	xxxx	2001-01-01	2001-01-01T00:00	2001-01-01T00:00	2001-01-01T00:00	char_min_value1                                   	char_min_value2                                                                                     	char_min_value3                                                                                                                                                                                                                                                	varchar_min_value1	varchar_min_value2	varchar_min_value3	{"min_key":"min_value"}	{"min_key":"min_value"}	{"a         ":"z         "}	{-1:-100}	{-1:-100000000000}	{-1.1:-10.1}	{-1.1:-10.1}	{0:1}	{-1.1:-1.1}	{-1.23:-1.23}	{-1.2345:-1.2345}	{-1.23456789:-1.23456789}	{-1.23456789:-1.23456789}	{-1.2345678901234568:-1.2345678901234568}	["min_string1", "min_string2"]	[-10, -5, -3]	[-100000000000, -200000000000]	[-1.1, -2.2]	[-1.123456789, -2.123456789]	[0, 1]	["min_varchar1", "min_varchar2"]	["min_char1 ", "min_char2 "]	[-1.1, -2.2]	[-1.23, -2.34]	[-1.2345, -2.3456]	[-1.23456789, -2.34567891]	[-1.23456789, -2.34567891]	[-1.2345678901234568, -2.3456789012345679]	{"s_bigint":-1234567890}	{"min_key":[{"s_int":-123}]}	{"struct_field":["min_value1", "min_value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"min_value"}	{"struct_field1":-123, "struct_field2":"min_value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}}	{"null_key":null}	[null, "min_value1", "min_value2"]	["min_value1", null, "min_value2"]	["min_value1", "min_value2", null]	[null, null, null]	dt1
+
+-- !q3 --
+1	true	127	32767	2147483647	9223372036854775807	123.45	123456.789	123456789	1234.5678	123456.789012	123456789.012345678901	string_value	binary_value	2024-03-20	2024-03-20T12:00	2024-03-20T12:00:00.123457	2024-03-20T12:00:00.123457	char_value1                                       	char_value2                                                                                         	char_value3                                                                                                                                                                                                                                                    	varchar_value1	varchar_value2	varchar_value3	{"key1":"value1"}	{"key1":"value1"}	{"a         ":"b         "}	{1:10}	{1:100000000000}	{1.1:10.1}	{1.1:10.1}	{1:0}	{1.1:1.1}	{1.23:1.23}	{1.2345:1.2345}	{1.23456789:1.23456789}	{1.23456789:1.23456789}	{1.2345678901234568:1.2345678901234568}	["string1", "string2"]	[1, 2, 3]	[100000000000, 200000000000]	[1.1, 2.2]	[1.123456789, 2.123456789]	[1, 0]	["varchar1", "varchar2"]	["char1     ", "char2     "]	[1.1, 2.2]	[1.23, 2.34]	[1.2345, 2.3456]	[1.23456789, 2.34567891]	[1.23456789, 2.34567891]	[1.2345678901234568, 2.3456789012345679]	{"s_bigint":1234567890}	{"key":[{"s_int":123}]}	{"struct_field":["value1", "value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"}	{"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}}	{"null_key":null}	[null, "value1", "value2"]	["value1", null, "value2"]	["value1", "value2", null]	[null, null, null]	dt1
+
+-- !q4 --
+123.45
+789.56
+-3.4028235e+38
+123.45
+
+-- !q5 --
+2	false	58	12345	2147483000	\N	789.56	654321.123	987654321	5678.1234	987654.321098	987654321.098765432109	changed_string	new_binary_value	2025-05-25	2025-05-25T15:30	2025-05-25T15:30:00.654322	2025-05-25T15:30:00.654322	char_new_value1                                   	char_new_value2                                                                                     	char_new_value3                                                                                                                                                                                                                                                	varchar_new_value1	varchar_new_value2	varchar_new_value3	{"key2":"value2"}	{"key2":"value2"}	{"x         ":"y         "}	{2:20}	{2:200000000000}	{2.2:20.2}	{2.2:20.2}	{0:1}	{2.2:2.2}	{2.34:2.34}	{2.3456:2.3456}	{2.34567891:2.34567891}	{2.34567891:2.34567891}	{2.3456789012345679:2.3456789012345679}	["string3", "string4"]	[4, 5, 6]	[300000000000, 400000000000]	[2.2, 3.3]	[2.123456789, 3.123456789]	[0, 1]	["varchar3", "varchar4"]	["char3     ", "char4     "]	[2.2, 3.3]	[2.34, 3.45]	[2.3456, 3.4567]	[2.34567891, 3.45678901]	[2.34567891, 3.45678901]	[2.3456789012345679, 3.4567890123456789]	{"s_bigint":9876543210}	{"key2":[{"s_int":456}]}	{"struct_field":["new_value1", "new_value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":456, "struct_non_nulls_after_nulls2":"new_value"}	{"struct_field1":456, "struct_field2":"new_value", "strict_field3":{"nested_struct_field1":456, "nested_struct_field2":"nested_value2"}}	{"null_key":null}	[null, "new_value1", "new_value2"]	["new_value1", null, "new_value2"]	["new_value1", "new_value2", null]	[null, null, null]	dt1
+
+-- !q6 --
+user1	{"name":"Alice", "age":28, "preferences":{"sports":{"preference_id":101, "preference_values":["soccer", "tennis"]}, "music":{"preference_id":102, "preference_values":["rock", "classical"]}}}	[{"activity_date":"2024-08-01", "activities":{"workout":{"details":"Morning run", "metrics":{"duration":30.5, "calories":200}}, "reading":{"details":"Read book on Hive", "metrics":{"pages":50, "time":2}}}}, {"activity_date":"2024-08-02", "activities":{"travel":{"details":"Flight to NY", "metrics":{"distance":500, "time":3}}, "meeting":{"details":"Project meeting", "metrics":{"duration":1.5, "participants":5}}}}]
+user2	{"name":"Bob", "age":32, "preferences":{"books":{"preference_id":201, "preference_values":["fiction", "non-fiction"]}, "travel":{"preference_id":202, "preference_values":["beaches", "mountains"]}}}	[{"activity_date":"2024-08-01", "activities":{"hiking":{"details":"Mountain trail", "metrics":{"distance":10, "elevation":500}}, "photography":{"details":"Wildlife photoshoot", "metrics":{"photos_taken":100, "time":4}}}}, {"activity_date":"2024-08-02", "activities":{"workshop":{"details":"Photography workshop", "metrics":{"duration":3, "participants":15}}, "shopping":{"details":"Bought camera gear", "metrics":{"items":5, "cost":1500}}}}]
+user3	{"name":"Carol", "age":24, "preferences":{"food":{"preference_id":301, "preference_values":["vegan", "desserts"]}, "movies":{"preference_id":302, "preference_values":["action", "comedy"]}}}	[{"activity_date":"2024-08-01", "activities":{"cooking":{"details":"Made vegan meal", "metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched action movie", "metrics":{"duration":2, "rating":8.5}}}}, {"activity_date":"2024-08-02", "activities":{"gym":{"details":"Strength training", "metrics":{"duration":1, "calories":300}}, "shopping":{"details":"Bought groceries", "metrics":{"items":10, "cost":100}}}}]
+user4	{"name":"Carol", "age":24, "preferences":{"food":{"preference_id":301, "preference_values":["vegan", "desserts"]}, "movies":{"preference_id":302, "preference_values":["action", "comedy"]}}}	[{"activity_date":"2024-08-01", "activities":{"cooking":{"details":"Made vegan meal", "metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched action movie", "metrics":{"duration":2, "rating":8.5}}}}, {"activity_date":"2024-08-02", "activities":{"gym":{"details":"Strength training", "metrics":{"duration":1, "calories":300}}, "shopping":{"details":"Bought groceries", "metrics":{"items":10, "cost":100}}}}]
+
+-- !q7 --
+user1	[{"activity_date":"2024-08-01", "activities":{"workout":{"details":"Morning run", "metrics":{"duration":30.5, "calories":200}}, "reading":{"details":"Read book on Hive", "metrics":{"pages":50, "time":2}}}}, {"activity_date":"2024-08-02", "activities":{"travel":{"details":"Flight to NY", "metrics":{"distance":500, "time":3}}, "meeting":{"details":"Project meeting", "metrics":{"duration":1.5, "participants":5}}}}]
+user2	[{"activity_date":"2024-08-01", "activities":{"hiking":{"details":"Mountain trail", "metrics":{"distance":10, "elevation":500}}, "photography":{"details":"Wildlife photoshoot", "metrics":{"photos_taken":100, "time":4}}}}, {"activity_date":"2024-08-02", "activities":{"workshop":{"details":"Photography workshop", "metrics":{"duration":3, "participants":15}}, "shopping":{"details":"Bought camera gear", "metrics":{"items":5, "cost":1500}}}}]
+user3	[{"activity_date":"2024-08-01", "activities":{"cooking":{"details":"Made vegan meal", "metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched action movie", "metrics":{"duration":2, "rating":8.5}}}}, {"activity_date":"2024-08-02", "activities":{"gym":{"details":"Strength training", "metrics":{"duration":1, "calories":300}}, "shopping":{"details":"Bought groceries", "metrics":{"items":10, "cost":100}}}}]
+user4	[{"activity_date":"2024-08-01", "activities":{"cooking":{"details":"Made vegan meal", "metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched action movie", "metrics":{"duration":2, "rating":8.5}}}}, {"activity_date":"2024-08-02", "activities":{"gym":{"details":"Strength training", "metrics":{"duration":1, "calories":300}}, "shopping":{"details":"Bought groceries", "metrics":{"items":10, "cost":100}}}}]
+
+-- !q8 --
+\N	\N	\N	\N
+\N	\N	\N	\N
+1	10	{"col2a":10, "col2b":"string1"}	{1:"string10"}
+2	20	{"col2a":20, "col2b":"string2"}	{2:"string2"}
+3	30	{"col2a":30, "col2b":"string3"}	\N
+4	40	{"col2a":40, "col2b":"string4"}	{4:"string4"}
+5	\N	\N	\N
+6	60	{"col2a":60, "col2b":"600"}	{6:"600"}
+7	70	{"col2a":70, "col2b":"string7"}	{7:"string7"}
+
+-- !q9 --
+\N	\N
+\N	\N
+\N	5
+10	1
+20	2
+30	3
+40	4
+60	6
+70	7
+
+-- !q1 --
+1	true	127	32767	2147483647	9223372036854775807	123.45	123456.789	123456789	1234.5678	123456.789012	123456789.012345678901	string_value	binary_value	2024-03-20	2024-03-20T12:00	2024-03-20T12:00:00.123457	2024-03-20T12:00:00.123457	char_value1                                       	char_value2                                                                                         	char_value3                                                                                                                                                                                                                                                    	varchar_value1	varchar_value2	varchar_value3	{"key1":"value1"}	{"key1":"value1"}	{"a         ":"b         "}	{1:10}	{1:100000000000}	{1.1:10.1}	{1.1:10.1}	{1:0}	{1.1:1.1}	{1.23:1.23}	{1.2345:1.2345}	{1.23456789:1.23456789}	{1.23456789:1.23456789}	{1.2345678901234568:1.2345678901234568}	["string1", "string2"]	[1, 2, 3]	[100000000000, 200000000000]	[1.1, 2.2]	[1.123456789, 2.123456789]	[1, 0]	["varchar1", "varchar2"]	["char1     ", "char2     "]	[1.1, 2.2]	[1.23, 2.34]	[1.2345, 2.3456]	[1.23456789, 2.34567891]	[1.23456789, 2.34567891]	[1.2345678901234568, 2.3456789012345679]	{"s_bigint":1234567890}	{"key":[{"s_int":123}]}	{"struct_field":["value1", "value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"}	{"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}}	{"null_key":null}	[null, "value1", "value2"]	["value1", null, "value2"]	["value1", "value2", null]	[null, null, null]	dt1
+2	false	58	12345	2147483000	\N	789.56	654321.123	987654321	5678.1234	987654.321098	987654321.098765432109	changed_string	new_binary_value	2025-05-25	2025-05-25T15:30	2025-05-25T15:30:00.654322	2025-05-25T15:30:00.654322	char_new_value1                                   	char_new_value2                                                                                     	char_new_value3                                                                                                                                                                                                                                                	varchar_new_value1	varchar_new_value2	varchar_new_value3	{"key2":"value2"}	{"key2":"value2"}	{"x         ":"y         "}	{2:20}	{2:200000000000}	{2.2:20.2}	{2.2:20.2}	{0:1}	{2.2:2.2}	{2.34:2.34}	{2.3456:2.3456}	{2.34567891:2.34567891}	{2.34567891:2.34567891}	{2.3456789012345679:2.3456789012345679}	["string3", "string4"]	[4, 5, 6]	[300000000000, 400000000000]	[2.2, 3.3]	[2.123456789, 3.123456789]	[0, 1]	["varchar3", "varchar4"]	["char3     ", "char4     "]	[2.2, 3.3]	[2.34, 3.45]	[2.3456, 3.4567]	[2.34567891, 3.45678901]	[2.34567891, 3.45678901]	[2.3456789012345679, 3.4567890123456789]	{"s_bigint":9876543210}	{"key2":[{"s_int":456}]}	{"struct_field":["new_value1", "new_value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":456, "struct_non_nulls_after_nulls2":"new_value"}	{"struct_field1":456, "struct_field2":"new_value", "strict_field3":{"nested_struct_field1":456, "nested_struct_field2":"nested_value2"}}	{"null_key":null}	[null, "new_value1", "new_value2"]	["new_value1", null, "new_value2"]	["new_value1", "new_value2", null]	[null, null, null]	dt1
+3	false	-128	-32768	-2147483648	-9223372036854775808	-3.4028235e+38	-1.7976931348623157E308	-999999999	-9999.9999	-999999999.999999	\N	min_string_value	xxxx	2001-01-01	2001-01-01T00:00	2001-01-01T00:00	2001-01-01T00:00	char_min_value1                                   	char_min_value2                                                                                     	char_min_value3                                                                                                                                                                                                                                                	varchar_min_value1	varchar_min_value2	varchar_min_value3	{"min_key":"min_value"}	{"min_key":"min_value"}	{"a         ":"z         "}	{-1:-100}	{-1:-100000000000}	{-1.1:-10.1}	{-1.1:-10.1}	{0:1}	{-1.1:-1.1}	{-1.23:-1.23}	{-1.2345:-1.2345}	{-1.23456789:-1.23456789}	{-1.23456789:-1.23456789}	{-1.2345678901234568:-1.2345678901234568}	["min_string1", "min_string2"]	[-10, -5, -3]	[-100000000000, -200000000000]	[-1.1, -2.2]	[-1.123456789, -2.123456789]	[0, 1]	["min_varchar1", "min_varchar2"]	["min_char1 ", "min_char2 "]	[-1.1, -2.2]	[-1.23, -2.34]	[-1.2345, -2.3456]	[-1.23456789, -2.34567891]	[-1.23456789, -2.34567891]	[-1.2345678901234568, -2.3456789012345679]	{"s_bigint":-1234567890}	{"min_key":[{"s_int":-123}]}	{"struct_field":["min_value1", "min_value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"min_value"}	{"struct_field1":-123, "struct_field2":"min_value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}}	{"null_key":null}	[null, "min_value1", "min_value2"]	["min_value1", null, "min_value2"]	["min_value1", "min_value2", null]	[null, null, null]	dt1
+4	\N	\N	\N	\N	\N	123.45	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	{1:10}	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	[1.2345, 2.3456]	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	[null, "value1", "value2"]	\N	\N	\N	dt2
+5	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	dt3
+6	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	dt3
+
+-- !q2 --
+3	false	-128	-32768	-2147483648	-9223372036854775808	-3.4028235e+38	-1.7976931348623157E308	-999999999	-9999.9999	-999999999.999999	\N	min_string_value	xxxx	2001-01-01	2001-01-01T00:00	2001-01-01T00:00	2001-01-01T00:00	char_min_value1                                   	char_min_value2                                                                                     	char_min_value3                                                                                                                                                                                                                                                	varchar_min_value1	varchar_min_value2	varchar_min_value3	{"min_key":"min_value"}	{"min_key":"min_value"}	{"a         ":"z         "}	{-1:-100}	{-1:-100000000000}	{-1.1:-10.1}	{-1.1:-10.1}	{0:1}	{-1.1:-1.1}	{-1.23:-1.23}	{-1.2345:-1.2345}	{-1.23456789:-1.23456789}	{-1.23456789:-1.23456789}	{-1.2345678901234568:-1.2345678901234568}	["min_string1", "min_string2"]	[-10, -5, -3]	[-100000000000, -200000000000]	[-1.1, -2.2]	[-1.123456789, -2.123456789]	[0, 1]	["min_varchar1", "min_varchar2"]	["min_char1 ", "min_char2 "]	[-1.1, -2.2]	[-1.23, -2.34]	[-1.2345, -2.3456]	[-1.23456789, -2.34567891]	[-1.23456789, -2.34567891]	[-1.2345678901234568, -2.3456789012345679]	{"s_bigint":-1234567890}	{"min_key":[{"s_int":-123}]}	{"struct_field":["min_value1", "min_value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"min_value"}	{"struct_field1":-123, "struct_field2":"min_value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}}	{"null_key":null}	[null, "min_value1", "min_value2"]	["min_value1", null, "min_value2"]	["min_value1", "min_value2", null]	[null, null, null]	dt1
+
+-- !q3 --
+1	true	127	32767	2147483647	9223372036854775807	123.45	123456.789	123456789	1234.5678	123456.789012	123456789.012345678901	string_value	binary_value	2024-03-20	2024-03-20T12:00	2024-03-20T12:00:00.123457	2024-03-20T12:00:00.123457	char_value1                                       	char_value2                                                                                         	char_value3                                                                                                                                                                                                                                                    	varchar_value1	varchar_value2	varchar_value3	{"key1":"value1"}	{"key1":"value1"}	{"a         ":"b         "}	{1:10}	{1:100000000000}	{1.1:10.1}	{1.1:10.1}	{1:0}	{1.1:1.1}	{1.23:1.23}	{1.2345:1.2345}	{1.23456789:1.23456789}	{1.23456789:1.23456789}	{1.2345678901234568:1.2345678901234568}	["string1", "string2"]	[1, 2, 3]	[100000000000, 200000000000]	[1.1, 2.2]	[1.123456789, 2.123456789]	[1, 0]	["varchar1", "varchar2"]	["char1     ", "char2     "]	[1.1, 2.2]	[1.23, 2.34]	[1.2345, 2.3456]	[1.23456789, 2.34567891]	[1.23456789, 2.34567891]	[1.2345678901234568, 2.3456789012345679]	{"s_bigint":1234567890}	{"key":[{"s_int":123}]}	{"struct_field":["value1", "value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"}	{"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}}	{"null_key":null}	[null, "value1", "value2"]	["value1", null, "value2"]	["value1", "value2", null]	[null, null, null]	dt1
+
+-- !q4 --
+123.45
+789.56
+-3.4028235e+38
+123.45
+
+-- !q5 --
+2	false	58	12345	2147483000	\N	789.56	654321.123	987654321	5678.1234	987654.321098	987654321.098765432109	changed_string	new_binary_value	2025-05-25	2025-05-25T15:30	2025-05-25T15:30:00.654322	2025-05-25T15:30:00.654322	char_new_value1                                   	char_new_value2                                                                                     	char_new_value3                                                                                                                                                                                                                                                	varchar_new_value1	varchar_new_value2	varchar_new_value3	{"key2":"value2"}	{"key2":"value2"}	{"x         ":"y         "}	{2:20}	{2:200000000000}	{2.2:20.2}	{2.2:20.2}	{0:1}	{2.2:2.2}	{2.34:2.34}	{2.3456:2.3456}	{2.34567891:2.34567891}	{2.34567891:2.34567891}	{2.3456789012345679:2.3456789012345679}	["string3", "string4"]	[4, 5, 6]	[300000000000, 400000000000]	[2.2, 3.3]	[2.123456789, 3.123456789]	[0, 1]	["varchar3", "varchar4"]	["char3     ", "char4     "]	[2.2, 3.3]	[2.34, 3.45]	[2.3456, 3.4567]	[2.34567891, 3.45678901]	[2.34567891, 3.45678901]	[2.3456789012345679, 3.4567890123456789]	{"s_bigint":9876543210}	{"key2":[{"s_int":456}]}	{"struct_field":["new_value1", "new_value2"]}	{"struct_field_null":null, "struct_field_null2":null}	{"struct_non_nulls_after_nulls1":456, "struct_non_nulls_after_nulls2":"new_value"}	{"struct_field1":456, "struct_field2":"new_value", "strict_field3":{"nested_struct_field1":456, "nested_struct_field2":"nested_value2"}}	{"null_key":null}	[null, "new_value1", "new_value2"]	["new_value1", null, "new_value2"]	["new_value1", "new_value2", null]	[null, null, null]	dt1
+
+-- !q6 --
+user1	{"name":"Alice", "age":28, "preferences":{"sports":{"preference_id":101, "preference_values":["soccer", "tennis"]}, "music":{"preference_id":102, "preference_values":["rock", "classical"]}}}	[{"activity_date":"2024-08-01", "activities":{"workout":{"details":"Morning run", "metrics":{"duration":30.5, "calories":200}}, "reading":{"details":"Read book on Hive", "metrics":{"pages":50, "time":2}}}}, {"activity_date":"2024-08-02", "activities":{"travel":{"details":"Flight to NY", "metrics":{"distance":500, "time":3}}, "meeting":{"details":"Project meeting", "metrics":{"duration":1.5, "participants":5}}}}]
+user2	{"name":"Bob", "age":32, "preferences":{"books":{"preference_id":201, "preference_values":["fiction", "non-fiction"]}, "travel":{"preference_id":202, "preference_values":["beaches", "mountains"]}}}	[{"activity_date":"2024-08-01", "activities":{"hiking":{"details":"Mountain trail", "metrics":{"distance":10, "elevation":500}}, "photography":{"details":"Wildlife photoshoot", "metrics":{"photos_taken":100, "time":4}}}}, {"activity_date":"2024-08-02", "activities":{"workshop":{"details":"Photography workshop", "metrics":{"duration":3, "participants":15}}, "shopping":{"details":"Bought camera gear", "metrics":{"items":5, "cost":1500}}}}]
+user3	{"name":"Carol", "age":24, "preferences":{"food":{"preference_id":301, "preference_values":["vegan", "desserts"]}, "movies":{"preference_id":302, "preference_values":["action", "comedy"]}}}	[{"activity_date":"2024-08-01", "activities":{"cooking":{"details":"Made vegan meal", "metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched action movie", "metrics":{"duration":2, "rating":8.5}}}}, {"activity_date":"2024-08-02", "activities":{"gym":{"details":"Strength training", "metrics":{"duration":1, "calories":300}}, "shopping":{"details":"Bought groceries", "metrics":{"items":10, "cost":100}}}}]
+user4	{"name":"Carol", "age":24, "preferences":{"food":{"preference_id":301, "preference_values":["vegan", "desserts"]}, "movies":{"preference_id":302, "preference_values":["action", "comedy"]}}}	[{"activity_date":"2024-08-01", "activities":{"cooking":{"details":"Made vegan meal", "metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched action movie", "metrics":{"duration":2, "rating":8.5}}}}, {"activity_date":"2024-08-02", "activities":{"gym":{"details":"Strength training", "metrics":{"duration":1, "calories":300}}, "shopping":{"details":"Bought groceries", "metrics":{"items":10, "cost":100}}}}]
+
+-- !q7 --
+user1	[{"activity_date":"2024-08-01", "activities":{"workout":{"details":"Morning run", "metrics":{"duration":30.5, "calories":200}}, "reading":{"details":"Read book on Hive", "metrics":{"pages":50, "time":2}}}}, {"activity_date":"2024-08-02", "activities":{"travel":{"details":"Flight to NY", "metrics":{"distance":500, "time":3}}, "meeting":{"details":"Project meeting", "metrics":{"duration":1.5, "participants":5}}}}]
+user2	[{"activity_date":"2024-08-01", "activities":{"hiking":{"details":"Mountain trail", "metrics":{"distance":10, "elevation":500}}, "photography":{"details":"Wildlife photoshoot", "metrics":{"photos_taken":100, "time":4}}}}, {"activity_date":"2024-08-02", "activities":{"workshop":{"details":"Photography workshop", "metrics":{"duration":3, "participants":15}}, "shopping":{"details":"Bought camera gear", "metrics":{"items":5, "cost":1500}}}}]
+user3	[{"activity_date":"2024-08-01", "activities":{"cooking":{"details":"Made vegan meal", "metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched action movie", "metrics":{"duration":2, "rating":8.5}}}}, {"activity_date":"2024-08-02", "activities":{"gym":{"details":"Strength training", "metrics":{"duration":1, "calories":300}}, "shopping":{"details":"Bought groceries", "metrics":{"items":10, "cost":100}}}}]
+user4	[{"activity_date":"2024-08-01", "activities":{"cooking":{"details":"Made vegan meal", "metrics":{"time_spent":1.5, "calories":500}}, "movie":{"details":"Watched action movie", "metrics":{"duration":2, "rating":8.5}}}}, {"activity_date":"2024-08-02", "activities":{"gym":{"details":"Strength training", "metrics":{"duration":1, "calories":300}}, "shopping":{"details":"Bought groceries", "metrics":{"items":10, "cost":100}}}}]
+
+-- !q8 --
+\N	\N	\N	\N
+\N	\N	\N	\N
+1	10	{"col2a":10, "col2b":"string1"}	{1:"string10"}
+2	20	{"col2a":20, "col2b":"string2"}	{2:"string2"}
+3	30	{"col2a":30, "col2b":"string3"}	\N
+4	40	{"col2a":40, "col2b":"string4"}	{4:"string4"}
+5	\N	\N	\N
+6	60	{"col2a":60, "col2b":"600"}	{6:"600"}
+7	70	{"col2a":70, "col2b":"string7"}	{7:"string7"}
+
+-- !q9 --
+\N	\N
+\N	\N
+\N	5
+10	1
+20	2
+30	3
+40	4
+60	6
+70	7
+
diff --git a/regression-test/suites/external_table_p0/hive/hive_json_basic_test.groovy b/regression-test/suites/external_table_p0/hive/hive_json_basic_test.groovy
new file mode 100644
index 00000000000000..9d05e1a4c7403d
--- /dev/null
+++ b/regression-test/suites/external_table_p0/hive/hive_json_basic_test.groovy
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("hive_json_basic_test",  "p0,external,hive,external_docker,external_docker_hive") {
+
+
+    String enabled = context.config.otherConfigs.get("enableHiveTest")
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+        logger.info("diable Hive test.")
+        return;
+    }
+
+    for (String hivePrefix : ["hive2", "hive3"]) {
+        try {
+            String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+            String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
+            String catalog_name = "${hivePrefix}_hive_json_basic_test"
+            String broker_name = "hdfs"
+
+            sql """drop catalog if exists ${catalog_name}"""
+            sql """create catalog if not exists ${catalog_name} properties (
+                'type'='hms',
+                'hive.metastore.uris'='thrift://${externalEnvIp}:${hms_port}'
+            );"""
+            sql """use `${catalog_name}`.`default`"""
+
+            String tb1 = """json_all_complex_types"""
+            String tb2 = """json_nested_complex_table"""
+            String tb3 = """json_load_data_table"""
+
+            def tables = sql """ show tables """
+            logger.info("tables = ${tables}")
+
+            qt_q1 """ select * from ${tb1} order by id """
+            qt_q2 """ select * from ${tb1} where tinyint_col < 0  order by id """
+            qt_q3 """ select * from ${tb1} where bigint_col > 0 order by id """
+            qt_q4 """ select float_col from ${tb1} where float_col is not null  order by id """
+            qt_q5 """ select * from ${tb1} where id = 2 order by id """
+
+
+
+            qt_q6 """ select * from  ${tb2} order by user_id"""
+            qt_q7 """ select user_id,activity_log from  ${tb2} order by user_id"""
+
+
+            order_qt_q8 """ select * from ${tb3} order by id """
+            
+            order_qt_q9 """ select col1,id from ${tb3} order by id """ 
+            
+
+
+
+            sql """drop catalog if exists ${catalog_name}"""
+        } finally {
+        }
+    }
+}

From 839208cbdda0b2531acb866da2ada98f0598e04e Mon Sep 17 00:00:00 2001
From: yujun <yujun@selectdb.com>
Date: Thu, 28 Nov 2024 22:43:33 +0800
Subject: [PATCH 068/399] [fix](nereids) fix UnknownValue's reference in 
 simplify range rule (#44637)

for sql 'a > 10 and b > 20' will generate an UnknowValue with
sourceValues = [ Range(a > 10), Range(b > 20) ].

in UnknowValue's constructor, it set reference =
sourceValues[0].reference, then its reference = a. This is an error.

when A union/interset B generating an UnknowValue, this UnknownValue's
reference should be:
1. A.reference if A.reference = B.reference;
2. expr(A union/interset B)  if A.reference != B.reference;
---
 .../rules/expression/rules/SimplifyRange.java | 235 ++++++++++--------
 .../rules/expression/SimplifyRangeTest.java   |   4 +
 .../nereids/rules/rewrite/OrToInTest.java     |   2 +-
 .../nereids_hint_tpcds_p0/shape/query41.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query47.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query88.out   |  16 +-
 .../shape/query41.out                         |   2 +-
 .../shape/query47.out                         |   2 +-
 .../shape/query88.out                         |  16 +-
 .../noStatsRfPrune/query41.out                |   2 +-
 .../noStatsRfPrune/query57.out                |   2 +-
 .../no_stats_shape/query41.out                |   2 +-
 .../no_stats_shape/query57.out                |   2 +-
 .../rf_prune/query41.out                      |   2 +-
 .../rf_prune/query57.out                      |   2 +-
 .../shape/query41.out                         |   2 +-
 .../shape/query57.out                         |   2 +-
 .../shape/query41.out                         |   2 +-
 .../shape/query47.out                         |   2 +-
 .../shape/query57.out                         |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query41.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query57.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query41.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query57.out    |   2 +-
 .../tpcds_sf100/rf_prune/query41.out          |   2 +-
 .../tpcds_sf100/rf_prune/query57.out          |   2 +-
 .../tpcds_sf100/shape/query41.out             |   2 +-
 .../tpcds_sf100/shape/query57.out             |   2 +-
 .../tpcds_sf1000/shape/query41.out            |   2 +-
 .../tpcds_sf1000/shape/query47.out            |   2 +-
 .../tpcds_sf1000/shape/query88.out            |  16 +-
 .../expression/test_simplify_range.groovy     |  28 +++
 32 files changed, 211 insertions(+), 156 deletions(-)
 create mode 100644 regression-test/suites/nereids_rules_p0/expression/test_simplify_range.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
index 297f11abd0e00e..ca50f7c4d49225 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
@@ -92,12 +92,12 @@ public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
     /** rewrite */
     public static Expression rewrite(CompoundPredicate expr, ExpressionRewriteContext context) {
         ValueDesc valueDesc = expr.accept(new RangeInference(), context);
-        Expression exprForNonNull = valueDesc.toExpressionForNonNull();
-        if (exprForNonNull == null) {
+        Expression toExpr = valueDesc.toExpression();
+        if (toExpr == null) {
             // this mean cannot simplify
-            return valueDesc.exprForNonNull;
+            return valueDesc.toExpr;
         }
-        return exprForNonNull;
+        return toExpr;
     }
 
     private static class RangeInference extends ExpressionVisitor<ValueDesc, ExpressionRewriteContext> {
@@ -197,18 +197,18 @@ private ValueDesc simplify(ExpressionRewriteContext context,
             }
 
             // use UnknownValue to wrap different references
-            return new UnknownValue(context, valuePerRefs, originExpr, exprOp);
+            return new UnknownValue(context, originExpr, valuePerRefs, exprOp);
         }
     }
 
     private abstract static class ValueDesc {
         ExpressionRewriteContext context;
-        Expression exprForNonNull;
+        Expression toExpr;
         Expression reference;
 
-        public ValueDesc(ExpressionRewriteContext context, Expression reference, Expression exprForNonNull) {
+        public ValueDesc(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
             this.context = context;
-            this.exprForNonNull = exprForNonNull;
+            this.toExpr = toExpr;
             this.reference = reference;
         }
 
@@ -220,28 +220,28 @@ public static ValueDesc union(ExpressionRewriteContext context,
             if (count == discrete.values.size()) {
                 return range;
             }
-            Expression exprForNonNull = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.or(range.exprForNonNull, discrete.exprForNonNull), context);
+            Expression toExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.or(range.toExpr, discrete.toExpr), context);
             List<ValueDesc> sourceValues = reverseOrder
                     ? ImmutableList.of(discrete, range)
                     : ImmutableList.of(range, discrete);
-            return new UnknownValue(context, sourceValues, exprForNonNull, ExpressionUtils::or);
+            return new UnknownValue(context, toExpr, sourceValues, ExpressionUtils::or);
         }
 
         public abstract ValueDesc intersect(ValueDesc other);
 
         public static ValueDesc intersect(ExpressionRewriteContext context, RangeValue range, DiscreteValue discrete) {
-            DiscreteValue result = new DiscreteValue(context, discrete.reference, discrete.exprForNonNull);
+            DiscreteValue result = new DiscreteValue(context, discrete.reference, discrete.toExpr);
             discrete.values.stream().filter(x -> range.range.contains(x)).forEach(result.values::add);
             if (!result.values.isEmpty()) {
                 return result;
             }
-            Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.and(range.exprForNonNull, discrete.exprForNonNull), context);
-            return new EmptyValue(context, range.reference, originExprForNonNull);
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.and(range.toExpr, discrete.toExpr), context);
+            return new EmptyValue(context, range.reference, originExpr);
         }
 
-        public abstract Expression toExpressionForNonNull();
+        public abstract Expression toExpression();
 
         public static ValueDesc range(ExpressionRewriteContext context, ComparisonPredicate predicate) {
             Literal value = (Literal) predicate.right();
@@ -271,8 +271,8 @@ public static ValueDesc discrete(ExpressionRewriteContext context, InPredicate i
 
     private static class EmptyValue extends ValueDesc {
 
-        public EmptyValue(ExpressionRewriteContext context, Expression reference, Expression exprForNonNull) {
-            super(context, reference, exprForNonNull);
+        public EmptyValue(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
+            super(context, reference, toExpr);
         }
 
         @Override
@@ -286,7 +286,7 @@ public ValueDesc intersect(ValueDesc other) {
         }
 
         @Override
-        public Expression toExpressionForNonNull() {
+        public Expression toExpression() {
             if (reference.nullable()) {
                 return new And(new IsNull(reference), new NullLiteral(BooleanType.INSTANCE));
             } else {
@@ -303,8 +303,8 @@ public Expression toExpressionForNonNull() {
     private static class RangeValue extends ValueDesc {
         Range<Literal> range;
 
-        public RangeValue(ExpressionRewriteContext context, Expression reference, Expression exprForNonNull) {
-            super(context, reference, exprForNonNull);
+        public RangeValue(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
+            super(context, reference, toExpr);
         }
 
         @Override
@@ -312,26 +312,25 @@ public ValueDesc union(ValueDesc other) {
             if (other instanceof EmptyValue) {
                 return other.union(this);
             }
-            try {
-                if (other instanceof RangeValue) {
-                    Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                            ExpressionUtils.or(exprForNonNull, other.exprForNonNull), context);
-                    RangeValue o = (RangeValue) other;
-                    if (range.isConnected(o.range)) {
-                        RangeValue rangeValue = new RangeValue(context, reference, originExprForNonNull);
-                        rangeValue.range = range.span(o.range);
-                        return rangeValue;
-                    }
-                    return new UnknownValue(context, ImmutableList.of(this, other),
-                            originExprForNonNull, ExpressionUtils::or);
+            if (other instanceof RangeValue) {
+                Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                        ExpressionUtils.or(toExpr, other.toExpr), context);
+                RangeValue o = (RangeValue) other;
+                if (range.isConnected(o.range)) {
+                    RangeValue rangeValue = new RangeValue(context, reference, originExpr);
+                    rangeValue.range = range.span(o.range);
+                    return rangeValue;
                 }
+                return new UnknownValue(context, originExpr,
+                        ImmutableList.of(this, other), ExpressionUtils::or);
+            }
+            if (other instanceof DiscreteValue) {
                 return union(context, this, (DiscreteValue) other, false);
-            } catch (Exception e) {
-                Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.or(exprForNonNull, other.exprForNonNull), context);
-                return new UnknownValue(context, ImmutableList.of(this, other),
-                        originExprForNonNull, ExpressionUtils::or);
             }
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.or(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), ExpressionUtils::or);
         }
 
         @Override
@@ -339,29 +338,28 @@ public ValueDesc intersect(ValueDesc other) {
             if (other instanceof EmptyValue) {
                 return other.intersect(this);
             }
-            try {
-                if (other instanceof RangeValue) {
-                    Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                            ExpressionUtils.and(exprForNonNull, other.exprForNonNull), context);
-                    RangeValue o = (RangeValue) other;
-                    if (range.isConnected(o.range)) {
-                        RangeValue rangeValue = new RangeValue(context, reference, originExprForNonNull);
-                        rangeValue.range = range.intersection(o.range);
-                        return rangeValue;
-                    }
-                    return new EmptyValue(context, reference, originExprForNonNull);
+            if (other instanceof RangeValue) {
+                Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                        ExpressionUtils.and(toExpr, other.toExpr), context);
+                RangeValue o = (RangeValue) other;
+                if (range.isConnected(o.range)) {
+                    RangeValue rangeValue = new RangeValue(context, reference, originExpr);
+                    rangeValue.range = range.intersection(o.range);
+                    return rangeValue;
                 }
+                return new EmptyValue(context, reference, originExpr);
+            }
+            if (other instanceof DiscreteValue) {
                 return intersect(context, this, (DiscreteValue) other);
-            } catch (Exception e) {
-                Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.and(exprForNonNull, other.exprForNonNull), context);
-                return new UnknownValue(context, ImmutableList.of(this, other),
-                        originExprForNonNull, ExpressionUtils::and);
             }
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.and(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), ExpressionUtils::and);
         }
 
         @Override
-        public Expression toExpressionForNonNull() {
+        public Expression toExpression() {
             List<Expression> result = Lists.newArrayList();
             if (range.hasLowerBound()) {
                 if (range.lowerBoundType() == BoundType.CLOSED) {
@@ -403,13 +401,13 @@ private static class DiscreteValue extends ValueDesc {
         Set<Literal> values;
 
         public DiscreteValue(ExpressionRewriteContext context,
-                Expression reference, Expression exprForNonNull, Literal... values) {
-            this(context, reference, exprForNonNull, Arrays.asList(values));
+                Expression reference, Expression toExpr, Literal... values) {
+            this(context, reference, toExpr, Arrays.asList(values));
         }
 
         public DiscreteValue(ExpressionRewriteContext context,
-                Expression reference, Expression exprForNonNull, Collection<Literal> values) {
-            super(context, reference, exprForNonNull);
+                Expression reference, Expression toExpr, Collection<Literal> values) {
+            super(context, reference, toExpr);
             this.values = Sets.newTreeSet(values);
         }
 
@@ -418,22 +416,21 @@ public ValueDesc union(ValueDesc other) {
             if (other instanceof EmptyValue) {
                 return other.union(this);
             }
-            try {
-                if (other instanceof DiscreteValue) {
-                    Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                            ExpressionUtils.or(exprForNonNull, other.exprForNonNull), context);
-                    DiscreteValue discreteValue = new DiscreteValue(context, reference, originExprForNonNull);
-                    discreteValue.values.addAll(((DiscreteValue) other).values);
-                    discreteValue.values.addAll(this.values);
-                    return discreteValue;
-                }
+            if (other instanceof DiscreteValue) {
+                Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                        ExpressionUtils.or(toExpr, other.toExpr), context);
+                DiscreteValue discreteValue = new DiscreteValue(context, reference, originExpr);
+                discreteValue.values.addAll(((DiscreteValue) other).values);
+                discreteValue.values.addAll(this.values);
+                return discreteValue;
+            }
+            if (other instanceof RangeValue) {
                 return union(context, (RangeValue) other, this, true);
-            } catch (Exception e) {
-                Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.or(exprForNonNull, other.exprForNonNull), context);
-                return new UnknownValue(context, ImmutableList.of(this, other),
-                        originExprForNonNull, ExpressionUtils::or);
             }
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.or(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), ExpressionUtils::or);
         }
 
         @Override
@@ -441,30 +438,29 @@ public ValueDesc intersect(ValueDesc other) {
             if (other instanceof EmptyValue) {
                 return other.intersect(this);
             }
-            try {
-                if (other instanceof DiscreteValue) {
-                    Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                            ExpressionUtils.and(exprForNonNull, other.exprForNonNull), context);
-                    DiscreteValue discreteValue = new DiscreteValue(context, reference, originExprForNonNull);
-                    discreteValue.values.addAll(((DiscreteValue) other).values);
-                    discreteValue.values.retainAll(this.values);
-                    if (discreteValue.values.isEmpty()) {
-                        return new EmptyValue(context, reference, originExprForNonNull);
-                    } else {
-                        return discreteValue;
-                    }
+            if (other instanceof DiscreteValue) {
+                Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                        ExpressionUtils.and(toExpr, other.toExpr), context);
+                DiscreteValue discreteValue = new DiscreteValue(context, reference, originExpr);
+                discreteValue.values.addAll(((DiscreteValue) other).values);
+                discreteValue.values.retainAll(this.values);
+                if (discreteValue.values.isEmpty()) {
+                    return new EmptyValue(context, reference, originExpr);
+                } else {
+                    return discreteValue;
                 }
+            }
+            if (other instanceof RangeValue) {
                 return intersect(context, (RangeValue) other, this);
-            } catch (Exception e) {
-                Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.and(exprForNonNull, other.exprForNonNull), context);
-                return new UnknownValue(context, ImmutableList.of(this, other),
-                        originExprForNonNull, ExpressionUtils::and);
             }
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.and(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), ExpressionUtils::and);
         }
 
         @Override
-        public Expression toExpressionForNonNull() {
+        public Expression toExpression() {
             // NOTICE: it's related with `InPredicateToEqualToRule`
             // They are same processes, so must change synchronously.
             if (values.size() == 1) {
@@ -498,41 +494,68 @@ private UnknownValue(ExpressionRewriteContext context, Expression expr) {
             mergeExprOp = null;
         }
 
-        public UnknownValue(ExpressionRewriteContext context,
-                List<ValueDesc> sourceValues, Expression exprForNonNull, BinaryOperator<Expression> mergeExprOp) {
-            super(context, sourceValues.get(0).reference, exprForNonNull);
+        public UnknownValue(ExpressionRewriteContext context, Expression toExpr,
+                List<ValueDesc> sourceValues, BinaryOperator<Expression> mergeExprOp) {
+            super(context, getReference(sourceValues, toExpr), toExpr);
             this.sourceValues = ImmutableList.copyOf(sourceValues);
             this.mergeExprOp = mergeExprOp;
         }
 
+        // reference is used to simplify multiple ValueDescs.
+        // when ValueDesc A op ValueDesc B, only A and B's references equals,
+        // can reduce them, like A op B = A.
+        // If A and B's reference not equal, A op B will always get UnknownValue(A op B).
+        //
+        // for example:
+        // 1. RangeValue(a < 10, reference=a) union RangeValue(a > 20, reference=a)
+        //    = UnknownValue1(a < 10 or a > 20, reference=a)
+        // 2. RangeValue(a < 10, reference=a) union RangeValue(b > 20, reference=b)
+        //    = UnknownValue2(a < 10 or b > 20, reference=(a < 10 or b > 20))
+        // then given EmptyValue(, reference=a) E,
+        // 1. since E and UnknownValue1's reference equals, then
+        //    E union UnknownValue1 = E.union(UnknownValue1) = UnknownValue1,
+        // 2. since E and UnknownValue2's reference not equals, then
+        //    E union UnknownValue2 = UnknownValue3(E union UnknownValue2, reference=E union UnknownValue2)
+        private static Expression getReference(List<ValueDesc> sourceValues, Expression toExpr) {
+            Expression reference = sourceValues.get(0).reference;
+            for (int i = 1; i < sourceValues.size(); i++) {
+                if (!reference.equals(sourceValues.get(i).reference)) {
+                    return toExpr;
+                }
+            }
+            return reference;
+        }
+
         @Override
         public ValueDesc union(ValueDesc other) {
-            Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.or(exprForNonNull, other.exprForNonNull), context);
-            return new UnknownValue(context, ImmutableList.of(this, other), originExprForNonNull, ExpressionUtils::or);
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.or(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), ExpressionUtils::or);
         }
 
         @Override
         public ValueDesc intersect(ValueDesc other) {
-            Expression originExprForNonNull = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.and(exprForNonNull, other.exprForNonNull), context);
-            return new UnknownValue(context, ImmutableList.of(this, other), originExprForNonNull, ExpressionUtils::and);
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.and(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), ExpressionUtils::and);
         }
 
         @Override
-        public Expression toExpressionForNonNull() {
+        public Expression toExpression() {
             if (sourceValues.isEmpty()) {
-                return exprForNonNull;
+                return toExpr;
             }
-            Expression result = sourceValues.get(0).toExpressionForNonNull();
+            Expression result = sourceValues.get(0).toExpression();
             for (int i = 1; i < sourceValues.size(); i++) {
-                result = mergeExprOp.apply(result, sourceValues.get(i).toExpressionForNonNull());
+                result = mergeExprOp.apply(result, sourceValues.get(i).toExpression());
             }
             result = FoldConstantRuleOnFE.evaluate(result, context);
             // ATTN: we must return original expr, because OrToIn is implemented with MutableState,
             //   newExpr will lose these states leading to dead loop by OrToIn -> SimplifyRange -> FoldConstantByFE
-            if (result.equals(exprForNonNull)) {
-                return exprForNonNull;
+            if (result.equals(toExpr)) {
+                return toExpr;
             }
             return result;
         }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
index 329efee77a8f57..d6a7b9e5585d14 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
@@ -101,6 +101,7 @@ public void testSimplify() {
         assertRewrite("TA > 5 + 1 and TA > 10", "cast(TA as smallint) > 6 and TA > 10");
         assertRewrite("(TA > 1 and TA > 10) or TA > 20", "TA > 10");
         assertRewrite("(TA > 1 or TA > 10) and TA > 20", "TA > 20");
+        assertRewrite("(TA < 1 and TA > 10) or TA = 20 and TB > 10", "(TA is null and null) or TA = 20 and TB > 10");
         assertRewrite("(TA + TB > 1 or TA + TB > 10) and TA + TB > 20", "TA + TB > 20");
         assertRewrite("TA > 10 or TA > 10", "TA > 10");
         assertRewrite("(TA > 10 or TA > 20) and (TB > 10 and TB < 20)", "TA > 10 and (TB > 10 and TB < 20) ");
@@ -131,7 +132,10 @@ public void testSimplify() {
         assertRewrite("TA in (1) and TA in (3)", "TA is null and null");
         assertRewrite("TA in (1) and TA in (1)", "TA = 1");
         assertRewriteNotNull("(TA > 3 and TA < 1) and TB < 5", "FALSE");
+        assertRewrite("(TA > 3 and TA < 1) and (TA > 5 and TA = 4)", "TA is null and null");
+        assertRewrite("(TA > 3 and TA < 1) or (TA > 5 and TA = 4)", "TA is null and null");
         assertRewrite("(TA > 3 and TA < 1) and TB < 5", "TA is null and null and TB < 5");
+        assertRewrite("(TA > 3 and TA < 1) and (TB < 5 and TB = 6)", "TA is null and null and TB is null");
         assertRewrite("TA > 3 and TB < 5 and TA < 1", "TA is null and null and TB < 5");
         assertRewrite("(TA > 3 and TA < 1) or TB < 5", "(TA is null and null) or TB < 5");
         assertRewrite("((IA = 1 AND SC ='1') OR SC = '1212') AND IA =1", "((IA = 1 AND SC ='1') OR SC = '1212') AND IA =1");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
index aaca3edf101949..5ce7871db144bc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
@@ -193,7 +193,7 @@ void test15() {
         String expr = "x=1 or (a=1 and b=2) or (a=2 and c=3)";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("((x = 1) OR (((a = 1) AND (b = 2)) OR ((a = 2) AND (c = 3))))",
+        Assertions.assertEquals("(((x = 1) OR ((a = 1) AND (b = 2))) OR ((a = 2) AND (c = 3)))",
                 rewritten.toSql());
     }
 
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query41.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query41.out
index c870e728a39819..ea5acccd883550 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query41.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke') AND i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound')) AND ((((((item.i_category = 'Women') AND i_color IN ('forest', 'lime')) AND i_units IN ('Pallet', 'Pound')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('navy', 'slate')) AND i_units IN ('Bunch', 'Gross')) AND i_size IN ('extra large', 'petite'))) OR (((((item.i_category = 'Men') AND i_color IN ('powder', 'sky')) AND i_units IN ('Dozen', 'Lb')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('maroon', 'smoke')) AND i_units IN ('Case', 'Ounce')) AND i_size IN ('economy', 'small'))))) OR ((i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna') AND i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'dark')) AND i_units IN ('Tbl', 'Ton')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('frosted', 'plum')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'petite'))) OR (((((item.i_category = 'Men') AND i_color IN ('papaya', 'peach')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('firebrick', 'sienna')) AND i_units IN ('Cup', 'Each')) AND i_size IN ('economy', 'small')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
+------------------------------filter((((i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke') AND i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound')) AND (((((((item.i_category = 'Women') AND i_color IN ('forest', 'lime')) AND i_units IN ('Pallet', 'Pound')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('navy', 'slate')) AND i_units IN ('Bunch', 'Gross')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('powder', 'sky')) AND i_units IN ('Dozen', 'Lb')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('maroon', 'smoke')) AND i_units IN ('Case', 'Ounce')) AND i_size IN ('economy', 'small')))) OR ((i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna') AND i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'dark')) AND i_units IN ('Tbl', 'Ton')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('frosted', 'plum')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('papaya', 'peach')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('firebrick', 'sienna')) AND i_units IN ('Cup', 'Each')) AND i_size IN ('economy', 'small'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query47.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query47.out
index 2bc34ca1076a83..5554797a7b9dc3 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query47.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2000) OR ((date_dim.d_year = 1999) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2001) AND (date_dim.d_moy = 1))) and d_year IN (1999, 2000, 2001) and d_year IN (1999, 2000, 2001))
+--------------------------------------filter((((date_dim.d_year = 2000) OR ((date_dim.d_year = 1999) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2001) AND (date_dim.d_moy = 1))) and d_year IN (1999, 2000, 2001))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out
index 01644900894142..49e27f92b64a61 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 --------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 ----------------------------------------PhysicalOlapScan[time_dim]
 --------------------------------PhysicalProject
-----------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+----------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((store.s_store_name = 'ese'))
@@ -45,7 +45,7 @@ PhysicalResultSink
 --------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 ----------------------------------------PhysicalOlapScan[time_dim]
 --------------------------------PhysicalProject
-----------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+----------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((store.s_store_name = 'ese'))
@@ -66,7 +66,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -87,7 +87,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -108,7 +108,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+----------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -129,7 +129,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -150,7 +150,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -171,7 +171,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+----------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query41.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query41.out
index c870e728a39819..ea5acccd883550 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke') AND i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound')) AND ((((((item.i_category = 'Women') AND i_color IN ('forest', 'lime')) AND i_units IN ('Pallet', 'Pound')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('navy', 'slate')) AND i_units IN ('Bunch', 'Gross')) AND i_size IN ('extra large', 'petite'))) OR (((((item.i_category = 'Men') AND i_color IN ('powder', 'sky')) AND i_units IN ('Dozen', 'Lb')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('maroon', 'smoke')) AND i_units IN ('Case', 'Ounce')) AND i_size IN ('economy', 'small'))))) OR ((i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna') AND i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'dark')) AND i_units IN ('Tbl', 'Ton')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('frosted', 'plum')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'petite'))) OR (((((item.i_category = 'Men') AND i_color IN ('papaya', 'peach')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('firebrick', 'sienna')) AND i_units IN ('Cup', 'Each')) AND i_size IN ('economy', 'small')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
+------------------------------filter((((i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke') AND i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound')) AND (((((((item.i_category = 'Women') AND i_color IN ('forest', 'lime')) AND i_units IN ('Pallet', 'Pound')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('navy', 'slate')) AND i_units IN ('Bunch', 'Gross')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('powder', 'sky')) AND i_units IN ('Dozen', 'Lb')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('maroon', 'smoke')) AND i_units IN ('Case', 'Ounce')) AND i_size IN ('economy', 'small')))) OR ((i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna') AND i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'dark')) AND i_units IN ('Tbl', 'Ton')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('frosted', 'plum')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('papaya', 'peach')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('firebrick', 'sienna')) AND i_units IN ('Cup', 'Each')) AND i_size IN ('economy', 'small'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out
index caec3b20581ed5..d51d48c5ab5677 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2000) OR ((date_dim.d_year = 1999) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2001) AND (date_dim.d_moy = 1))) and d_year IN (1999, 2000, 2001) and d_year IN (1999, 2000, 2001))
+--------------------------------------filter((((date_dim.d_year = 2000) OR ((date_dim.d_year = 1999) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2001) AND (date_dim.d_moy = 1))) and d_year IN (1999, 2000, 2001))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out
index dc3f1ece14f64d..3c8f0335f1f415 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+----------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+----------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query41.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query41.out
index ff811ef082cc23..c9c1ffff499741 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium'))))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND ((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out
index a05f8e84e68830..842b18a4eac9e6 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query41.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query41.out
index ff811ef082cc23..c9c1ffff499741 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium'))))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND ((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out
index 9b61dc54e2a91e..e640ec24d7b3e3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query41.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query41.out
index ff811ef082cc23..c9c1ffff499741 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium'))))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND ((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out
index 053e8bfc4c7f54..4a7de98eb56b50 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+--------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query41.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query41.out
index ff811ef082cc23..c9c1ffff499741 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium'))))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND ((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out
index 7bbcf019701d4d..a0157318139d88 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+--------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query41.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query41.out
index a470709da6831c..16a3a620b77c8e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('black', 'chocolate', 'cornflower', 'firebrick', 'frosted', 'magenta', 'rose', 'slate') AND i_units IN ('Box', 'Bundle', 'Carton', 'Dram', 'Gross', 'Lb', 'Oz', 'Pound')) AND ((((((item.i_category = 'Women') AND i_color IN ('frosted', 'rose')) AND i_units IN ('Gross', 'Lb')) AND i_size IN ('large', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('black', 'chocolate')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('economy', 'petite'))) OR (((((item.i_category = 'Men') AND i_color IN ('magenta', 'slate')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'small')) OR ((((item.i_category = 'Men') AND i_color IN ('cornflower', 'firebrick')) AND i_units IN ('Oz', 'Pound')) AND i_size IN ('large', 'medium'))))) OR ((i_color IN ('almond', 'aquamarine', 'cyan', 'lavender', 'maroon', 'papaya', 'purple', 'steel') AND i_units IN ('Bunch', 'Case', 'Cup', 'Each', 'Gram', 'N/A', 'Pallet', 'Tsp')) AND ((((((item.i_category = 'Women') AND i_color IN ('almond', 'steel')) AND i_units IN ('Case', 'Tsp')) AND i_size IN ('large', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'purple')) AND i_units IN ('Bunch', 'Gram')) AND i_size IN ('economy', 'petite'))) OR (((((item.i_category = 'Men') AND i_color IN ('lavender', 'papaya')) AND i_units IN ('Cup', 'Pallet')) AND i_size IN ('N/A', 'small')) OR ((((item.i_category = 'Men') AND i_color IN ('cyan', 'maroon')) AND i_units IN ('Each', 'N/A')) AND i_size IN ('large', 'medium')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'large', 'medium', 'petite', 'small'))
+------------------------------filter((((i_color IN ('black', 'chocolate', 'cornflower', 'firebrick', 'frosted', 'magenta', 'rose', 'slate') AND i_units IN ('Box', 'Bundle', 'Carton', 'Dram', 'Gross', 'Lb', 'Oz', 'Pound')) AND (((((((item.i_category = 'Women') AND i_color IN ('frosted', 'rose')) AND i_units IN ('Gross', 'Lb')) AND i_size IN ('large', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('black', 'chocolate')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('economy', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('magenta', 'slate')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('cornflower', 'firebrick')) AND i_units IN ('Oz', 'Pound')) AND i_size IN ('large', 'medium')))) OR ((i_color IN ('almond', 'aquamarine', 'cyan', 'lavender', 'maroon', 'papaya', 'purple', 'steel') AND i_units IN ('Bunch', 'Case', 'Cup', 'Each', 'Gram', 'N/A', 'Pallet', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('almond', 'steel')) AND i_units IN ('Case', 'Tsp')) AND i_size IN ('large', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'purple')) AND i_units IN ('Bunch', 'Gram')) AND i_size IN ('economy', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('lavender', 'papaya')) AND i_units IN ('Cup', 'Pallet')) AND i_size IN ('N/A', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('cyan', 'maroon')) AND i_units IN ('Each', 'N/A')) AND i_size IN ('large', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'large', 'medium', 'petite', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query47.out
index 591df70c5abce8..a3205932929569 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query47.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query47.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query57.out
index 9b61dc54e2a91e..e640ec24d7b3e3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.out
index ff811ef082cc23..c9c1ffff499741 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium'))))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND ((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.out
index a05f8e84e68830..842b18a4eac9e6 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.out
index ff811ef082cc23..c9c1ffff499741 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium'))))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND ((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.out
index 9b61dc54e2a91e..e640ec24d7b3e3 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query41.out
index ff811ef082cc23..c9c1ffff499741 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium'))))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND ((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query57.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query57.out
index 053e8bfc4c7f54..4a7de98eb56b50 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query57.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+--------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query41.out
index ff811ef082cc23..c9c1ffff499741 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium'))))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND ((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR (((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query57.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query57.out
index 7bbcf019701d4d..a0157318139d88 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query57.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000) and d_year IN (1998, 1999, 2000))
+--------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query41.out
index c870e728a39819..ea5acccd883550 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke') AND i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound')) AND ((((((item.i_category = 'Women') AND i_color IN ('forest', 'lime')) AND i_units IN ('Pallet', 'Pound')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('navy', 'slate')) AND i_units IN ('Bunch', 'Gross')) AND i_size IN ('extra large', 'petite'))) OR (((((item.i_category = 'Men') AND i_color IN ('powder', 'sky')) AND i_units IN ('Dozen', 'Lb')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('maroon', 'smoke')) AND i_units IN ('Case', 'Ounce')) AND i_size IN ('economy', 'small'))))) OR ((i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna') AND i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton')) AND ((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'dark')) AND i_units IN ('Tbl', 'Ton')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('frosted', 'plum')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'petite'))) OR (((((item.i_category = 'Men') AND i_color IN ('papaya', 'peach')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'large')) OR ((((item.i_category = 'Men') AND i_color IN ('firebrick', 'sienna')) AND i_units IN ('Cup', 'Each')) AND i_size IN ('economy', 'small')))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
+------------------------------filter((((i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke') AND i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound')) AND (((((((item.i_category = 'Women') AND i_color IN ('forest', 'lime')) AND i_units IN ('Pallet', 'Pound')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('navy', 'slate')) AND i_units IN ('Bunch', 'Gross')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('powder', 'sky')) AND i_units IN ('Dozen', 'Lb')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('maroon', 'smoke')) AND i_units IN ('Case', 'Ounce')) AND i_size IN ('economy', 'small')))) OR ((i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna') AND i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'dark')) AND i_units IN ('Tbl', 'Ton')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('frosted', 'plum')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('papaya', 'peach')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('firebrick', 'sienna')) AND i_units IN ('Cup', 'Each')) AND i_size IN ('economy', 'small'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query47.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query47.out
index caec3b20581ed5..d51d48c5ab5677 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query47.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2000) OR ((date_dim.d_year = 1999) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2001) AND (date_dim.d_moy = 1))) and d_year IN (1999, 2000, 2001) and d_year IN (1999, 2000, 2001))
+--------------------------------------filter((((date_dim.d_year = 2000) OR ((date_dim.d_year = 1999) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2001) AND (date_dim.d_moy = 1))) and d_year IN (1999, 2000, 2001))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out
index dc3f1ece14f64d..3c8f0335f1f415 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+----------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+----------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3) and hd_dep_count IN (-1, 0, 3))
+--------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/suites/nereids_rules_p0/expression/test_simplify_range.groovy b/regression-test/suites/nereids_rules_p0/expression/test_simplify_range.groovy
new file mode 100644
index 00000000000000..c6d12f3e6bc0be
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/expression/test_simplify_range.groovy
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('test_simplify_range') {
+    def tbl_1 = 'test_simplify_range_tbl_1'
+    sql "DROP TABLE IF EXISTS  ${tbl_1} FORCE"
+    sql "CREATE TABLE ${tbl_1}(a DECIMAL(16,8), b INT) PROPERTIES ('replication_num' = '1')"
+    sql "INSERT INTO ${tbl_1} VALUES(null, 10)"
+    test {
+        sql "SELECT a BETWEEN 100.02 and 40.123 OR a IN (54.0402) AND b < 10 FROM ${tbl_1}"
+        result([[null]])
+    }
+    sql "DROP TABLE IF EXISTS  ${tbl_1} FORCE"
+}

From 5641c2dcf7e08bda64b5118fca7a1905af89f56b Mon Sep 17 00:00:00 2001
From: Tiewei Fang <fangtiewei@selectdb.com>
Date: Thu, 28 Nov 2024 23:27:10 +0800
Subject: [PATCH 069/399] [fix](Outfile) Fix the data type mapping for complex
 types in Doris to the ORC and Parquet file formats. (#44041)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What problem does this PR solve?

Problem Summary:

As before, the behavior of exporting of complex data types in Doris is
as follows:
  | orc type | parquet type | csv
-- | -- | -- | --
bitmap | string | Not Supported | Not Supported
quantile_state | Not Supported | Not Supported | Not Supported
hll | string | string | invisible string
jsonb | Not Supported | string | string
variant | Not Supported | string | string

What's more, there are some issues when exporting complex data types to
the ORC file format.

This PR does two things:
1. Fix the problem with exporting complex data types from Doris.
2. Support exporting these three complex types to both the ORC and the
Parquet file format.


  | orc type | parquet type | csv
-- | -- | -- | --
bitmap | binary | binary | "NULL"
quantile_state | binary | binary | "NULL"
hll | binary | binary | "NULL"
jsonb | string | string | string
variant | string | string | string

### Release note

[fix](Outfile) Fix the data type mapping for complex types in Doris to the ORC and Parquet file formats.
---
 be/src/util/arrow/row_batch.cpp               |   9 +-
 .../serde/data_type_bitmap_serde.cpp          |  59 +++++++++-
 .../data_types/serde/data_type_bitmap_serde.h |  13 +--
 .../serde/data_type_date64_serde.cpp          |  20 +---
 .../data_types/serde/data_type_hll_serde.cpp  |  48 ++++----
 .../data_types/serde/data_type_ipv6_serde.cpp |  37 ++----
 .../serde/data_type_jsonb_serde.cpp           |  25 ++++-
 .../serde/data_type_number_serde.cpp          |  36 ++----
 .../serde/data_type_object_serde.cpp          |  33 ++++++
 .../data_types/serde/data_type_object_serde.h |   4 +-
 .../serde/data_type_quantilestate_serde.h     |  56 ++++++++-
 be/src/vec/data_types/serde/data_type_serde.h |  12 ++
 .../apache/doris/analysis/OutFileClause.java  |  21 ++--
 .../outfile/test_outfile_complex_type.out     |  25 +++++
 .../test_outfile_jsonb_and_variant.out        |  25 +++++
 .../outfile/test_outfile_complex_type.groovy  | 106 ++++++++++++++++++
 .../test_outfile_jsonb_and_variant.groovy     | 104 +++++++++++++++++
 17 files changed, 501 insertions(+), 132 deletions(-)
 create mode 100644 regression-test/data/export_p0/outfile/test_outfile_complex_type.out
 create mode 100644 regression-test/data/export_p0/outfile/test_outfile_jsonb_and_variant.out
 create mode 100644 regression-test/suites/export_p0/outfile/test_outfile_complex_type.groovy
 create mode 100644 regression-test/suites/export_p0/outfile/test_outfile_jsonb_and_variant.groovy

diff --git a/be/src/util/arrow/row_batch.cpp b/be/src/util/arrow/row_batch.cpp
index dd11d5ae46f740..a0cd77aee41931 100644
--- a/be/src/util/arrow/row_batch.cpp
+++ b/be/src/util/arrow/row_batch.cpp
@@ -24,6 +24,7 @@
 #include <arrow/result.h>
 #include <arrow/status.h>
 #include <arrow/type.h>
+#include <arrow/type_fwd.h>
 #include <glog/logging.h>
 #include <stdint.h>
 
@@ -84,12 +85,10 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
     case TYPE_LARGEINT:
     case TYPE_VARCHAR:
     case TYPE_CHAR:
-    case TYPE_HLL:
     case TYPE_DATE:
     case TYPE_DATETIME:
     case TYPE_STRING:
     case TYPE_JSONB:
-    case TYPE_OBJECT:
         *result = arrow::utf8();
         break;
     case TYPE_DATEV2:
@@ -150,6 +149,12 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
         *result = arrow::utf8();
         break;
     }
+    case TYPE_QUANTILE_STATE:
+    case TYPE_OBJECT:
+    case TYPE_HLL: {
+        *result = arrow::binary();
+        break;
+    }
     default:
         return Status::InvalidArgument("Unknown primitive type({}) convert to Arrow type",
                                        type.type);
diff --git a/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp b/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp
index a4a2367aded65e..5d024a418340e8 100644
--- a/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp
@@ -17,6 +17,7 @@
 
 #include "data_type_bitmap_serde.h"
 
+#include <arrow/array/builder_binary.h>
 #include <gen_cpp/types.pb.h>
 
 #include <string>
@@ -27,6 +28,7 @@
 #include "vec/columns/column_const.h"
 #include "vec/common/arena.h"
 #include "vec/common/assert_cast.h"
+#include "vec/data_types/serde/data_type_nullable_serde.h"
 
 namespace doris {
 
@@ -34,6 +36,29 @@ namespace vectorized {
 class IColumn;
 #include "common/compile_check_begin.h"
 
+Status DataTypeBitMapSerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx,
+                                                     int64_t end_idx, BufferWritable& bw,
+                                                     FormatOptions& options) const {
+    SERIALIZE_COLUMN_TO_JSON();
+}
+
+Status DataTypeBitMapSerDe::serialize_one_cell_to_json(const IColumn& column, int64_t row_num,
+                                                       BufferWritable& bw,
+                                                       FormatOptions& options) const {
+    /**
+    * For null values in ordinary types, we use \N to represent them;
+    * for null values in nested types, we use null to represent them, just like the json format.
+    */
+    if (_nesting_level >= 2) {
+        bw.write(DataTypeNullableSerDe::NULL_IN_COMPLEX_TYPE.c_str(),
+                 strlen(NULL_IN_COMPLEX_TYPE.c_str()));
+    } else {
+        bw.write(DataTypeNullableSerDe::NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str(),
+                 strlen(NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str()));
+    }
+    return Status::OK();
+}
+
 Status DataTypeBitMapSerDe::deserialize_column_from_json_vector(
         IColumn& column, std::vector<Slice>& slices, int* num_deserialized,
         const FormatOptions& options) const {
@@ -96,6 +121,26 @@ void DataTypeBitMapSerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWr
     result.writeEndBinary();
 }
 
+void DataTypeBitMapSerDe::write_column_to_arrow(const IColumn& column, const NullMap* null_map,
+                                                arrow::ArrayBuilder* array_builder, int64_t start,
+                                                int64_t end, const cctz::time_zone& ctz) const {
+    const auto& col = assert_cast<const ColumnBitmap&>(column);
+    auto& builder = assert_cast<arrow::BinaryBuilder&>(*array_builder);
+    for (size_t string_i = start; string_i < end; ++string_i) {
+        if (null_map && (*null_map)[string_i]) {
+            checkArrowStatus(builder.AppendNull(), column.get_name(),
+                             array_builder->type()->name());
+        } else {
+            auto& bitmap_value = const_cast<BitmapValue&>(col.get_element(string_i));
+            std::string memory_buffer(bitmap_value.getSizeInBytes(), '0');
+            bitmap_value.write_to(memory_buffer.data());
+            checkArrowStatus(
+                    builder.Append(memory_buffer.data(), static_cast<int>(memory_buffer.size())),
+                    column.get_name(), array_builder->type()->name());
+        }
+    }
+}
+
 void DataTypeBitMapSerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const {
     auto& col = reinterpret_cast<ColumnBitmap&>(column);
     auto blob = static_cast<const JsonbBlobVal*>(arg);
@@ -148,11 +193,19 @@ Status DataTypeBitMapSerDe::write_column_to_orc(const std::string& timezone, con
     auto& col_data = assert_cast<const ColumnBitmap&>(column);
     orc::StringVectorBatch* cur_batch = dynamic_cast<orc::StringVectorBatch*>(orc_col_batch);
 
+    INIT_MEMORY_FOR_ORC_WRITER()
+
     for (size_t row_id = start; row_id < end; row_id++) {
         if (cur_batch->notNull[row_id] == 1) {
-            const auto& ele = col_data.get_data_at(row_id);
-            cur_batch->data[row_id] = const_cast<char*>(ele.data);
-            cur_batch->length[row_id] = ele.size;
+            auto bitmap_value = const_cast<BitmapValue&>(col_data.get_element(row_id));
+            size_t len = bitmap_value.getSizeInBytes();
+
+            REALLOC_MEMORY_FOR_ORC_WRITER()
+
+            bitmap_value.write_to(const_cast<char*>(bufferRef.data) + offset);
+            cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + offset;
+            cur_batch->length[row_id] = len;
+            offset += len;
         }
     }
 
diff --git a/be/src/vec/data_types/serde/data_type_bitmap_serde.h b/be/src/vec/data_types/serde/data_type_bitmap_serde.h
index a4be5b8ec204f1..24c2e6f930d203 100644
--- a/be/src/vec/data_types/serde/data_type_bitmap_serde.h
+++ b/be/src/vec/data_types/serde/data_type_bitmap_serde.h
@@ -36,14 +36,10 @@ class DataTypeBitMapSerDe : public DataTypeSerDe {
     DataTypeBitMapSerDe(int nesting_level = 1) : DataTypeSerDe(nesting_level) {};
 
     Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw,
-                                      FormatOptions& options) const override {
-        return Status::NotSupported("serialize_one_cell_to_json with type [{}]", column.get_name());
-    }
+                                      FormatOptions& options) const override;
 
     Status serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx,
-                                    BufferWritable& bw, FormatOptions& options) const override {
-        return Status::NotSupported("serialize_column_to_json with type [{}]", column.get_name());
-    }
+                                    BufferWritable& bw, FormatOptions& options) const override;
 
     Status deserialize_one_cell_from_json(IColumn& column, Slice& slice,
                                           const FormatOptions& options) const override;
@@ -63,10 +59,7 @@ class DataTypeBitMapSerDe : public DataTypeSerDe {
 
     void write_column_to_arrow(const IColumn& column, const NullMap* null_map,
                                arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
-                               const cctz::time_zone& ctz) const override {
-        throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
-                               "write_column_to_arrow with type " + column.get_name());
-    }
+                               const cctz::time_zone& ctz) const override;
 
     void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
                                 int end, const cctz::time_zone& ctz) const override {
diff --git a/be/src/vec/data_types/serde/data_type_date64_serde.cpp b/be/src/vec/data_types/serde/data_type_date64_serde.cpp
index 8e102ec6e3a729..c91db85be5bcda 100644
--- a/be/src/vec/data_types/serde/data_type_date64_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_date64_serde.cpp
@@ -289,16 +289,7 @@ Status DataTypeDate64SerDe::write_column_to_orc(const std::string& timezone, con
     auto& col_data = static_cast<const ColumnVector<Int64>&>(column).get_data();
     orc::StringVectorBatch* cur_batch = dynamic_cast<orc::StringVectorBatch*>(orc_col_batch);
 
-    char* ptr = (char*)malloc(BUFFER_UNIT_SIZE);
-    if (!ptr) {
-        return Status::InternalError(
-                "malloc memory error when write largeint column data to orc file.");
-    }
-    StringRef bufferRef;
-    bufferRef.data = ptr;
-    bufferRef.size = BUFFER_UNIT_SIZE;
-    size_t offset = 0;
-    const size_t begin_off = offset;
+    INIT_MEMORY_FOR_ORC_WRITER()
 
     for (size_t row_id = start; row_id < end; row_id++) {
         if (cur_batch->notNull[row_id] == 0) {
@@ -310,18 +301,11 @@ Status DataTypeDate64SerDe::write_column_to_orc(const std::string& timezone, con
 
         REALLOC_MEMORY_FOR_ORC_WRITER()
 
+        cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + offset;
         cur_batch->length[row_id] = len;
         offset += len;
     }
-    size_t data_off = 0;
-    for (size_t row_id = start; row_id < end; row_id++) {
-        if (cur_batch->notNull[row_id] == 1) {
-            cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + begin_off + data_off;
-            data_off += cur_batch->length[row_id];
-        }
-    }
 
-    buffer_list.emplace_back(bufferRef);
     cur_batch->numElements = end - start;
     return Status::OK();
 }
diff --git a/be/src/vec/data_types/serde/data_type_hll_serde.cpp b/be/src/vec/data_types/serde/data_type_hll_serde.cpp
index c22bb31862e41c..42260b092605e1 100644
--- a/be/src/vec/data_types/serde/data_type_hll_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_hll_serde.cpp
@@ -21,6 +21,7 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include <memory>
 #include <string>
 
 #include "arrow/array/builder_binary.h"
@@ -48,28 +49,17 @@ Status DataTypeHLLSerDe::serialize_column_to_json(const IColumn& column, int64_t
 Status DataTypeHLLSerDe::serialize_one_cell_to_json(const IColumn& column, int64_t row_num,
                                                     BufferWritable& bw,
                                                     FormatOptions& options) const {
-    if (!options._output_object_data) {
-        /**
-         * For null values in ordinary types, we use \N to represent them;
-         * for null values in nested types, we use null to represent them, just like the json format.
-         */
-        if (_nesting_level >= 2) {
-            bw.write(DataTypeNullableSerDe::NULL_IN_COMPLEX_TYPE.c_str(),
-                     strlen(NULL_IN_COMPLEX_TYPE.c_str()));
-        } else {
-            bw.write(DataTypeNullableSerDe::NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str(),
-                     strlen(NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str()));
-        }
-        return Status::OK();
+    /**
+    * For null values in ordinary types, we use \N to represent them;
+    * for null values in nested types, we use null to represent them, just like the json format.
+    */
+    if (_nesting_level >= 2) {
+        bw.write(DataTypeNullableSerDe::NULL_IN_COMPLEX_TYPE.c_str(),
+                 strlen(NULL_IN_COMPLEX_TYPE.c_str()));
+    } else {
+        bw.write(DataTypeNullableSerDe::NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str(),
+                 strlen(NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str()));
     }
-    auto col_row = check_column_const_set_readability(column, row_num);
-    ColumnPtr ptr = col_row.first;
-    row_num = col_row.second;
-    auto& data = const_cast<HyperLogLog&>(assert_cast<const ColumnHLL&>(*ptr).get_element(row_num));
-    std::unique_ptr<char[]> buf =
-            std::make_unique_for_overwrite<char[]>(data.max_serialized_size());
-    size_t size = data.serialize((uint8*)buf.get());
-    bw.write(buf.get(), size);
     return Status::OK();
 }
 
@@ -139,7 +129,7 @@ void DataTypeHLLSerDe::write_column_to_arrow(const IColumn& column, const NullMa
                                              arrow::ArrayBuilder* array_builder, int64_t start,
                                              int64_t end, const cctz::time_zone& ctz) const {
     const auto& col = assert_cast<const ColumnHLL&>(column);
-    auto& builder = assert_cast<arrow::StringBuilder&>(*array_builder);
+    auto& builder = assert_cast<arrow::BinaryBuilder&>(*array_builder);
     for (size_t string_i = start; string_i < end; ++string_i) {
         if (null_map && (*null_map)[string_i]) {
             checkArrowStatus(builder.AppendNull(), column.get_name(),
@@ -198,11 +188,19 @@ Status DataTypeHLLSerDe::write_column_to_orc(const std::string& timezone, const
     auto& col_data = assert_cast<const ColumnHLL&>(column);
     orc::StringVectorBatch* cur_batch = dynamic_cast<orc::StringVectorBatch*>(orc_col_batch);
 
+    INIT_MEMORY_FOR_ORC_WRITER()
+
     for (size_t row_id = start; row_id < end; row_id++) {
         if (cur_batch->notNull[row_id] == 1) {
-            const auto& ele = col_data.get_data_at(row_id);
-            cur_batch->data[row_id] = const_cast<char*>(ele.data);
-            cur_batch->length[row_id] = ele.size;
+            auto hll_value = const_cast<HyperLogLog&>(col_data.get_element(row_id));
+            size_t len = hll_value.max_serialized_size();
+
+            REALLOC_MEMORY_FOR_ORC_WRITER()
+
+            hll_value.serialize((uint8_t*)(bufferRef.data) + offset);
+            cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + offset;
+            cur_batch->length[row_id] = len;
+            offset += len;
         }
     }
 
diff --git a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp
index ac4dbc030432b8..e899de93c90ce0 100644
--- a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp
@@ -187,38 +187,23 @@ Status DataTypeIPv6SerDe::write_column_to_orc(const std::string& timezone, const
                                               std::vector<StringRef>& buffer_list) const {
     const auto& col_data = assert_cast<const ColumnIPv6&>(column).get_data();
     orc::StringVectorBatch* cur_batch = assert_cast<orc::StringVectorBatch*>(orc_col_batch);
-    char* ptr = (char*)malloc(BUFFER_UNIT_SIZE);
-    if (!ptr) {
-        return Status::InternalError(
-                "malloc memory error when write largeint column data to orc file.");
-    }
-    StringRef bufferRef;
-    bufferRef.data = ptr;
-    bufferRef.size = BUFFER_UNIT_SIZE;
-    size_t offset = 0;
-    const size_t begin_off = offset;
-
-    for (size_t row_id = start; row_id < end; row_id++) {
-        if (cur_batch->notNull[row_id] == 0) {
-            continue;
-        }
-        std::string ipv6_str = IPv6Value::to_string(col_data[row_id]);
-        size_t len = ipv6_str.size();
 
-        REALLOC_MEMORY_FOR_ORC_WRITER()
+    INIT_MEMORY_FOR_ORC_WRITER()
 
-        strcpy(const_cast<char*>(bufferRef.data) + offset, ipv6_str.c_str());
-        offset += len;
-        cur_batch->length[row_id] = len;
-    }
-    size_t data_off = 0;
     for (size_t row_id = start; row_id < end; row_id++) {
         if (cur_batch->notNull[row_id] == 1) {
-            cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + begin_off + data_off;
-            data_off += cur_batch->length[row_id];
+            std::string ipv6_str = IPv6Value::to_string(col_data[row_id]);
+            size_t len = ipv6_str.size();
+
+            REALLOC_MEMORY_FOR_ORC_WRITER()
+
+            strcpy(const_cast<char*>(bufferRef.data) + offset, ipv6_str.c_str());
+            cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + offset;
+            cur_batch->length[row_id] = len;
+            offset += len;
         }
     }
-    buffer_list.emplace_back(bufferRef);
+
     cur_batch->numElements = end - start;
     return Status::OK();
 }
diff --git a/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp b/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
index 10218e4164d5c1..adc041f511198e 100644
--- a/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
@@ -23,6 +23,7 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <memory>
 
 #include "arrow/array/builder_binary.h"
 #include "common/exception.h"
@@ -142,7 +143,29 @@ Status DataTypeJsonbSerDe::write_column_to_orc(const std::string& timezone, cons
                                                orc::ColumnVectorBatch* orc_col_batch, int64_t start,
                                                int64_t end,
                                                std::vector<StringRef>& buffer_list) const {
-    return Status::NotSupported("write_column_to_orc with type [{}]", column.get_name());
+    auto* cur_batch = dynamic_cast<orc::StringVectorBatch*>(orc_col_batch);
+    const auto& string_column = assert_cast<const ColumnString&>(column);
+
+    INIT_MEMORY_FOR_ORC_WRITER()
+
+    for (size_t row_id = start; row_id < end; row_id++) {
+        if (cur_batch->notNull[row_id] == 1) {
+            std::string_view string_ref = string_column.get_data_at(row_id).to_string_view();
+            auto serialized_value = std::make_unique<std::string>(
+                    JsonbToJson::jsonb_to_json_string(string_ref.data(), string_ref.size()));
+            auto len = serialized_value->size();
+
+            REALLOC_MEMORY_FOR_ORC_WRITER()
+
+            memcpy(const_cast<char*>(bufferRef.data) + offset, serialized_value->data(), len);
+            cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + offset;
+            cur_batch->length[row_id] = len;
+            offset += len;
+        }
+    }
+
+    cur_batch->numElements = end - start;
+    return Status::OK();
 }
 
 void convert_jsonb_to_rapidjson(const JsonbValue& val, rapidjson::Value& target,
diff --git a/be/src/vec/data_types/serde/data_type_number_serde.cpp b/be/src/vec/data_types/serde/data_type_number_serde.cpp
index 9416fc9a8b3020..55c7b2c9505dae 100644
--- a/be/src/vec/data_types/serde/data_type_number_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_number_serde.cpp
@@ -342,38 +342,22 @@ Status DataTypeNumberSerDe<T>::write_column_to_orc(const std::string& timezone,
     if constexpr (std::is_same_v<T, Int128>) { // largeint
         orc::StringVectorBatch* cur_batch = dynamic_cast<orc::StringVectorBatch*>(orc_col_batch);
 
-        char* ptr = (char*)malloc(BUFFER_UNIT_SIZE);
-        if (!ptr) {
-            return Status::InternalError(
-                    "malloc memory error when write largeint column data to orc file.");
-        }
-        StringRef bufferRef;
-        bufferRef.data = ptr;
-        bufferRef.size = BUFFER_UNIT_SIZE;
-        size_t offset = 0;
-        const size_t begin_off = offset;
+        INIT_MEMORY_FOR_ORC_WRITER()
 
         for (size_t row_id = start; row_id < end; row_id++) {
-            if (cur_batch->notNull[row_id] == 0) {
-                continue;
-            }
-            std::string value_str = fmt::format("{}", col_data[row_id]);
-            size_t len = value_str.size();
+            if (cur_batch->notNull[row_id] == 1) {
+                std::string value_str = fmt::format("{}", col_data[row_id]);
+                size_t len = value_str.size();
 
-            REALLOC_MEMORY_FOR_ORC_WRITER()
+                REALLOC_MEMORY_FOR_ORC_WRITER()
 
-            strcpy(const_cast<char*>(bufferRef.data) + offset, value_str.c_str());
-            offset += len;
-            cur_batch->length[row_id] = len;
-        }
-        size_t data_off = 0;
-        for (size_t row_id = start; row_id < end; row_id++) {
-            if (cur_batch->notNull[row_id] == 1) {
-                cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + begin_off + data_off;
-                data_off += cur_batch->length[row_id];
+                strcpy(const_cast<char*>(bufferRef.data) + offset, value_str.c_str());
+                cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + offset;
+                cur_batch->length[row_id] = len;
+                offset += len;
             }
         }
-        buffer_list.emplace_back(bufferRef);
+
         cur_batch->numElements = end - start;
     } else if constexpr (std::is_same_v<T, Int8> || std::is_same_v<T, UInt8>) { // tinyint/boolean
         WRITE_INTEGRAL_COLUMN_TO_ORC(orc::ByteVectorBatch)
diff --git a/be/src/vec/data_types/serde/data_type_object_serde.cpp b/be/src/vec/data_types/serde/data_type_object_serde.cpp
index f67194372857cf..fc536d9ef0df7b 100644
--- a/be/src/vec/data_types/serde/data_type_object_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_object_serde.cpp
@@ -20,6 +20,7 @@
 #include <rapidjson/stringbuffer.h>
 
 #include <cstdint>
+#include <string>
 
 #include "common/exception.h"
 #include "common/status.h"
@@ -164,6 +165,38 @@ void DataTypeObjectSerDe::write_column_to_arrow(const IColumn& column, const Nul
     }
 }
 
+Status DataTypeObjectSerDe::write_column_to_orc(const std::string& timezone, const IColumn& column,
+                                                const NullMap* null_map,
+                                                orc::ColumnVectorBatch* orc_col_batch,
+                                                int64_t start, int64_t end,
+                                                std::vector<StringRef>& buffer_list) const {
+    const auto* var = check_and_get_column<ColumnObject>(column);
+    orc::StringVectorBatch* cur_batch = dynamic_cast<orc::StringVectorBatch*>(orc_col_batch);
+
+    INIT_MEMORY_FOR_ORC_WRITER()
+
+    for (size_t row_id = start; row_id < end; row_id++) {
+        if (cur_batch->notNull[row_id] == 1) {
+            auto serialized_value = std::make_unique<std::string>();
+            if (!var->serialize_one_row_to_string(row_id, serialized_value.get())) {
+                throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Failed to serialize variant {}",
+                                       var->dump_structure());
+            }
+            auto len = serialized_value->length();
+
+            REALLOC_MEMORY_FOR_ORC_WRITER()
+
+            memcpy(const_cast<char*>(bufferRef.data) + offset, serialized_value->data(), len);
+            cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + offset;
+            cur_batch->length[row_id] = len;
+            offset += len;
+        }
+    }
+
+    cur_batch->numElements = end - start;
+    return Status::OK();
+}
+
 } // namespace vectorized
 
 } // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_object_serde.h b/be/src/vec/data_types/serde/data_type_object_serde.h
index 414755ef0f8d04..c08d4d0af0d2c3 100644
--- a/be/src/vec/data_types/serde/data_type_object_serde.h
+++ b/be/src/vec/data_types/serde/data_type_object_serde.h
@@ -89,9 +89,7 @@ class DataTypeObjectSerDe : public DataTypeSerDe {
     Status write_column_to_orc(const std::string& timezone, const IColumn& column,
                                const NullMap* null_map, orc::ColumnVectorBatch* orc_col_batch,
                                int64_t start, int64_t end,
-                               std::vector<StringRef>& buffer_list) const override {
-        return Status::NotSupported("write_column_to_orc with type " + column.get_name());
-    }
+                               std::vector<StringRef>& buffer_list) const override;
 
 private:
     template <bool is_binary_format>
diff --git a/be/src/vec/data_types/serde/data_type_quantilestate_serde.h b/be/src/vec/data_types/serde/data_type_quantilestate_serde.h
index 9608bf1cb7899a..d3526ba389925f 100644
--- a/be/src/vec/data_types/serde/data_type_quantilestate_serde.h
+++ b/be/src/vec/data_types/serde/data_type_quantilestate_serde.h
@@ -17,6 +17,7 @@
 
 #pragma once
 
+#include <arrow/array/builder_binary.h>
 #include <gen_cpp/types.pb.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -32,6 +33,7 @@
 #include "vec/columns/column_const.h"
 #include "vec/common/arena.h"
 #include "vec/common/string_ref.h"
+#include "vec/data_types/serde/data_type_nullable_serde.h"
 
 namespace doris {
 
@@ -43,12 +45,23 @@ class DataTypeQuantileStateSerDe : public DataTypeSerDe {
 
     Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw,
                                       FormatOptions& options) const override {
-        return Status::NotSupported("serialize_one_cell_to_json with type [{}]", column.get_name());
+        /**
+        * For null values in ordinary types, we use \N to represent them;
+        * for null values in nested types, we use null to represent them, just like the json format.
+        */
+        if (_nesting_level >= 2) {
+            bw.write(DataTypeNullableSerDe::NULL_IN_COMPLEX_TYPE.c_str(),
+                     strlen(NULL_IN_COMPLEX_TYPE.c_str()));
+        } else {
+            bw.write(DataTypeNullableSerDe::NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str(),
+                     strlen(NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str()));
+        }
+        return Status::OK();
     }
 
     Status serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx,
                                     BufferWritable& bw, FormatOptions& options) const override {
-        return Status::NotSupported("serialize_column_to_json with type [{}]", column.get_name());
+        SERIALIZE_COLUMN_TO_JSON();
     }
     Status deserialize_one_cell_from_json(IColumn& column, Slice& slice,
                                           const FormatOptions& options) const override {
@@ -102,8 +115,21 @@ class DataTypeQuantileStateSerDe : public DataTypeSerDe {
     void write_column_to_arrow(const IColumn& column, const NullMap* null_map,
                                arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
                                const cctz::time_zone& ctz) const override {
-        throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
-                               "write_column_to_arrow with type " + column.get_name());
+        const auto& col = assert_cast<const ColumnQuantileState&>(column);
+        auto& builder = assert_cast<arrow::BinaryBuilder&>(*array_builder);
+        for (size_t string_i = start; string_i < end; ++string_i) {
+            if (null_map && (*null_map)[string_i]) {
+                checkArrowStatus(builder.AppendNull(), column.get_name(),
+                                 array_builder->type()->name());
+            } else {
+                auto& quantile_state_value = const_cast<QuantileState&>(col.get_element(string_i));
+                std::string memory_buffer(quantile_state_value.get_serialized_size(), '0');
+                quantile_state_value.serialize((uint8_t*)memory_buffer.data());
+                checkArrowStatus(builder.Append(memory_buffer.data(),
+                                                static_cast<int>(memory_buffer.size())),
+                                 column.get_name(), array_builder->type()->name());
+            }
+        }
     }
     void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
                                 int end, const cctz::time_zone& ctz) const override {
@@ -126,7 +152,27 @@ class DataTypeQuantileStateSerDe : public DataTypeSerDe {
                                const NullMap* null_map, orc::ColumnVectorBatch* orc_col_batch,
                                int64_t start, int64_t end,
                                std::vector<StringRef>& buffer_list) const override {
-        return Status::NotSupported("write_column_to_orc with type [{}]", column.get_name());
+        auto& col_data = assert_cast<const ColumnQuantileState&>(column);
+        orc::StringVectorBatch* cur_batch = dynamic_cast<orc::StringVectorBatch*>(orc_col_batch);
+
+        INIT_MEMORY_FOR_ORC_WRITER()
+
+        for (size_t row_id = start; row_id < end; row_id++) {
+            if (cur_batch->notNull[row_id] == 1) {
+                auto quantilestate_value = const_cast<QuantileState&>(col_data.get_element(row_id));
+                size_t len = quantilestate_value.get_serialized_size();
+
+                REALLOC_MEMORY_FOR_ORC_WRITER()
+
+                quantilestate_value.serialize((uint8_t*)(bufferRef.data) + offset);
+                cur_batch->data[row_id] = const_cast<char*>(bufferRef.data) + offset;
+                cur_batch->length[row_id] = len;
+                offset += len;
+            }
+        }
+
+        cur_batch->numElements = end - start;
+        return Status::OK();
     }
 
 private:
diff --git a/be/src/vec/data_types/serde/data_type_serde.h b/be/src/vec/data_types/serde/data_type_serde.h
index bec0fabfcca9c3..1a089bb73fe99c 100644
--- a/be/src/vec/data_types/serde/data_type_serde.h
+++ b/be/src/vec/data_types/serde/data_type_serde.h
@@ -77,6 +77,18 @@ struct ColumnVectorBatch;
         ++*num_deserialized;                                                             \
     }
 
+#define INIT_MEMORY_FOR_ORC_WRITER()                                                 \
+    char* ptr = (char*)malloc(BUFFER_UNIT_SIZE);                                     \
+    if (!ptr) {                                                                      \
+        return Status::InternalError(                                                \
+                "malloc memory error when write largeint column data to orc file."); \
+    }                                                                                \
+    StringRef bufferRef;                                                             \
+    bufferRef.data = ptr;                                                            \
+    bufferRef.size = BUFFER_UNIT_SIZE;                                               \
+    size_t offset = 0;                                                               \
+    buffer_list.emplace_back(bufferRef);
+
 #define REALLOC_MEMORY_FOR_ORC_WRITER()                                                  \
     while (bufferRef.size - BUFFER_RESERVED_SIZE < offset + len) {                       \
         char* new_ptr = (char*)malloc(bufferRef.size + BUFFER_UNIT_SIZE);                \
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index ef65b405853765..026e4da29b59da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -36,7 +36,6 @@
 import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.property.PropertyConverter;
 import org.apache.doris.datasource.property.constants.S3Properties;
-import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.thrift.TFileCompressType;
 import org.apache.doris.thrift.TFileFormatType;
 import org.apache.doris.thrift.TParquetCompressionType;
@@ -302,11 +301,8 @@ private String dorisTypeToOrcTypeMap(Type dorisType) throws AnalysisException {
                 break;
             case HLL:
             case BITMAP:
-                if (!(ConnectContext.get() != null && ConnectContext.get()
-                        .getSessionVariable().isReturnObjectDataAsBinary())) {
-                    break;
-                }
-                orcType = "string";
+            case QUANTILE_STATE:
+                orcType = "binary";
                 break;
             case DATEV2:
                 orcType = "date";
@@ -327,6 +323,8 @@ private String dorisTypeToOrcTypeMap(Type dorisType) throws AnalysisException {
             case DATE:
             case DATETIME:
             case IPV6:
+            case VARIANT:
+            case JSONB:
                 orcType = "string";
                 break;
             case DECIMALV2:
@@ -445,6 +443,8 @@ private void analyzeForOrcFormat(List<Expr> resultExprs, List<String> colLabels)
                 case DATE:
                 case DATETIME:
                 case IPV6:
+                case VARIANT:
+                case JSONB:
                     checkOrcType(schema.second, "string", true, resultType.getPrimitiveType().toString());
                     break;
                 case DECIMAL32:
@@ -455,13 +455,8 @@ private void analyzeForOrcFormat(List<Expr> resultExprs, List<String> colLabels)
                     break;
                 case HLL:
                 case BITMAP:
-                    if (ConnectContext.get() != null && ConnectContext.get()
-                            .getSessionVariable().isReturnObjectDataAsBinary()) {
-                        checkOrcType(schema.second, "string", true, resultType.getPrimitiveType().toString());
-                    } else {
-                        throw new AnalysisException("Orc format does not support column type: "
-                                + resultType.getPrimitiveType());
-                    }
+                case QUANTILE_STATE:
+                    checkOrcType(schema.second, "binary", true, resultType.getPrimitiveType().toString());
                     break;
                 case STRUCT:
                     checkOrcType(schema.second, "struct", false, resultType.getPrimitiveType().toString());
diff --git a/regression-test/data/export_p0/outfile/test_outfile_complex_type.out b/regression-test/data/export_p0/outfile/test_outfile_complex_type.out
new file mode 100644
index 00000000000000..cd6f000b6c5052
--- /dev/null
+++ b/regression-test/data/export_p0/outfile/test_outfile_complex_type.out
@@ -0,0 +1,25 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_load_parquet --
+20220201	0	0000004501000000000000F03F	0101675D86AC33FA8CD6
+20220201	1	0000004501000000000000F0BF	01010B3C52B765A11A2F
+20220201	2	00000045010000000000000000	0101DDEA60F9C89AA329
+20220201	3	0000004501000000000000F03F	0101EF81F59130F8B748
+20220201	4	00000045010000000000000040	010114CAA737BD54146E
+20220201	5	00000045010000000000000840	0101DCBC5BA258F9602C
+
+-- !select_load_orc --
+20220201	0	0000004501000000000000F03F	0101675D86AC33FA8CD6
+20220201	1	0000004501000000000000F0BF	01010B3C52B765A11A2F
+20220201	2	00000045010000000000000000	0101DDEA60F9C89AA329
+20220201	3	0000004501000000000000F03F	0101EF81F59130F8B748
+20220201	4	00000045010000000000000040	010114CAA737BD54146E
+20220201	5	00000045010000000000000840	0101DCBC5BA258F9602C
+
+-- !select_load_csv --
+20220201	0	\N	\N	\N
+20220201	1	\N	\N	\N
+20220201	2	\N	\N	\N
+20220201	3	\N	\N	\N
+20220201	4	\N	\N	\N
+20220201	5	\N	\N	\N
+
diff --git a/regression-test/data/export_p0/outfile/test_outfile_jsonb_and_variant.out b/regression-test/data/export_p0/outfile/test_outfile_jsonb_and_variant.out
new file mode 100644
index 00000000000000..d25830939645f6
--- /dev/null
+++ b/regression-test/data/export_p0/outfile/test_outfile_jsonb_and_variant.out
@@ -0,0 +1,25 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_load_parquet --
+20220201	0	{"k1":"100"}	{"k1":"100"}
+20220201	1	{"k1":"100","k2":"123"}	{"k1":"100","k2":"123"}
+20220201	2	{"k1":"100","abc":"567"}	{"abc":"567","k1":"100"}
+20220201	3	{"k1":"100","k3":123}	{"k1":"100","k3":123}
+20220201	4	{"k1":"100","doris":"nereids"}	{"doris":"nereids","k1":"100"}
+20220201	5	{"k1":"100","doris":"pipeline"}	{"doris":"pipeline","k1":"100"}
+
+-- !select_load_orc --
+20220201	0	{"k1":"100"}	{"k1":"100"}
+20220201	1	{"k1":"100","k2":"123"}	{"k1":"100","k2":"123"}
+20220201	2	{"k1":"100","abc":"567"}	{"abc":"567","k1":"100"}
+20220201	3	{"k1":"100","k3":123}	{"k1":"100","k3":123}
+20220201	4	{"k1":"100","doris":"nereids"}	{"doris":"nereids","k1":"100"}
+20220201	5	{"k1":"100","doris":"pipeline"}	{"doris":"pipeline","k1":"100"}
+
+-- !select_load_orc --
+20220201	0	{"k1":"100"}	{"k1":"100"}
+20220201	1	{"k1":"100","k2":"123"}	{"k1":"100","k2":"123"}
+20220201	2	{"k1":"100","abc":"567"}	{"abc":"567","k1":"100"}
+20220201	3	{"k1":"100","k3":123}	{"k1":"100","k3":123}
+20220201	4	{"k1":"100","doris":"nereids"}	{"doris":"nereids","k1":"100"}
+20220201	5	{"k1":"100","doris":"pipeline"}	{"doris":"pipeline","k1":"100"}
+
diff --git a/regression-test/suites/export_p0/outfile/test_outfile_complex_type.groovy b/regression-test/suites/export_p0/outfile/test_outfile_complex_type.groovy
new file mode 100644
index 00000000000000..49f81732791126
--- /dev/null
+++ b/regression-test/suites/export_p0/outfile/test_outfile_complex_type.groovy
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_outfile_complex_type", "p0") {
+    String ak = getS3AK()
+    String sk = getS3SK()
+    String s3_endpoint = getS3Endpoint()
+    String region = getS3Region()
+    String bucket = context.config.otherConfigs.get("s3BucketName");
+
+    def export_table_name = "test_outfile_complex_type_table"
+    def outFilePath = "${bucket}/outfile/complex_type/exp_"
+
+    def outfile_to_S3 = { format ->
+        // select ... into outfile ...
+        def res = sql """
+            SELECT * FROM ${export_table_name} t
+            INTO OUTFILE "s3://${outFilePath}"
+            FORMAT AS ${format}
+            PROPERTIES (
+                "s3.endpoint" = "${s3_endpoint}",
+                "s3.region" = "${region}",
+                "s3.secret_key"="${sk}",
+                "s3.access_key" = "${ak}"
+            );
+        """
+
+        return res[0][3]
+    }
+
+    sql """ DROP TABLE IF EXISTS ${export_table_name} """
+    sql """
+        CREATE TABLE `${export_table_name}` (
+            `dt` int(11) NULL COMMENT "",
+            `id` int(11) NULL COMMENT "",
+            `price` quantile_state QUANTILE_UNION NOT NULL COMMENT "",
+            `hll_t` hll hll_union,
+            `device_id` bitmap BITMAP_UNION
+        ) ENGINE=OLAP
+        AGGREGATE KEY(`dt`, `id`)
+        DISTRIBUTED BY HASH(`dt`)
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+        """
+
+    sql """
+        INSERT INTO `${export_table_name}` values
+        (20220201,0, to_quantile_state(1, 2048), hll_hash(1), to_bitmap(243)),
+        (20220201,1, to_quantile_state(-1, 2048), hll_hash(2), bitmap_from_array([1,2,3,4,5,434543])),
+        (20220201,2, to_quantile_state(0, 2048), hll_hash(3), to_bitmap(1234566)),
+        (20220201,3, to_quantile_state(1, 2048), hll_hash(4), to_bitmap(8888888888888)),
+        (20220201,4, to_quantile_state(2, 2048), hll_hash(5), to_bitmap(98392819412234)),
+        (20220201,5, to_quantile_state(3, 2048), hll_hash(6), to_bitmap(253234234));
+        """
+
+    // parquet file format
+    def format = "parquet"
+    def outfile_url = outfile_to_S3("${format}")
+    qt_select_load_parquet """ SELECT dt, id, hex(price), hex(hll_t) FROM S3 (
+                "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+                "ACCESS_KEY"= "${ak}",
+                "SECRET_KEY" = "${sk}",
+                "format" = "${format}",
+                "region" = "${region}"
+            );
+            """
+
+    // orc file foramt
+    format = "orc"
+    outfile_url = outfile_to_S3("${format}")
+    qt_select_load_orc """ SELECT dt, id, hex(price), hex(hll_t) FROM S3 (
+                "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+                "ACCESS_KEY"= "${ak}",
+                "SECRET_KEY" = "${sk}",
+                "format" = "${format}",
+                "region" = "${region}"
+            );
+            """
+
+    // csv file foramt
+    format = "csv"
+    outfile_url = outfile_to_S3("${format}")
+    qt_select_load_csv """ SELECT * FROM S3 (
+                "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+                "ACCESS_KEY"= "${ak}",
+                "SECRET_KEY" = "${sk}",
+                "format" = "${format}",
+                "region" = "${region}"
+            );
+            """
+}
\ No newline at end of file
diff --git a/regression-test/suites/export_p0/outfile/test_outfile_jsonb_and_variant.groovy b/regression-test/suites/export_p0/outfile/test_outfile_jsonb_and_variant.groovy
new file mode 100644
index 00000000000000..ed3019436aef29
--- /dev/null
+++ b/regression-test/suites/export_p0/outfile/test_outfile_jsonb_and_variant.groovy
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_outfile_jsonb_and_variant", "p0") {
+    String ak = getS3AK()
+    String sk = getS3SK()
+    String s3_endpoint = getS3Endpoint()
+    String region = getS3Region()
+    String bucket = context.config.otherConfigs.get("s3BucketName");
+
+    def export_table_name = "test_outfile_jsonb_and_variant_table"
+    def outFilePath = "${bucket}/outfile/jsonb_and_variant/exp_"
+
+    def outfile_to_S3 = { format ->
+        // select ... into outfile ...
+        def res = sql """
+            SELECT * FROM ${export_table_name} t
+            INTO OUTFILE "s3://${outFilePath}"
+            FORMAT AS ${format}
+            PROPERTIES (
+                "s3.endpoint" = "${s3_endpoint}",
+                "s3.region" = "${region}",
+                "s3.secret_key"="${sk}",
+                "s3.access_key" = "${ak}"
+            );
+        """
+
+        return res[0][3]
+    }
+
+    sql """ DROP TABLE IF EXISTS ${export_table_name} """
+    sql """
+        CREATE TABLE `${export_table_name}` (
+            `dt` int(11) NULL COMMENT "",
+            `id` int(11) NULL COMMENT "",
+            `json_col` JSON NULL COMMENT "",
+            `variant_col` variant NULL COMMENT ""
+        ) ENGINE=OLAP
+        DISTRIBUTED BY HASH(`dt`)
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+        """
+
+    sql """
+        INSERT INTO `${export_table_name}` values
+        (20220201,0, '{"k1": "100"}', '{"k1": "100"}'),
+        (20220201,1, '{"k1": "100", "k2": "123"}', '{"k1": "100", "k2": "123"}'),
+        (20220201,2, '{"k1": "100", "abc": "567"}', '{"k1": "100", "abc": "567"}'),
+        (20220201,3, '{"k1": "100", "k3": 123}', '{"k1": "100", "k3": 123}'),
+        (20220201,4, '{"k1": "100", "doris": "nereids"}', '{"k1": "100", "doris": "nereids"}'),
+        (20220201,5, '{"k1": "100", "doris": "pipeline"}', '{"k1": "100", "doris": "pipeline"}');
+        """
+
+    // parquet file format
+    def format = "parquet"
+    def outfile_url = outfile_to_S3("${format}")
+    qt_select_load_parquet """ SELECT * FROM S3 (
+                "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+                "ACCESS_KEY"= "${ak}",
+                "SECRET_KEY" = "${sk}",
+                "format" = "${format}",
+                "region" = "${region}"
+            );
+            """
+
+    // orc file foramt
+    format = "orc"
+    outfile_url = outfile_to_S3("${format}")
+    qt_select_load_orc """ SELECT * FROM S3 (
+                "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+                "ACCESS_KEY"= "${ak}",
+                "SECRET_KEY" = "${sk}",
+                "format" = "${format}",
+                "region" = "${region}"
+            );
+            """
+
+    // orc file foramt
+    format = "csv"
+    outfile_url = outfile_to_S3("${format}")
+    qt_select_load_orc """ SELECT * FROM S3 (
+                "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+                "ACCESS_KEY"= "${ak}",
+                "SECRET_KEY" = "${sk}",
+                "format" = "${format}",
+                "region" = "${region}"
+            );
+            """
+}
\ No newline at end of file

From 24f1eeffa3cb5533d28095a6b4d9434a7e3e5d10 Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyongkang@selectdb.com>
Date: Thu, 28 Nov 2024 23:27:41 +0800
Subject: [PATCH 070/399] [Enhancement](jdbc catalog) Support reading some
 array types of PostgreSQL (#44160)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

This PR supports reading some PostgreSQL array types, including:

```
{
                "int2", "int4", "int8", "smallserial", "serial",
                "bigserial", "float4", "float8", "timestamp", "timestamptz",
                "date", "bool", "bpchar", "varchar", "text"
        }
```


### Release note

[feat](jdbc catalog) Support reading some array types of PostgreSQL
---
 .../postgresql/init/02-create-table.sql       |  88 ++++++++
 .../postgresql/init/04-insert.sql             | 210 ++++++++++++++++++
 .../doris/jdbc/PostgreSQLJdbcExecutor.java    |  85 ++++++-
 .../java/org/apache/doris/common/Config.java  |   2 +-
 .../jdbc/client/JdbcPostgreSQLClient.java     |  92 +++++++-
 .../datasource/jdbc/util/JdbcFieldSchema.java |  15 ++
 .../jdbc/test_pg_jdbc_catalog.out             |  11 +
 .../jdbc/test_pg_jdbc_catalog.groovy          |   5 +
 8 files changed, 504 insertions(+), 4 deletions(-)

diff --git a/docker/thirdparties/docker-compose/postgresql/init/02-create-table.sql b/docker/thirdparties/docker-compose/postgresql/init/02-create-table.sql
index 7983d55a018917..bafdd615839e14 100644
--- a/docker/thirdparties/docker-compose/postgresql/init/02-create-table.sql
+++ b/docker/thirdparties/docker-compose/postgresql/init/02-create-table.sql
@@ -263,3 +263,91 @@ CREATE TABLE catalog_pg_test.num_zero (
     id varchar(20) NULL,
     num numeric NULL
 );
+
+CREATE TABLE catalog_pg_test.test_all_types_array (
+	id int4 NOT NULL,
+	char_value _bpchar NULL,
+	varchar_value _varchar NULL,
+	date_value _date NULL,
+	smallint_value _int2 NULL,
+	int_value _int4 NULL,
+	bigint_value _int8 NULL,
+	timestamp_value _timestamp NULL,
+	decimal_value _numeric NULL,
+	bit_value _bit NULL,
+	real_value _float4 NULL,
+	cidr_value _cidr NULL,
+	inet_value _inet NULL,
+	macaddr_value _macaddr NULL,
+	bitn_value _bit NULL,
+	bitnv_value _varbit NULL,
+	jsonb_value _jsonb NULL,
+	point_value _point NULL,
+	line_value _line NULL,
+	lseg_value _lseg NULL,
+	box_value _box NULL,
+	path_value _path NULL,
+	polygon_value _polygon NULL,
+	circle_value _circle NULL
+);
+
+
+CREATE TABLE catalog_pg_test.test_all_types_2d_array (
+	id int4 NOT NULL,
+	char_value _bpchar NULL,
+	varchar_value _varchar NULL,
+	date_value _date NULL,
+	smallint_value _int2 NULL,
+	int_value _int4 NULL,
+	bigint_value _int8 NULL,
+	timestamp_value _timestamp NULL,
+	decimal_value _numeric NULL,
+	bit_value _bit NULL,
+	real_value _float4 NULL,
+	cidr_value _cidr NULL,
+	inet_value _inet NULL,
+	macaddr_value _macaddr NULL,
+	bitn_value _bit NULL,
+	bitnv_value _varbit NULL,
+	jsonb_value _jsonb NULL,
+	point_value _point NULL,
+	line_value _line NULL,
+	lseg_value _lseg NULL,
+	box_value _box NULL,
+	path_value _path NULL,
+	polygon_value _polygon NULL,
+	circle_value _circle NULL
+);
+
+CREATE TABLE catalog_pg_test.test_all_support_types_array (
+        id int4 NOT NULL,
+        char_value bpchar[] NULL,
+        varchar_value varchar[] NULL,
+        text_value text[] null,
+        date_value date[] NULL,
+        smallint_value int2[] NULL,
+        int_value int4[] NULL,
+        bigint_value int8[] NULL,
+        timestamp_value timestamp[] NULL,
+        timestamptz_value timestamptz[] null,
+        bool_value bool[] NULL,
+        float4_value float4[] null,
+        float8_value float8[] null
+);
+
+CREATE TABLE catalog_pg_test.test_all_support_types_array_2d (
+    id int4 NOT NULL,
+    char_value bpchar[][] NULL,
+    varchar_value varchar[][] NULL,
+    text_value text[][] NULL,
+    date_value date[][] NULL,
+    smallint_value int2[][] NULL,
+    int_value int4[][] NULL,
+    bigint_value int8[][] NULL,
+    timestamp_value timestamp[][] NULL,
+    timestamptz_value timestamptz[][] NULL,
+    bool_value bool[][] NULL,
+    float4_value float4[][] NULL,
+    float8_value float8[][] NULL
+);
+
diff --git a/docker/thirdparties/docker-compose/postgresql/init/04-insert.sql b/docker/thirdparties/docker-compose/postgresql/init/04-insert.sql
index ae3570dfc186ce..b2a14148ead53e 100644
--- a/docker/thirdparties/docker-compose/postgresql/init/04-insert.sql
+++ b/docker/thirdparties/docker-compose/postgresql/init/04-insert.sql
@@ -2730,3 +2730,213 @@ values('1','A','2020-09-02 00:00:00')
 INSERT INTO catalog_pg_test.num_zero (id, num) VALUES ('001', 123.45);
 INSERT INTO catalog_pg_test.num_zero (id, num) VALUES ('002', 678.90);
 INSERT INTO catalog_pg_test.num_zero (id, num) VALUES ('003', 1011.12);
+
+INSERT INTO catalog_pg_test.test_all_types_2d_array VALUES
+(1,
+    ARRAY[ARRAY['abc', 'def']::text[], ARRAY['ghi', 'jkl']::text[]],
+    ARRAY[ARRAY['abc', 'def']::text[], ARRAY['ghi', 'jkl']::text[]],
+    ARRAY[ARRAY['2022-10-11'::date, '2022-11-12'::date], ARRAY['2023-01-01'::date, '2023-02-02'::date]],
+    ARRAY[ARRAY[1, 2], ARRAY[3, 4]],
+    ARRAY[ARRAY[1, 2], ARRAY[2, 3]],
+    ARRAY[ARRAY[3, 4], ARRAY[4, 5]],
+    ARRAY[ARRAY['2022-10-22 10:59:59'::timestamp, '2023-01-01 12:00:00'::timestamp],
+          ARRAY['2023-02-01 14:30:00'::timestamp, '2023-02-10 16:45:00'::timestamp]],
+    ARRAY[ARRAY[34.123, 45.678], ARRAY[56.789, 67.890]],
+    ARRAY[ARRAY[0::bit, 1::bit], ARRAY[1::bit, 0::bit]],
+    ARRAY[ARRAY[12.123456, 34.567890], ARRAY[56.789123, 45.678345]],
+    ARRAY[ARRAY['10.16.10.14'::cidr, '192.168.1.1'::cidr],
+          ARRAY['172.16.0.1'::cidr, '192.168.2.2'::cidr]],
+    ARRAY[ARRAY['10.16.10.14'::inet, '192.168.1.1'::inet],
+          ARRAY['172.16.0.1'::inet, '192.168.2.2'::inet]],
+    ARRAY[ARRAY['ff:ff:ff:ff:ff:AA'::macaddr, '00:14:22:01:23:45'::macaddr],
+          ARRAY['11:22:33:44:55:66'::macaddr, '77:88:99:00:11:22'::macaddr]],
+    ARRAY[ARRAY['1010101010'::bit(10), '1100101010'::bit(10)],
+          ARRAY['1110001111'::bit(10), '0001110000'::bit(10)]],
+    ARRAY[ARRAY[cast(10 as bit(5)), cast(20 as bit(5))],
+          ARRAY[cast(30 as bit(5)), cast(40 as bit(5))]],
+    ARRAY[ARRAY['{"id":1}'::jsonb, '{"id":2}'::jsonb],
+          ARRAY['{"id":3}'::jsonb, '{"id":4}'::jsonb]],
+    ARRAY[ARRAY['(1,1)'::point, '(2,2)'::point],
+          ARRAY['(3,3)'::point, '(4,4)'::point]],
+    ARRAY[ARRAY['{1,1,1}'::line, '{2,2,2}'::line],
+          ARRAY['{3,3,3}'::line, '{4,4,4}'::line]],
+    ARRAY[ARRAY['(1,1),(2,2)'::lseg, '(3,3),(4,4)'::lseg],
+          ARRAY['(5,5),(6,6)'::lseg, '(7,7),(8,8)'::lseg]],
+    ARRAY[ARRAY['(1,1),(2,2)'::box, '(5,5),(6,6)'::box],
+          ARRAY['(7,7),(8,8)'::box, '(9,9),(10,10)'::box]],
+    ARRAY[ARRAY['(1,1),(2,2),(2,1)'::path, '(3,3),(4,4),(5,5)'::path],
+          ARRAY['(6,6),(7,7),(8,8)'::path, '(9,9),(10,10),(11,11)'::path]],
+    ARRAY[ARRAY['((1,1),(2,2),(2,1))'::polygon, '((3,3),(4,4),(5,5))'::polygon],
+          ARRAY['((6,6),(7,7),(8,8))'::polygon, '((9,9),(10,10),(11,11))'::polygon]],
+    ARRAY[ARRAY['<(0,0),1>'::circle, '<(5,5),10>'::circle],
+          ARRAY['<(2,2),3>'::circle, '<(7,7),14>'::circle]]
+);
+
+INSERT INTO catalog_pg_test.test_all_types_array VALUES
+(1,
+    ARRAY['abc', 'def'],
+    ARRAY['ghi', 'jkl'],
+    ARRAY['2022-10-11'::date, '2022-11-12'::date],
+    ARRAY[1, 2],
+    ARRAY[2, 3],
+    ARRAY[3, 4],
+    ARRAY['2022-10-22 10:59:59'::timestamp, '2023-01-01 12:00:00'::timestamp],
+    ARRAY[34.123, 45.678],
+    ARRAY[0::bit, 1::bit],
+    ARRAY[12.123456, 34.567890],
+    ARRAY['10.16.10.14'::cidr, '192.168.1.1'::cidr],
+    ARRAY['10.16.10.14'::inet, '192.168.0.1'::inet],
+    ARRAY['ff:ff:ff:ff:ff:AA'::macaddr, '00:14:22:01:23:45'::macaddr],
+    ARRAY['1010101010'::bit(10), '1100101010'::bit(10)],
+    ARRAY[cast('00010' as bit(5)), cast('10100' as bit(5))],  -- bit(5) values need 5 bits
+    ARRAY['{"id":1}'::jsonb, '{"id":2}'::jsonb],
+    ARRAY['(1,1)'::point, '(2,2)'::point],
+    ARRAY['{1,1,1}'::line, '{2,2,2}'::line],
+    ARRAY['(1,1),(2,2)'::lseg, '(3,3),(4,4)'::lseg],
+    ARRAY['(1,1),(2,2)'::box, '(5,5),(6,6)'::box],
+    ARRAY['(1,1),(2,2),(2,1)'::path, '(3,3),(4,4),(5,5)'::path],
+    ARRAY['((1,1),(2,2),(2,1))'::polygon, '((3,3),(4,4),(5,5))'::polygon],
+    ARRAY['<(0,0),1>'::circle, '<(5,5),10>'::circle]
+);
+
+INSERT INTO catalog_pg_test.test_all_support_types_array (
+    id, char_value, varchar_value, text_value, date_value,
+    smallint_value, int_value, bigint_value, timestamp_value,
+    timestamptz_value, bool_value, float4_value, float8_value
+) VALUES (
+    1,
+    ARRAY['A', 'B', 'C'],
+    ARRAY['Hello', 'World'],
+    ARRAY['Text1', 'Text2'],
+    ARRAY['2024-01-01'::date, '2024-01-02'::date],
+    ARRAY[10, 20],
+    ARRAY[100, 200],
+    ARRAY[1000, 2000],
+    ARRAY['2024-01-01 10:00:00'::timestamp, '2024-01-02 12:00:00'::timestamp],
+    ARRAY['2024-01-01 10:00:00+00'::timestamptz, '2024-01-02 12:00:00+00'::timestamptz],
+    ARRAY[true, false],
+    ARRAY[1.1, 2.2],
+    ARRAY[3.3, 4.4]
+);
+
+INSERT INTO catalog_pg_test.test_all_support_types_array (
+    id, char_value, varchar_value, text_value, date_value,
+    smallint_value, int_value, bigint_value, timestamp_value,
+    timestamptz_value, bool_value, float4_value, float8_value
+) VALUES (
+    2,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null
+);
+
+INSERT INTO catalog_pg_test.test_all_support_types_array (
+    id, char_value, varchar_value, text_value, date_value,
+    smallint_value, int_value, bigint_value, timestamp_value,
+    timestamptz_value, bool_value, float4_value, float8_value
+) VALUES (
+    3,
+    ARRAY['A', 'B', 'C', null],
+    ARRAY['Hello', 'World', null],
+    ARRAY['Text1', 'Text2', null],
+    ARRAY['2024-01-01'::date, '2024-01-02'::date, null],
+    ARRAY[10, 20, null],
+    ARRAY[100, 200, null],
+    ARRAY[1000, 2000, null],
+    ARRAY['2024-01-01 10:00:00'::timestamp, '2024-01-02 12:00:00'::timestamp, null],
+    ARRAY['2024-01-01 10:00:00+00'::timestamptz, '2024-01-02 12:00:00+00'::timestamptz, null],
+    ARRAY[true, false, null],
+    ARRAY[1.1, 2.2, null],
+    ARRAY[3.3, 4.4, null]
+);
+
+INSERT INTO catalog_pg_test.test_all_support_types_array_2d (
+    id, char_value, varchar_value, text_value, date_value,
+    smallint_value, int_value, bigint_value, timestamp_value,
+    timestamptz_value, bool_value, float4_value, float8_value
+) VALUES (
+    1,
+    ARRAY[ARRAY['A', 'B'], ARRAY['C', 'D']],
+    ARRAY[ARRAY['Hello', 'World'], ARRAY['Foo', 'Bar']],
+    ARRAY[ARRAY['Text1', 'Text2'], ARRAY['Text3', 'Text4']],
+    ARRAY[ARRAY['2024-01-01'::date, '2024-01-02'::date], ARRAY['2024-02-01'::date, '2024-02-02'::date]],
+    ARRAY[ARRAY[10, 20], ARRAY[30, 40]],
+    ARRAY[ARRAY[100, 200], ARRAY[300, 400]],
+    ARRAY[ARRAY[1000, 2000], ARRAY[3000, 4000]],
+    ARRAY[ARRAY['2024-01-01 10:00:00'::timestamp, '2024-01-02 12:00:00'::timestamp], ARRAY['2024-03-01 10:00:00'::timestamp, '2024-03-02 12:00:00'::timestamp]],
+    ARRAY[ARRAY['2024-01-01 10:00:00+00'::timestamptz, '2024-01-02 12:00:00+00'::timestamptz], ARRAY['2024-03-01 10:00:00+00'::timestamptz, '2024-03-02 12:00:00+00'::timestamptz]],
+    ARRAY[ARRAY[true, false], ARRAY[false, true]],
+    ARRAY[ARRAY[1.1, 2.2], ARRAY[3.3, 4.4]],
+    ARRAY[ARRAY[5.5, 6.6], ARRAY[7.7, 8.8]]
+);
+
+INSERT INTO catalog_pg_test.test_all_support_types_array_2d (
+    id, char_value, varchar_value, text_value, date_value,
+    smallint_value, int_value, bigint_value, timestamp_value,
+    timestamptz_value, bool_value, float4_value, float8_value
+) VALUES (
+    2,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null
+);
+
+INSERT INTO catalog_pg_test.test_all_support_types_array_2d (
+    id, char_value, varchar_value, text_value, date_value,
+    smallint_value, int_value, bigint_value, timestamp_value,
+    timestamptz_value, bool_value, float4_value, float8_value
+) VALUES (
+    3,
+    ARRAY[ARRAY['A', null], ARRAY['C', 'D']],
+    ARRAY[ARRAY['Hello', null], ARRAY['Foo', 'Bar']],
+    ARRAY[ARRAY['Text1', null], ARRAY['Text3', 'Text4']],
+    ARRAY[ARRAY['2024-01-01'::date, null], ARRAY['2024-02-01'::date, '2024-02-02'::date]],
+    ARRAY[ARRAY[10, null], ARRAY[30, 40]],
+    ARRAY[ARRAY[100, null], ARRAY[300, 400]],
+    ARRAY[ARRAY[1000, null], ARRAY[3000, 4000]],
+    ARRAY[ARRAY['2024-01-01 10:00:00'::timestamp, null], ARRAY['2024-03-01 10:00:00'::timestamp, '2024-03-02 12:00:00'::timestamp]],
+    ARRAY[ARRAY['2024-01-01 10:00:00+00'::timestamptz, null], ARRAY['2024-03-01 10:00:00+00'::timestamptz, '2024-03-02 12:00:00+00'::timestamptz]],
+    ARRAY[ARRAY[true,null], ARRAY[false, true]],
+    ARRAY[ARRAY[1.1, null], ARRAY[3.3, 4.4]],
+    ARRAY[ARRAY[5.5, null], ARRAY[7.7, 8.8]]
+);
+
+
+INSERT INTO catalog_pg_test.test_all_support_types_array_2d (
+    id, char_value, varchar_value, text_value, date_value,
+    smallint_value, int_value, bigint_value, timestamp_value,
+    timestamptz_value, bool_value, float4_value, float8_value
+) VALUES (
+    4,
+    ARRAY[ARRAY['A', 'B'], ARRAY[NULL, NULL]],
+    ARRAY[ARRAY['Hello', 'World'], ARRAY[NULL, NULL]],
+    ARRAY[ARRAY['Text1', 'Text2'], ARRAY[NULL, NULL]],
+    ARRAY[ARRAY['2024-01-01'::date, '2024-01-02'::date], ARRAY[NULL::date, NULL::date]],
+    ARRAY[ARRAY[10, 20], ARRAY[NULL::smallint, NULL::smallint]],
+    ARRAY[ARRAY[100, 200], ARRAY[NULL::int, NULL::int]],
+    ARRAY[ARRAY[1000, 2000], ARRAY[NULL::bigint, NULL::bigint]],
+    ARRAY[ARRAY['2024-01-01 10:00:00'::timestamp, '2024-01-02 12:00:00'::timestamp], ARRAY[NULL::timestamp, NULL::timestamp]],
+    ARRAY[ARRAY['2024-01-01 10:00:00+00'::timestamptz, '2024-01-02 12:00:00+00'::timestamptz], ARRAY[NULL::timestamptz, NULL::timestamptz]],
+    ARRAY[ARRAY[true, false], ARRAY[NULL::boolean, NULL::boolean]],
+    ARRAY[ARRAY[1.1, 2.2], ARRAY[NULL::real, NULL::real]],
+    ARRAY[ARRAY[5.5, 6.6], ARRAY[NULL::double precision, NULL::double precision]]
+);
+
diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/PostgreSQLJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/PostgreSQLJdbcExecutor.java
index 89fdb0ba997773..c25bcec9a497f1 100644
--- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/PostgreSQLJdbcExecutor.java
+++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/PostgreSQLJdbcExecutor.java
@@ -22,14 +22,22 @@
 import org.apache.doris.common.jni.vec.ColumnValueConverter;
 import org.apache.doris.common.jni.vec.VectorTable;
 
+import com.google.common.collect.Lists;
+import org.apache.log4j.Logger;
+
 import java.math.BigDecimal;
+import java.sql.Date;
 import java.sql.SQLException;
 import java.sql.Timestamp;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.List;
 
 public class PostgreSQLJdbcExecutor extends BaseJdbcExecutor {
+    private static final Logger LOG = Logger.getLogger(PostgreSQLJdbcExecutor.class);
+
     public PostgreSQLJdbcExecutor(byte[] thriftParams) throws Exception {
         super(thriftParams);
     }
@@ -41,7 +49,8 @@ protected void initializeBlock(int columnCount, String[] replaceStringList, int
             if (outputTable.getColumnType(i).getType() == Type.DATETIME
                     || outputTable.getColumnType(i).getType() == Type.DATETIMEV2) {
                 block.add(new Object[batchSizeNum]);
-            } else if (outputTable.getColumnType(i).getType() == Type.STRING) {
+            } else if (outputTable.getColumnType(i).getType() == Type.STRING
+                    || outputTable.getColumnType(i).getType() == Type.ARRAY) {
                 block.add(new Object[batchSizeNum]);
             } else {
                 block.add(outputTable.getColumn(i).newObjectContainerArray(batchSizeNum));
@@ -78,6 +87,9 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
             case VARCHAR:
             case STRING:
                 return resultSet.getObject(columnIndex + 1);
+            case ARRAY:
+                java.sql.Array array = resultSet.getArray(columnIndex + 1);
+                return array == null ? null : convertArrayToList(array.getArray());
             default:
                 throw new IllegalArgumentException("Unsupported column type: " + type.getType());
         }
@@ -110,6 +122,10 @@ protected ColumnValueConverter getOutputConverter(ColumnType columnType, String
                         return input.toString();
                     }
                 }, String.class);
+            case ARRAY:
+                return createConverter(
+                        (Object input) -> convertArray((List<?>) input, columnType.getChildTypes().get(0)),
+                        List.class);
             default:
                 return null;
         }
@@ -122,4 +138,71 @@ private static String pgByteArrayToHexString(byte[] bytes) {
         }
         return hexString.toString();
     }
+
+    private List<Object> convertArrayToList(Object array) {
+        if (array == null) {
+            return null;
+        }
+
+        int length = java.lang.reflect.Array.getLength(array);
+        List<Object> list = new ArrayList<>(length);
+
+        for (int i = 0; i < length; i++) {
+            Object element = java.lang.reflect.Array.get(array, i);
+            list.add(element);
+        }
+
+        return list;
+    }
+
+    private List<?> convertArray(List<?> array, ColumnType type) {
+        if (array == null) {
+            return null;
+        }
+        switch (type.getType()) {
+            case DATE:
+            case DATEV2: {
+                List<LocalDate> result = new ArrayList<>();
+                for (Object element : array) {
+                    if (element == null) {
+                        result.add(null);
+                    } else {
+                        result.add(((Date) element).toLocalDate());
+                    }
+                }
+                return result;
+            }
+            case DATETIME:
+            case DATETIMEV2: {
+                List<LocalDateTime> result = new ArrayList<>();
+                for (Object element : array) {
+                    if (element == null) {
+                        result.add(null);
+                    } else {
+                        if (element instanceof Timestamp) {
+                            result.add(((Timestamp) element).toLocalDateTime());
+                        } else if (element instanceof OffsetDateTime) {
+                            result.add(((OffsetDateTime) element).toLocalDateTime());
+                        } else {
+                            result.add((LocalDateTime) element);
+                        }
+                    }
+                }
+                return result;
+            }
+            case ARRAY:
+                List<List<?>> resultArray = Lists.newArrayList();
+                for (Object element : array) {
+                    if (element == null) {
+                        resultArray.add(null);
+                    } else {
+                        List<?> nestedList = convertArrayToList(element);
+                        resultArray.add(convertArray(nestedList, type.getChildTypes().get(0)));
+                    }
+                }
+                return resultArray;
+            default:
+                return array;
+        }
+    }
 }
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 66dbff5abed8e6..3caa137c858440 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -164,7 +164,7 @@ public class Config extends ConfigBase {
             "MySQL Jdbc Catalog mysql does not support pushdown functions"})
     public static String[] jdbc_mysql_unsupported_pushdown_functions = {"date_trunc", "money_format", "negative"};
 
-    @ConfField(description = {"强制 SQLServer Jdbc Catalog 加密为 false",
+    @ConfField(mutable = true, masterOnly = true, description = {"强制 SQLServer Jdbc Catalog 加密为 false",
             "Force SQLServer Jdbc Catalog encrypt to false"})
     public static boolean force_sqlserver_jdbc_encrypt_false = false;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
index e6694ffdc674bb..481e5ea5e490c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
@@ -17,17 +17,75 @@
 
 package org.apache.doris.datasource.jdbc.client;
 
+import org.apache.doris.catalog.ArrayType;
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
+import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
 
+import com.google.common.collect.Lists;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
 public class JdbcPostgreSQLClient extends JdbcClient {
 
+    private static final String[] supportedInnerType = new String[] {
+            "int2", "int4", "int8", "smallserial", "serial",
+            "bigserial", "float4", "float8", "timestamp", "timestamptz",
+            "date", "bool", "bpchar", "varchar", "text"
+    };
+
     protected JdbcPostgreSQLClient(JdbcClientConfig jdbcClientConfig) {
         super(jdbcClientConfig);
     }
 
+    @Override
+    public List<JdbcFieldSchema> getJdbcColumnsInfo(String localDbName, String localTableName) {
+        Connection conn = null;
+        ResultSet rs = null;
+        List<JdbcFieldSchema> tableSchema = Lists.newArrayList();
+        String remoteDbName = getRemoteDatabaseName(localDbName);
+        String remoteTableName = getRemoteTableName(localDbName, localTableName);
+        try {
+            conn = getConnection();
+            DatabaseMetaData databaseMetaData = conn.getMetaData();
+            String catalogName = getCatalogName(conn);
+            rs = getRemoteColumns(databaseMetaData, catalogName, remoteDbName, remoteTableName);
+            while (rs.next()) {
+                int dataType = rs.getInt("DATA_TYPE");
+                int arrayDimensions = 0;
+                if (dataType == Types.ARRAY) {
+                    String columnName = rs.getString("COLUMN_NAME");
+                    try (PreparedStatement pstmt = conn.prepareStatement(
+                            String.format("SELECT array_ndims(%s) FROM %s.%s LIMIT 1",
+                                    columnName, remoteDbName, remoteTableName))) {
+                        try (ResultSet arrayRs = pstmt.executeQuery()) {
+                            if (arrayRs.next()) {
+                                arrayDimensions = arrayRs.getInt(1);
+                            }
+                        }
+                    }
+                }
+                tableSchema.add(new JdbcFieldSchema(rs, arrayDimensions));
+            }
+        } catch (SQLException e) {
+            throw new JdbcClientException("failed to get jdbc columns info for remote table `%s.%s`: %s",
+                    remoteDbName, remoteTableName, Util.getRootCauseMessage(e));
+        } finally {
+            close(rs, conn);
+        }
+        return tableSchema;
+    }
+
     @Override
     protected String[] getTableTypes() {
         return new String[] {"TABLE", "PARTITIONED TABLE", "VIEW", "MATERIALIZED VIEW", "FOREIGN TABLE"};
@@ -99,8 +157,38 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
             case "json":
             case "jsonb":
                 return ScalarType.createStringType();
-            default:
-                return Type.UNSUPPORTED;
+            default: {
+                if (fieldSchema.getDataType() == Types.ARRAY && pgType.startsWith("_")) {
+                    return convertArrayType(fieldSchema);
+                } else {
+                    return Type.UNSUPPORTED;
+                }
+            }
+        }
+    }
+
+    private Type convertArrayType(JdbcFieldSchema fieldSchema) {
+        int arrayDimensions = fieldSchema.getArrayDimensions().orElse(0);
+        if (arrayDimensions == 0) {
+            return Type.UNSUPPORTED;
+        }
+
+        String innerType = fieldSchema.getDataTypeName().orElse("unknown").substring(1);
+
+        boolean isSupported = Arrays.asList(supportedInnerType).contains(innerType);
+        if (!isSupported) {
+            return Type.UNSUPPORTED;
+        }
+        if (innerType.equals("bpchar")) {
+            innerType = "text";
+        }
+        JdbcFieldSchema innerFieldSchema = new JdbcFieldSchema(fieldSchema);
+        innerFieldSchema.setDataTypeName(Optional.of(innerType));
+        Type arrayInnerType = jdbcTypeToDoris(innerFieldSchema);
+        Type arrayType = ArrayType.create(arrayInnerType, true);
+        for (int i = 1; i < arrayDimensions; i++) {
+            arrayType = ArrayType.create(arrayType, true);
         }
+        return arrayType;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java
index 7d643fac25c7e4..8cbd27ed684072 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java
@@ -37,6 +37,7 @@ public class JdbcFieldSchema {
     // For NUMERIC/DECIMAL, columnSize means precision.
     protected Optional<Integer> columnSize;
     protected Optional<Integer> decimalDigits;
+    protected Optional<Integer> arrayDimensions;
     // Base number (usually 10 or 2)
     protected int numPrecRadix;
     // column description
@@ -53,6 +54,7 @@ public JdbcFieldSchema(JdbcFieldSchema other) {
         this.dataTypeName = other.dataTypeName;
         this.columnSize = other.columnSize;
         this.decimalDigits = other.decimalDigits;
+        this.arrayDimensions = other.arrayDimensions;
         this.numPrecRadix = other.numPrecRadix;
         this.remarks = other.remarks;
         this.charOctetLength = other.charOctetLength;
@@ -71,6 +73,19 @@ public JdbcFieldSchema(ResultSet rs) throws SQLException {
         this.charOctetLength = rs.getInt("CHAR_OCTET_LENGTH");
     }
 
+    public JdbcFieldSchema(ResultSet rs, int arrayDimensions) throws SQLException {
+        this.columnName = rs.getString("COLUMN_NAME");
+        this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null"));
+        this.dataTypeName = Optional.ofNullable(rs.getString("TYPE_NAME"));
+        this.columnSize = getInteger(rs, "COLUMN_SIZE");
+        this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS");
+        this.numPrecRadix = rs.getInt("NUM_PREC_RADIX");
+        this.isAllowNull = rs.getInt("NULLABLE") != DatabaseMetaData.columnNoNulls;
+        this.remarks = rs.getString("REMARKS");
+        this.charOctetLength = rs.getInt("CHAR_OCTET_LENGTH");
+        this.arrayDimensions = Optional.of(arrayDimensions);
+    }
+
     public JdbcFieldSchema(ResultSet rs, Map<String, String> dataTypeOverrides) throws SQLException {
         this.columnName = rs.getString("COLUMN_NAME");
         this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null"));
diff --git a/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out
index 1950235745749e..e2993ffbaa7fcd 100644
--- a/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out
+++ b/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out
@@ -2234,6 +2234,17 @@ doris3	20
 1	abc	def	2022-10-11	1	2	3	2022-10-22T10:59:59	34.123	false	12.123456	10.16.10.14/32	10.16.10.14	ff:ff:ff:ff:ff:aa	1010101010	01010	1	{"id": 1}	(1.0,1.0)	{1.0,1.0,1.0}	[(1.0,1.0),(2.0,2.0)]	(2.0,2.0),(1.0,1.0)	((1.0,1.0),(2.0,2.0),(2.0,1.0))	((1.0,1.0),(2.0,2.0),(2.0,1.0))	<(0.0,0.0),1.0>
 2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	2	\N	\N	\N	\N	\N	\N	\N	\N
 
+-- !select_all_arr_types --
+1	["A", "B", "C"]	["Hello", "World"]	["Text1", "Text2"]	["2024-01-01", "2024-01-02"]	[10, 20]	[100, 200]	[1000, 2000]	["2024-01-01 10:00:00.000000", "2024-01-02 12:00:00.000000"]	["2024-01-01 18:00:00.000000", "2024-01-02 20:00:00.000000"]	[1, 0]	[1.1, 2.2]	[3.3, 4.4]
+2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
+3	["A", "B", "C", null]	["Hello", "World", null]	["Text1", "Text2", null]	["2024-01-01", "2024-01-02", null]	[10, 20, null]	[100, 200, null]	[1000, 2000, null]	["2024-01-01 10:00:00.000000", "2024-01-02 12:00:00.000000", null]	["2024-01-01 18:00:00.000000", "2024-01-02 20:00:00.000000", null]	[1, 0, null]	[1.1, 2.2, null]	[3.3, 4.4, null]
+
+-- !select_all_arr2d_types --
+1	[["A", "B"], ["C", "D"]]	[["Hello", "World"], ["Foo", "Bar"]]	[["Text1", "Text2"], ["Text3", "Text4"]]	[["2024-01-01", "2024-01-02"], ["2024-02-01", "2024-02-02"]]	[[10, 20], [30, 40]]	[[100, 200], [300, 400]]	[[1000, 2000], [3000, 4000]]	[["2024-01-01 10:00:00.000000", "2024-01-02 12:00:00.000000"], ["2024-03-01 10:00:00.000000", "2024-03-02 12:00:00.000000"]]	[["2024-01-01 18:00:00.000000", "2024-01-02 20:00:00.000000"], ["2024-03-01 18:00:00.000000", "2024-03-02 20:00:00.000000"]]	[[1, 0], [0, 1]]	[[1.1, 2.2], [3.3, 4.4]]	[[5.5, 6.6], [7.7, 8.8]]
+2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
+3	[["A", null], ["C", "D"]]	[["Hello", null], ["Foo", "Bar"]]	[["Text1", null], ["Text3", "Text4"]]	[["2024-01-01", null], ["2024-02-01", "2024-02-02"]]	[[10, null], [30, 40]]	[[100, null], [300, 400]]	[[1000, null], [3000, 4000]]	[["2024-01-01 10:00:00.000000", null], ["2024-03-01 10:00:00.000000", "2024-03-02 12:00:00.000000"]]	[["2024-01-01 18:00:00.000000", null], ["2024-03-01 18:00:00.000000", "2024-03-02 20:00:00.000000"]]	[[1, null], [0, 1]]	[[1.1, null], [3.3, 4.4]]	[[5.5, null], [7.7, 8.8]]
+4	[["A", "B"], [null, null]]	[["Hello", "World"], [null, null]]	[["Text1", "Text2"], [null, null]]	[["2024-01-01", "2024-01-02"], [null, null]]	[[10, 20], [null, null]]	[[100, 200], [null, null]]	[[1000, 2000], [null, null]]	[["2024-01-01 10:00:00.000000", "2024-01-02 12:00:00.000000"], [null, null]]	[["2024-01-01 18:00:00.000000", "2024-01-02 20:00:00.000000"], [null, null]]	[[1, 0], [null, null]]	[[1.1, 2.2], [null, null]]	[[5.5, 6.6], [null, null]]
+
 -- !ctas --
 1	abc	def	2022-10-11	1	2	3	2022-10-22T10:59:59	34.123	false	12.123456	10.16.10.14/32	10.16.10.14	ff:ff:ff:ff:ff:aa	1010101010	01010	1	{"id": 1}	(1.0,1.0)	{1.0,1.0,1.0}	[(1.0,1.0),(2.0,2.0)]	(2.0,2.0),(1.0,1.0)	((1.0,1.0),(2.0,2.0),(2.0,1.0))	((1.0,1.0),(2.0,2.0),(2.0,1.0))	<(0.0,0.0),1.0>
 2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	2	\N	\N	\N	\N	\N	\N	\N	\N
diff --git a/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy
index 7f597493b9183d..c898f31ace65b3 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy
@@ -146,6 +146,11 @@ suite("test_pg_jdbc_catalog", "p0,external,pg,external_docker,external_docker_pg
         // test select all types
         order_qt_select_all_types """select * from ${test_all_types}; """
 
+        // test select all array types
+        order_qt_select_all_arr_types """select *  from test_all_support_types_array order by 1;"""
+
+        order_qt_select_all_arr2d_types """select *  from test_all_support_types_array_2d order by 1;"""
+
         // test test ctas
         sql """ drop table if exists internal.${internal_db_name}.${test_ctas} """
         sql """ create table internal.${internal_db_name}.${test_ctas}

From 8492f02559aa5b489fa4f927a0a12e950c1f1597 Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyongkang@selectdb.com>
Date: Thu, 28 Nov 2024 23:49:29 +0800
Subject: [PATCH 071/399] [fix](jdbc catalog) set
 `enable_jdbc_cast_predicate_push_down` default true (#44548)

### What problem does this PR solve?

Related PR: #42102

Problem Summary:

We found that after disabling pushdown of predicates with implicit
casts, some users experienced slower queries, so we temporarily changed
this parameter back to the default behavior.
---
 .../src/main/java/org/apache/doris/qe/SessionVariable.java  | 2 +-
 .../jdbc/test_clickhouse_jdbc_catalog.groovy                | 6 ------
 .../jdbc/test_jdbc_catalog_push_cast.groovy                 | 2 ++
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index f977c3ccb05f17..3c180be9d42802 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -726,7 +726,7 @@ public class SessionVariable implements Serializable, Writable {
     @VariableMgr.VarAttr(name = ENABLE_JDBC_CAST_PREDICATE_PUSH_DOWN, needForward = true,
             description = {"是否允许将带有 CAST 表达式的谓词下推到 JDBC 外部表。",
                     "Whether to allow predicates with CAST expressions to be pushed down to JDBC external tables."})
-    public boolean enableJdbcCastPredicatePushDown = false;
+    public boolean enableJdbcCastPredicatePushDown = true;
 
     @VariableMgr.VarAttr(name = ROUND_PRECISE_DECIMALV2_VALUE)
     public boolean roundPreciseDecimalV2Value = false;
diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
index fba50c3341e8a8..7875ac4244e372 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
@@ -83,17 +83,11 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex
             sql("select * from ts where from_unixtime(ts,'yyyyMMdd') >= '2022-01-01';")
             contains """QUERY: SELECT "id", "ts" FROM "doris_test"."ts" WHERE ((FROM_UNIXTIME("ts", '%Y%m%d') >= '2022-01-01'))"""
         }
-        explain {
-            sql("select * from ts where nvl(ts,null) >= '1';")
-            contains """QUERY: SELECT "id", "ts" FROM "doris_test"."ts"""
-        }
         order_qt_func_push2 """select * from ts where ts <= unix_timestamp(from_unixtime(ts,'yyyyMMdd'));"""
-        sql "set enable_jdbc_cast_predicate_push_down = true;"
         explain {
             sql("select * from ts where ts <= unix_timestamp(from_unixtime(ts,'yyyy-MM-dd'));")
             contains """QUERY: SELECT "id", "ts" FROM "doris_test"."ts" WHERE (("ts" <= toUnixTimestamp(FROM_UNIXTIME("ts", '%Y-%m-%d'))))"""
         }
-        sql "set enable_jdbc_cast_predicate_push_down = false;"
 
         order_qt_dt_with_tz """ select * from dt_with_tz order by id; """
 
diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_catalog_push_cast.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_catalog_push_cast.groovy
index 6271a1619a5fac..3585f5fbb50941 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_catalog_push_cast.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_catalog_push_cast.groovy
@@ -34,6 +34,8 @@ suite("test_jdbc_catalog_push_cast", "p0,external,mysql,external_docker,external
             "driver_class" = "com.mysql.cj.jdbc.Driver"
         );"""
 
+        sql "set enable_jdbc_cast_predicate_push_down = false;"
+
         sql "use jdbc_catalog_push_cast.doris_test"
 
         qt_sql """select * from test_cast where date(datetime_c) = '2022-01-01';"""

From 2a5e992a3a67662d40c81933eb0568a7429eb026 Mon Sep 17 00:00:00 2001
From: abmdocrt <Yukang.Lian2022@gmail.com>
Date: Fri, 29 Nov 2024 10:22:33 +0800
Subject: [PATCH 072/399] [Enhancement](cloud) Add concurrency for tablet stats
 processing (#43903)

---
 .../java/org/apache/doris/common/Config.java  |  4 ++
 .../doris/catalog/CloudTabletStatMgr.java     | 54 ++++++++++++-------
 2 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 3caa137c858440..2ef391bb7ab390 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3222,6 +3222,10 @@ public static int metaServiceRpcRetryTimes() {
             + "model is set to 300 times, which is approximately 5 minutes by default."})
     public static int auto_start_wait_to_resume_times = 300;
 
+    @ConfField(description = {"Get tablet stat task的最大并发数。",
+        "Maximal concurrent num of get tablet stat job."})
+    public static int max_get_tablet_stat_task_threads_num = 4;
+
     // ATTN: DONOT add any config not related to cloud mode here
     // ATTN: DONOT add any config not related to cloud mode here
     // ATTN: DONOT add any config not related to cloud mode here
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
index 0ba2686060e1f5..309b433572c942 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
@@ -36,6 +36,10 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 /*
  * CloudTabletStatMgr is for collecting tablet(replica) statistics from backends.
@@ -47,6 +51,9 @@ public class CloudTabletStatMgr extends MasterDaemon {
     // <(dbId, tableId) -> OlapTable.Statistics>
     private volatile Map<Pair<Long, Long>, OlapTable.Statistics> cloudTableStatsMap = new HashMap<>();
 
+    private static final ExecutorService GET_TABLET_STATS_THREAD_POOL = Executors.newFixedThreadPool(
+            Config.max_get_tablet_stat_task_threads_num);
+
     public CloudTabletStatMgr() {
         super("cloud tablet stat mgr", Config.tablet_stat_update_interval_second * 1000);
     }
@@ -103,28 +110,37 @@ protected void runAfterCatalogReady() {
             reqList.add(builder.build());
         }
 
+        List<Future<Void>> futures = new ArrayList<>();
         for (GetTabletStatsRequest req : reqList) {
-            GetTabletStatsResponse resp;
-            try {
-                resp = getTabletStats(req);
-            } catch (RpcException e) {
-                LOG.info("get tablet stats exception:", e);
-                continue;
-            }
-
-            if (resp.getStatus().getCode() != MetaServiceCode.OK) {
-                continue;
-            }
-
-            if (LOG.isDebugEnabled()) {
-                int i = 0;
-                for (TabletIndexPB idx : req.getTabletIdxList()) {
-                    LOG.debug("db_id: {} table_id: {} index_id: {} tablet_id: {} size: {}",
-                            idx.getDbId(), idx.getTableId(), idx.getIndexId(), idx.getTabletId(),
-                            resp.getTabletStats(i++).getDataSize());
+            futures.add(GET_TABLET_STATS_THREAD_POOL.submit(() -> {
+                GetTabletStatsResponse resp = GetTabletStatsResponse.newBuilder().build();
+                try {
+                    resp = getTabletStats(req);
+                } catch (RpcException e) {
+                    LOG.warn("get tablet stats exception:", e);
+                }
+                if (resp.getStatus().getCode() != MetaServiceCode.OK) {
+                    LOG.warn("get tablet stats return failed.");
+                }
+                if (LOG.isDebugEnabled()) {
+                    int i = 0;
+                    for (TabletIndexPB idx : req.getTabletIdxList()) {
+                        LOG.debug("db_id: {} table_id: {} index_id: {} tablet_id: {} size: {}",
+                                idx.getDbId(), idx.getTableId(), idx.getIndexId(),
+                                idx.getTabletId(), resp.getTabletStats(i++).getDataSize());
+                    }
                 }
+                updateTabletStat(resp);
+                return null;
+            }));
+        }
+
+        try {
+            for (Future<Void> future : futures) {
+                future.get();
             }
-            updateTabletStat(resp);
+        } catch (InterruptedException | ExecutionException e) {
+            LOG.error("Error waiting for get tablet stats tasks to complete", e);
         }
 
         LOG.info("finished to get tablet stat of all backends. cost: {} ms",

From bc67fc9ef0e39507697c01cb74771f8a59b83acd Mon Sep 17 00:00:00 2001
From: zhengyu <zhangzhengyu@selectdb.com>
Date: Fri, 29 Nov 2024 10:40:01 +0800
Subject: [PATCH 073/399] [fix](cloud) serialize cache init to avoid unstable
 cache pick (#44429)

The original paralleled cache init will causing unstable pick of cache
base path because the choice depends on the order of init which could be
different after each BE reboot. Thus, cause cache missing and duplicate
cache block across multiple caches (disk space waste).

This commit will serialize the init process of multiple cache and using
fixed order, i.e. the order explicitly declared in be conf:
file_cache_path.

Signed-off-by: zhengyu <zhangzhengyu@selectdb.com>
---
 be/src/runtime/exec_env_init.cpp | 26 +++++++-------------------
 1 file changed, 7 insertions(+), 19 deletions(-)

diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 706fd7efd07d0f..b7f926cc3b4512 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -425,36 +425,24 @@ void ExecEnv::init_file_cache_factory(std::vector<doris::CachePath>& cache_paths
                    << ", reason=" << rest.msg();
         exit(-1);
     }
-    std::vector<std::thread> file_cache_init_threads;
 
-    std::list<doris::Status> cache_status;
+    doris::Status cache_status;
     for (auto& cache_path : cache_paths) {
         if (cache_path_set.find(cache_path.path) != cache_path_set.end()) {
             LOG(WARNING) << fmt::format("cache path {} is duplicate", cache_path.path);
             continue;
         }
 
-        file_cache_init_threads.emplace_back([&, status = &cache_status.emplace_back()]() {
-            *status = doris::io::FileCacheFactory::instance()->create_file_cache(
-                    cache_path.path, cache_path.init_settings());
-        });
-
-        cache_path_set.emplace(cache_path.path);
-    }
-
-    for (std::thread& thread : file_cache_init_threads) {
-        if (thread.joinable()) {
-            thread.join();
-        }
-    }
-    for (const auto& status : cache_status) {
-        if (!status.ok()) {
+        cache_status = doris::io::FileCacheFactory::instance()->create_file_cache(
+                cache_path.path, cache_path.init_settings());
+        if (!cache_status.ok()) {
             if (!doris::config::ignore_broken_disk) {
-                LOG(FATAL) << "failed to init file cache, err: " << status;
+                LOG(FATAL) << "failed to init file cache, err: " << cache_status;
                 exit(-1);
             }
-            LOG(WARNING) << "failed to init file cache, err: " << status;
+            LOG(WARNING) << "failed to init file cache, err: " << cache_status;
         }
+        cache_path_set.emplace(cache_path.path);
     }
 }
 

From 797ac3b27e7aba275609a566d646fc56d925b958 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Fri, 29 Nov 2024 11:46:32 +0800
Subject: [PATCH 074/399] [Chore](runtime-filter) improve rf::debug_string and
 add some check about rf (#44697)

improve rf::debug_string and add some check about rf
---
 be/src/exprs/runtime_filter.cpp              | 14 +++++++-------
 be/src/exprs/runtime_filter.h                |  9 ++++++---
 be/src/exprs/runtime_filter_slots.h          |  9 ++-------
 be/src/pipeline/exec/hashjoin_build_sink.cpp |  7 ++++++-
 4 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index efe96ae3395432..19789fbd743cbc 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -1535,11 +1535,13 @@ void IRuntimeFilter::update_runtime_filter_type_to_profile(uint64_t local_merge_
 
 std::string IRuntimeFilter::debug_string() const {
     return fmt::format(
-            "RuntimeFilter: (id = {}, type = {}, is_broadcast: {}, "
-            "build_bf_cardinality: {}, ignored: {},  error_msg: {}",
+            "RuntimeFilter: (id = {}, type = {}, is_broadcast: {}, ignored: {}"
+            "build_bf_cardinality: {}, dependency: {}, synced_size: {}, has_local_target: {}, "
+            "has_remote_target: {},error_msg: [{}]",
             _filter_id, to_string(_runtime_filter_type), _is_broadcast_join,
-            _wrapper->get_build_bf_cardinality(), _wrapper->is_ignored(),
-            _wrapper->_context->err_msg);
+            _wrapper->_context->ignored, _wrapper->get_build_bf_cardinality(),
+            _dependency ? _dependency->debug_string() : "none", _synced_size, _has_local_target,
+            _has_remote_target, _wrapper->_context->err_msg);
 }
 
 Status IRuntimeFilter::merge_from(const RuntimePredicateWrapper* wrapper) {
@@ -1592,9 +1594,7 @@ RuntimeFilterType IRuntimeFilter::get_real_type() {
 }
 
 bool IRuntimeFilter::need_sync_filter_size() {
-    return (type() == RuntimeFilterType::IN_OR_BLOOM_FILTER ||
-            type() == RuntimeFilterType::BLOOM_FILTER) &&
-           _wrapper->get_build_bf_cardinality() && !_is_broadcast_join;
+    return _wrapper->get_build_bf_cardinality() && !_is_broadcast_join;
 }
 
 void IRuntimeFilter::update_filter(std::shared_ptr<RuntimePredicateWrapper> wrapper,
diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index 50ee52865be6d6..a1fdfbf07d239d 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -354,9 +354,12 @@ class IRuntimeFilter {
     void set_finish_dependency(
             const std::shared_ptr<pipeline::CountedFinishDependency>& dependency);
 
-    int64_t get_synced_size() const { return _synced_size; }
-
-    bool isset_synced_size() const { return _synced_size != -1; }
+    int64_t get_synced_size() const {
+        if (_synced_size == -1) {
+            throw Status::InternalError("sync filter size meet error, filter: {}", debug_string());
+        }
+        return _synced_size;
+    }
 
 protected:
     // serialize _wrapper to protobuf
diff --git a/be/src/exprs/runtime_filter_slots.h b/be/src/exprs/runtime_filter_slots.h
index 3c18735e4e82ce..7eb8c131c8a303 100644
--- a/be/src/exprs/runtime_filter_slots.h
+++ b/be/src/exprs/runtime_filter_slots.h
@@ -62,9 +62,8 @@ class VRuntimeFilterSlots {
     }
 
     // use synced size when this rf has global merged
-    static uint64_t get_real_size(IRuntimeFilter* runtime_filter, uint64_t hash_table_size) {
-        return runtime_filter->isset_synced_size() ? runtime_filter->get_synced_size()
-                                                   : hash_table_size;
+    static uint64_t get_real_size(IRuntimeFilter* filter, uint64_t hash_table_size) {
+        return filter->need_sync_filter_size() ? filter->get_synced_size() : hash_table_size;
     }
 
     Status ignore_filters(RuntimeState* state) {
@@ -119,10 +118,6 @@ class VRuntimeFilterSlots {
             }
 
             if (filter->get_real_type() == RuntimeFilterType::BLOOM_FILTER) {
-                if (filter->need_sync_filter_size() != filter->isset_synced_size()) {
-                    return Status::InternalError("sync filter size meet error, filter: {}",
-                                                 filter->debug_string());
-                }
                 RETURN_IF_ERROR(filter->init_bloom_filter(
                         get_real_size(filter.get(), local_hash_table_size)));
             }
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index 5c1fa9daa0d837..a15dbb8dd03111 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -112,13 +112,14 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
     if (_closed) {
         return Status::OK();
     }
+    auto p = _parent->cast<HashJoinBuildSinkOperatorX>();
     Defer defer {[&]() {
         if (!_should_build_hash_table) {
             return;
         }
         // The build side hash key column maybe no need output, but we need to keep the column in block
         // because it is used to compare with probe side hash key column
-        auto p = _parent->cast<HashJoinBuildSinkOperatorX>();
+
         if (p._should_keep_hash_key_column && _build_col_ids.size() == 1) {
             p._should_keep_column_flags[_build_col_ids[0]] = true;
         }
@@ -143,6 +144,10 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
             RETURN_IF_ERROR(_runtime_filter_slots->send_filter_size(state, 0, _finish_dependency));
             RETURN_IF_ERROR(_runtime_filter_slots->ignore_all_filters());
         } else {
+            if (p._shared_hashtable_controller &&
+                !p._shared_hash_table_context->complete_build_stage) {
+                return Status::InternalError("close before sink meet eos");
+            }
             auto* block = _shared_state->build_block.get();
             uint64_t hash_table_size = block ? block->rows() : 0;
             {

From c6cb3b646704acc6f52a7b837c24c555c85afc30 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Fri, 29 Nov 2024 11:48:55 +0800
Subject: [PATCH 075/399] [fix](clone) Fix wrong clone file path (#44746)

The argument `clone_file` is a basename, not a full path
---
 be/src/olap/task/engine_clone_task.cpp | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp
index 75cbcf68e956c1..bea1d3b1a91e89 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -82,8 +82,9 @@ namespace {
 /// if not equal, then return error
 /// return value: if binlog file not exist, then return to binlog file path
 Result<std::string> check_dest_binlog_valid(const std::string& tablet_dir,
+                                            const std::string& clone_dir,
                                             const std::string& clone_file, bool* skip_link_file) {
-    std::string to;
+    std::string from, to;
     std::string new_clone_file = clone_file;
     if (clone_file.ends_with(".binlog")) {
         // change clone_file suffix from .binlog to .dat
@@ -92,6 +93,7 @@ Result<std::string> check_dest_binlog_valid(const std::string& tablet_dir,
         // change clone_file suffix from .binlog-index to .idx
         new_clone_file.replace(clone_file.size() - 13, 13, ".idx");
     }
+    from = fmt::format("{}/{}", clone_dir, clone_file);
     to = fmt::format("{}/_binlog/{}", tablet_dir, new_clone_file);
 
     // check to to file exist
@@ -106,10 +108,10 @@ Result<std::string> check_dest_binlog_valid(const std::string& tablet_dir,
     }
 
     LOG(WARNING) << "binlog file already exist. "
-                 << "tablet_dir=" << tablet_dir << ", clone_file=" << clone_file;
+                 << "tablet_dir=" << tablet_dir << ", clone_file=" << from << ", to=" << to;
 
     std::string clone_file_md5sum;
-    status = io::global_local_filesystem()->md5sum(clone_file, &clone_file_md5sum);
+    status = io::global_local_filesystem()->md5sum(from, &clone_file_md5sum);
     if (!status.ok()) {
         return ResultError(std::move(status));
     }
@@ -697,7 +699,6 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d
             continue;
         }
 
-        auto from = fmt::format("{}/{}", clone_dir, clone_file);
         std::string to;
         if (clone_file.ends_with(".binlog") || clone_file.ends_with(".binlog-index")) {
             if (!contain_binlog) {
@@ -706,7 +707,8 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d
                 break;
             }
 
-            if (auto&& result = check_dest_binlog_valid(tablet_dir, clone_file, &skip_link_file);
+            if (auto&& result =
+                        check_dest_binlog_valid(tablet_dir, clone_dir, clone_file, &skip_link_file);
                 result) {
                 to = std::move(result.value());
             } else {
@@ -718,6 +720,7 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d
         }
 
         if (!skip_link_file) {
+            auto from = fmt::format("{}/{}", clone_dir, clone_file);
             status = io::global_local_filesystem()->link_file(from, to);
             if (!status.ok()) {
                 return status;

From 9daa3b77dc895d879e5ab62ff91b58d8473e6ffb Mon Sep 17 00:00:00 2001
From: seawinde <wusi@selectdb.com>
Date: Fri, 29 Nov 2024 14:42:38 +0800
Subject: [PATCH 076/399] [test](mtmv) Add inject statistic when mv rewrite
 regression test to make sure rewrite result stable (#43785)

### What problem does this PR solve?

The result of successful rewriting by the cbo optimizer depends on the
statistics.
The priority of the optimizer consumption statistics in descending order
is
1. the injected statistics
2. the statistics reported by be
3. and the statistics analyzed actively.

When the pipeline runs the case, the statistics reported by be may not
be timely. Therefore, the outcome that leads to the cbo optimizer's
successful selection of overwrites is not very certain, so the
statistics are currently injected manually in the test cases
---
 ...AbstractMaterializedViewAggregateRule.java |  28 ++-
 .../data/mv_p0/ssb/q_1_1/q_1_1.out            |   7 +-
 .../doris/regression/suite/Suite.groovy       |   3 +-
 .../agg_have_dup_base.groovy                  |   2 +
 .../diffrent_serialize.groovy                 |   2 +
 .../agg_state/test_agg_state_max_by.groovy    |   3 +
 .../mv_p0/case_ignore/case_ignore.groovy      |   3 +
 .../contain_cast/mv_contains_cast.groovy      |   2 +
 .../suites/mv_p0/count_star/count_star.groovy |   2 +
 .../suites/mv_p0/dis_26495/dis_26495.groovy   |   2 +
 .../suites/mv_p0/k1ap2spa/k1ap2spa.groovy     |   2 +
 .../suites/mv_p0/k1s2m3/k1s2m3.groovy         |   1 +
 .../k1s2m3_auto_inc/k1s2m3_auto_inc.groovy    |   1 +
 .../multi_agg_with_same_slot.groovy           |   1 +
 .../multi_slot_k123p/multi_slot_k123p.groovy  |   1 +
 .../multi_slot_k1a2p2ap3p.groovy              |   1 +
 .../multi_slot_k1a2p2ap3ps.groovy             |   1 +
 .../multi_slot_k1p2ap3p.groovy                |   1 +
 .../multi_slot_k1p2ap3ps.groovy               |   1 +
 .../multi_slot_multi_mv.groovy                |   1 +
 .../mv_p0/mv_with_view/mv_with_view.groovy    |   1 +
 .../mv_p0/null_insert/null_insert.groovy      |   1 +
 .../routine_load_hll/routine_load_hll.groovy  |   1 +
 .../ssb/multiple_ssb/multiple_ssb.groovy      |   1 +
 .../multiple_ssb_between.groovy               |   2 +
 .../suites/mv_p0/ssb/q_1_1/q_1_1.groovy       |  20 +-
 .../suites/mv_p0/ssb/q_2_1/q_2_1.groovy       |   2 +
 .../suites/mv_p0/ssb/q_3_1/q_3_1.groovy       |   2 +
 .../suites/mv_p0/ssb/q_4_1/q_4_1.groovy       |   2 +
 .../suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy |   1 +
 .../suites/mv_p0/sum_count/sum_count.groovy   |   1 +
 .../sum_divede_count/sum_devide_count.groovy  |   2 +
 .../suites/mv_p0/test_28741/test_28741.groovy |   2 +-
 .../test_approx_count_distinct.groovy         |   1 +
 .../suites/mv_p0/test_base/test_base.groovy   |   1 +
 .../mv_p0/test_casewhen/test_casewhen.groovy  |   1 +
 .../test_create_mv/test_create_mv.groovy      |   2 +
 .../test_create_mv_complex_type.groovy        |   2 +
 .../mv_p0/test_doc_e4/test_doc_e4.groovy      |   1 +
 .../test_dup_group_by_mv_abs.groovy           |   1 +
 .../test_dup_group_by_mv_plus.groovy          |   1 +
 .../test_dup_mv_abs/test_dup_mv_abs.groovy    |   1 +
 .../test_dup_mv_bin/test_dup_mv_bin.groovy    |   1 +
 .../test_dup_mv_bitmap_hash.groovy            |   3 +
 .../test_dup_mv_plus/test_dup_mv_plus.groovy  |   2 +-
 .../test_dup_mv_repeat.groovy                 |   1 +
 .../test_dup_mv_year/test_dup_mv_year.groovy  |   1 +
 .../test_duplicate_mv.groovy                  |   2 +
 .../suites/mv_p0/test_mv_dp/test_mv_dp.groovy |   2 +
 .../mv_p0/test_mv_mor/test_mv_mor.groovy      |   2 +
 .../mv_p0/test_mv_mow/test_mv_mow.groovy      |   1 +
 .../suites/mv_p0/test_ndv/test_ndv.groovy     |   1 +
 .../suites/mv_p0/test_nvl/test_nvl.groovy     |   1 +
 .../suites/mv_p0/test_o2/test_o2.groovy       |   1 +
 .../mv_p0/test_substr/test_substr.groovy      |   1 +
 .../mv_p0/test_tbl_name/test_tbl_name.groovy  |   1 +
 .../test_upper_alias/test_upper_alias.groovy  |   1 +
 .../test_user_activity.groovy                 |   1 +
 .../suites/mv_p0/unique/unique.groovy         |   1 +
 .../testAggQueryOnAggMV1.groovy               |   1 +
 .../testAggQueryOnAggMV10.groovy              |   1 +
 .../testAggQueryOnAggMV11.groovy              |   1 +
 .../testAggQueryOnAggMV2.groovy               |   1 +
 .../testAggQueryOnAggMV3.groovy               |   2 +-
 .../testAggQuqeryOnAggMV5.groovy              |   1 +
 .../testAggQuqeryOnAggMV6.groovy              |   1 +
 .../testAggQuqeryOnAggMV7.groovy              |   2 +
 ...stAggTableCountDistinctInBitmapType.groovy |   1 +
 ...testAggregateMVCalcAggFunctionQuery.groovy |   1 +
 .../testBitmapUnionInQuery.groovy             |   2 +-
 .../testCountDistinctToBitmap.groovy          |   4 +
 .../testIncorrectMVRewriteInSubquery.groovy   |   1 +
 .../testIncorrectRewriteCountDistinct.groovy  |   1 +
 .../testJoinOnLeftProjectToJoin.groovy        |   3 +
 .../mv_p0/ut/testNDVToHll/testNDVToHll.groovy |   1 +
 .../testOrderByQueryOnProjectView.groovy      |   1 +
 .../testProjectionMV1.groovy                  |   1 +
 .../testProjectionMV2.groovy                  |   1 +
 .../testProjectionMV3.groovy                  |   1 +
 .../testProjectionMV4.groovy                  |   1 +
 .../ut/testQueryOnStar/testQueryOnStar.groovy |   1 +
 .../testSelectMVWithTableAlias.groovy         |   2 +
 .../testSingleMVMultiUsage.groovy             |   1 +
 .../mv_p0/ut/testSubQuery/testSubQuery.groovy |   2 +-
 .../testUnionDistinct.groovy                  |   2 +
 .../mv/agg_on_none_agg/agg_on_none_agg.groovy |   6 +
 .../agg_optimize_when_uniform.groovy          |   4 +
 .../mv/agg_variety/agg_variety.groovy         |   6 +
 .../aggregate_with_roll_up.groovy             |   3 +
 .../aggregate_without_roll_up.groovy          |   6 +
 .../mv/availability/grace_period.groovy       |   2 +
 .../materialized_view_switch.groovy           |   5 +
 .../range_date_datetrunc_part_up.groovy       |   1 +
 .../range_date_part_up.groovy                 |   2 +
 .../range_date_part_up_rewrite.groovy         |   3 +
 .../range_datetime_part_up_rewrite.groovy     |   4 +
 .../mv/dimension/dimension_1.groovy           |   3 +
 .../mv/dimension/dimension_2_3.groovy         |   3 +
 .../mv/dimension/dimension_2_4.groovy         |   4 +
 .../mv/dimension/dimension_2_5.groovy         |   4 +
 .../mv/dimension/dimension_2_6.groovy         |   6 +
 .../mv/dimension/dimension_2_full_join.groovy |   2 +
 .../dimension/dimension_2_inner_join.groovy   |   3 +
 .../dimension_2_left_anti_join.groovy         |   3 +
 .../mv/dimension/dimension_2_left_join.groovy |   3 +
 .../dimension_2_left_semi_join.groovy         |   3 +
 .../dimension_2_right_anti_join.groovy        |   3 +
 .../dimension/dimension_2_right_join.groovy   |   3 +
 .../dimension_2_right_semi_join.groovy        |   3 +
 .../mv/dimension/dimension_self_conn.groovy   |   3 +
 .../dimension_2_join_agg.groovy               |  15 +-
 .../dimension_join_agg_negative.groovy        |   3 +
 .../filter_equal_or_notequal.groovy           |   3 +
 .../full_join_filter.groovy                   |   3 +
 .../inner_join_filter.groovy                  |   3 +
 .../left_anti_join_filter.groovy              |   3 +
 .../left_join_filter.groovy                   |   3 +
 .../left_semi_join_filter.groovy              |   3 +
 .../right_anti_join_filter.groovy             |   3 +
 .../right_join_filter.groovy                  |   3 +
 .../right_semi_join_filter.groovy             |   3 +
 .../mv/direct_query/direct_query.groovy       |   3 +
 .../mv/grouping_sets/grouping_sets.groovy     |   3 +
 .../join/dphyp_inner/inner_join_dphyp.groovy  |   4 +
 .../join/dphyp_outer/outer_join_dphyp.groovy  |   4 +
 .../mv/join/inner/inner_join.groovy           |   4 +
 .../mv/join/left_outer/outer_join.groovy      |  11 +
 .../inner_join_infer_and_derive.groovy        |   3 +
 .../left_join_infer_and_derive.groovy         |   2 +
 .../right_join_infer_and_derive.groovy        |   3 +
 .../mv/negative/negative_test.groovy          |   3 +
 .../mv/nested/nested_materialized_view.groovy |   4 +
 .../mv/nested_mtmv/nested_mtmv.groovy         | 202 +++++++-----------
 .../nested_mtmv_rewrite_switch.groovy         |   3 +
 .../anti/other_join_conjuncts_anti.groovy     |   4 +
 .../inner/other_join_conjuncts_inner.groovy   |   4 +
 .../outer/other_join_conjuncts_outer.groovy   |   4 +
 .../semi/other_join_conjuncts_semi.groovy     |   4 +
 .../mv/partition_mv_rewrite.groovy            |   4 +
 .../mv/same_name/sync_async_same_name.groovy  |   2 +
 .../mv/scan/scan_table.groovy                 |   4 +
 .../single_table_without_aggregate.groovy     |   1 +
 .../union_all_compensate.groovy               |   3 +
 .../partition_curd_union_rewrite.groovy       |   3 +
 .../partition_curd_union_rewrite_hive.groovy  |   3 +
 .../usercase_union_rewrite.groovy             |   3 +
 .../unsafe_equals/null_un_safe_equals.groovy  |   2 +
 .../mv/variant/variant_mv.groovy              |   3 +
 .../with_auth/with_select_table_auth.groovy   |   3 +
 .../query_with_sql_limit.groovy               |   4 +
 .../mv/newMv/aggHaveDupBase.groovy            |   1 +
 .../mv/newMv/case_ignore.groovy               |   2 +
 .../mv/newMv/dup_gb_mv_abs.groovy             |   1 +
 .../mv/newMv/dup_gb_mv_plus.groovy            |   1 +
 .../mv/newMv/dup_mv_abs.groovy                |   2 +-
 .../mv/newMv/dup_mv_bin.groovy                |   1 +
 .../mv/newMv/dup_mv_bm_hash.groovy            |   1 +
 .../mv/newMv/dup_mv_plus.groovy               |   2 +-
 .../mv/newMv/dup_mv_year.groovy               |   1 +
 .../mv/newMv/multi_slot1.groovy               |   1 +
 .../mv/newMv/multi_slot2.groovy               |   1 +
 .../mv/newMv/multi_slot3.groovy               |   1 +
 .../mv/newMv/multi_slot4.groovy               |   1 +
 .../mv/newMv/multi_slot5.groovy               |   1 +
 .../mv/newMv/multi_slot6.groovy               |   1 +
 .../mv/newMv/mv_with_view.groovy              |   1 +
 .../mv/newMv/single_slot.groovy               |   2 +-
 .../mv/newMv/sum_devide_count.groovy          |   2 +-
 .../mv/newMv/unique_mv.groovy                 |   1 +
 .../mv/ut/MVMultiUsage.groovy                 |   3 +
 .../nereids_syntax_p0/mv/ut/MVWithAs.groovy   |   3 +
 .../mv/ut/aggCDInBitmap.groovy                |   4 +-
 .../mv/ut/aggMVCalcAggFun.groovy              |   2 +-
 .../mv/ut/aggOnAggMV1.groovy                  |   1 +
 .../mv/ut/aggOnAggMV10.groovy                 |   1 +
 .../mv/ut/aggOnAggMV11.groovy                 |   3 +
 .../mv/ut/aggOnAggMV2.groovy                  |   1 +
 .../mv/ut/aggOnAggMV3.groovy                  |   1 +
 .../mv/ut/aggOnAggMV5.groovy                  |   2 +
 .../mv/ut/aggOnAggMV6.groovy                  |   1 +
 .../mv/ut/aggOnAggMV7.groovy                  |   1 +
 .../mv/ut/bitmapUnionIn.groovy                |   3 +
 .../mv/ut/distinctQuery.groovy                |   2 +
 .../mv/ut/incMVReInSub.groovy                 |   3 +
 .../mv/ut/incRewriteCD.groovy                 |   3 +
 .../mv/ut/joinOnCalcToJoin.groovy             |   5 +
 .../mv/ut/joinOnLeftPToJoin.groovy            |   5 +
 .../mv/ut/onlyGroupBy.groovy                  |   3 +
 .../mv/ut/orderByOnPView.groovy               |   2 +
 .../nereids_syntax_p0/mv/ut/projectMV1.groovy |   3 +
 .../nereids_syntax_p0/mv/ut/projectMV2.groovy |   3 +
 .../nereids_syntax_p0/mv/ut/projectMV3.groovy |   3 +
 .../nereids_syntax_p0/mv/ut/projectMV4.groovy |   3 +
 .../nereids_syntax_p0/mv/ut/subQuery.groovy   |   2 +-
 .../nereids_syntax_p0/mv/ut/unionDis.groovy   |   2 +
 .../nereids_syntax_p0/rollup/agg.groovy       |   1 +
 .../nereids_syntax_p0/rollup/agg_date.groovy  |   1 +
 .../nereids_syntax_p0/rollup/bitmap.groovy    |   1 +
 .../nereids_syntax_p0/rollup/date.groovy      |   1 +
 .../nereids_syntax_p0/rollup/hll/hll.groovy   |   1 +
 .../hll_with_light_sc.groovy                  |   1 +
 201 files changed, 532 insertions(+), 170 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
index 743003dca6bb81..5a854f085b2f77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
@@ -487,27 +487,37 @@ private static boolean isGroupByEqualsByFunctionDependency(
         if (!viewShuttledExprQueryBasedSet.containsAll(queryGroupShuttledExpression)) {
             return false;
         }
-        Set<Expression> viewShouldUniformExpressionSet = new HashSet<>();
+        Set<Expression> viewShouldRemovedExpressionSet = new HashSet<>();
+        Set<Expression> viewScanShouldReservedExpressionSet = new HashSet<>();
         // calc the group by expr which is needed to roll up and should be uniform
-        for (Map.Entry<Expression, Expression> expressionEntry :
+        for (Map.Entry<Expression, Expression> expressionMappingEntry :
                 viewShuttledExprQueryBasedToViewGroupByExprMap.entrySet()) {
-            if (queryGroupShuttledExpression.contains(expressionEntry.getKey())) {
-                // the group expr which query has, do not require uniform
-                continue;
+            if (queryGroupShuttledExpression.contains(expressionMappingEntry.getKey())) {
+                // the group expr which query has, do not require eliminate
+                viewScanShouldReservedExpressionSet.add(
+                        viewShuttledExprToScanExprMapping.get(expressionMappingEntry.getValue()));
+            } else {
+                // the view expression which is more than query's expression, should try to eliminate
+                viewShouldRemovedExpressionSet.add(expressionMappingEntry.getValue());
             }
-            viewShouldUniformExpressionSet.add(expressionEntry.getValue());
         }
 
         DataTrait dataTrait = tempRewrittenPlan.computeDataTrait();
-        for (Expression shouldUniformExpr : viewShouldUniformExpressionSet) {
-            Expression viewScanExpression = viewShuttledExprToScanExprMapping.get(shouldUniformExpr);
+        for (Expression viewShouldRemovedExpr : viewShouldRemovedExpressionSet) {
+            Expression viewScanExpression = viewShuttledExprToScanExprMapping.get(viewShouldRemovedExpr);
             if (viewScanExpression == null) {
                 return false;
             }
             if (!(viewScanExpression instanceof Slot)) {
                 return false;
             }
-            if (!dataTrait.isUniform((Slot) viewScanExpression)) {
+            if (!dataTrait.isUniform((Slot) viewScanExpression)
+                    && Sets.intersection(dataTrait.calEqualSet((Slot) viewScanExpression),
+                    viewScanShouldReservedExpressionSet).isEmpty()) {
+                // Such as query is l_orderkey#0, l_linenumber#1, o_custkey#17, l_partkey#2
+                // view is ps_partkey#25, o_orderkey#16, l_orderkey#0, l_linenumber#1, o_custkey#17, l_partkey#2
+                // If want to check the group by equality, ps_partkey#25, o_orderkey#16 should be uniform
+                // or should be equal any of [ l_orderkey#0, l_linenumber#1, o_custkey#17, l_partkey#2]
                 return false;
             }
         }
diff --git a/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out b/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out
index 1da845620e8dd7..c00bbc9377409e 100644
--- a/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out
+++ b/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out
@@ -6,10 +6,13 @@
 19930101	1	1	1	1	1	1	1	1	1	1	100	1	1	1	2023-06-09	shipmode	name	address	city	nation	AMERICA	phone	mktsegment	name	address	city	nation	AMERICA	phone	name	MFGR#1	category	brand	color	type	4	container
 19930101	1	1	1	1	1	1	1	1	1	1	100	1	1	1	2023-06-09	shipmode	name	address	city	nation	AMERICA	phone	mktsegment	name	address	city	nation	AMERICA	phone	name	MFGR#1	category	brand	color	type	4	container
 19930101	2	2	2	2	2	2	2	2	2	2	2	2	2	2	2023-06-09	shipmode	name	address	city	nation	region	phone	mktsegment	name	address	city	nation	region	phone	name	mfgr	category	brand	color	type	4	container
+19930101	2	2	2	2	2	2	2	2	2	2	2	2	2	2	2023-06-09	shipmode	name	address	city	nation	region	phone	mktsegment	name	address	city	nation	region	phone	name	mfgr	category	brand	color	type	4	container
+19930101	2	2	2	2	2	2	2	2	2	2	2	2	2	2	2023-06-09	shipmode	name	address	city	nation	region	phone	mktsegment	name	address	city	nation	region	phone	name	mfgr	category	brand	color	type	4	container
+19930101	2	2	2	2	2	2	2	2	2	2	2	2	2	2	2023-06-09	shipmode	name	address	city	nation	region	phone	mktsegment	name	address	city	nation	region	phone	name	mfgr	category	brand	color	type	4	container
 
 -- !select_mv --
-4
+16
 
 -- !select --
-4
+16
 
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 2a5ba5acc414b7..62e6032578fd6a 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1983,9 +1983,8 @@ class Suite implements GroovyInterceptable {
             check { result ->
                 boolean success = true;
                 for (String mv_name : mv_names) {
-                    success = success && result.contains("${mv_name} chose")
+                    Assert.assertEquals(true, result.contains("${mv_name} chose"))
                 }
-                Assert.assertEquals(true, success)
             }
         }
     }
diff --git a/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy b/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy
index 8f3ec4171e32b0..ece526b1e24e7a 100644
--- a/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy
+++ b/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy
@@ -60,6 +60,8 @@ suite ("agg_have_dup_base") {
     qt_select_mv "select unix_timestamp(k1) tmp,sum(k2) from d_table group by tmp order by tmp;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k4 set stats ('row_count'='5');"""
+
     mv_rewrite_success("select k1,sum(k2),max(k2) from d_table group by k1;", "k12s3m")
 
     mv_rewrite_success("select k1,sum(k2) from d_table group by k1;", "k12s3m")
diff --git a/regression-test/suites/mv_p0/agg_state/diffrent_serialize/diffrent_serialize.groovy b/regression-test/suites/mv_p0/agg_state/diffrent_serialize/diffrent_serialize.groovy
index 78d26f476fa7a8..005646b89e47b2 100644
--- a/regression-test/suites/mv_p0/agg_state/diffrent_serialize/diffrent_serialize.groovy
+++ b/regression-test/suites/mv_p0/agg_state/diffrent_serialize/diffrent_serialize.groovy
@@ -37,6 +37,8 @@ suite ("diffrent_serialize") {
     sql "insert into d_table select 2,2,2,'b';"
     sql "insert into d_table select 3,3,null,'c';"
 
+    sql """alter table d_table modify column k4 set stats ('row_count'='7');"""
+
     createMV("create materialized view mv1_1 as select k1,bitmap_intersect(to_bitmap(k2)) from d_table group by k1;")
     createMV("create materialized view mv1 as select k1,bitmap_agg(k2) from d_table group by k1;")
     createMV("create materialized view mv1_2 as select k1, multi_distinct_group_concat(k4) from d_table group by k1 order by k1;")
diff --git a/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy b/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy
index 1e34c70ccec900..29190396230dce 100644
--- a/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy
+++ b/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy
@@ -69,6 +69,7 @@ suite ("test_agg_state_max_by") {
     qt_select_star "select * from d_table order by 1,2;"
     mv_rewrite_success("select k1,max_by(k2,k3) from d_table group by k1 order by 1,2;", "k1mb")
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k4 set stats ('row_count'='8');"""
     mv_rewrite_success("select k1,max_by(k2,k3) from d_table group by k1 order by 1,2;", "k1mb")
     qt_select_mv "select k1,max_by(k2,k3) from d_table group by k1 order by 1,2;"
 
@@ -101,6 +102,7 @@ suite ("test_agg_state_max_by") {
     qt_select_star "select * from d_table order by 1,2;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k4 set stats ('row_count'='8');"""
     sql "analyze table d_table with sync;"
     sql """set enable_stats=false;"""
 
@@ -114,6 +116,7 @@ suite ("test_agg_state_max_by") {
     qt_select_mv "select k1,max_by(k2,abs(k3)) from d_table group by k1 order by 1,2;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k4 set stats ('row_count'='8');"""
     mv_rewrite_success("select k1,max_by(k2+k3,abs(k3)) from d_table group by k1 order by 1,2;", "k1mbcp1")
     mv_rewrite_success("select k1,max_by(k2+k3,k3) from d_table group by k1 order by 1,2;", "k1mbcp2")
     mv_rewrite_success("select k1,max_by(k2,abs(k3)) from d_table group by k1 order by 1,2;", "k1mbcp3")
diff --git a/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy b/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy
index 1134d930185b9a..271eb0cb5ed190 100644
--- a/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy
+++ b/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy
@@ -37,6 +37,8 @@ suite ("case_ignore") {
     sql "insert into d_table select 2,2,2,'b';"
     sql "insert into d_table select 3,-3,null,'c';"
 
+    sql """alter table d_table modify column k4 set stats ('row_count'='4');"""
+
     createMV ("create materialized view k12a as select K1,abs(K2) from d_table;")
 
     sql "insert into d_table select -4,-4,-4,'d';"
@@ -53,6 +55,7 @@ suite ("case_ignore") {
     qt_select_mv "select K1,abs(K2) from d_table order by K1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k4 set stats ('row_count'='8');"""
     mv_rewrite_success("select k1,abs(k2) from d_table order by k1;", "k12a")
     mv_rewrite_success("select K1,abs(K2) from d_table order by K1;", "k12a")
 
diff --git a/regression-test/suites/mv_p0/contain_cast/mv_contains_cast.groovy b/regression-test/suites/mv_p0/contain_cast/mv_contains_cast.groovy
index e45798ba7984cd..af81035c3f9a6f 100644
--- a/regression-test/suites/mv_p0/contain_cast/mv_contains_cast.groovy
+++ b/regression-test/suites/mv_p0/contain_cast/mv_contains_cast.groovy
@@ -91,6 +91,8 @@ suite("mv_contains_cast") {
 
     order_qt_query_before "${query_sql}"
 
+    sql """alter table test modify column event_type set stats ('row_count'='10');"""
+
     createMV("""
     CREATE MATERIALIZED VIEW sync_mv
     AS
diff --git a/regression-test/suites/mv_p0/count_star/count_star.groovy b/regression-test/suites/mv_p0/count_star/count_star.groovy
index 52edda58276ef6..4d951d76fa98a9 100644
--- a/regression-test/suites/mv_p0/count_star/count_star.groovy
+++ b/regression-test/suites/mv_p0/count_star/count_star.groovy
@@ -64,6 +64,8 @@ suite ("count_star") {
     qt_select_mv "select count(*) from d_table where k3=1;"
 
     sql """set enable_stats=true;"""
+
+    sql """alter table d_table modify column k4 set stats ('row_count'='8');"""
     mv_rewrite_success("select k1,k4,count(*) from d_table group by k1,k4;", "kstar")
     mv_rewrite_success("select k1,k4,count(*) from d_table where k1=1 group by k1,k4;", "kstar")
     mv_rewrite_fail("select k1,k4,count(*) from d_table where k3=1 group by k1,k4;", "kstar")
diff --git a/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy b/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy
index e30af70b0041b3..14ebb67ddc8168 100644
--- a/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy
+++ b/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy
@@ -28,6 +28,8 @@ suite ("dis_26495") {
 
     sql """insert into doris_test values (1,2,max_by_state(1,2));"""
 
+    sql """alter table doris_test modify column agg_st_1 set stats ('row_count'='1');"""
+
     streamLoad {
         table "doris_test"
         set 'column_separator', ','
diff --git a/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy b/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy
index cfe2a5307d7ed4..b4c2921669663a 100644
--- a/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy
+++ b/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy
@@ -52,6 +52,8 @@ suite ("k1ap2spa") {
 
     sql """set enable_stats=true;"""
 
+    sql """alter table d_table modify column k1 set stats ('row_count'='5');"""
+
     mv_rewrite_success("select abs(k1)+1 t,sum(abs(k2+1)) from d_table group by t order by t;", "k1ap2spa")
 
 }
diff --git a/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy b/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy
index ecae837b4fe33c..25319e78807d7e 100644
--- a/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy
+++ b/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy
@@ -37,6 +37,7 @@ suite ("k1s2m3") {
     sql "insert into d_table select 2,2,2,'b';"
     sql "insert into d_table select 3,-3,null,'c';"
 
+    sql """alter table d_table modify column k1 set stats ('row_count'='6');"""
     createMV("create materialized view k1s2m3 as select k1,sum(k2*k3) from d_table group by k1;")
 
     sql "insert into d_table select -4,-4,-4,'d';"
diff --git a/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy b/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy
index 367cd9863c1e3e..17583741e68b91 100644
--- a/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy
+++ b/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy
@@ -57,5 +57,6 @@ suite ("k1s2m3_auto_inc") {
     qt_select_mv "select k3,sum(abs(k2+1)) from d_table group by k3 order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='2');"""
     mv_rewrite_success("select k3,sum(abs(k2+1)) from d_table group by k3 order by 1;", "k3ap2spa")
 }
diff --git a/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy b/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy
index a1d254b333e211..538d050fec8a9d 100644
--- a/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy
+++ b/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy
@@ -64,6 +64,7 @@ suite ("multi_agg_with_same_slot") {
 
     sql """set enable_stats=true;"""
 
+    sql """alter table d_table modify column k1 set stats ('row_count'='5');"""
     mv_rewrite_success("select k1,k2,avg(k3),max(k3) from d_table group by k1,k2 order by 1,2;", "kmv")
     mv_rewrite_success("select k1,k2,avg(k3)+max(k3) from d_table group by k1,k2 order by 1,2;", "kmv")
     mv_rewrite_success("select k1,k2,avg(k3)+max(k3) from d_table group by grouping sets((k1),(k1,k2),()) order by 1,2;", "kmv")
diff --git a/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy b/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy
index a4a0be3ca8df65..154939a6bb5ef3 100644
--- a/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy
+++ b/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy
@@ -63,5 +63,6 @@ suite ("multi_slot_k123p") {
     qt_select_mv "select k1,version() from d_table order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='5');"""
     mv_rewrite_success("select k1,k2+k3 from d_table order by k1;", "k123p")
 }
diff --git a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy
index e64adbb06b2988..fc722c5a07fbe7 100644
--- a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy
+++ b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy
@@ -51,6 +51,7 @@ suite ("multi_slot_k1a2p2ap3p") {
     qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from d_table order by abs(k1)+k2+1,abs(k2+2)+k3+3;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='7');"""
     mv_rewrite_success("select abs(k1)+k2+1,abs(k2+2)+k3+3 from d_table order by abs(k1)+k2+1,abs(k2+2)+k3+3", "k1a2p2ap3p")
 
 }
diff --git a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy
index 3a2f58f38d65a9..b2292084504199 100644
--- a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy
+++ b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy
@@ -61,6 +61,7 @@ suite ("multi_slot_k1a2p2ap3ps") {
     qt_select_base "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2 order by abs(k1)+k2;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by abs(k1)+k2+1", "k1a2p2ap3ps")
 
     mv_rewrite_fail("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2 order by abs(k1)+k2", "k1a2p2ap3ps")
diff --git a/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy b/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy
index fa67fd3889d93e..75bbd3fb4b7bb8 100644
--- a/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy
+++ b/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy
@@ -50,5 +50,6 @@ suite ("multi_slot_k1p2ap3p") {
     qt_select_mv "select k1+1,abs(k2+2)+k3+3 from d_table order by k1+1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1+1,abs(k2+2)+k3+3 from d_table order by k1+1;", "k1p2ap3p")
 }
diff --git a/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy b/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy
index 9895bc6872e740..2ed75247e8ea29 100644
--- a/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy
+++ b/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy
@@ -50,5 +50,6 @@ suite ("multi_slot_k1p2ap3ps") {
     qt_select_mv "select k1+1,sum(abs(k2+2)+k3+3) from d_table group by k1+1 order by k1+1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='5');"""
     mv_rewrite_success("select k1+1,sum(abs(k2+2)+k3+3) from d_table group by k1+1 order by k1+1;", "k1p2ap3ps")
 }
diff --git a/regression-test/suites/mv_p0/multi_slot_multi_mv/multi_slot_multi_mv.groovy b/regression-test/suites/mv_p0/multi_slot_multi_mv/multi_slot_multi_mv.groovy
index 5527a511e9df2a..172e3217dafe4d 100644
--- a/regression-test/suites/mv_p0/multi_slot_multi_mv/multi_slot_multi_mv.groovy
+++ b/regression-test/suites/mv_p0/multi_slot_multi_mv/multi_slot_multi_mv.groovy
@@ -82,6 +82,7 @@ suite ("multi_slot_multi_mv") {
     qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from d_table order by abs(k1)+k2+1,abs(k2+2)+k3+3;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='5');"""
     for (def i = 0; i < retry_times; ++i) {
         boolean is_k1a2p2ap3p = false
         boolean is_k1a2p2ap3ps = false
diff --git a/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy b/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy
index 236bde3bf6c828..f2e389e867a564 100644
--- a/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy
+++ b/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy
@@ -67,6 +67,7 @@ suite ("mv_with_view") {
     qt_select_mv "select * from v_k124 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from d_table order by k1;", "k312")
 
     sql """
diff --git a/regression-test/suites/mv_p0/null_insert/null_insert.groovy b/regression-test/suites/mv_p0/null_insert/null_insert.groovy
index 015406d80a3796..7b6ea058f2d90b 100644
--- a/regression-test/suites/mv_p0/null_insert/null_insert.groovy
+++ b/regression-test/suites/mv_p0/null_insert/null_insert.groovy
@@ -76,6 +76,7 @@ suite ("null_insert") {
                     GROUP BY date,vid,os,ver,ip_country;"""
 
     sql """set enable_stats=true;"""
+    sql """alter table test modify column date set stats ('row_count'='3');"""
     mv_rewrite_success("""SELECT date, vid, os, ver, ip_country, hll_union(hll_hash(uid))
                 FROM test
                 GROUP BY date,vid,os,ver,ip_country;""", "mv_test")
diff --git a/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy b/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy
index a872e23efc4cce..d34024ebdf5068 100644
--- a/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy
+++ b/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy
@@ -58,5 +58,6 @@ suite ("routine_load_hll") {
     qt_select_mv "select time_stamp, hll_union_agg(device_id) from test group by time_stamp order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table test modify column event_id set stats ('row_count'='2');"""
     mv_rewrite_success("select time_stamp, hll_union_agg(device_id) from test group by time_stamp order by 1;", "m_view")
 }
diff --git a/regression-test/suites/mv_p0/ssb/multiple_ssb/multiple_ssb.groovy b/regression-test/suites/mv_p0/ssb/multiple_ssb/multiple_ssb.groovy
index eb8cfd1618fea3..e0dc916c05b66a 100644
--- a/regression-test/suites/mv_p0/ssb/multiple_ssb/multiple_ssb.groovy
+++ b/regression-test/suites/mv_p0/ssb/multiple_ssb/multiple_ssb.groovy
@@ -240,6 +240,7 @@ suite ("multiple_ssb") {
     qt_select_count_3 "select LO_ORDERPRIORITY, count(1) from lineorder_flat where LO_ORDERPRIORITY in ('1','2','3') group by LO_ORDERPRIORITY order by 1,2;"
 
     sql """set enable_stats=true;"""
+    sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');"""
     mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue
                 FROM lineorder_flat
                 WHERE
diff --git a/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy b/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy
index 71b737279f35bf..4dfdd748a5765b 100644
--- a/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy
+++ b/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy
@@ -151,6 +151,8 @@ suite ("multiple_ssb_between") {
 
     sql "analyze table lineorder_flat with sync;"
     sql """set enable_stats=true;"""
+
+    sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');"""
     
     mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue
                 FROM lineorder_flat
diff --git a/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy b/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy
index 30959801e72af1..c7aa6e67c6ee8e 100644
--- a/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy
+++ b/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy
@@ -75,10 +75,12 @@ suite ("mv_ssb_q_1_1") {
     // Add more data when insert into firstly to make sure use mv easier
     sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) 
             VALUES (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'),
-            (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'),
-            (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'),
-            (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'),
-            (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');"""
+             (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'),
+             (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'),
+             (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'),
+             (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');"""
+
+    sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='6');"""
 
     createMV ("""create materialized view lineorder_q_1_1 as 
                 SELECT LO_ORDERKEY, SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue
@@ -88,12 +90,14 @@ suite ("mv_ssb_q_1_1") {
                     AND LO_ORDERDATE <= 19931231
                     AND LO_DISCOUNT >= 1 AND LO_DISCOUNT <= 3
                     AND LO_QUANTITY < 25
-                GROUP BY
-                    LO_ORDERKEY;""")
+                GROUP BY LO_ORDERKEY;""")
 
     sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) 
-            VALUES (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');"""
-
+            VALUES 
+            (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container'),
+            (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container'),
+            (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container'),
+            (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');"""
     qt_select_star "select * from lineorder_flat order by 1,2, P_MFGR;"
 
     sql "analyze table lineorder_flat with sync;"
diff --git a/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy b/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy
index 6ead8380eed59b..0ac5f0245ef8ae 100644
--- a/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy
+++ b/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy
@@ -93,6 +93,8 @@ suite ("mv_ssb_q_2_1") {
 
     qt_select_star "select * from lineorder_flat order by 1,2,P_MFGR;"
 
+  sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');"""
+
     mv_rewrite_success("""SELECT
                 SUM(LO_REVENUE), (LO_ORDERDATE DIV 10000) AS YEAR,
                 P_BRAND
diff --git a/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy b/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy
index 2b170fe3b2fefd..6b333d97050630 100644
--- a/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy
+++ b/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy
@@ -98,6 +98,8 @@ suite ("mv_ssb_q_3_1") {
 
     sql """analyze table lineorder_flat with sync;"""
 
+    sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');"""
+
     mv_rewrite_success("""SELECT
                 C_NATION,
                 S_NATION, (LO_ORDERDATE DIV 10000) AS YEAR,
diff --git a/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy b/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy
index c094e8e00c9eaf..361f60997e3da6 100644
--- a/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy
+++ b/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy
@@ -92,6 +92,8 @@ suite ("mv_ssb_q_4_1") {
 
     sql """analyze table lineorder_flat with sync;"""
 
+    sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');"""
+
     mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR,
                 C_NATION,
                 SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit
diff --git a/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy b/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy
index 1baf21b547f062..ebeb23d96e75e5 100644
--- a/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy
+++ b/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy
@@ -119,6 +119,7 @@ suite ("q_4_1_r1") {
                 GROUP BY YEAR, C_NATION
                 ORDER BY YEAR ASC, C_NATION ASC;"""
     sql """set enable_stats=true;"""
+    sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');"""
     mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR,
             C_NATION,
             SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit
diff --git a/regression-test/suites/mv_p0/sum_count/sum_count.groovy b/regression-test/suites/mv_p0/sum_count/sum_count.groovy
index b47e51999b7c42..137aabc4e16a89 100644
--- a/regression-test/suites/mv_p0/sum_count/sum_count.groovy
+++ b/regression-test/suites/mv_p0/sum_count/sum_count.groovy
@@ -51,6 +51,7 @@ suite ("sum_count") {
     sql "analyze table d_table with sync;"
     sql """set enable_stats=true;"""
 
+    sql """alter table d_table modify column k1 set stats ('row_count'='8');"""
     mv_rewrite_success("select k1,k4,sum(k2),count(k2) from d_table group by k1,k4;", "kavg")
     
     qt_select_mv "select k1,k4,sum(k2),count(k2) from d_table group by k1,k4 order by 1,2;"
diff --git a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy
index 4e6374e300b09e..85a3f7137b9e40 100644
--- a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy
+++ b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy
@@ -46,6 +46,7 @@ suite ("sum_devide_count") {
     sql """analyze table d_table with sync;"""
     sql """set enable_stats=false;"""
 
+
     mv_rewrite_success("select k1,k4,sum(k2)/count(k2) from d_table group by k1,k4 order by k1,k4;", "kavg")
     qt_select_mv "select k1,k4,sum(k2)/count(k2) from d_table group by k1,k4 order by k1,k4;"
 
@@ -59,6 +60,7 @@ suite ("sum_devide_count") {
     qt_select_mv "select sum(k2)/count(k2) from d_table;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='5');"""
     mv_rewrite_success("select k1,k4,sum(k2)/count(k2) from d_table group by k1,k4 order by k1,k4;", "kavg")
 
     mv_rewrite_success("select k1,sum(k2)/count(k2) from d_table group by k1 order by k1;", "kavg")
diff --git a/regression-test/suites/mv_p0/test_28741/test_28741.groovy b/regression-test/suites/mv_p0/test_28741/test_28741.groovy
index 9fabca41b943fc..88fb3c057da092 100644
--- a/regression-test/suites/mv_p0/test_28741/test_28741.groovy
+++ b/regression-test/suites/mv_p0/test_28741/test_28741.groovy
@@ -69,8 +69,8 @@ suite ("test_28741") {
     sql """set enable_stats=false;"""
 
     mv_rewrite_fail("select b1 from test where t >= '2023-12-20 17:21:00'", "mv_test")
-    qt_select "select b1 from test where t >= '2023-12-20 17:21:00'"
 
     sql """set enable_stats=true;"""
+    sql """alter table test modify column a set stats ('row_count'='2');"""
     mv_rewrite_fail("select b1 from test where t >= '2023-12-20 17:21:00'", "mv_test")
 }
diff --git a/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy b/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy
index 752cf40d9e5f6a..68952da602aa81 100644
--- a/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy
+++ b/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy
@@ -50,6 +50,7 @@ suite ("test_approx_count_distinct") {
     qt_select_mv "select user_id, approx_count_distinct(tag_id) a from user_tags group by user_id order by user_id;"
 
     sql """set enable_stats=true;"""
+    sql """alter table user_tags modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv")
 
     mv_rewrite_success("select user_id, ndv(tag_id) a from user_tags group by user_id order by user_id;", "user_tags_mv")
diff --git a/regression-test/suites/mv_p0/test_base/test_base.groovy b/regression-test/suites/mv_p0/test_base/test_base.groovy
index d2c018febd5ada..97ece8cd6cca68 100644
--- a/regression-test/suites/mv_p0/test_base/test_base.groovy
+++ b/regression-test/suites/mv_p0/test_base/test_base.groovy
@@ -53,6 +53,7 @@ suite ("test_base") {
     qt_select_mv "SELECT id,created_at FROM dwd order by 1, 2;"
 
     sql """set enable_stats=true;"""
+    sql """alter table dwd modify column id set stats ('row_count'='2');"""
     mv_rewrite_success("SELECT created_at, id  FROM dwd where created_at = '2020-09-09 00:00:00' order by 1, 2;", "dwd_mv")
 
     mv_rewrite_success("SELECT id,created_at  FROM dwd where id is not null order by 1, 2;", "dwd_mv")
diff --git a/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy b/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy
index eac65eeef22adf..0c9e0026cb7fbd 100644
--- a/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy
+++ b/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy
@@ -40,5 +40,6 @@ suite ("test_casewhen") {
     qt_select_mv "select store_id, sum(case when sale_amt>10 then 1 else 2 end) from sales_records group by store_id order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table sales_records modify column record_id set stats ('row_count'='4');"""
     mv_rewrite_success("select store_id, sum(case when sale_amt>10 then 1 else 2 end) from sales_records group by store_id order by 1;", "store_amt")
 }
diff --git a/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy
index 776bf2ecf0bd17..8ee24cb5ba2bd9 100644
--- a/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy
+++ b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy
@@ -49,6 +49,8 @@ suite("test_create_mv") {
 
     sql """ insert into ${tableName} values ('2024-03-20 10:00:00', 'a', 'b', 1) """
 
+    sql """alter table test_mv_10010 modify column load_time set stats ('row_count'='1');"""
+
     sql """
         create materialized view mv_1 as
         select 
diff --git a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
index 12790500d2f98e..1e07d69ba82871 100644
--- a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
+++ b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
@@ -37,6 +37,8 @@ suite ("create_mv_complex_type") {
 
     sql """insert into base_table select 1, 100000, 1.0, '{"jsonk1": 123}', [100, 200], {"k1": 10}, {1, 2};"""
 
+    sql """alter table base_table modify column c_int set stats ('row_count'='1');"""
+
     def success = false
 
     // 1. special column - mv dup key
diff --git a/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy b/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy
index 5e99464018e8d7..517b4415bd69ea 100644
--- a/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy
+++ b/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy
@@ -57,6 +57,7 @@ suite ("test_doc_e4") {
     qt_select_mv "select year(k4)+month(k4) from d_table where year(k4) = 2020 order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='3');"""
     mv_rewrite_success("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by 1,2;", "k1a2p2ap3ps")
 
     mv_rewrite_success("select bin(abs(k1)+k2+1),sum(abs(k2+2)+k3+3) from d_table group by bin(abs(k1)+k2+1);", "k1a2p2ap3ps")
diff --git a/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy b/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy
index 5e917e9b0f5bd0..23aeeb207f6898 100644
--- a/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy
+++ b/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy
@@ -53,6 +53,7 @@ suite ("test_dup_group_by_mv_abs") {
     qt_select_mv_sub "select sum(abs(k2)) from d_table group by k1 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,sum(abs(k2)) from d_table group by k1;", "k12sa")
     mv_rewrite_success("select sum(abs(k2)) from d_table group by k1;", "k12sa")
 }
diff --git a/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy b/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy
index ba9f60f6df5026..3551bb8a7fb359 100644
--- a/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy
+++ b/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy
@@ -53,6 +53,7 @@ suite ("test_dup_group_by_mv_plus") {
     qt_select_mv_sub "select sum(k2+1) from d_table group by k1 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,sum(k2+1) from d_table group by k1;", "k12sp")
     mv_rewrite_success("select sum(k2+1) from d_table group by k1;", "k12sp")
 
diff --git a/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy b/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy
index 89088ffb9ea317..f44c9d134f91f0 100644
--- a/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy
+++ b/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy
@@ -65,6 +65,7 @@ suite ("test_dup_mv_abs") {
     qt_select_group_mv_not "select sum(abs(k2)) from d_table group by k3 order by k3;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,abs(k2) from d_table order by k1;", "k12a")
 
     mv_rewrite_success("select abs(k2) from d_table order by k1;", "k12a")
diff --git a/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy b/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy
index a7db2617fe6067..4b8fa5aa6359e8 100644
--- a/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy
+++ b/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy
@@ -64,6 +64,7 @@ suite ("test_dup_mv_bin") {
     qt_select_group_mv_not "select group_concat(bin(k2)) from d_table group by k3 order by k3;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,bin(k2) from d_table order by k1;", "k12b")
 
     mv_rewrite_success("select bin(k2) from d_table order by k1;", "k12b")
diff --git a/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy b/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy
index 659f44de1e93c4..6530fd84cac7a8 100644
--- a/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy
+++ b/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy
@@ -46,6 +46,7 @@ suite ("test_dup_mv_bitmap_hash") {
     mv_rewrite_success("select bitmap_union_count(to_bitmap(k2)) from d_table group by k1 order by k1;", "k1g2bm")
     qt_select_mv "select bitmap_union_count(to_bitmap(k2)) from d_table group by k1 order by k1;"
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select bitmap_union_count(to_bitmap(k2)) from d_table group by k1 order by k1;", "k1g2bm")
 
     createMV "create materialized view k1g3bm as select k1,bitmap_union(bitmap_hash(k3)) from d_table group by k1;"
@@ -58,11 +59,13 @@ suite ("test_dup_mv_bitmap_hash") {
     qt_select_star "select * from d_table order by k1,k2,k3;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     sql """analyze table d_table with sync;"""
     sql """set enable_stats=false;"""
 
     mv_rewrite_success("select k1,bitmap_union_count(bitmap_hash(k3)) from d_table group by k1;", "k1g3bm")
     qt_select_mv_sub "select k1,bitmap_union_count(bitmap_hash(k3)) from d_table group by k1 order by k1;"
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,bitmap_union_count(bitmap_hash(k3)) from d_table group by k1;", "k1g3bm")
 }
diff --git a/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy b/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy
index 5abe03e736b641..ad437db464b3bb 100644
--- a/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy
+++ b/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy
@@ -70,7 +70,7 @@ suite ("test_dup_mv_plus") {
     qt_select_mv "select k1,k2+1 from d_table order by k2;"
 
     sql """set enable_stats=true;"""
-
+    sql """alter table d_table modify column k4 set stats ('row_count'='3');"""
     mv_rewrite_success("select k1,k2+1 from d_table order by k1;", "k12p")
 
     mv_rewrite_success("select k2+1 from d_table order by k1;", "k12p")
diff --git a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy
index 562fba25b2e8a9..5ccdae1bd8a7c5 100644
--- a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy
+++ b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy
@@ -47,6 +47,7 @@ suite ("test_dup_mv_repeat") {
     qt_select_mv "SELECT s AS s, sum(n) / count(DISTINCT dt) AS n FROM  db1 GROUP BY  GROUPING SETS((s)) order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table db1 modify column dt set stats ('row_count'='2');"""
     mv_rewrite_success("SELECT s AS s, sum(n) / count(DISTINCT dt) AS n FROM  db1 GROUP BY  GROUPING SETS((s)) order by 1;",
             "dbviwe")
 }
diff --git a/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy b/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy
index 92a8d6520f3122..18f798f8b6aac3 100644
--- a/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy
+++ b/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy
@@ -45,6 +45,7 @@ suite ("test_dup_mv_year") {
     qt_select_mv "select k1,year(k2) from d_table order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,year(k2) from d_table order by k1;", "k12y")
 
     createMV "create materialized view k13y as select k1,year(k3) from d_table;"
diff --git a/regression-test/suites/mv_p0/test_duplicate_mv/test_duplicate_mv.groovy b/regression-test/suites/mv_p0/test_duplicate_mv/test_duplicate_mv.groovy
index ccd87d792348c3..c1a495a74e20bc 100644
--- a/regression-test/suites/mv_p0/test_duplicate_mv/test_duplicate_mv.groovy
+++ b/regression-test/suites/mv_p0/test_duplicate_mv/test_duplicate_mv.groovy
@@ -39,6 +39,8 @@ suite ("test_duplicate_mv") {
    sql "insert into duplicate_table select 2,3,4,5;"
    sql "insert into duplicate_table select 1,2,3,4;"
 
+    sql """alter table duplicate_table modify column k1 set stats ('row_count'='3');"""
+
     qt_select_star "select * from duplicate_table order by k1;"
 
     mv_rewrite_success("select k1, k2, k3, k4 from duplicate_table group by k1, k2, k3, k4;", "deduplicate")
diff --git a/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy b/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy
index 50f1fd61806f2e..2fadc0b0da91a0 100644
--- a/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy
+++ b/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy
@@ -56,6 +56,7 @@ suite ("test_mv_dp") {
         time 10000 // limit inflight 10s
     }
 */
+
     mv_rewrite_success("""select d,
                         bitmap_union_count(bitmap_from_array(cast(uid_list as array<bigint>))),
                         bitmap_union_count(bitmap_from_array(if(status='success', cast(uid_list as array<bigint>), array())))
@@ -68,6 +69,7 @@ suite ("test_mv_dp") {
                     from dp
                     group by d order by 1;"""
     sql """set enable_stats=true;"""
+    sql """alter table dp modify column d set stats ('row_count'='4');"""
     mv_rewrite_success("""select d,
                         bitmap_union_count(bitmap_from_array(cast(uid_list as array<bigint>))),
                         bitmap_union_count(bitmap_from_array(if(status='success', cast(uid_list as array<bigint>), array())))
diff --git a/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy b/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy
index b7f95123f5f9dd..072c7d905cbea0 100644
--- a/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy
+++ b/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy
@@ -39,6 +39,8 @@ suite ("test_mv_mor") {
     sql "insert into u_table select 1,1,1,2;"
     sql "insert into u_table select 1,2,1,2;"
 
+    sql """alter table u_table modify column k1 set stats ('row_count'='2');"""
+
     // do not match mv coz preagg is off, mv need contains all key column to make row count correct
     mv_rewrite_success("select k1,k2+k3 from u_table order by k1;", "k123p")
     qt_select_mv "select k1,k2+k3 from u_table order by k1;"
diff --git a/regression-test/suites/mv_p0/test_mv_mow/test_mv_mow.groovy b/regression-test/suites/mv_p0/test_mv_mow/test_mv_mow.groovy
index f584730ad1fca3..3fe10c2e50a33a 100644
--- a/regression-test/suites/mv_p0/test_mv_mow/test_mv_mow.groovy
+++ b/regression-test/suites/mv_p0/test_mv_mow/test_mv_mow.groovy
@@ -50,5 +50,6 @@ suite ("test_mv_mow") {
     qt_select_mv "select mv_k1 from `u_table` index `k123p` order by 1;"
     qt_select_mv "select `mv_(k2 + k3)` from `u_table` index `k123p` order by 1;"
     sql """set enable_stats=true;"""
+    sql """alter table u_table modify column k1 set stats ('row_count'='2');"""
     mv_rewrite_success("select k1,k2+k3 from u_table order by k1;", "k123p")
 }
diff --git a/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy b/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy
index d2c9921edefe7a..2d43785ba6fafb 100644
--- a/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy
+++ b/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy
@@ -50,6 +50,7 @@ suite ("test_ndv") {
     qt_select_mv "select user_id, approx_count_distinct(tag_id) a from user_tags group by user_id order by user_id;"
 
     sql """set enable_stats=true;"""
+    sql """alter table user_tags modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv")
 
     mv_rewrite_success("select user_id, ndv(tag_id) a from user_tags group by user_id order by user_id;", "user_tags_mv")
diff --git a/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy b/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy
index c6b1dd2d8a9b1c..12f5765666ef28 100644
--- a/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy
+++ b/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy
@@ -51,6 +51,7 @@ suite ("test_nvl") {
     qt_select_mv "select ifnull(id,0) from dwd order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table dwd modify column id set stats ('row_count'='2');"""
     mv_rewrite_success("select nvl(id,0) from dwd order by 1;", "dwd_mv")
 
     mv_rewrite_success("select ifnull(id,0) from dwd order by 1;", "dwd_mv")
diff --git a/regression-test/suites/mv_p0/test_o2/test_o2.groovy b/regression-test/suites/mv_p0/test_o2/test_o2.groovy
index 665156056c0096..34d31a08cfae7b 100644
--- a/regression-test/suites/mv_p0/test_o2/test_o2.groovy
+++ b/regression-test/suites/mv_p0/test_o2/test_o2.groovy
@@ -60,6 +60,7 @@ suite ("test_o2") {
     qt_select_mv "select ts,metric_name,platform,sum(count_value) from o2_order_events group by ts,metric_name,platform;"
 
     sql """set enable_stats=true;"""
+    sql """alter table o2_order_events modify column ts set stats ('row_count'='2');"""
     mv_rewrite_success("select ts,metric_name,platform,sum(count_value) from o2_order_events group by ts,metric_name,platform;",
             "o2_order_events_mv")
 }
diff --git a/regression-test/suites/mv_p0/test_substr/test_substr.groovy b/regression-test/suites/mv_p0/test_substr/test_substr.groovy
index c43c3ae6483acd..7d6ae82634fd8c 100644
--- a/regression-test/suites/mv_p0/test_substr/test_substr.groovy
+++ b/regression-test/suites/mv_p0/test_substr/test_substr.groovy
@@ -56,6 +56,7 @@ suite ("test_substr") {
     qt_select_mv "SELECT substr(created_at,1,10) as statistic_date, max(dt) as dt FROM dwd  group by substr(created_at,1,10);"
 
     sql """set enable_stats=true;"""
+    sql """alter table dwd modify column id set stats ('row_count'='2');"""
     mv_rewrite_success("SELECT substr(created_at,1,10) as statistic_date, max(dt) as dt FROM dwd  group by substr(created_at,1,10);",
             "dwd_mv")
 }
diff --git a/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy b/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy
index f65e36484e7de9..bbfa8ea6a54d17 100644
--- a/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy
+++ b/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy
@@ -66,6 +66,7 @@ suite ("test_tbl_name") {
         group by id order by 1,2;
         """
     sql """set enable_stats=true;"""
+    sql """alter table functionality_olap modify column id set stats ('row_count'='2');"""
     mv_rewrite_success("""select 
             functionality_olap.id as id,
             sum(functionality_olap.score) as score_max
diff --git a/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy b/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy
index b62ec696224197..69264d838e4d24 100644
--- a/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy
+++ b/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy
@@ -62,6 +62,7 @@ suite ("test_upper_alias") {
     qt_select_mv "SELECT d_a AS d_b FROM test_0401 order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table test_0401 modify column d_b set stats ('row_count'='3');"""
     mv_rewrite_any_success("SELECT upper(d_b) AS d_b FROM test_0401 GROUP BY upper(d_b) order by 1;",
             ["test_0401_mv", "test_0401_mv2"])
 
diff --git a/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy b/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy
index 3d139cbc01bbe7..f971673971a2aa 100644
--- a/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy
+++ b/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy
@@ -53,6 +53,7 @@ suite ("test_user_activity") {
     qt_select_group_mv "select n_dx, percentile_approx(n_duration, 0.5) as p50, percentile_approx(n_duration, 0.90) as p90 FROM u_axx GROUP BY n_dx;"
 
     sql """set enable_stats=true;"""
+    sql """alter table u_axx modify column r_xx set stats ('row_count'='3');"""
     mv_rewrite_success("select n_dx, percentile_approx(n_duration, 0.5) as p50, percentile_approx(n_duration, 0.90) as p90 FROM u_axx GROUP BY n_dx;",
             "session_distribution_2")
 }
diff --git a/regression-test/suites/mv_p0/unique/unique.groovy b/regression-test/suites/mv_p0/unique/unique.groovy
index e2a78c60bc2b0f..d054803678da4f 100644
--- a/regression-test/suites/mv_p0/unique/unique.groovy
+++ b/regression-test/suites/mv_p0/unique/unique.groovy
@@ -73,6 +73,7 @@ suite ("unique") {
     qt_select_star "select * from u_table order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table u_table modify column k1 set stats ('row_count'='3');"""
     mv_rewrite_success("select k3,length(k1),k2 from u_table order by 1,2,3;", "k31l42")
 
     // todo: support match query
diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy
index 70b016de2da80f..f39e8df6cabe6a 100644
--- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy
@@ -36,6 +36,7 @@ suite ("testAggQueryOnAggMV1") {
     sql """insert into emps values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into emps values("2020-01-03",3,"c",3,3,3);"""
 
+sql """alter table emps modify column time_col set stats ('row_count'='4');"""
 
     createMV("create materialized view emps_mv as select deptno, sum(salary), max(commission) from emps group by deptno;")
     createMV("create materialized view emps_mv_count_key as select deptno, count(deptno) from emps group by deptno;")
diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy
index a25ac24404e181..739dde33372cb6 100644
--- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy
@@ -51,6 +51,7 @@ suite ("testAggQueryOnAggMV10") {
     qt_select_mv "select deptno, commission, sum(salary) + 1 from emps group by rollup (deptno, commission) order by 1,2;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select deptno, commission, sum(salary) + 1 from emps group by rollup (deptno, commission);",
diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy
index c26db8e034e66a..a4d69136bc03ca 100644
--- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy
@@ -50,6 +50,7 @@ suite ("testAggQueryOnAggMV11") {
     qt_select_mv "select deptno, count(salary) + count(1) from emps group by deptno order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_fail("select deptno, count(salary) + count(1) from emps group by deptno;", "emps_mv")
diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy
index 945e4e9194910f..5cf7d257fe8b73 100644
--- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy
@@ -53,6 +53,7 @@ suite ("testAggQueryOnAggMV2") {
     qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;",
diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy
index 46457f0d0a81d4..d2f61e04762b2f 100644
--- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy
@@ -38,7 +38,6 @@ suite ("testAggQueryOnAggMV3") {
     sql """insert into emps values("2020-01-04",4,"d",21,4,4);"""
 
 
-
     createMV("create materialized view emps_mv as select deptno, commission, sum(salary) from emps group by deptno, commission;")
 
     sql "analyze table emps with sync;"
@@ -56,6 +55,7 @@ suite ("testAggQueryOnAggMV3") {
     qt_select_mv "select commission, sum(salary) from emps where commission = 100 group by commission order by commission;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select commission, sum(salary) from emps where deptno > 0 and commission * (deptno + commission) = 100 group by commission order by commission;",
diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy
index 1fa4342952cad4..1f14728e3b7b2c 100644
--- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy
@@ -51,6 +51,7 @@ suite ("testAggQuqeryOnAggMV5") {
     qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where sum_salary>10 order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where sum_salary>0;",
diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy
index 3cca0273051f0a..4cafeaa185f4a3 100644
--- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy
@@ -51,6 +51,7 @@ suite ("testAggQuqeryOnAggMV6") {
     qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps where deptno>=20 group by deptno) a where sum_salary>10 order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from emps where deptno>=0 group by deptno) a where sum_salary>10;",
diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy
index 1a606141e38f04..d43b5796e6d7f9 100644
--- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy
@@ -50,6 +50,8 @@ suite ("testAggQuqeryOnAggMV7") {
     qt_select_mv "select deptno, sum(salary) from emps where deptno>=20 group by deptno order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select deptno, sum(salary) from emps where deptno>=20 group by deptno;", "emps_mv")
diff --git a/regression-test/suites/mv_p0/ut/testAggTableCountDistinctInBitmapType/testAggTableCountDistinctInBitmapType.groovy b/regression-test/suites/mv_p0/ut/testAggTableCountDistinctInBitmapType/testAggTableCountDistinctInBitmapType.groovy
index 7e8199b9932e72..26f1fdb0219a2d 100644
--- a/regression-test/suites/mv_p0/ut/testAggTableCountDistinctInBitmapType/testAggTableCountDistinctInBitmapType.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggTableCountDistinctInBitmapType/testAggTableCountDistinctInBitmapType.groovy
@@ -42,6 +42,7 @@ suite ("testAggTableCountDistinctInBitmapType") {
     qt_select_mv "select k1, count(distinct v1) from test_tb group by k1 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table test_tb modify column k1 set stats ('row_count'='3');"""
     explain {
         sql("select k1, count(distinct v1) from test_tb group by k1;")
         contains "bitmap_union_count"
diff --git a/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy b/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy
index 7f2427e275de47..6dc78784956d32 100644
--- a/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy
@@ -50,6 +50,7 @@ suite ("testAggregateMVCalcAggFunctionQuery") {
     qt_select_mv "select deptno, sum(salary + 1) from emps where deptno > 10 group by deptno order by deptno;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_fail("select deptno, sum(salary + 1) from emps where deptno > 10 group by deptno;", "emps_mv")
diff --git a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy
index 69410ae7be5d0d..c1287bfa94bcc4 100644
--- a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy
+++ b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy
@@ -51,7 +51,7 @@ suite ("testBitmapUnionInQuery") {
     qt_select_mv "select user_id, bitmap_count(bitmap_union(to_bitmap(tag_id))) a from user_tags group by user_id having a>1 order by a;"
 
     sql """set enable_stats=true;"""
-
+    sql """alter table user_tags modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv")
 
     mv_rewrite_success("select user_id, bitmap_union_count(to_bitmap(tag_id)) a from user_tags group by user_id having a>1 order by a;",
diff --git a/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy b/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy
index 12152aad369e1d..5cc05db62be1a7 100644
--- a/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy
+++ b/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy
@@ -66,6 +66,10 @@ suite ("testCountDistinctToBitmap") {
     sql """insert into user_tags2 values("2020-01-01",1,"a",1);"""
     sql """insert into user_tags2 values("2020-01-02",2,"b",2);"""
 
+
+    sql """alter table user_tags modify column time_col set stats ('row_count'='3');"""
+    sql """alter table user_tags2 modify column time_col set stats ('row_count'='3');"""
+
     createMV("create materialized view user_tags_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags2 group by user_id;")
 
     sql """insert into user_tags2 values("2020-01-01",1,"a",2);"""
diff --git a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy
index c671e106a10648..698490f0943e88 100644
--- a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy
+++ b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy
@@ -47,6 +47,7 @@ suite ("testIncorrectMVRewriteInSubquery") {
     qt_select_mv "select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags where user_name in (select user_name from user_tags group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;"
 
     sql """set enable_stats=true;"""
+    sql """alter table user_tags modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv")
 
     mv_rewrite_fail("select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags where user_name in (select user_name from user_tags group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;",
diff --git a/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy b/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy
index dbece0c9592364..45230c5b5587d8 100644
--- a/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy
+++ b/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy
@@ -45,6 +45,7 @@ suite ("testIncorrectRewriteCountDistinct") {
     qt_select_mv "select user_name, count(distinct tag_id) from user_tags group by user_name order by user_name;"
 
     sql """set enable_stats=true;"""
+    sql """alter table user_tags modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv")
 
     mv_rewrite_fail("select user_name, count(distinct tag_id) from user_tags group by user_name;", "user_tags_mv")
diff --git a/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy b/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy
index 3caf0000c1824d..8f41fc1ffb5e74 100644
--- a/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy
+++ b/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy
@@ -43,6 +43,8 @@ suite ("testJoinOnLeftProjectToJoin") {
         partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+    sql """alter table depts modify column time_col set stats ('row_count'='3');"""
+
     sql """insert into depts values("2020-01-02",2,"b",2);"""
     sql """insert into depts values("2020-01-03",3,"c",3);"""
     sql """insert into depts values("2020-01-02",2,"b",1);"""
@@ -59,6 +61,7 @@ suite ("testJoinOnLeftProjectToJoin") {
     qt_select_mv "select * from (select deptno , sum(salary) from emps group by deptno) A join (select deptno, max(cost) from depts group by deptno ) B on A.deptno = B.deptno order by A.deptno;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_all_success("select * from (select deptno , sum(salary) from emps group by deptno) A join (select deptno, max(cost) from depts group by deptno ) B on A.deptno = B.deptno;",
             ["emps_mv", "depts_mv"])
 }
diff --git a/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy b/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy
index a9456cb4e79c96..c77f58609ca0cb 100644
--- a/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy
+++ b/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy
@@ -50,6 +50,7 @@ suite ("testNDVToHll") {
     qt_select_mv "select user_id, approx_count_distinct(tag_id) a from user_tags group by user_id order by user_id;"
 
     sql """set enable_stats=true;"""
+    sql """alter table user_tags modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv")
 
     mv_rewrite_success("select user_id, ndv(tag_id) a from user_tags group by user_id order by user_id;", "user_tags_mv")
diff --git a/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy b/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy
index 2aa88d84f3d433..41cf480e880cd0 100644
--- a/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy
+++ b/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy
@@ -49,6 +49,7 @@ suite ("testOrderByQueryOnProjectView") {
     qt_select_mv "select empid from emps order by deptno;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select empid from emps where deptno > 0 order by deptno;", "emps_mv")
diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy
index d95c3e2fc6de88..d578cbddba945d 100644
--- a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy
+++ b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy
@@ -61,6 +61,7 @@ suite ("testProjectionMV1") {
     qt_select_mv "select deptno, sum(empid) from emps group by deptno order by deptno;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select empid, deptno from emps where deptno > 0 order by empid;", "emps_mv")
diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy
index 2f7548b14c8635..b663bfcd2b1738 100644
--- a/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy
+++ b/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy
@@ -51,6 +51,7 @@ suite ("testProjectionMV2") {
     qt_select_base "select name from emps where deptno -1 = 0 order by empid;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select empid + 1 from emps where deptno = 1 order by empid;", "emps_mv")
diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy
index e4ea1aa4de023e..378ec80d6cfd72 100644
--- a/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy
+++ b/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy
@@ -53,6 +53,7 @@ suite ("testProjectionMV3") {
     qt_select_mv2 "select name from emps where deptno = 1 order by empid;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_success("select empid + 1, name from emps where deptno = 1 order by empid;", "emps_mv")
diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy
index 0c6f737deabb29..37a57f80d03cae 100644
--- a/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy
+++ b/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy
@@ -52,6 +52,7 @@ suite ("testProjectionMV4") {
     qt_select_base "select empid from emps where deptno > 1 and empid > 1 order by empid;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     mv_rewrite_fail("select empid from emps where deptno > 1 and empid > 1 order by empid;", "emps_mv")
diff --git a/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy b/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy
index d81129d520729f..7d563883514f72 100644
--- a/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy
+++ b/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy
@@ -58,4 +58,5 @@ suite ("testQueryOnStar") {
     sql """insert into tpch_tiny_region values(1,'a','a');"""
 
     qt_select_mv "select ref_1.`empid` as c0 from tpch_tiny_region as ref_0 left join emps as ref_1 on (ref_0.`r_comment` = ref_1.`name` ) where true order by ref_0.`r_regionkey`,ref_0.`r_regionkey` desc ,ref_0.`r_regionkey`,ref_0.`r_regionkey`;"    
+
 }
diff --git a/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy b/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy
index 0cfd80fcb06b4e..9e3bd886b31ea5 100644
--- a/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy
+++ b/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy
@@ -29,6 +29,8 @@ suite ("testSelectMVWithTableAlias") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+    sql """alter table user_tags modify column time_col set stats ('row_count'='3');"""
+
     sql """insert into user_tags values("2020-01-01",1,"a",1);"""
     sql """insert into user_tags values("2020-01-02",2,"b",2);"""
 
diff --git a/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy b/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy
index 006b72885ec112..73ddb21109bb91 100644
--- a/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy
+++ b/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy
@@ -52,6 +52,7 @@ suite ("testSingleMVMultiUsage") {
     }
     qt_select_mv "select * from (select deptno, empid from emps where deptno>100) A join (select deptno, empid from emps where deptno >200) B using (deptno) order by 1;"
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     explain {
diff --git a/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy b/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy
index 6c1747059d1ae3..cd134a5d48ea40 100644
--- a/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy
+++ b/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy
@@ -38,7 +38,6 @@ suite ("testSubQuery") {
     sql """insert into emps values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into emps values("2020-01-03",3,"c",3,3,3);"""
 
-
     createMV("create materialized view emps_mv as select deptno, empid from emps;")
 
     sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
@@ -52,5 +51,6 @@ suite ("testSubQuery") {
     qt_select_mv "select empid, deptno, salary from emps e1 where empid = (select max(empid) from emps where deptno = e1.deptno) order by deptno;"
 
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 }
diff --git a/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy b/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy
index f504c81f214e91..c6243a3f923616 100644
--- a/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy
+++ b/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy
@@ -35,6 +35,7 @@ suite ("testUnionDistinct") {
     sql """insert into emps values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into emps values("2020-01-03",3,"c",3,3,3);"""
 
+
     createMV("create materialized view emps_mv as select empid, deptno from emps order by empid, deptno;")
 
     sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
@@ -52,6 +53,7 @@ suite ("testUnionDistinct") {
     }
     qt_select_mv "select * from (select empid, deptno from emps where empid >1 union select empid, deptno from emps where empid <0) t order by 1;"
     sql """set enable_stats=true;"""
+    sql """alter table emps modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from emps order by empid;", "emps_mv")
 
     explain {
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy
index d10498705f60c6..db72f1174fdff2 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy
@@ -57,6 +57,8 @@ suite("agg_on_none_agg") {
     (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi');  
     """
 
+    sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');"""
+
     sql """
     drop table if exists lineitem
     """
@@ -95,6 +97,8 @@ suite("agg_on_none_agg") {
     (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');
     """
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+
     sql """
     drop table if exists partsupp
     """
@@ -120,6 +124,8 @@ suite("agg_on_none_agg") {
     (2, 3, 10, 11.01, 'supply2');
     """
 
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     sql """analyze table orders with sync;"""
     sql """analyze table lineitem with sync;"""
     sql """analyze table partsupp with sync;"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_optimize_when_uniform/agg_optimize_when_uniform.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_optimize_when_uniform/agg_optimize_when_uniform.groovy
index 355165055c8ef5..e0f33a2b0db8e5 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_optimize_when_uniform/agg_optimize_when_uniform.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_optimize_when_uniform/agg_optimize_when_uniform.groovy
@@ -120,6 +120,10 @@ suite("agg_optimize_when_uniform") {
     (2, 3, 10, 11.01, 'supply2');
     """
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     sql """analyze table lineitem with sync;"""
     sql """analyze table orders with sync;"""
     sql """analyze table partsupp with sync;"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy
index 3fa6dffca01367..2de47b4bbc2f8e 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy
@@ -95,6 +95,8 @@ suite("agg_variety") {
     (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');
     """
 
+
+
     sql """
     drop table if exists partsupp
     """
@@ -124,6 +126,10 @@ suite("agg_variety") {
     sql """analyze table lineitem with sync;"""
     sql """analyze table partsupp with sync;"""
 
+    sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     def check_rewrite_but_not_chose = { mv_sql, query_sql, mv_name ->
 
         sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
index cda9b21fc4e51d..5b8aa6e00d2308 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
@@ -132,6 +132,9 @@ suite("aggregate_with_roll_up") {
     sql """analyze table partsupp with sync"""
     sql """analyze table lineitem with sync"""
     sql """analyze table orders with sync"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
 
     // multi table
     // filter inside + left + use roll up dimension
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
index 9d60280503cf74..c43bde59acaa3d 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
@@ -121,6 +121,12 @@ suite("aggregate_without_roll_up") {
     (2, 3, 10, 11.01, 'supply2');
     """
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // single table
     // with filter
     def mv1_0 = """
diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy
index 5a6c75f6c59c6e..b9e671f2636b82 100644
--- a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy
@@ -49,6 +49,7 @@ suite("grace_period") {
       "replication_num" = "1"
     );
     """
+    sql """alter table orders_partition modify column o_comment set stats ('row_count'='3');"""
 
     sql """
     drop table if exists lineitem_partition
@@ -81,6 +82,7 @@ suite("grace_period") {
       "replication_num" = "1"
     );
     """
+    sql """alter table lineitem_partition modify column l_comment set stats ('row_count'='3');"""
 
     sql """
     insert into orders_partition values 
diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy
index ae349c6514c469..4af1d778c002b2 100644
--- a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy
@@ -124,6 +124,11 @@ suite("materialized_view_switch") {
       analyze table orders with sync;
       analyze table partsupp with sync;
     """
+
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     def mv_name = """
         select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey
         from lineitem
diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy
index c86f5d6a141f7f..a2fcb2eba15913 100644
--- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_datetrunc_part_up.groovy
@@ -71,6 +71,7 @@ suite("mtmv_range_date_datetrunc_date_part_up") {
     (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29 03:00:00'),
     (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-29 04:00:00');
     """
+    sql """alter table ${tb_name} modify column l_comment set stats ('row_count'='5');"""
 
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up.groovy
index 5be6b1afdcdc44..4c5b01ff59fd3a 100644
--- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up.groovy
@@ -119,6 +119,8 @@ suite("mtmv_range_date_part_up") {
     (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29');
     """
 
+    sql """alter table lineitem_range_date modify column l_comment set stats ('row_count'='7');"""
+
     def get_part = { def mv_name ->
         def part_res = sql """show partitions from ${mv_name}"""
         return part_res.size()
diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy
index 8bf629d93841fb..88d8ad6ea2d84d 100644
--- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy
@@ -113,6 +113,9 @@ suite("mtmv_range_date_part_up_rewrite") {
     (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-31'); 
     """
 
+    sql """alter table lineitem_range_date_union modify column l_comment set stats ('row_count'='7');"""
+    sql """alter table orders_range_date_union modify column o_comment set stats ('row_count'='10');"""
+
     sql """DROP MATERIALIZED VIEW if exists ${mv_prefix}_mv1;"""
     sql """CREATE MATERIALIZED VIEW ${mv_prefix}_mv1 BUILD IMMEDIATE REFRESH AUTO ON MANUAL partition by(date_trunc(`col1`, 'month')) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS  
         select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey from lineitem_range_date_union as t1 left join orders_range_date_union as t2 on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey;"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
index b7e9562249d789..f8e601e64f5819 100644
--- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
@@ -98,6 +98,7 @@ suite("mtmv_range_datetime_part_up_rewrite") {
     (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-29 02:00:00'),
     (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29 00:00:00');
     """
+    sql """alter table lineitem_range_datetime_union modify column l_comment set stats ('row_count'='7');"""
 
     sql """
     insert into orders_range_datetime_union values 
@@ -112,6 +113,7 @@ suite("mtmv_range_datetime_part_up_rewrite") {
     (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29 00:00:00'),
     (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29 02:00:00'); 
     """
+    sql """alter table orders_range_datetime_union modify column o_comment set stats ('row_count'='10');"""
 
     sql """DROP MATERIALIZED VIEW if exists ${mv_prefix}_mv1;"""
     sql """CREATE MATERIALIZED VIEW ${mv_prefix}_mv1 BUILD IMMEDIATE REFRESH AUTO ON MANUAL partition by(date_trunc(`col1`, 'month')) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS  
@@ -174,6 +176,7 @@ suite("mtmv_range_datetime_part_up_rewrite") {
     sql """alter table lineitem_range_datetime_union add partition p4 values [("2023-11-29 03:00:00"), ("2023-11-29 04:00:00"));"""
     sql """insert into lineitem_range_datetime_union values 
         (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-29 03:00:00')"""
+    sql """alter table lineitem_range_datetime_union modify column l_comment set stats ('row_count'='8');"""
     for (int i = 0; i < mv_name_list.size(); i++) {
         mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
         compare_res(query_stmt_list[i] + " order by 1,2,3")
@@ -187,6 +190,7 @@ suite("mtmv_range_datetime_part_up_rewrite") {
 
     sql """insert into lineitem_range_datetime_union values 
         (3, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-29 03:00:00');"""
+    sql """alter table lineitem_range_datetime_union modify column l_comment set stats ('row_count'='9');"""
     for (int i = 0; i < mv_name_list.size(); i++) {
         mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
         compare_res(query_stmt_list[i] + " order by 1,2,3")
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy
index f2b5ad09c0b17a..3aed3b0f9e24df 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy
@@ -98,6 +98,9 @@ suite("partition_mv_rewrite_dimension_1") {
     (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
     """
 
+    sql """alter table orders_1 modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_1 modify column l_comment set stats ('row_count'='7');"""
+
     sql """analyze table orders_1 with sync;"""
     sql """analyze table lineitem_1 with sync;"""
 
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy
index 53f8777a1589d7..c7ee359cdef2e4 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy
@@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_3") {
     sql """analyze table orders_2_3 with sync;"""
     sql """analyze table lineitem_2_3 with sync;"""
 
+    sql """alter table orders_2_3 modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_3 modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy
index deb6255119a26e..e59b2771dd4e57 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy
@@ -124,6 +124,10 @@ suite("partition_mv_rewrite_dimension_2_4") {
     (3, null, 1, 99.5, 'yy'); 
     """
 
+    sql """alter table orders_2_4 modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_4 modify column l_comment set stats ('row_count'='7');"""
+    sql """alter table partsupp_2_4 modify column ps_comment set stats ('row_count'='3');"""
+
     sql """analyze table orders_2_4 with sync;"""
     sql """analyze table lineitem_2_4 with sync;"""
     sql """analyze table partsupp_2_4 with sync;"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy
index 5d8a6b5270285b..e5e43073bede3e 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy
@@ -124,6 +124,10 @@ suite("partition_mv_rewrite_dimension_2_5") {
     (3, null, 1, 99.5, 'yy'); 
     """
 
+    sql """alter table orders_2_5 modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_5 modify column l_comment set stats ('row_count'='7');"""
+    sql """alter table partsupp_2_5 modify column ps_comment set stats ('row_count'='3');"""
+
     sql """analyze table orders_2_5 with sync;"""
     sql """analyze table lineitem_2_5 with sync;"""
     sql """analyze table partsupp_2_5 with sync;"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy
index 1831e8de9a7a9d..f3289b007a0505 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy
@@ -124,6 +124,12 @@ suite("partition_mv_rewrite_dimension_2_6") {
     (3, null, 1, 99.5, 'yy'); 
     """
 
+    sql """alter table orders_2_6 modify column o_comment set stats ('row_count'='10');"""
+
+    sql """alter table lineitem_2_6 modify column l_comment set stats ('row_count'='7');"""
+
+    sql """alter table partsupp_2_6 modify column ps_comment set stats ('row_count'='3');"""
+
     sql """analyze table orders_2_6 with sync;"""
     sql """analyze table lineitem_2_6 with sync;"""
     sql """analyze table partsupp_2_6 with sync;"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy
index 6def436e11c5c4..e36a5e277daca7 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy
@@ -102,6 +102,8 @@ suite("partition_mv_rewrite_dimension_2_full_join") {
     sql """analyze table orders_2_full_join with sync;"""
     sql """analyze table lineitem_2_full_join with sync;"""
 
+    sql """alter table orders_2_full_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_full_join modify column l_comment set stats ('row_count'='7');"""
 
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy
index 526e05f0692fbe..a615c7316bdb56 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy
@@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_2") {
     sql """analyze table orders_2_2 with sync;"""
     sql """analyze table lineitem_2_2 with sync;"""
 
+    sql """alter table orders_2_2 modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_2 modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy
index 0db7b3bf008c68..35d48270574a6f 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy
@@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_left_anti_join") {
     sql """analyze table orders_2_left_anti_join with sync;"""
     sql """analyze table lineitem_2_left_anti_join with sync;"""
 
+    sql """alter table orders_2_left_anti_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_left_anti_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy
index f59dffd0091bb4..616707412954e4 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy
@@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_1") {
     sql """analyze table orders_2_1 with sync;"""
     sql """analyze table lineitem_2_1 with sync;"""
 
+    sql """alter table orders_2_1 modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_1 modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy
index 5e60250ef91d26..f203db1134f2f2 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy
@@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_left_semi_join") {
     sql """analyze table orders_2_left_semi_join with sync;"""
     sql """analyze table lineitem_2_left_semi_join with sync;"""
 
+    sql """alter table orders_2_left_semi_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_left_semi_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy
index 289ed57287f1d8..c6097d845f41c9 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy
@@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_right_anti_join") {
     sql """analyze table orders_2_right_anti_join with sync;"""
     sql """analyze table lineitem_2_right_anti_join with sync;"""
 
+    sql """alter table orders_2_right_anti_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_right_anti_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy
index 8d6abb6dc561f8..850de42fc696eb 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy
@@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_right_join") {
     sql """analyze table orders_2_right_join with sync;"""
     sql """analyze table lineitem_2_right_join with sync;"""
 
+    sql """alter table orders_2_right_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_right_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy
index 8f1f6f48a0ae91..97ddcd0eb1da29 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy
@@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_right_semi_join") {
     sql """analyze table orders_2_right_semi_join with sync;"""
     sql """analyze table lineitem_2_right_semi_join with sync;"""
 
+    sql """alter table orders_2_right_semi_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_2_right_semi_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
index f433e18f48c062..776f1b996f1826 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
@@ -101,6 +101,9 @@ suite("partition_mv_rewrite_dimension_self_conn") {
     sql """analyze table orders_self_conn with sync;"""
     sql """analyze table lineitem_self_conn with sync;"""
 
+    sql """alter table orders_self_conn modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_self_conn modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy
index dee43f20f4926a..fd41789d02661e 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy
@@ -101,6 +101,9 @@ suite("dimension_2_join_agg_replenish") {
     sql """analyze table orders with sync;"""
     sql """analyze table lineitem with sync;"""
 
+    sql """alter table orders modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
@@ -649,19 +652,19 @@ suite("dimension_2_join_agg_replenish") {
             """
     def right_semi_mv_stmt_8 = """select t1.l_orderkey, t1.l_shipdate, t1.l_partkey 
             from orders 
-            right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 
+            right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem where l_shipdate >= "2023-10-17" group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 
             on t1.l_orderkey = orders.o_orderkey
             group by
             t1.l_orderkey, t1.l_shipdate, t1.l_partkey 
             """
     def right_semi_mv_stmt_9 = """select t1.l_orderkey, t1.l_shipdate, t1.l_partkey 
             from orders 
-            right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 
+            right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem where l_shipdate >= "2023-10-17" group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 
             on t1.l_orderkey = orders.o_orderkey
             group by
             t1.l_orderkey, t1.l_shipdate, t1.l_partkey 
             """
-    def right_semi_mv_stmt_10 = """select t1.l_orderkey 
+    def right_semi_mv_stmt_10 = """select t1.sum_total, max_total+min_total as col3, count_all  
             from orders 
             right semi join (select l_orderkey, sum(l_quantity) as sum_total,
             max(l_quantity) as max_total,
@@ -670,10 +673,10 @@ suite("dimension_2_join_agg_replenish") {
             sum(l_quantity) + count(*) as col5, 
             bitmap_union(to_bitmap(case when l_quantity > 1 and l_orderkey IN (1, 3) then l_partkey else null end)) as cnt_1,
             bitmap_union(to_bitmap(case when l_quantity > 2 and l_orderkey IN (2) then l_partkey else null end)) as cnt_2  
-            from lineitem where l_orderkey > 1 + 1 group by l_orderkey) as t1 
+            from lineitem where l_shipdate >= "2023-10-17" group by l_orderkey) as t1 
             on t1.l_orderkey = orders.o_orderkey
             group by
-            t1.l_orderkey
+            t1.sum_total, col3, count_all 
             """
 
     def left_anti_mv_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey 
@@ -790,7 +793,7 @@ suite("dimension_2_join_agg_replenish") {
             bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 
             from orders where o_orderkey >= 1 + 1 group by o_orderkey) as t1
             right anti join lineitem on lineitem.l_orderkey = t1.o_orderkey 
-            group by l_orderkey, l_shipdate, l_partkey
+            group by l_orderkey, l_shipdate, l_partkey 
             """
 
     def right_anti_mv_stmt_6 = """select t1.l_shipdate, t1.l_quantity, t1.l_orderkey 
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy
index 04bd6ac5c6ac8d..3afbe42fd2a1c3 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy
@@ -101,6 +101,9 @@ suite("dimension_join_agg_negative") {
     sql """analyze table orders_negative with sync;"""
     sql """analyze table lineitem_negative with sync;"""
 
+    sql """alter table orders_negative modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_negative modify column l_comment set stats ('row_count'='7');"""
+
     // left join + agg (function + group by + +-*/ + filter)
     def left_mv_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey 
             from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority) as t1
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy
index 763880ea94be44..2ca4a524989c99 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy
@@ -99,6 +99,9 @@ suite("filter_equal_or_notequal_case") {
     sql """analyze table orders_1 with sync;"""
     sql """analyze table lineitem_1 with sync;"""
 
+    sql """alter table orders_1 modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_1 modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/full_join_filter.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/full_join_filter.groovy
index ce30fab1b6c71e..751e5d0056726f 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/full_join_filter.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/full_join_filter.groovy
@@ -105,6 +105,9 @@ suite("full_join_filter") {
     sql """analyze table orders_full_join with sync;"""
     sql """analyze table lineitem_full_join with sync;"""
 
+    sql """alter table orders_full_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_full_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/inner_join_filter.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/inner_join_filter.groovy
index 932c2608ad408a..8e6f06036aa7da 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/inner_join_filter.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/inner_join_filter.groovy
@@ -105,6 +105,9 @@ suite("inner_join_filter") {
     sql """analyze table orders_inner_join with sync;"""
     sql """analyze table lineitem_inner_join with sync;"""
 
+    sql """alter table orders_inner_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_inner_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_anti_join_filter.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_anti_join_filter.groovy
index d813e55f33286a..6522d7f2dc248a 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_anti_join_filter.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_anti_join_filter.groovy
@@ -107,6 +107,9 @@ suite("left_anti_join_filter") {
     sql """analyze table orders_left_anti_join with sync;"""
     sql """analyze table lineitem_left_anti_join with sync;"""
 
+    sql """alter table orders_left_anti_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_left_anti_join modify column l_comment set stats ('row_count'='9');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_join_filter.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_join_filter.groovy
index 608aec88c63621..37c9d3bfde4e47 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_join_filter.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_join_filter.groovy
@@ -105,6 +105,9 @@ suite("left_join_filter") {
     sql """analyze table orders_left_join with sync;"""
     sql """analyze table lineitem_left_join with sync;"""
 
+    sql """alter table orders_left_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_left_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_semi_join_filter.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_semi_join_filter.groovy
index da12c8fbeaef3f..2c9854eca82106 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_semi_join_filter.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/left_semi_join_filter.groovy
@@ -105,6 +105,9 @@ suite("left_semi_join_filter") {
     sql """analyze table orders_left_semi_join with sync;"""
     sql """analyze table lineitem_left_semi_join with sync;"""
 
+    sql """alter table orders_left_semi_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_left_semi_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_anti_join_filter.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_anti_join_filter.groovy
index b920b766b07aa4..547a36c424c8cb 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_anti_join_filter.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_anti_join_filter.groovy
@@ -107,6 +107,9 @@ suite("right_anti_join_filter") {
     sql """analyze table orders_right_anti_join with sync;"""
     sql """analyze table lineitem_right_anti_join with sync;"""
 
+    sql """alter table orders_right_anti_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_right_anti_join modify column l_comment set stats ('row_count'='9');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_join_filter.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_join_filter.groovy
index f3223855173f48..c29effd36cb701 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_join_filter.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_join_filter.groovy
@@ -105,6 +105,9 @@ suite("right_join_filter") {
     sql """analyze table orders_right_join with sync;"""
     sql """analyze table lineitem_right_join with sync;"""
 
+    sql """alter table orders_right_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_right_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_semi_join_filter.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_semi_join_filter.groovy
index bbf4e60afb64ec..6dc16554b26ca8 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_semi_join_filter.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_predicate/right_semi_join_filter.groovy
@@ -107,6 +107,9 @@ suite("right_semi_join_filter") {
     sql """analyze table orders_right_semi_join with sync;"""
     sql """analyze table lineitem_right_semi_join with sync;"""
 
+    sql """alter table orders_right_semi_join modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_right_semi_join modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy
index f4fc91db26c3b8..0aaaebb64cf3a3 100644
--- a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy
@@ -131,6 +131,9 @@ suite("direct_query_mv") {
     sql """analyze table orders with sync;"""
     sql """analyze table partsupp with sync;"""
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column o_comment set stats ('row_count'='15');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
 
     create_async_mv(db, "mv1_0",
             """
diff --git a/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy b/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy
index ec8cecbaf517d0..4655f9d004bc7a 100644
--- a/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy
@@ -127,6 +127,9 @@ suite("materialized_view_grouping_sets") {
     sql """analyze table orders with sync;"""
     sql """analyze table partsupp with sync;"""
 
+    sql """alter table orders modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='7');"""
+
     // query has group sets, and mv doesn't
     // single table grouping sets without grouping scalar function
     def mv1_0 =
diff --git a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy
index 2b018ebc37373b..8c0674faeb8c32 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy
@@ -123,6 +123,10 @@ suite("inner_join_dphyp") {
     sql """analyze table orders with sync;"""
     sql """analyze table partsupp with sync;"""
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // without filter
     def mv1_0 = "select  lineitem.L_LINENUMBER, orders.O_CUSTKEY " +
             "from lineitem " +
diff --git a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy
index 45d9aee067c798..71154fbeb276cf 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy
@@ -123,6 +123,10 @@ suite("outer_join_dphyp") {
     sql """analyze table orders with sync;"""
     sql """analyze table partsupp with sync;"""
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // without filter
     def mv1_0 = "select  lineitem.L_LINENUMBER, orders.O_CUSTKEY " +
             "from lineitem " +
diff --git a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
index b745f7d0197c28..3aebf92b7cec71 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
@@ -122,6 +122,10 @@ suite("inner_join") {
     sql """analyze table orders with sync;"""
     sql """analyze table partsupp with sync;"""
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // without filter
     def mv1_0 =
             """
diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
index faa2c747a837c5..cdc4632ff7379b 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
@@ -278,9 +278,20 @@ suite("outer_join") {
     sql """analyze table lineitem with sync;"""
     sql """analyze table orders with sync;"""
     sql """analyze table partsupp with sync;"""
+    sql """analyze table lineitem_null with sync;"""
+    sql """analyze table orders_null with sync;"""
     sql """analyze table orders_same_col with sync;"""
     sql """analyze table lineitem_same_col with sync;"""
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table lineitem_same_col modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+    sql """alter table orders_same_col modify column o_comment set stats ('row_count'='18');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+    sql """alter table lineitem_null modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders_null modify column o_comment set stats ('row_count'='5');"""
+
+
     // without filter
     def mv1_0 = "select  lineitem.L_LINENUMBER, orders.O_CUSTKEY " +
             "from lineitem " +
diff --git a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy
index 5688c2bbe21d6a..cce090b2a940fb 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy
@@ -106,6 +106,9 @@ suite("inner_join_infer_and_derive") {
     sql """analyze table orders_inner with sync;"""
     sql """analyze table lineitem_inner with sync;"""
 
+    sql """alter table orders_inner modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_inner modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy
index dbfaa783383af8..7e51847c3b1a07 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy
@@ -106,6 +106,8 @@ suite("left_join_infer_and_derive") {
     sql """analyze table orders_left with sync;"""
     sql """analyze table lineitem_left with sync;"""
 
+    sql """alter table orders_left modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_left modify column l_comment set stats ('row_count'='7');"""
 
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
diff --git a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy
index 4b5bdf467a5d02..1761efa5ef4475 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy
@@ -106,6 +106,9 @@ suite("right_join_infer_and_derive") {
     sql """analyze table orders_right with sync;"""
     sql """analyze table lineitem_right with sync;"""
 
+    sql """alter table orders_right modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_right modify column l_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
diff --git a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy
index ba23c29ab9abc4..ea45b806529d8d 100644
--- a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy
@@ -127,6 +127,9 @@ suite("negative_partition_mv_rewrite") {
     sql """analyze table lineitem_1 with sync;"""
     sql """analyze table partsupp_1 with sync;"""
 
+    sql """alter table orders_1 modify column o_comment set stats ('row_count'='10');"""
+    sql """alter table lineitem_1 modify column l_comment set stats ('row_count'='7');"""
+
     def mv_name = "mv_1"
     def mtmv_sql = """
         select l_shipdate, o_orderdate, l_partkey, l_suppkey 
diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy
index 3a130d489eb036..692ea03fc61806 100644
--- a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy
@@ -199,6 +199,10 @@ suite("nested_materialized_view") {
     (2, 3, 10, 11.01, 'supply2');
     """
 
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // simple nested materialized view
     def mv1_0_inner_mv = """
             select
diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
index 700c17606d2539..4870ec99e659c0 100644
--- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
@@ -96,12 +96,20 @@ suite("nested_mtmv") {
     insert into orders_1 values 
     (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
     (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
     (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
     (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
     (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
     (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
     (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
     (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
     (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
     (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
     """
@@ -109,22 +117,29 @@ suite("nested_mtmv") {
     sql """
     insert into lineitem_1 values 
     (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
     (1, 1, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
     (3, 3, 3, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (3, 3, 3, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
     (1, 2, 3, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
     (2, 1, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (2, 1, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
     (3, 1, 3, 1, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
     (1, 2, 1, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
     (2, 2, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
     (3, 3, 3, 3, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (3, 3, 3, 3, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
     (1, 1, 1, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
     """
 
     sql"""
     insert into partsupp_1 values 
     (1, 1, 1, 99.5, 'yy'),
+    (1, 1, 1, 99.5, 'yy'),
+    (2, 2, 2, 109.2, 'mm'),
     (2, 2, 2, 109.2, 'mm'),
     (3, 3, 1, 99.5, 'yy'),
+    (3, 3, 1, 99.5, 'yy'),
     (3, null, 1, 99.5, 'yy'); 
     """
 
@@ -132,6 +147,10 @@ suite("nested_mtmv") {
     sql """analyze table lineitem_1 with sync;"""
     sql """analyze table partsupp_1 with sync;"""
 
+    sql """alter table orders_1 modify column o_orderdate set stats ('row_count'='17');"""
+    sql """alter table lineitem_1 modify column l_shipdate set stats ('row_count'='14');"""
+    sql """alter table partsupp_1 modify column ps_comment set stats ('row_count'='7');"""
+
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
         def origin_res = sql stmt
@@ -659,154 +678,77 @@ suite("nested_mtmv") {
         left join (
             select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 
             from (
-                select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 
+                select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, 
+                t.agg1 as agg1, 
+                t.sum_total as agg3,
+                t.max_total as agg4,
+                t.min_total as agg5,
+                t.count_all as agg6,
+                cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2
                 from (
-                    select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, 
-                    t.agg1 as agg1, 
-                    t.sum_total as agg3,
-                    t.max_total as agg4,
-                    t.min_total as agg5,
-                    t.count_all as agg6,
-                    cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2
-                    from (
-                        select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1,
-                        sum(o_totalprice) as sum_total, 
-                        max(o_totalprice) as max_total, 
-                        min(o_totalprice) as min_total, 
-                        count(*) as count_all, 
-                        bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, 
-                        bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 
-                        from lineitem_1
-                        inner join orders_1
-                        on lineitem_1.l_orderkey = orders_1.o_orderkey
-                        where lineitem_1.l_shipdate >= "2023-10-17"
-                        group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey
-                    ) as t
-                    inner join partsupp_1
-                    on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey
-                    where partsupp_1.ps_suppkey > 1
-                    group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6
-                ) as t1
-                left join (
-                    select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, 
-                    t.agg1 as agg1, 
-                    t.sum_total as agg3,
-                    t.max_total as agg4,
-                    t.min_total as agg5,
-                    t.count_all as agg6,
-                    cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2
-                    from (
-                        select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1,
-                        sum(o_totalprice) as sum_total, 
-                        max(o_totalprice) as max_total, 
-                        min(o_totalprice) as min_total, 
-                        count(*) as count_all, 
-                        bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, 
-                        bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 
-                        from lineitem_1
-                        inner join orders_1
-                        on lineitem_1.l_orderkey = orders_1.o_orderkey
-                        where lineitem_1.l_shipdate >= "2023-10-17"
-                        group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey
-                    ) as t
-                    inner join partsupp_1
-                    on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey
-                    where partsupp_1.ps_suppkey > 1
-                    group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6
-                ) as t2
-                on t1.l_orderkey = t2.l_orderkey
-                where t1.l_orderkey > 1
-                group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6
+                    select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1,
+                    sum(o_totalprice) as sum_total, 
+                    max(o_totalprice) as max_total, 
+                    min(o_totalprice) as min_total, 
+                    count(*) as count_all, 
+                    bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, 
+                    bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 
+                    from lineitem_1
+                    inner join orders_1
+                    on lineitem_1.l_orderkey = orders_1.o_orderkey
+                    where lineitem_1.l_shipdate >= "2023-10-17"
+                    group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey
+                ) as t
+                inner join partsupp_1
+                on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey
+                where partsupp_1.ps_suppkey > 1
+                group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6
             ) as t1
             left join (
-                select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 
+                select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, 
+                t.agg1 as agg1, 
+                t.sum_total as agg3,
+                t.max_total as agg4,
+                t.min_total as agg5,
+                t.count_all as agg6,
+                cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2
                 from (
-                    select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, 
-                    t.agg1 as agg1, 
-                    t.sum_total as agg3,
-                    t.max_total as agg4,
-                    t.min_total as agg5,
-                    t.count_all as agg6,
-                    cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2
-                    from (
-                        select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1,
-                        sum(o_totalprice) as sum_total, 
-                        max(o_totalprice) as max_total, 
-                        min(o_totalprice) as min_total, 
-                        count(*) as count_all, 
-                        bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, 
-                        bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 
-                        from lineitem_1
-                        inner join orders_1
-                        on lineitem_1.l_orderkey = orders_1.o_orderkey
-                        where lineitem_1.l_shipdate >= "2023-10-17"
-                        group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey
-                    ) as t
-                    inner join partsupp_1
-                    on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey
-                    where partsupp_1.ps_suppkey > 1
-                    group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6
-                ) as t1
-                left join (
-                    select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, 
-                    t.agg1 as agg1, 
-                    t.sum_total as agg3,
-                    t.max_total as agg4,
-                    t.min_total as agg5,
-                    t.count_all as agg6,
-                    cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2
-                    from (
-                        select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1,
-                        sum(o_totalprice) as sum_total, 
-                        max(o_totalprice) as max_total, 
-                        min(o_totalprice) as min_total, 
-                        count(*) as count_all, 
-                        bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, 
-                        bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 
-                        from lineitem_1
-                        inner join orders_1
-                        on lineitem_1.l_orderkey = orders_1.o_orderkey
-                        where lineitem_1.l_shipdate >= "2023-10-17"
-                        group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey
-                    ) as t
-                    inner join partsupp_1
-                    on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey
-                    where partsupp_1.ps_suppkey > 1
-                    group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6
-                ) as t2
-                on t1.l_orderkey = t2.l_orderkey
-                where t1.l_orderkey > 1
-                group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6
+                    select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1,
+                    sum(o_totalprice) as sum_total, 
+                    max(o_totalprice) as max_total, 
+                    min(o_totalprice) as min_total, 
+                    count(*) as count_all, 
+                    bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, 
+                    bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 
+                    from lineitem_1
+                    inner join orders_1
+                    on lineitem_1.l_orderkey = orders_1.o_orderkey
+                    where lineitem_1.l_shipdate >= "2023-10-17"
+                    group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey
+                ) as t
+                inner join partsupp_1
+                on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey
+                where partsupp_1.ps_suppkey > 1
+                group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6
             ) as t2
             on t1.l_orderkey = t2.l_orderkey
+            where t1.l_orderkey > 1
             group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6
         ) as t2
         on t1.l_orderkey = t2.l_orderkey
         group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6
         """
 
-    explain {
-        sql("${sql_2}")
-        contains "${mv_2}(${mv_2})"
-    }
+    mv_rewrite_any_success(sql_2, [mv_1, mv_2])
     compare_res(sql_2 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
 
-    explain {
-        sql("${sql_3}")
-        contains "${mv_3}(${mv_3})"
-    }
+    mv_rewrite_any_success(sql_3, [mv_3, mv_4])
     compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
 
-    explain {
-        sql("${sql_4}")
-        contains "${mv_4}(${mv_4})"
-    }
+    mv_rewrite_any_success(sql_4, [mv_3, mv_4])
     compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
 
-    explain {
-        sql("${sql_5}")
-        contains "${mv_5}(${mv_5})"
-    }
+    mv_rewrite_any_success(sql_5, [mv_3, mv_4, mv_5])
     compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
 
 }
diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy
index f9a84fa5250179..64354267a039bd 100644
--- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy
@@ -99,6 +99,9 @@ suite("nested_mtmv_rewrite_switch") {
     sql """analyze table orders_2 with sync;"""
     sql """analyze table lineitem_2 with sync;"""
 
+    sql """alter table orders_2 modify column o_orderdate set stats ('row_count'='10');"""
+    sql """alter table lineitem_2 modify column l_shipdate set stats ('row_count'='7');"""
+
 
     def compare_res = { def stmt ->
         sql "SET enable_materialized_view_rewrite=false"
diff --git a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.groovy b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.groovy
index 68337b00d09da8..be5301cdf2ec17 100644
--- a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.groovy
@@ -140,6 +140,10 @@ suite("other_join_conjuncts_anti") {
     sql """analyze table lineitem with sync"""
     sql """analyze table orders with sync"""
 
+    sql """alter table orders modify column lo_orderdate set stats ('row_count'='18');"""
+   sql """alter table lineitem modify column lo_orderdate set stats ('row_count'='10');"""
+sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // =, !=, >, <, <=, >=
     // left anti join other conjuncts in join condition
     def mv1_0 =
diff --git a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.groovy b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.groovy
index b79e5808932044..0f07b7390cb3d7 100644
--- a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.groovy
@@ -140,6 +140,10 @@ suite("other_join_conjuncts_inner") {
     sql """analyze table lineitem with sync"""
     sql """analyze table orders with sync"""
 
+    sql """alter table orders modify column lo_orderdate set stats ('row_count'='18');"""
+    sql """alter table lineitem modify column lo_orderdate set stats ('row_count'='10');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // =, !=, >, <, <=, >=
     // other conjuncts in join condition
     def mv1_0 =
diff --git a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.groovy b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.groovy
index 8a434364c89168..87fd620c704baf 100644
--- a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.groovy
@@ -140,6 +140,10 @@ suite("other_join_conjuncts_outer") {
     sql """analyze table lineitem with sync"""
     sql """analyze table orders with sync"""
 
+    sql """alter table orders modify column lo_orderdate set stats ('row_count'='18');"""
+    sql """alter table lineitem modify column lo_orderdate set stats ('row_count'='10');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // =, !=, >, <, <=, >=
     // left outer join
     // other conjuncts in join condition
diff --git a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.groovy b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.groovy
index 30c21c77269e1c..4bbfa7e7c0a15c 100644
--- a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.groovy
@@ -140,6 +140,10 @@ suite("other_join_conjuncts_semi") {
     sql """analyze table lineitem with sync"""
     sql """analyze table orders with sync"""
 
+    sql """alter table orders modify column lo_orderdate set stats ('row_count'='18');"""
+   sql """alter table lineitem modify column lo_orderdate set stats ('row_count'='10');"""
+sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // left self join
     def mv1_0 =
             """
diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
index 35a983c264c0e2..f8023a9e1e4c49 100644
--- a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
@@ -480,6 +480,10 @@ suite("partition_mv_rewrite") {
         analyze table orders with sync;
         """
 
+    sql """alter table orders modify column o_comment set stats ('row_count'='3');"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='6');"""
+    sql """alter table lineitem_static modify column l_comment set stats ('row_count'='4');"""
+
     // should rewrite successful when union rewrite enalbe if base table add new partition
     mv_rewrite_success(roll_up_all_partition_sql, "mv_10086", true,
             is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"]))
diff --git a/regression-test/suites/nereids_rules_p0/mv/same_name/sync_async_same_name.groovy b/regression-test/suites/nereids_rules_p0/mv/same_name/sync_async_same_name.groovy
index e518c016d2b641..4dda0f9c08414b 100644
--- a/regression-test/suites/nereids_rules_p0/mv/same_name/sync_async_same_name.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/same_name/sync_async_same_name.groovy
@@ -57,6 +57,8 @@ suite("sync_async_same_name") {
     (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi');  
     """
 
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+
     sql """analyze table orders with sync;"""
 
     def common_mv_name = 'common_mv_name'
diff --git a/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy
index 6b5c2dc45e875f..c42888e35abc0a 100644
--- a/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy
@@ -123,6 +123,10 @@ suite("mv_scan_table") {
     sql """analyze table lineitem with sync;"""
     sql """analyze table partsupp with sync;"""
 
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+   sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
     // with filter
     def mv1_0 =
             """
diff --git a/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy b/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy
index b6f44bba80139c..450fb9c0ea3187 100644
--- a/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy
@@ -52,6 +52,7 @@ suite("single_table_without_aggregate") {
     """
 
     sql "analyze table orders with sync;"
+    sql """alter table orders modify column o_comment set stats ('row_count'='2');"""
     sql """set enable_stats=false;"""
 
     def check_rewrite = { mv_sql, query_sql, mv_name ->
diff --git a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy
index e72e7575053b5b..192f9c420f18b5 100644
--- a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy
@@ -180,6 +180,9 @@ suite("union_all_compensate") {
     mv_rewrite_fail(query1_0, "test_agg_mv")
     order_qt_query1_1_after "${query1_0}"
 
+    sql """alter table test_table1 modify column num set stats ('row_count'='20');"""
+    sql """alter table test_table2 modify column num set stats ('row_count'='16');"""
+
 
     // Aggregate, if query group by expression doesn't use the partition column, but the invalid partition is in the
     // grace_period, should not compensate union all, but should rewritten successfully
diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy
index 4f6717dfe59255..28a8f2b60f653c 100644
--- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy
@@ -78,6 +78,9 @@ suite ("partition_curd_union_rewrite") {
     );
     """
 
+    sql """alter table orders modify column o_comment set stats ('row_count'='3');"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='3');"""
+
     sql"""
     insert into orders values 
     (1, 1, 'ok', 99.5, '2023-10-17', 'a', 'b', 1, 'yy'),
diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite_hive.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite_hive.groovy
index 14170bf21f40fa..d82eaefde509e6 100644
--- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite_hive.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite_hive.groovy
@@ -155,6 +155,9 @@ suite ("partition_curd_union_rewrite_hive") {
         sql """create database if not exists ${db}"""
         sql """use ${db}"""
 
+        sql """alter table ${orders_tb_name} modify column o_comment set stats ('row_count'='3');"""
+sql """alter table ${lineitem_tb_name} modify column l_comment set stats ('row_count'='3');"""
+
         def mv_def_sql = """
             select l_shipdate, o_orderdate, l_partkey,
             l_suppkey, sum(o_totalprice) as sum_total
diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy
index 702431e5e391ae..4e74090b1a2bb2 100644
--- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy
@@ -90,6 +90,9 @@ suite ("usercase_union_rewrite") {
     sql """analyze table orders_user with sync;"""
     sql """analyze table lineitem_user with sync;"""
 
+    sql """alter table orders_user modify column o_comment set stats ('row_count'='4');"""
+    sql """alter table lineitem_user modify column l_comment set stats ('row_count'='3');"""
+
     def create_mv_orders = { mv_name, mv_sql ->
         sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
         sql """DROP TABLE IF EXISTS ${mv_name}"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy
index 5c64379950f942..cba2509483b6dc 100644
--- a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy
@@ -57,6 +57,8 @@ suite("null_unsafe_equals") {
     (5, 2, 'o', 1.2, '2023-12-12', 'c','d', null, 'mi');  
     """
 
+    sql """alter table orders modify column o_comment set stats ('row_count'='8');"""
+
     def mv1_0 =
             """
             select count(*), o_orderstatus, o_comment
diff --git a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
index 866f82af156c69..fae32e914ed031 100644
--- a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
@@ -87,6 +87,9 @@ suite("variant_mv") {
     sql """analyze table github_events1 with sync;"""
     sql """analyze table github_events2 with sync;"""
 
+    sql """alter table github_events1 modify column created_at set stats ('row_count'='3');"""
+    sql """alter table github_events2 modify column created_at set stats ('row_count'='3');"""
+
     // variant appear in where both slot and in expression
     def mv1_0 = """
     SELECT
diff --git a/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy b/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy
index 400e113b6f3ae5..c51dea2f5a7403 100644
--- a/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy
@@ -112,6 +112,9 @@ suite("with_select_table_auth","p0,auth") {
     sql """analyze table lineitem with sync"""
     sql """analyze table orders with sync"""
 
+    sql """alter table orders modify column o_comment set stats ('row_count'='18');"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+
     sql """grant select_priv on ${db}.orders to ${user_name}"""
     sql """grant select_priv on ${db}.lineitem to ${user_name}"""
     sql """grant select_priv on regression_test to ${user_name}"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy b/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy
index b7c6ecbd8ae008..757a4c42e5ad32 100644
--- a/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy
@@ -143,6 +143,10 @@ suite("query_with_sql_limit") {
     sql """analyze table lineitem with sync"""
     sql """analyze table orders with sync"""
 
+    sql """alter table orders modify column o_comment set stats ('row_count'='18');"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+ sql """alter table partsupp modify column ps_comment set stats ('row_count'='3');"""
+
     // test sql_select_limit default, default 9223372036854775807
     sql """set sql_select_limit = 2;"""
     def mv1_0 =
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy
index 76696e1c62b305..54ad59943d77bd 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy
@@ -36,6 +36,7 @@ suite ("aggHaveDupBase") {
             distributed BY hash(k1) buckets 3
             properties("replication_num" = "1");
         """
+    sql """alter table agg_have_dup_base modify column k1 set stats ('row_count'='5');"""
 
     sql "insert into agg_have_dup_base select 1,1,1,'a';"
     sql "insert into agg_have_dup_base select 2,2,2,'b';"
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy
index 91293bb747aa64..e2f334566e9c24 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy
@@ -36,6 +36,7 @@ suite ("case_ignore") {
     sql "insert into case_ignore select 2,2,2,'b';"
     sql "insert into case_ignore select 3,-3,null,'c';"
 
+
     createMV ("create materialized view k12a as select K1,abs(K2) from case_ignore;")
     sleep(3000)
 
@@ -55,6 +56,7 @@ suite ("case_ignore") {
     order_qt_select_mv "select K1,abs(K2) from case_ignore order by K1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table case_ignore modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,abs(k2) from case_ignore order by k1;", "k12a")
 
     mv_rewrite_success("select K1,abs(K2) from case_ignore order by K1;", "k12a")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy
index 132a4c3f054571..3537777d312c58 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy
@@ -57,6 +57,7 @@ suite ("dup_gb_mv_abs") {
     order_qt_select_mv_sub "select sum(abs(k2)) from dup_gb_mv_abs group by k1 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table dup_gb_mv_abs modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,sum(abs(k2)) from dup_gb_mv_abs group by k1;", "k12sa")
 
     mv_rewrite_success("select sum(abs(k2)) from dup_gb_mv_abs group by k1;", "k12sa")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy
index 45271415b98ef5..2e6e9c6b627850 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy
@@ -57,6 +57,7 @@ suite ("dup_gb_mv_plus") {
     order_qt_select_mv_sub "select sum(k2+1) from dup_gb_mv_plus group by k1 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table dup_gb_mv_plus modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,sum(k2+1) from dup_gb_mv_plus group by k1;", "k12sp")
 
     mv_rewrite_success("select sum(k2+1) from dup_gb_mv_plus group by k1;", "k12sp")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy
index c4770d6dcd0262..6cef52fd96edc7 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy
@@ -31,7 +31,6 @@ suite ("dup_mv_abs") {
             distributed BY hash(k1) buckets 3
             properties("replication_num" = "1");
         """
-
     sql "insert into dup_mv_abs select 1,1,1,'a';"
     sql "insert into dup_mv_abs select 2,2,2,'b';"
     sql "insert into dup_mv_abs select 3,-3,null,'c';"
@@ -69,6 +68,7 @@ suite ("dup_mv_abs") {
     order_qt_select_group_mv_not "select sum(abs(k2)) from dup_mv_abs group by k3 order by k3;"
 
     sql """set enable_stats=true;"""
+    sql """alter table dup_mv_abs modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,abs(k2) from dup_mv_abs order by k1;", "k12a")
 
     mv_rewrite_success("select abs(k2) from dup_mv_abs order by k1;", "k12a")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy
index 788647ae224d98..7fd22f5b71892c 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy
@@ -69,6 +69,7 @@ suite ("dup_mv_bin") {
     order_qt_select_group_mv_not "select group_concat(bin(k2)) from dup_mv_bin group by k3 order by k3;"
 
     sql """set enable_stats=true;"""
+    sql """alter table dup_mv_bin modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,bin(k2) from dup_mv_bin order by k1;", "k12b")
 
     mv_rewrite_success("select bin(k2) from dup_mv_bin order by k1;", "k12b")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy
index d049a0aeb7c1b0..f5784e7a5fdaa6 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy
@@ -45,6 +45,7 @@ suite ("dup_mv_bm_hash") {
     order_qt_select_mv "select bitmap_union_count(to_bitmap(k2)) from dup_mv_bm_hash group by k1 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table dup_mv_bm_hash modify column k1 set stats ('row_count'='5');"""
     mv_rewrite_success("select bitmap_union_count(to_bitmap(k2)) from dup_mv_bm_hash group by k1 order by k1;", "dup_mv_bm_hash_mv1")
 
     createMV("create materialized view dup_mv_bm_hash_mv2 as select k1,bitmap_union(bitmap_hash(k3)) from dup_mv_bm_hash group by k1;")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy
index b661e75ad967ab..d96d2e07843dd9 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy
@@ -46,7 +46,6 @@ suite ("dup_mv_plus") {
     sql "analyze table dup_mv_plus with sync;"
     sql """set enable_stats=false;"""
 
-
     order_qt_select_star "select * from dup_mv_plus order by k1;"
 
     mv_rewrite_success("select k1,k2+1 from dup_mv_plus order by k1;", "k12p")
@@ -86,6 +85,7 @@ suite ("dup_mv_plus") {
     order_qt_select_mv "select k1,k2+1 from dup_mv_plus order by k2;"
 
     sql """set enable_stats=true;"""
+    sql """alter table dup_mv_plus modify column k1 set stats ('row_count'='4');"""
 
     mv_rewrite_success("select k1,k2+1 from dup_mv_plus order by k1;", "k12p")
 
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy
index c5098c55074c7d..95902c27866481 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy
@@ -45,6 +45,7 @@ suite ("dup_mv_year") {
     order_qt_select_mv "select k1,year(k2) from dup_mv_year order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table dup_mv_year modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1,year(k2) from dup_mv_year order by k1;", "k12y")
 
     createMV "create materialized view k13y as select k1,year(k3) from dup_mv_year;"
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy
index 7e317d501bea07..d0fa214e98f042 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy
@@ -52,5 +52,6 @@ suite ("multi_slot1") {
     order_qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot1 order by abs(k1)+k2+1,abs(k2+2)+k3+3;"
 
     sql """set enable_stats=true;"""
+    sql """alter table multi_slot1 modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot1 order by abs(k1)+k2+1,abs(k2+2)+k3+3", "k1a2p2ap3p")
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy
index 6bbded534a77ee..0fa36b77b44114 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy
@@ -65,6 +65,7 @@ suite ("multi_slot2") {
     order_qt_select_base "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2 order by abs(k1)+k2;"
 
     sql """set enable_stats=true;"""
+    sql """alter table multi_slot2 modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2+1 order by abs(k1)+k2+1",
             "k1a2p2ap3ps")
 
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy
index c6016296e8b352..51cc90aaf6ae26 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy
@@ -53,5 +53,6 @@ suite ("multi_slot3") {
     order_qt_select_mv "select k1+1,abs(k2+2)+k3+3 from multi_slot3 order by k1+1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table multi_slot3 modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select k1+1,abs(k2+2)+k3+3 from multi_slot3 order by k1+1;", "k1p2ap3p")
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
index a09f3937aee302..49aba77aad05ff 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
@@ -56,5 +56,6 @@ suite ("multi_slot4") {
     order_qt_select_mv "select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1 order by k1+1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table multi_slot4 modify column k1 set stats ('row_count'='5');"""
     mv_rewrite_success("select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1 order by k1+1;", "k1p2ap3ps")
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy
index 55ea991913cb3e..ebe6c8e57800a4 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy
@@ -58,5 +58,6 @@ suite ("multi_slot5") {
     order_qt_select_mv "select k1,version() from multi_slot5 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table multi_slot5 modify column k1 set stats ('row_count'='5');"""
     mv_rewrite_success("select k1,k2+k3 from multi_slot5 order by k1;", "k123p")
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy
index 9201a1cea4d321..2ed66fc8477a37 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy
@@ -84,5 +84,6 @@ suite ("multi_slot6") {
     order_qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot6 order by abs(k1)+k2+1,abs(k2+2)+k3+3;"
 
     sql """set enable_stats=true;"""
+    sql """alter table multi_slot6 modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot6 order by abs(k1)+k2+1,abs(k2+2)+k3+3", "k1a2p2ap3p")
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/mv_with_view.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/mv_with_view.groovy
index 236bde3bf6c828..60e7f6fefed232 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/mv_with_view.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/mv_with_view.groovy
@@ -67,6 +67,7 @@ suite ("mv_with_view") {
     qt_select_mv "select * from v_k124 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table d_table modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from d_table order by k1;", "k312")
 
     sql """
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy
index d6cd0d5a24421a..387c155a5e45c8 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy
@@ -47,7 +47,6 @@ suite ("single_slot") {
     sql "analyze table single_slot with sync;"
     sql """set enable_stats=false;"""
 
-
     order_qt_select_star "select * from single_slot order by k1;"
 
     explain {
@@ -58,5 +57,6 @@ suite ("single_slot") {
     order_qt_select_mv "select abs(k1)+1 t,sum(abs(k2+1)) from single_slot group by t order by t;"
 
     sql """set enable_stats=true;"""
+    sql """alter table single_slot modify column k1 set stats ('row_count'='4');"""
     mv_rewrite_success("select abs(k1)+1 t,sum(abs(k2+1)) from single_slot group by t order by t;", "k1ap2spa")
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy
index 031560a41ef502..be8bc50532ff81 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy
@@ -62,7 +62,7 @@ suite ("sum_devide_count") {
     order_qt_select_mv "select sum(k2)/count(k2) from sum_devide_count;"
 
     sql """set enable_stats=true;"""
-
+    sql """alter table sum_devide_count modify column k1 set stats ('row_count'='5');"""
     mv_rewrite_success("select k1,k4,sum(k2)/count(k2) from sum_devide_count group by k1,k4 order by k1,k4;", "kavg")
 
     mv_rewrite_success("select k1,sum(k2)/count(k2) from sum_devide_count group by k1 order by k1;", "kavg")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy
index df390b26551054..1ebaa8884ab383 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy
@@ -47,6 +47,7 @@ suite ("unique_mv") {
     mv_rewrite_success("SELECT * FROM c5816_t WHERE call_uuid='adc';", "mv_1")
 
     sql """set enable_stats=true;"""
+    sql """alter table c5816_t modify column org_id set stats ('row_count'='1');"""
     mv_rewrite_success("SELECT * FROM c5816_t WHERE call_uuid='adc';", "mv_1")
 
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy
index 9a5e42034be3cb..2b6b0dd354e4cd 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy
@@ -37,6 +37,7 @@ suite ("MVMultiUsage") {
     sql """insert into MVMultiUsage values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into MVMultiUsage values("2020-01-03",3,"c",3,3,3);"""
 
+
     createMV("create materialized view MVMultiUsage_mv as select deptno, empid, salary from MVMultiUsage order by deptno;")
 
     sleep(3000)
@@ -58,6 +59,8 @@ suite ("MVMultiUsage") {
     order_qt_select_mv "select * from (select deptno, empid from MVMultiUsage where deptno>100) A join (select deptno, empid from MVMultiUsage where deptno >200) B using (deptno) order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table MVMultiUsage modify column time_col set stats ('row_count'='4');"""
+
     mv_rewrite_fail("select * from MVMultiUsage order by empid;", "MVMultiUsage_mv")
     explain {
         sql("select * from (select deptno, empid from MVMultiUsage where deptno>100) A join (select deptno, empid from MVMultiUsage where deptno >200) B using (deptno);")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy
index 246c97022e9e85..86a85dcc647443 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy
@@ -30,6 +30,7 @@ suite ("MVWithAs") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into MVWithAs values("2020-01-01",1,"a",1);"""
     sql """insert into MVWithAs values("2020-01-01",1,"a",1);"""
     sql """insert into MVWithAs values("2020-01-01",1,"a",1);"""
@@ -53,6 +54,8 @@ suite ("MVWithAs") {
     order_qt_select_mv "select count(tag_id) from MVWithAs t;"
 
     sql """set enable_stats=true;"""
+    sql """alter table MVWithAs modify column time_col set stats ('row_count'='7');"""
+
     mv_rewrite_fail("select * from MVWithAs order by time_col;", "MVWithAs_mv")
 
     mv_rewrite_success("select count(tag_id) from MVWithAs t;", "MVWithAs_mv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy
index 79ede78f0d3752..f45d523d423527 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy
@@ -34,7 +34,6 @@ suite ("aggCDInBitmap") {
     sql "analyze table aggCDInBitmap with sync;"
     sql """set enable_stats=false;"""
 
-
     order_qt_select_star "select * from aggCDInBitmap order by 1;"
 
 
@@ -45,9 +44,10 @@ suite ("aggCDInBitmap") {
     order_qt_select_mv "select k1, count(distinct v1) from aggCDInBitmap group by k1 order by k1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table aggCDInBitmap modify column k1 set stats ('row_count'='3');"""
     explain {
         sql("select k1, count(distinct v1) from aggCDInBitmap group by k1;")
         contains "bitmap_union_count"
     }
-
+    
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy
index 1a7ccd851503d8..a234661150e732 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy
@@ -38,7 +38,6 @@ suite ("aggMVCalcAggFun") {
     sql """insert into aggMVCalcAggFun values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into aggMVCalcAggFun values("2020-01-03",3,"c",3,3,3);"""
 
-
     createMV("create materialized view aggMVCalcAggFunMv as select deptno, empid, sum(salary) from aggMVCalcAggFun group by empid, deptno;")
 
     sleep(3000)
@@ -55,6 +54,7 @@ suite ("aggMVCalcAggFun") {
     order_qt_select_mv "select deptno, sum(salary + 1) from aggMVCalcAggFun where deptno > 10 group by deptno order by deptno;"
 
     sql """set enable_stats=true;"""
+    sql """alter table aggMVCalcAggFun modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from aggMVCalcAggFun order by empid;", "aggMVCalcAggFunMv")\
 
     mv_rewrite_fail("select deptno, sum(salary + 1) from aggMVCalcAggFun where deptno > 10 group by deptno;", "aggMVCalcAggFunMv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy
index 0cb12cbcde0b51..c29eec40c8da93 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy
@@ -54,6 +54,7 @@ suite ("aggOnAggMV1") {
     order_qt_select_mv "select sum(salary), deptno from aggOnAggMV1 group by deptno order by deptno;"
 
     sql """set enable_stats=true;"""
+    sql """alter table aggOnAggMV1 modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from aggOnAggMV1 order by empid;", "aggOnAggMV1_mv")
 
     mv_rewrite_success("select sum(salary), deptno from aggOnAggMV1 group by deptno order by deptno;", "aggOnAggMV1_mv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy
index bd826537ff88f7..e38a27c1db3c53 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy
@@ -54,6 +54,7 @@ suite ("aggOnAggMV10") {
     order_qt_select_mv "select deptno, commission, sum(salary) + 1 from aggOnAggMV10 group by rollup (deptno, commission) order by 1,2;"
 
     sql """set enable_stats=true;"""
+    sql """alter table aggOnAggMV10 modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from aggOnAggMV10 order by empid;", "aggOnAggMV10_mv")
 
     mv_rewrite_success("select deptno, commission, sum(salary) + 1 from aggOnAggMV10 group by rollup (deptno, commission);",
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy
index 793389ec68e4ae..b0f379b19476b8 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy
@@ -33,6 +33,7 @@ suite ("aggOnAggMV11") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into aggOnAggMV11 values("2020-01-01",1,"a",1,1,1);"""
     sql """insert into aggOnAggMV11 values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into aggOnAggMV11 values("2020-01-03",3,"c",3,3,3);"""
@@ -54,6 +55,8 @@ suite ("aggOnAggMV11") {
     order_qt_select_mv "select deptno, count(salary) + count(1) from aggOnAggMV11 group by deptno order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table aggOnAggMV11 modify column time_col set stats ('row_count'='4');"""
+
     mv_rewrite_fail("select * from aggOnAggMV11 order by empid;", "aggOnAggMV11_mv")
 
     mv_rewrite_fail("select deptno, count(salary) + count(1) from aggOnAggMV11 group by deptno;",
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy
index 32d897939ea652..179e22584d32a1 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy
@@ -56,6 +56,7 @@ suite ("aggOnAggMV2") {
     order_qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV2 group by deptno) a where (sum_salary * 2) > 3 order by deptno ;"
 
     sql """set enable_stats=true;"""
+    sql """alter table aggOnAggMV2 modify column time_col set stats ('row_count'='3');"""
     mv_rewrite_fail("select * from aggOnAggMV2 order by empid;", "aggOnAggMV2_mv")
 
     mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV2 group by deptno) a where (sum_salary * 2) > 3 order by deptno ;", "aggOnAggMV2_mv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy
index dfd0ce3e67503c..0b54345099bcb9 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy
@@ -56,6 +56,7 @@ suite ("aggOnAggMV3") {
     order_qt_select_mv "select commission, sum(salary) from aggOnAggMV3 where commission * (deptno + commission) = 100 group by commission order by commission;"
 
     sql """set enable_stats=true;"""
+    sql """alter table aggOnAggMV3 modify column time_col set stats ('row_count'='5');"""
     mv_rewrite_fail("select * from aggOnAggMV3 order by empid;", "aggOnAggMV3_mv")
 
     mv_rewrite_success("select commission, sum(salary) from aggOnAggMV3 where commission * (deptno + commission) = 100 group by commission order by commission;",
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy
index 481d8f19b5a133..d90057b8bf5120 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy
@@ -33,6 +33,8 @@ suite ("aggOnAggMV5") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+    sql """alter table aggOnAggMV5 modify column time_col set stats ('row_count'='4');"""
+
     sql """insert into aggOnAggMV5 values("2020-01-01",1,"a",1,1,1);"""
     sql """insert into aggOnAggMV5 values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into aggOnAggMV5 values("2020-01-03",3,"c",3,3,3);"""
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy
index abd66e1ab13ccd..7e2e87718b8e76 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy
@@ -54,6 +54,7 @@ suite ("aggOnAggMV6") {
     order_qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV6 where deptno>=20 group by deptno) a where sum_salary>10 order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table aggOnAggMV6 modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from aggOnAggMV6 order by empid;", "aggOnAggMV6_mv")
 
     mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV6 where deptno>=20 group by deptno) a where sum_salary>10;",
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy
index 53b75d03d85167..386871a8a5a98f 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy
@@ -54,6 +54,7 @@ suite ("aggOnAggMV7") {
     order_qt_select_mv "select deptno, sum(salary) from aggOnAggMV7 where deptno>=20 group by deptno order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table aggOnAggMV7 modify column time_col set stats ('row_count'='4');"""
     mv_rewrite_fail("select * from aggOnAggMV7 order by empid;", "aggOnAggMV7_mv")
 
     mv_rewrite_success("select deptno, sum(salary) from aggOnAggMV7 where deptno>=20 group by deptno;",
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy
index 13e2097cdb46ba..8b2697cb1f78f7 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy
@@ -30,6 +30,7 @@ suite ("bitmapUnionIn") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into bitmapUnionIn values("2020-01-01",1,"a",1);"""
     sql """insert into bitmapUnionIn values("2020-01-02",2,"b",2);"""
 
@@ -50,6 +51,8 @@ suite ("bitmapUnionIn") {
     order_qt_select_mv "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from bitmapUnionIn group by user_id having a>1 order by a;"
 
     sql """set enable_stats=true;"""
+    sql """alter table bitmapUnionIn modify column time_col set stats ('row_count'='3');"""
+
     mv_rewrite_fail("select * from bitmapUnionIn order by time_col;", "bitmapUnionIn_mv")
 
     mv_rewrite_success("select user_id, bitmap_union_count(to_bitmap(tag_id)) a from bitmapUnionIn group by user_id having a>1 order by a;",
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/distinctQuery.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/distinctQuery.groovy
index bfd7c159dade89..eb314fecccbb4b 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/distinctQuery.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/distinctQuery.groovy
@@ -37,6 +37,8 @@ suite ("distinctQuery") {
     sql """insert into distinctQuery values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into distinctQuery values("2020-01-03",3,"c",3,3,3);"""
 
+    sql """alter table distinctQuery modify column time_col set stats ('row_count'='5');"""
+
     createMV("create materialized view distinctQuery_mv as select deptno, count(salary) from distinctQuery group by deptno;")
 
     createMV("create materialized view distinctQuery_mv2 as select empid, deptno, count(salary) from distinctQuery group by empid, deptno;")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy
index c28dac1d227aab..52abff46678bb0 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy
@@ -31,6 +31,7 @@ suite ("incMVReInSub") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into incMVReInSub values("2020-01-01",1,"a",1);"""
     sql """insert into incMVReInSub values("2020-01-02",2,"b",2);"""
 
@@ -52,6 +53,8 @@ suite ("incMVReInSub") {
     order_qt_select_mv "select user_id, bitmap_union(to_bitmap(tag_id)) from incMVReInSub where user_name in (select user_name from incMVReInSub group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;"
 
     sql """set enable_stats=true;"""
+    sql """alter table incMVReInSub modify column time_col set stats ('row_count'='3');"""
+
     mv_rewrite_fail("select * from incMVReInSub order by time_col;", "incMVReInSub_mv")
 
     mv_rewrite_fail("select user_id, bitmap_union(to_bitmap(tag_id)) from incMVReInSub where user_name in (select user_name from incMVReInSub group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;",
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy
index ee8e688c0e7313..6172773ca7a43b 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy
@@ -31,6 +31,7 @@ suite ("incRewriteCD") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into incRewriteCD values("2020-01-01",1,"a",1);"""
     sql """insert into incRewriteCD values("2020-01-02",2,"b",2);"""
 
@@ -50,6 +51,8 @@ suite ("incRewriteCD") {
     order_qt_select_mv "select user_name, count(distinct tag_id) from incRewriteCD group by user_name order by user_name;"
 
     sql """set enable_stats=true;"""
+    sql """alter table incRewriteCD modify column time_col set stats ('row_count'='3');"""
+
     mv_rewrite_fail("select * from incRewriteCD order by time_col;", "incRewriteCD_mv")
 
     mv_rewrite_fail("select user_name, count(distinct tag_id) from incRewriteCD group by user_name;", "incRewriteCD_mv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnCalcToJoin.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnCalcToJoin.groovy
index 8ed3e09bdf0df3..378731dbb97a62 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnCalcToJoin.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnCalcToJoin.groovy
@@ -35,6 +35,8 @@ suite ("joinOnCalcToJoin") {
     sql """insert into joinOnCalcToJoin values("2020-01-03",3,"c",3,3,3);"""
     sql """insert into joinOnCalcToJoin values("2020-01-02",2,"b",2,7,2);"""
 
+    sql """alter table joinOnCalcToJoin modify column time_col set stats ('row_count'='3');"""
+
     sql """ DROP TABLE IF EXISTS joinOnCalcToJoin_1; """
     sql """
         create table joinOnCalcToJoin_1 (
@@ -49,6 +51,7 @@ suite ("joinOnCalcToJoin") {
     sql """insert into joinOnCalcToJoin_1 values("2020-01-03",3,"c",3);"""
     sql """insert into joinOnCalcToJoin_1 values("2020-01-02",2,"b",1);"""
 
+
     createMV("create materialized view joinOnLeftPToJoin_mv as select empid, deptno from joinOnCalcToJoin;")
     sleep(3000)
     createMV("create materialized view joinOnLeftPToJoin_1_mv as select deptno, cost from joinOnCalcToJoin_1;")
@@ -62,6 +65,8 @@ suite ("joinOnCalcToJoin") {
             ["joinOnLeftPToJoin_mv", "joinOnLeftPToJoin_1_mv"])
 
     sql """set enable_stats=true;"""
+    sql """alter table joinOnCalcToJoin_1 modify column time_col set stats ('row_count'='3');"""
+
     mv_rewrite_all_success("select * from (select empid, deptno from joinOnCalcToJoin where empid = 0) A join (select deptno, cost from joinOnCalcToJoin_1 where deptno > 0) B on A.deptno = B.deptno;",
             ["joinOnLeftPToJoin_mv", "joinOnLeftPToJoin_1_mv"])
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy
index f2f7a014d98fc8..69d1c39931c17f 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy
@@ -31,6 +31,8 @@ suite ("joinOnLeftPToJoin") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+    sql """alter table joinOnLeftPToJoin modify column time_col set stats ('row_count'='3');"""
+
     sql """insert into joinOnLeftPToJoin values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into joinOnLeftPToJoin values("2020-01-03",3,"c",3,3,3);"""
     sql """insert into joinOnLeftPToJoin values("2020-01-02",2,"b",2,7,2);"""
@@ -45,6 +47,7 @@ suite ("joinOnLeftPToJoin") {
         partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into joinOnLeftPToJoin_1 values("2020-01-02",2,"b",2);"""
     sql """insert into joinOnLeftPToJoin_1 values("2020-01-03",3,"c",3);"""
     sql """insert into joinOnLeftPToJoin_1 values("2020-01-02",2,"b",1);"""
@@ -64,6 +67,8 @@ suite ("joinOnLeftPToJoin") {
     order_qt_select_mv "select * from (select deptno , sum(salary) from joinOnLeftPToJoin group by deptno) A join (select deptno, max(cost) from joinOnLeftPToJoin_1 group by deptno ) B on A.deptno = B.deptno order by A.deptno;"
 
     sql """set enable_stats=true;"""
+    sql """alter table joinOnLeftPToJoin_1 modify column time_col set stats ('row_count'='3');"""
+
     mv_rewrite_all_success("select * from (select deptno , sum(salary) from joinOnLeftPToJoin group by deptno) A join (select deptno, max(cost) from joinOnLeftPToJoin_1 group by deptno ) B on A.deptno = B.deptno;",
             ["joinOnLeftPToJoin_mv", "joinOnLeftPToJoin_1_mv"])
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/onlyGroupBy.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/onlyGroupBy.groovy
index 698f4ae13f328d..e36ae20e9bcb11 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/onlyGroupBy.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/onlyGroupBy.groovy
@@ -33,6 +33,7 @@ suite ("onlyGroupBy") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into onlyGroupBy values("2020-01-01",1,"a",1,1,1);"""
     sql """insert into onlyGroupBy values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into onlyGroupBy values("2020-01-03",3,"c",3,3,3);"""
@@ -47,5 +48,7 @@ suite ("onlyGroupBy") {
     mv_rewrite_success("select deptno from onlyGroupBy group by deptno;", "onlyGroupBy_mv")
 
     sql """set enable_stats=true;"""
+    sql """alter table onlyGroupBy modify column time_col set stats ('row_count'='4');"""
+
     mv_rewrite_success("select deptno from onlyGroupBy group by deptno;", "onlyGroupBy_mv")
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy
index 7f455da429e6c3..21864f973ee8c0 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy
@@ -54,6 +54,8 @@ suite ("orderByOnPView") {
     order_qt_select_mv "select empid from orderByOnPView order by deptno;"
 
     sql """set enable_stats=true;"""
+    sql """alter table orderByOnPView modify column time_col set stats ('row_count'='4');"""
+
     mv_rewrite_fail("select * from orderByOnPView where time_col='2020-01-01' order by empid;", "orderByOnPView_mv")
 
     mv_rewrite_success("select empid from orderByOnPView where deptno = 0 order by deptno;", "orderByOnPView_mv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy
index 384137471d827f..157a15498e717c 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy
@@ -33,6 +33,7 @@ suite ("projectMV1") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into projectMV1 values("2020-01-01",1,"a",1,1,1);"""
     sql """insert into projectMV1 values("2020-01-02",2,"b",2,2,2);"""
 
@@ -52,6 +53,8 @@ suite ("projectMV1") {
     order_qt_select_mv "select empid, deptno from projectMV1 order by empid;"
 
     sql """set enable_stats=true;"""
+    sql """alter table projectMV1 modify column time_col set stats ('row_count'='3');"""
+
     mv_rewrite_fail("select * from projectMV1 where time_col='2020-01-01' order by empid;", "projectMV1_mv")
 
     mv_rewrite_success("select empid, deptno from projectMV1 where deptno=0 order by empid;", "projectMV1_mv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy
index 672bda198b01aa..4d85fcc9c8aeae 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy
@@ -33,6 +33,7 @@ suite ("projectMV2") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into projectMV2 values("2020-01-01",1,"a",1,1,1);"""
     sql """insert into projectMV2 values("2020-01-02",2,"b",2,2,2);"""
 
@@ -55,6 +56,8 @@ suite ("projectMV2") {
     order_qt_select_base "select name from projectMV2 where deptno -1 = 0 order by empid;"
 
     sql """set enable_stats=true;"""
+    sql """alter table projectMV2 modify column time_col set stats ('row_count'='3');"""
+
     mv_rewrite_fail("select * from projectMV2 order by empid;", "projectMV2_mv")
 
     mv_rewrite_success("select empid + 1 from projectMV2 where deptno = 1 order by empid;", "projectMV2_mv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy
index 37b73a9a81aff4..9ad301fbbc6a97 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy
@@ -33,6 +33,7 @@ suite ("projectMV3") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into projectMV3 values("2020-01-01",1,"a",1,1,1);"""
     sql """insert into projectMV3 values("2020-01-02",2,"b",2,2,2);"""
 
@@ -57,6 +58,8 @@ suite ("projectMV3") {
     order_qt_select_mv2 "select name from projectMV3 where deptno -1 = 0 order by empid;"
 
     sql """set enable_stats=true;"""
+    sql """alter table projectMV3 modify column time_col set stats ('row_count'='3');"""
+
     mv_rewrite_fail("select * from projectMV3 order by empid;", "projectMV3_mv")
 
     mv_rewrite_success("select empid + 1, name from projectMV3 where deptno = 1 order by empid;", "projectMV3_mv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy
index ea87551dfa1249..ddcdada549ad8f 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy
@@ -33,6 +33,7 @@ suite ("projectMV4") {
             partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1');
         """
 
+
     sql """insert into projectMV4 values("2020-01-01",1,"a",1,1,1);"""
     sql """insert into projectMV4 values("2020-01-02",2,"b",2,2,2);"""
 
@@ -57,6 +58,8 @@ suite ("projectMV4") {
     order_qt_select_base "select empid from projectMV4 where deptno > 1 and empid > 1 order by empid;"
 
     sql """set enable_stats=true;"""
+    sql """alter table projectMV4 modify column time_col set stats ('row_count'='3');"""
+
     mv_rewrite_fail("select * from projectMV4 order by empid;", "projectMV4_mv")
 
     mv_rewrite_success("select name from projectMV4 where deptno > 1 and salary > 1 and name = 'a' order by name;", "projectMV4_mv")
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy
index 25300c9d602681..67312eed088ed4 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy
@@ -37,7 +37,6 @@ suite ("subQuery") {
     sql """insert into subQuery values("2020-01-02",2,"b",2,2,2);"""
     sql """insert into subQuery values("2020-01-03",3,"c",3,3,3);"""
 
-
     createMV("create materialized view subQuery_mv as select deptno, empid from subQuery;")
 
     sleep(3000)
@@ -60,6 +59,7 @@ suite ("subQuery") {
      */
 
      sql """set enable_stats=true;"""
+     sql """alter table subQuery modify column time_col set stats ('row_count'='4');"""
 
     mv_rewrite_fail("select * from subQuery order by empid;", "subQuery_mv")
 }
diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy
index 0f57b5c6c1687e..9feb1aa1c975f5 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy
@@ -58,6 +58,8 @@ suite ("unionDis") {
     order_qt_select_mv "select * from (select empid, deptno from unionDis where empid >1 union select empid, deptno from unionDis where empid <0) t order by 1;"
 
     sql """set enable_stats=true;"""
+    sql """alter table unionDis modify column time_col set stats ('row_count'='4');"""
+
     mv_rewrite_fail("select * from unionDis order by empid;", "unionDis_mv")
 
     explain {
diff --git a/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy b/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy
index 0f682cacd8be96..3d65f3a85063d4 100644
--- a/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy
+++ b/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy
@@ -44,6 +44,7 @@ suite("agg") {
             AGGREGATE KEY (siteid,citycode,username)
             DISTRIBUTED BY HASH(siteid) BUCKETS 5 properties("replication_num" = "1");
         """
+    sql """alter table test_rollup_agg1 modify column siteid set stats ('row_count'='3');"""
     sql """ALTER TABLE ${tbName} ADD ROLLUP rollup_city(citycode, pv);"""
     int max_try_secs = 60
     String res = "NOT_FINISHED"
diff --git a/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy b/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy
index b711728f769129..07af0da35098e1 100644
--- a/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy
+++ b/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy
@@ -44,6 +44,7 @@ suite("agg_date", "rollup") {
             AGGREGATE KEY (datek1, datetimek1, datetimek2, datetimek3)
             DISTRIBUTED BY HASH(datek1) BUCKETS 5 properties("replication_num" = "1");
         """
+    sql """alter table test_rollup_agg_date1 modify column datek1 set stats ('row_count'='2');"""
     sql """ALTER TABLE ${tbName} ADD ROLLUP rollup_date(datek1,datetimek2,datetimek1,datetimek3,datev1,datetimev1,datetimev2,datetimev3);"""
     int max_try_secs = 60
     while (max_try_secs--) {
diff --git a/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy
index a3917dee28a75a..2407c70d8a451c 100644
--- a/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy
+++ b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy
@@ -34,6 +34,7 @@ suite("bitmap", "rollup") {
             ) 
             DISTRIBUTED BY HASH(k1) properties("replication_num" = "1");
         """
+    sql """alter table test_materialized_view_bitmap1 modify column k1 set stats ('row_count'='2');"""
 
     sql "CREATE MATERIALIZED VIEW test_neg as select k1,bitmap_union(to_bitmap(k2)), bitmap_union(to_bitmap(k3)) FROM ${tbName1} GROUP BY k1;"
     max_try_secs = 60
diff --git a/regression-test/suites/nereids_syntax_p0/rollup/date.groovy b/regression-test/suites/nereids_syntax_p0/rollup/date.groovy
index 1415a442952ab6..138b8775778a6c 100644
--- a/regression-test/suites/nereids_syntax_p0/rollup/date.groovy
+++ b/regression-test/suites/nereids_syntax_p0/rollup/date.groovy
@@ -61,6 +61,7 @@ suite("date", "rollup") {
 
     mv_rewrite_success("SELECT store_id, max(sale_datetime3) FROM ${tbName1} GROUP BY store_id", "amt_max4")
     sql """set enable_stats=true;"""
+    sql """alter table test_materialized_view_date1 modify column record_id set stats ('row_count'='2');"""
     mv_rewrite_success("SELECT store_id, max(sale_date1) FROM ${tbName1} GROUP BY store_id", "amt_max1")
 
     mv_rewrite_success("SELECT store_id, max(sale_datetime1) FROM ${tbName1} GROUP BY store_id", "amt_max2")
diff --git a/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy b/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy
index 2ba3540832945a..5002c509a20617 100644
--- a/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy
+++ b/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy
@@ -46,6 +46,7 @@ suite("hll", "rollup") {
             "amt_count")
 
     sql """set enable_stats=true;"""
+    sql """alter table test_materialized_view_hll1 modify column record_id set stats ('row_count'='2');"""
     mv_rewrite_success("SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM test_materialized_view_hll1 GROUP BY store_id;",
             "amt_count")
 }
diff --git a/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy b/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy
index 51599f294d1e62..d7dec7696d95eb 100644
--- a/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy
+++ b/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy
@@ -43,6 +43,7 @@ suite("hll_with_light_sc", "rollup") {
             "amt_count1")
 
     sql """set enable_stats=true;"""
+    sql """alter table test_materialized_view_hll_with_light_sc1 modify column record_id set stats ('row_count'='2');"""
     mv_rewrite_success("SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM test_materialized_view_hll_with_light_sc1 GROUP BY store_id;",
             "amt_count1")
 }

From b9afb48383f5e10b228f4e8d64cb16ce332ef6bf Mon Sep 17 00:00:00 2001
From: zhangstar333 <zhangsida@selectdb.com>
Date: Fri, 29 Nov 2024 15:12:43 +0800
Subject: [PATCH 077/399] [log](function) print type name when input is invaild
 (#44758)

### What problem does this PR solve?
Problem Summary:
add log to print data type when input is invalid
---
 .../vec/functions/function_date_or_datetime_computation.h  | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h b/be/src/vec/functions/function_date_or_datetime_computation.h
index 90221e66c213a9..330ea75cba96c8 100644
--- a/be/src/vec/functions/function_date_or_datetime_computation.h
+++ b/be/src/vec/functions/function_date_or_datetime_computation.h
@@ -684,9 +684,10 @@ struct DateTimeAddIntervalImpl {
                 block.replace_by_position(result, std::move(col_to));
             }
         } else {
-            return Status::RuntimeError("Illegal column {} of first argument of function {}",
-                                        block.get_by_position(arguments[0]).column->get_name(),
-                                        Transform::name);
+            return Status::RuntimeError(
+                    "Illegal column {} of first argument and type {} of function {}",
+                    block.get_by_position(arguments[0]).column->get_name(),
+                    block.get_by_position(arguments[0]).type->get_name(), Transform::name);
         }
         return Status::OK();
     }

From cb04281d63f475902062311c5daf6e155204856e Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Fri, 29 Nov 2024 15:35:29 +0800
Subject: [PATCH 078/399] [minor](column) remove function
 "remove_low_cardinality" (#44766)

remove function "remove_low_cardinality"
---
 be/src/vec/columns/column.h         | 4 ----
 be/src/vec/columns/column_const.cpp | 4 ----
 be/src/vec/columns/column_const.h   | 2 --
 be/src/vec/functions/function.cpp   | 8 --------
 be/src/vec/functions/nullif.cpp     | 8 --------
 5 files changed, 26 deletions(-)

diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index 96408579a843c4..593666568aa375 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -100,10 +100,6 @@ class IColumn : public COW<IColumn> {
   */
     virtual Ptr convert_column_if_overflow() { return get_ptr(); }
 
-    /// If column isn't ColumnLowCardinality, return itself.
-    /// If column is ColumnLowCardinality, transforms is to full column.
-    virtual Ptr convert_to_full_column_if_low_cardinality() const { return get_ptr(); }
-
     /// If column isn't ColumnDictionary, return itself.
     /// If column is ColumnDictionary, transforms is to predicate column.
     virtual MutablePtr convert_to_predicate_column_if_dictionary() { return get_ptr(); }
diff --git a/be/src/vec/columns/column_const.cpp b/be/src/vec/columns/column_const.cpp
index fd05127f6d72aa..feeaec497295ba 100644
--- a/be/src/vec/columns/column_const.cpp
+++ b/be/src/vec/columns/column_const.cpp
@@ -72,10 +72,6 @@ ColumnPtr ColumnConst::convert_to_full_column() const {
     return data->replicate(Offsets(1, cast_set<Offset>(s)));
 }
 
-ColumnPtr ColumnConst::remove_low_cardinality() const {
-    return ColumnConst::create(data->convert_to_full_column_if_low_cardinality(), s);
-}
-
 ColumnPtr ColumnConst::filter(const Filter& filt, ssize_t /*result_size_hint*/) const {
     column_match_filter_size(s, filt.size());
 
diff --git a/be/src/vec/columns/column_const.h b/be/src/vec/columns/column_const.h
index 0fa22ca23bf20b..69177eb2ca377a 100644
--- a/be/src/vec/columns/column_const.h
+++ b/be/src/vec/columns/column_const.h
@@ -118,8 +118,6 @@ class ColumnConst final : public COWHelper<IColumn, ColumnConst> {
 
     bool is_variable_length() const override { return data->is_variable_length(); }
 
-    ColumnPtr remove_low_cardinality() const;
-
     std::string get_name() const override { return "Const(" + data->get_name() + ")"; }
 
     void resize(size_t new_size) override { s = new_size; }
diff --git a/be/src/vec/functions/function.cpp b/be/src/vec/functions/function.cpp
index 5b935444710e25..851e430d2f0407 100644
--- a/be/src/vec/functions/function.cpp
+++ b/be/src/vec/functions/function.cpp
@@ -285,14 +285,6 @@ DataTypePtr FunctionBuilderImpl::get_return_type_without_low_cardinality(
 DataTypePtr FunctionBuilderImpl::get_return_type(const ColumnsWithTypeAndName& arguments) const {
     if (use_default_implementation_for_low_cardinality_columns()) {
         ColumnsWithTypeAndName args_without_low_cardinality(arguments);
-
-        for (ColumnWithTypeAndName& arg : args_without_low_cardinality) {
-            bool is_const = arg.column && is_column_const(*arg.column);
-            if (is_const) {
-                arg.column = assert_cast<const ColumnConst&>(*arg.column).remove_low_cardinality();
-            }
-        }
-
         auto type_without_low_cardinality =
                 get_return_type_without_low_cardinality(args_without_low_cardinality);
 
diff --git a/be/src/vec/functions/nullif.cpp b/be/src/vec/functions/nullif.cpp
index 4d85cab6b1cbc7..d8ab344acc47a4 100644
--- a/be/src/vec/functions/nullif.cpp
+++ b/be/src/vec/functions/nullif.cpp
@@ -67,14 +67,6 @@ class FunctionNullIf : public IFunction {
 
     static DataTypePtr get_return_type_for_equal(const ColumnsWithTypeAndName& arguments) {
         ColumnsWithTypeAndName args_without_low_cardinality(arguments);
-
-        for (ColumnWithTypeAndName& arg : args_without_low_cardinality) {
-            bool is_const = arg.column && is_column_const(*arg.column);
-            if (is_const) {
-                arg.column = assert_cast<const ColumnConst&>(*arg.column).remove_low_cardinality();
-            }
-        }
-
         if (!arguments.empty()) {
             if (have_null_column(arguments)) {
                 return make_nullable(std::make_shared<doris::vectorized::DataTypeUInt8>());

From 08b187b2687837921dddc22cd2235e186c428f0a Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Fri, 29 Nov 2024 16:15:04 +0800
Subject: [PATCH 079/399] [fix](orc) check all the cases before
 build_search_argument (#44615)

### What problem does this PR solve?
In the old logic, the `check_expr_can_push_down` function does not check
whether the `orc::Literal` are constructed successfully, but only checks
during `build_search_argument`. However, if it is found that the
`orc::Literal` fails to be constructed after `builder->startNot`, it
will fail because the builder cannot end `startNot`.
Therefore, we advance the behavior of constructing `orc::Literal` to the
`check_expr_can_push_down` function and save the result to the map, so
that it will never fail in the `build_search_argument` phase.

Related PR: #43255
---
 be/src/vec/exec/format/orc/vorc_reader.cpp | 176 ++++++++++-----------
 be/src/vec/exec/format/orc/vorc_reader.h   |  18 ++-
 be/test/vec/exec/orc_reader_test.cpp       |  57 +++----
 3 files changed, 131 insertions(+), 120 deletions(-)

diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index 70f3f6f003f611..df8ec52efc2896 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -34,6 +34,7 @@
 #include <memory>
 #include <ostream>
 #include <tuple>
+#include <utility>
 
 #include "cctz/civil_time.h"
 #include "cctz/time_zone.h"
@@ -567,12 +568,14 @@ std::tuple<bool, orc::Literal> convert_to_orc_literal(const orc::Type* type,
 
 std::tuple<bool, orc::Literal, orc::PredicateDataType> OrcReader::_make_orc_literal(
         const VSlotRef* slot_ref, const VLiteral* literal) {
+    DCHECK(_col_name_to_file_col_name_low_case.contains(slot_ref->expr_name()));
     auto file_col_name_low_case = _col_name_to_file_col_name_low_case[slot_ref->expr_name()];
     if (!_type_map.contains(file_col_name_low_case)) {
         // TODO: this is for acid table
         LOG(WARNING) << "Column " << slot_ref->expr_name() << " not found in _type_map";
         return std::make_tuple(false, orc::Literal(false), orc::PredicateDataType::LONG);
     }
+    DCHECK(_type_map.contains(file_col_name_low_case));
     const auto* orc_type = _type_map[file_col_name_low_case];
     if (!TYPEKIND_TO_PREDICATE_TYPE.contains(orc_type->getKind())) {
         LOG(WARNING) << "Unsupported Push Down Orc Type [TypeKind=" << orc_type->getKind() << "]";
@@ -624,15 +627,37 @@ std::tuple<bool, orc::Literal, orc::PredicateDataType> OrcReader::_make_orc_lite
     }
 }
 
-// check if the slot of expr can be pushed down to orc reader
+// check if the slot of expr can be pushed down to orc reader and make orc predicate type
 bool OrcReader::_check_slot_can_push_down(const VExprSPtr& expr) {
     if (!expr->children()[0]->is_slot_ref()) {
         return false;
     }
     const auto* slot_ref = static_cast<const VSlotRef*>(expr->children()[0].get());
     // check if the slot exists in orc file and not partition column
-    return _col_name_to_file_col_name.contains(slot_ref->expr_name()) &&
-           !_lazy_read_ctx.predicate_partition_columns.contains(slot_ref->expr_name());
+    if (!_col_name_to_file_col_name.contains(slot_ref->expr_name()) ||
+        _lazy_read_ctx.predicate_partition_columns.contains(slot_ref->expr_name())) {
+        return false;
+    }
+    auto [valid, _, predicate_type] = _make_orc_literal(slot_ref, nullptr);
+    if (valid) {
+        _vslot_ref_to_orc_predicate_data_type[slot_ref] = predicate_type;
+    }
+    return valid;
+}
+
+// check if the literal of expr can be pushed down to orc reader and make orc literal
+bool OrcReader::_check_literal_can_push_down(const VExprSPtr& expr, uint16_t child_id) {
+    if (!expr->children()[child_id]->is_literal()) {
+        return false;
+    }
+    // the slot has been checked in _check_slot_can_push_down before calling this function
+    const auto* slot_ref = static_cast<const VSlotRef*>(expr->children()[0].get());
+    const auto* literal = static_cast<const VLiteral*>(expr->children()[child_id].get());
+    auto [valid, orc_literal, _] = _make_orc_literal(slot_ref, literal);
+    if (valid) {
+        _vliteral_to_orc_literal.insert(std::make_pair(literal, orc_literal));
+    }
+    return valid;
 }
 
 // check if there are rest children of expr can be pushed down to orc reader
@@ -642,7 +667,7 @@ bool OrcReader::_check_rest_children_can_push_down(const VExprSPtr& expr) {
     }
 
     for (size_t i = 1; i < expr->children().size(); ++i) {
-        if (!expr->children()[i]->is_literal()) {
+        if (!_check_literal_can_push_down(expr, i)) {
             return false;
         }
     }
@@ -651,7 +676,10 @@ bool OrcReader::_check_rest_children_can_push_down(const VExprSPtr& expr) {
 
 // check if the expr can be pushed down to orc reader
 bool OrcReader::_check_expr_can_push_down(const VExprSPtr& expr) {
-    DCHECK(expr != nullptr);
+    if (expr == nullptr) {
+        return false;
+    }
+
     switch (expr->op()) {
     case TExprOpcode::COMPOUND_AND:
         // at least one child can be pushed down
@@ -693,198 +721,167 @@ bool OrcReader::_check_expr_can_push_down(const VExprSPtr& expr) {
     }
 }
 
-bool OrcReader::_build_less_than(const VExprSPtr& expr,
+void OrcReader::_build_less_than(const VExprSPtr& expr,
                                  std::unique_ptr<orc::SearchArgumentBuilder>& builder) {
     DCHECK(expr->children().size() == 2);
     DCHECK(expr->children()[0]->is_slot_ref());
     DCHECK(expr->children()[1]->is_literal());
     const auto* slot_ref = static_cast<const VSlotRef*>(expr->children()[0].get());
     const auto* literal = static_cast<const VLiteral*>(expr->children()[1].get());
-    auto [valid, orc_literal, predicate_type] = _make_orc_literal(slot_ref, literal);
-    if (!valid) {
-        return false;
-    }
+    DCHECK(_vslot_ref_to_orc_predicate_data_type.contains(slot_ref));
+    auto predicate_type = _vslot_ref_to_orc_predicate_data_type[slot_ref];
+    DCHECK(_vliteral_to_orc_literal.contains(literal));
+    auto orc_literal = _vliteral_to_orc_literal.find(literal)->second;
     builder->lessThan(slot_ref->expr_name(), predicate_type, orc_literal);
-    return true;
 }
 
-bool OrcReader::_build_less_than_equals(const VExprSPtr& expr,
+void OrcReader::_build_less_than_equals(const VExprSPtr& expr,
                                         std::unique_ptr<orc::SearchArgumentBuilder>& builder) {
     DCHECK(expr->children().size() == 2);
     DCHECK(expr->children()[0]->is_slot_ref());
     DCHECK(expr->children()[1]->is_literal());
     const auto* slot_ref = static_cast<const VSlotRef*>(expr->children()[0].get());
     const auto* literal = static_cast<const VLiteral*>(expr->children()[1].get());
-    auto [valid, orc_literal, predicate_type] = _make_orc_literal(slot_ref, literal);
-    if (!valid) {
-        return false;
-    }
+    DCHECK(_vslot_ref_to_orc_predicate_data_type.contains(slot_ref));
+    auto predicate_type = _vslot_ref_to_orc_predicate_data_type[slot_ref];
+    DCHECK(_vliteral_to_orc_literal.contains(literal));
+    auto orc_literal = _vliteral_to_orc_literal.find(literal)->second;
     builder->lessThanEquals(slot_ref->expr_name(), predicate_type, orc_literal);
-    return true;
 }
 
-bool OrcReader::_build_equals(const VExprSPtr& expr,
+void OrcReader::_build_equals(const VExprSPtr& expr,
                               std::unique_ptr<orc::SearchArgumentBuilder>& builder) {
     DCHECK(expr->children().size() == 2);
     DCHECK(expr->children()[0]->is_slot_ref());
     DCHECK(expr->children()[1]->is_literal());
     const auto* slot_ref = static_cast<const VSlotRef*>(expr->children()[0].get());
     const auto* literal = static_cast<const VLiteral*>(expr->children()[1].get());
-    auto [valid, orc_literal, predicate_type] = _make_orc_literal(slot_ref, literal);
-    if (!valid) {
-        return false;
-    }
+    DCHECK(_vslot_ref_to_orc_predicate_data_type.contains(slot_ref));
+    auto predicate_type = _vslot_ref_to_orc_predicate_data_type[slot_ref];
+    DCHECK(_vliteral_to_orc_literal.contains(literal));
+    auto orc_literal = _vliteral_to_orc_literal.find(literal)->second;
     builder->equals(slot_ref->expr_name(), predicate_type, orc_literal);
-    return true;
 }
 
-bool OrcReader::_build_filter_in(const VExprSPtr& expr,
+void OrcReader::_build_filter_in(const VExprSPtr& expr,
                                  std::unique_ptr<orc::SearchArgumentBuilder>& builder) {
     DCHECK(expr->children().size() >= 2);
     DCHECK(expr->children()[0]->is_slot_ref());
     const auto* slot_ref = static_cast<const VSlotRef*>(expr->children()[0].get());
     std::vector<orc::Literal> literals;
-    orc::PredicateDataType predicate_type = orc::PredicateDataType::LONG;
+    DCHECK(_vslot_ref_to_orc_predicate_data_type.contains(slot_ref));
+    orc::PredicateDataType predicate_type = _vslot_ref_to_orc_predicate_data_type[slot_ref];
     for (size_t i = 1; i < expr->children().size(); ++i) {
         DCHECK(expr->children()[i]->is_literal());
         const auto* literal = static_cast<const VLiteral*>(expr->children()[i].get());
-        auto [valid, orc_literal, type] = _make_orc_literal(slot_ref, literal);
-        if (!valid) {
-            return false;
-        }
+        DCHECK(_vliteral_to_orc_literal.contains(literal));
+        auto orc_literal = _vliteral_to_orc_literal.find(literal)->second;
         literals.emplace_back(orc_literal);
-        predicate_type = type;
     }
     DCHECK(!literals.empty());
     builder->in(slot_ref->expr_name(), predicate_type, literals);
-    return true;
 }
 
-bool OrcReader::_build_is_null(const VExprSPtr& expr,
+void OrcReader::_build_is_null(const VExprSPtr& expr,
                                std::unique_ptr<orc::SearchArgumentBuilder>& builder) {
     DCHECK(expr->children().size() == 1);
     DCHECK(expr->children()[0]->is_slot_ref());
     const auto* slot_ref = static_cast<const VSlotRef*>(expr->children()[0].get());
-    auto [valid, _, predicate_type] = _make_orc_literal(slot_ref, nullptr);
+    DCHECK(_vslot_ref_to_orc_predicate_data_type.contains(slot_ref));
+    auto predicate_type = _vslot_ref_to_orc_predicate_data_type[slot_ref];
     builder->isNull(slot_ref->expr_name(), predicate_type);
-    return true;
 }
 
 bool OrcReader::_build_search_argument(const VExprSPtr& expr,
                                        std::unique_ptr<orc::SearchArgumentBuilder>& builder) {
-    if (expr == nullptr) {
-        return false;
-    }
-
-    // if expr can not be pushed down, skip it and continue to next expr
+    // OPTIMIZE: check expr only once
     if (!_check_expr_can_push_down(expr)) {
         return false;
     }
-
     switch (expr->op()) {
     case TExprOpcode::COMPOUND_AND: {
-        bool at_least_one_can_push_down = false;
         builder->startAnd();
+        bool at_least_one_can_push_down = false;
         for (const auto& child : expr->children()) {
             if (_build_search_argument(child, builder)) {
                 at_least_one_can_push_down = true;
             }
         }
-        if (!at_least_one_can_push_down) {
-            // if all exprs can not be pushed down, builder->end() will throw exception
-            return false;
-        }
+        DCHECK(at_least_one_can_push_down);
         builder->end();
         break;
     }
-    case TExprOpcode::COMPOUND_OR:
+    case TExprOpcode::COMPOUND_OR: {
         builder->startOr();
+        bool all_can_push_down = true;
         for (const auto& child : expr->children()) {
             if (!_build_search_argument(child, builder)) {
-                return false;
+                all_can_push_down = false;
             }
         }
+        DCHECK(all_can_push_down);
         builder->end();
         break;
-    case TExprOpcode::COMPOUND_NOT:
-        builder->startNot();
+    }
+    case TExprOpcode::COMPOUND_NOT: {
         DCHECK_EQ(expr->children().size(), 1);
-        if (!_build_search_argument(expr->children()[0], builder)) {
-            return false;
-        }
+        builder->startNot();
+        auto res = _build_search_argument(expr->children()[0], builder);
+        DCHECK(res);
         builder->end();
         break;
+    }
     case TExprOpcode::GE:
         builder->startNot();
-        if (!_build_less_than(expr, builder)) {
-            return false;
-        }
+        _build_less_than(expr, builder);
         builder->end();
         break;
     case TExprOpcode::GT:
         builder->startNot();
-        if (!_build_less_than_equals(expr, builder)) {
-            return false;
-        }
+        _build_less_than_equals(expr, builder);
         builder->end();
         break;
     case TExprOpcode::LE:
-        if (!_build_less_than_equals(expr, builder)) {
-            return false;
-        }
+        _build_less_than_equals(expr, builder);
         break;
     case TExprOpcode::LT:
-        if (!_build_less_than(expr, builder)) {
-            return false;
-        }
+        _build_less_than(expr, builder);
         break;
     case TExprOpcode::EQ:
-        if (!_build_equals(expr, builder)) {
-            return false;
-        }
+        _build_equals(expr, builder);
         break;
     case TExprOpcode::NE:
         builder->startNot();
-        if (!_build_equals(expr, builder)) {
-            return false;
-        }
+        _build_equals(expr, builder);
         builder->end();
         break;
     case TExprOpcode::FILTER_IN:
-        if (!_build_filter_in(expr, builder)) {
-            return false;
-        }
+        _build_filter_in(expr, builder);
         break;
     case TExprOpcode::FILTER_NOT_IN:
         builder->startNot();
-        if (!_build_filter_in(expr, builder)) {
-            return false;
-        }
+        _build_filter_in(expr, builder);
         builder->end();
         break;
     // is null and is not null is represented as function call
-    case TExprOpcode::INVALID_OPCODE: {
+    case TExprOpcode::INVALID_OPCODE:
         DCHECK(expr->node_type() == TExprNodeType::FUNCTION_CALL);
         if (expr->fn().name.function_name == "is_null_pred") {
-            if (!_build_is_null(expr, builder)) {
-                return false;
-            }
+            _build_is_null(expr, builder);
         } else if (expr->fn().name.function_name == "is_not_null_pred") {
             builder->startNot();
-            if (!_build_is_null(expr, builder)) {
-                return false;
-            }
+            _build_is_null(expr, builder);
             builder->end();
         } else {
+            // should not reach here, because _check_expr_can_push_down has already checked
             __builtin_unreachable();
         }
         break;
-    }
-    default: {
+
+    default:
         // should not reach here, because _check_expr_can_push_down has already checked
         __builtin_unreachable();
     }
-    }
     return true;
 }
 
@@ -898,6 +895,8 @@ bool OrcReader::_init_search_argument(const VExprContextSPtrs& conjuncts) {
     bool at_least_one_can_push_down = false;
     builder->startAnd();
     for (const auto& expr_ctx : conjuncts) {
+        _vslot_ref_to_orc_predicate_data_type.clear();
+        _vliteral_to_orc_literal.clear();
         if (_build_search_argument(expr_ctx->root(), builder)) {
             at_least_one_can_push_down = true;
         }
@@ -943,7 +942,7 @@ Status OrcReader::set_fill_columns(
                     visit_slot(child.get());
                 }
             } else if (VInPredicate* in_predicate = typeid_cast<VInPredicate*>(filter_impl)) {
-                if (in_predicate->get_num_children() > 0) {
+                if (!in_predicate->children().empty()) {
                     visit_slot(in_predicate->children()[0].get());
                 }
             } else {
@@ -1179,7 +1178,8 @@ Status OrcReader::_fill_partition_columns(
         if (num_deserialized != rows) {
             return Status::InternalError(
                     "Failed to fill partition column: {}={} ."
-                    "Number of rows expected to be written : {}, number of rows actually written : "
+                    "Number of rows expected to be written : {}, number of rows actually "
+                    "written : "
                     "{}",
                     slot_desc->col_name(), value, num_deserialized, rows);
         }
diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h
index 8c73957e79e4e0..0dd19077bcf0af 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.h
+++ b/be/src/vec/exec/format/orc/vorc_reader.h
@@ -41,6 +41,7 @@
 #include "orc/Reader.hh"
 #include "orc/Type.hh"
 #include "orc/Vector.hh"
+#include "orc/sargs/Literal.hh"
 #include "runtime/types.h"
 #include "util/runtime_profile.h"
 #include "vec/aggregate_functions/aggregate_function.h"
@@ -288,23 +289,27 @@ class OrcReader : public GenericReader {
                         bool* is_hive1_orc);
     static bool _check_acid_schema(const orc::Type& type);
     static const orc::Type& _remove_acid(const orc::Type& type);
+
+    // functions for building search argument until _init_search_argument
     std::tuple<bool, orc::Literal, orc::PredicateDataType> _make_orc_literal(
             const VSlotRef* slot_ref, const VLiteral* literal);
     bool _check_slot_can_push_down(const VExprSPtr& expr);
+    bool _check_literal_can_push_down(const VExprSPtr& expr, uint16_t child_id);
     bool _check_rest_children_can_push_down(const VExprSPtr& expr);
     bool _check_expr_can_push_down(const VExprSPtr& expr);
-    bool _build_less_than(const VExprSPtr& expr,
+    void _build_less_than(const VExprSPtr& expr,
                           std::unique_ptr<orc::SearchArgumentBuilder>& builder);
-    bool _build_less_than_equals(const VExprSPtr& expr,
+    void _build_less_than_equals(const VExprSPtr& expr,
                                  std::unique_ptr<orc::SearchArgumentBuilder>& builder);
-    bool _build_equals(const VExprSPtr& expr, std::unique_ptr<orc::SearchArgumentBuilder>& builder);
-    bool _build_filter_in(const VExprSPtr& expr,
+    void _build_equals(const VExprSPtr& expr, std::unique_ptr<orc::SearchArgumentBuilder>& builder);
+    void _build_filter_in(const VExprSPtr& expr,
                           std::unique_ptr<orc::SearchArgumentBuilder>& builder);
-    bool _build_is_null(const VExprSPtr& expr,
+    void _build_is_null(const VExprSPtr& expr,
                         std::unique_ptr<orc::SearchArgumentBuilder>& builder);
     bool _build_search_argument(const VExprSPtr& expr,
                                 std::unique_ptr<orc::SearchArgumentBuilder>& builder);
     bool _init_search_argument(const VExprContextSPtrs& conjuncts);
+
     void _init_bloom_filter(
             std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range);
     void _init_system_properties();
@@ -644,6 +649,9 @@ class OrcReader : public GenericReader {
     std::unordered_map<std::string, std::string> _table_col_to_file_col;
     //support iceberg position delete .
     std::vector<int64_t>* _position_delete_ordered_rowids = nullptr;
+    std::unordered_map<const VSlotRef*, orc::PredicateDataType>
+            _vslot_ref_to_orc_predicate_data_type;
+    std::unordered_map<const VLiteral*, orc::Literal> _vliteral_to_orc_literal;
 };
 
 class ORCFileInputStream : public orc::InputStream, public ProfileCollector {
diff --git a/be/test/vec/exec/orc_reader_test.cpp b/be/test/vec/exec/orc_reader_test.cpp
index ec5bd9b519de63..f2bba434368ae1 100644
--- a/be/test/vec/exec/orc_reader_test.cpp
+++ b/be/test/vec/exec/orc_reader_test.cpp
@@ -94,33 +94,34 @@ class OrcReaderTest : public testing::Test {
 
 TEST_F(OrcReaderTest, test_build_search_argument) {
     ExecEnv::GetInstance()->set_orc_memory_pool(new ORCMemoryPool());
-    std::
-            vector<std::string>
-                    exprs =
-                            {
-                                    // select count(o_orderkey) from tpch1_orc.orders where o_orderkey < 100 or o_orderkey > 5999900 or o_orderkey in (1000000, 2000000, 3000000);
-                                    R"|({"1":{"lst":["rec",13,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":100}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":13},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"gt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"gt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":5999900}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":11},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":5},"4":{"i32":4},"11":{"rec":{"1":{"tf":0}}},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":3000000}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // select count(o_orderkey) from tpch1_orc.orders where o_orderkey is null or (o_orderkey between 100 and 1000 and o_orderkey not in (200, 300, 400));
-                                    R"|({"1":{"lst":["rec",16,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":1},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"is_null_pred"}}},"2":{"i32":0},"3":{"lst":["rec",1,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"is_null_pred(int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":0}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":100}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":12},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"le"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"le(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":11},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":6},"4":{"i32":4},"11":{"rec":{"1":{"tf":1}}},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":200}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":300}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":400}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // select count(o_orderkey) from tpch1_orc.orders where o_orderkey is null or (o_orderkey between 1000000 and 1200000 and o_orderkey != 1100000);
-                                    R"|({"1":{"lst":["rec",14,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":1},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"is_null_pred"}}},"2":{"i32":0},"3":{"lst":["rec",1,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"is_null_pred(int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":0}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":12},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"le"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"le(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1200000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":10},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ne"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ne(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1100000}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // SELECT count(o_orderkey) FROM tpch1_orc.orders WHERE o_orderkey IN (1000000, 2000000, 3000000) OR (o_orderdate >= '1994-01-01' AND o_orderdate <= '1994-12-31');
-                                    R"|({"1":{"lst":["rec",13,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":11},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":5},"4":{"i32":4},"11":{"rec":{"1":{"tf":0}}},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":3000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(datev2, datev2)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":26},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":4},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderdate"}},{"1":{"i32":7},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"8":{"rec":{"1":{"str":"1994-01-01"}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":12},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"le"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"le(datev2, datev2)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":26},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":4},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderdate"}},{"1":{"i32":7},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"8":{"rec":{"1":{"str":"1994-12-31"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // select count(o_orderkey) from tpch1_orc.orders where o_orderkey < 2 or (o_comment like '%delayed%' and o_orderpriority = '1-URGENT');
-                                    R"|({"1":{"lst":["rec",11,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"like"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":9}}}}]},"3":{"i64":9}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"like(text, varchar(9))"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":8},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_comment"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"%delayed%"}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":9},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"eq"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"eq(text, varchar(65533))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":23},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":5},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderpriority"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"1-URGENT"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // select count(o_orderkey) from tpch1_orc.orders where o_orderkey < 1 + 1;
-                                    R"|({"1":{"lst":["rec",3,{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // SELECT count(o_orderkey) FROM tpch1_orc.orders WHERE o_comment LIKE '%delayed%' OR o_orderpriority = '1-URGENT';
-                                    R"|({"1":{"lst":["rec",7,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"like"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":9}}}}]},"3":{"i64":9}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"like(text, varchar(9))"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":8},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_comment"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"%delayed%"}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":9},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"eq"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"eq(text, varchar(65533))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":23},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":5},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderpriority"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"1-URGENT"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // select count(o_orderkey) from tpch1_orc.orders where o_orderkey between 1 and 100 or random() > 0.5;
-                                    R"|({"1":{"lst":["rec",11,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":12},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"le"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"le(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":100}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":13},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"gt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"gt(double, double)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":8},"29":{"tf":0}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"random"}}},"2":{"i32":0},"3":{"lst":["rec",0]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"random()"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":0.5}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // select count(o_orderkey) from tpch1_orc.orders where lower(o_orderpriority) = '1-urgent';
-                                    R"|({"1":{"lst":["rec",4,{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":9},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"eq"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"eq(varchar(65533), varchar(65533))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":15},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":1},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lower"}}},"2":{"i32":0},"3":{"lst":["rec",1,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lower(text)"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":5},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderpriority"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"1-urgent"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // select count(o_orderkey) from tpch1_orc.orders where o_orderkey * 2 < 60;
-                                    R"|({"1":{"lst":["rec",5,{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(bigint, bigint)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":6},"29":{"tf":1}},{"1":{"i32":1},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}}},"3":{"i32":55},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"multiply"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":3}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"multiply(int, tinyint)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":3}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":60}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                                    // select count(o_orderkey) from tpch1_orc.orders where o_orderdate is not null;
-                                    R"|({"1":{"lst":["rec",4,{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":9},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"eq"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"eq(datetimev2(0), datetimev2(0))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":27},"29":{"tf":1}},{"1":{"i32":5},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}}},"3":{"i32":4},"4":{"i32":1},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"casttodatetimev2"}}},"2":{"i32":0},"3":{"lst":["rec",1,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"casttodatetimev2(datev2)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":4},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderdate"}},{"1":{"i32":7},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"8":{"rec":{"1":{"str":"2024-11-12 21:13:02"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
-                            };
+    std::vector<std::string>
+            exprs =
+                    {
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderkey < 100 or o_orderkey > 5999900 or o_orderkey in (1000000, 2000000, 3000000);
+                            R"|({"1":{"lst":["rec",13,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":100}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":13},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"gt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"gt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":5999900}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":11},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":5},"4":{"i32":4},"11":{"rec":{"1":{"tf":0}}},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":3000000}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderkey is null or (o_orderkey between 100 and 1000 and o_orderkey not in (200, 300, 400));
+                            R"|({"1":{"lst":["rec",16,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":1},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"is_null_pred"}}},"2":{"i32":0},"3":{"lst":["rec",1,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"is_null_pred(int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":0}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":100}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":12},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"le"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"le(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":11},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":6},"4":{"i32":4},"11":{"rec":{"1":{"tf":1}}},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":200}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":300}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":400}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderkey is null or (o_orderkey between 1000000 and 1200000 and o_orderkey != 1100000);
+                            R"|({"1":{"lst":["rec",14,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":1},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"is_null_pred"}}},"2":{"i32":0},"3":{"lst":["rec",1,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"is_null_pred(int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":0}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":12},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"le"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"le(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1200000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":10},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ne"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ne(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1100000}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // SELECT count(o_orderkey) FROM tpch1_orc.orders WHERE o_orderkey IN (1000000, 2000000, 3000000) OR (o_orderdate >= '1994-01-01' AND o_orderdate <= '1994-12-31');
+                            R"|({"1":{"lst":["rec",13,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":11},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":5},"4":{"i32":4},"11":{"rec":{"1":{"tf":0}}},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":3000000}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(datev2, datev2)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":26},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":4},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderdate"}},{"1":{"i32":7},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"8":{"rec":{"1":{"str":"1994-01-01"}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":12},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"le"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"le(datev2, datev2)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":26},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":4},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderdate"}},{"1":{"i32":7},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"8":{"rec":{"1":{"str":"1994-12-31"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderkey < 2 or (o_comment like '%delayed%' and o_orderpriority = '1-URGENT');
+                            R"|({"1":{"lst":["rec",11,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"like"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":9}}}}]},"3":{"i64":9}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"like(text, varchar(9))"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":8},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_comment"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"%delayed%"}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":9},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"eq"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"eq(text, varchar(65533))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":23},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":5},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderpriority"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"1-URGENT"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderkey < 2 or (o_totalprice < 173665.47 and o_custkey >= 36901);
+                            R"|({"1":{"lst":["rec",11,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":30},"3":{"i32":12},"4":{"i32":2}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":30},"3":{"i32":12},"4":{"i32":2}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(decimalv3(12,2), decimalv3(12,2))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":30},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":30},"3":{"i32":12},"4":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":3},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_totalprice"}},{"1":{"i32":10},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":30},"3":{"i32":12},"4":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"19":{"rec":{"1":{"str":"173665.47"}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":1},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_custkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":36901}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderkey < 1 + 1;
+                            R"|({"1":{"lst":["rec",3,{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // SELECT count(o_orderkey) FROM tpch1_orc.orders WHERE o_comment LIKE '%delayed%' OR o_orderpriority = '1-URGENT';
+                            R"|({"1":{"lst":["rec",7,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"like"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":9}}}}]},"3":{"i64":9}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"like(text, varchar(9))"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":8},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_comment"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"%delayed%"}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":9},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"eq"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"eq(text, varchar(65533))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":23},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":5},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderpriority"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"1-URGENT"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderkey between 1 and 100 or random() > 0.5;
+                            R"|({"1":{"lst":["rec",11,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":1}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":12},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"le"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"le(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":100}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":13},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"gt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"gt(double, double)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":8},"29":{"tf":0}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"random"}}},"2":{"i32":0},"3":{"lst":["rec",0]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"random()"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":0}},{"1":{"i32":8},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":8}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"9":{"rec":{"1":{"dbl":0.5}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where lower(o_orderpriority) = '1-urgent';
+                            R"|({"1":{"lst":["rec",4,{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":9},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"eq"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"eq(varchar(65533), varchar(65533))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":15},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":1},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lower"}}},"2":{"i32":0},"3":{"lst":["rec",1,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lower(text)"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":5},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderpriority"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"1-urgent"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderkey * 2 < 60;
+                            R"|({"1":{"lst":["rec",5,{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(bigint, bigint)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":6},"29":{"tf":1}},{"1":{"i32":1},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}}},"3":{"i32":55},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"multiply"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":3}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"multiply(int, tinyint)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":3}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":6}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":60}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderdate is not null;
+                            R"|({"1":{"lst":["rec",4,{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":9},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"eq"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"eq(datetimev2(0), datetimev2(0))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":27},"29":{"tf":1}},{"1":{"i32":5},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}}},"3":{"i32":4},"4":{"i32":1},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"casttodatetimev2"}}},"2":{"i32":0},"3":{"lst":["rec",1,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"casttodatetimev2(datev2)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":26}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":4},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderdate"}},{"1":{"i32":7},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":27},"3":{"i32":18},"4":{"i32":0}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"8":{"rec":{"1":{"str":"2024-11-12 21:13:02"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                    };
     std::vector<std::string> result_search_arguments = {
             "leaf-0 = (o_orderkey < 100), leaf-1 = (o_orderkey <= 5999900), leaf-2 "
             "= (o_orderkey "
@@ -139,6 +140,8 @@ TEST_F(OrcReaderTest, test_build_search_argument) {
             "(or leaf-0 leaf-2))",
             "leaf-0 = (o_orderkey < 2), leaf-1 = (o_orderpriority = 1-URGENT), expr = (or leaf-0 "
             "leaf-1)",
+            "leaf-0 = (o_orderkey < 2), leaf-1 = (o_custkey < 36901), expr = (or leaf-0 (not "
+            "leaf-1))",
             "leaf-0 = (o_orderkey < 2), expr = leaf-0",
             CANNOT_PUSH_DOWN_ERROR,
             CANNOT_PUSH_DOWN_ERROR,

From 46575e59bac2c39088ecd925d1d20097d86667a3 Mon Sep 17 00:00:00 2001
From: HappenLee <happenlee@selectdb.com>
Date: Fri, 29 Nov 2024 17:52:49 +0800
Subject: [PATCH 080/399] [Refactor](vec) Remove the unless api in IColumn
 (#44619)

Remove the unless api in IColumn
---
 be/src/vec/columns/column.h                 | 12 +-----------
 be/src/vec/columns/column_complex.h         |  5 -----
 be/src/vec/columns/column_const.h           |  1 -
 be/src/vec/columns/column_decimal.h         |  3 ---
 be/src/vec/columns/column_nullable.h        |  3 ---
 be/src/vec/columns/column_vector.h          |  2 --
 be/src/vec/columns/predicate_column.h       |  2 --
 be/src/vec/data_types/data_type.h           |  1 +
 be/src/vec/functions/functions_comparison.h |  6 +++---
 9 files changed, 5 insertions(+), 30 deletions(-)

diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index 593666568aa375..24653f878555f5 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -565,10 +565,6 @@ class IColumn : public COW<IColumn> {
     /// It's true for ColumnNullable, can be true or false for ColumnConst, etc.
     virtual bool is_concrete_nullable() const { return false; }
 
-    virtual bool is_bitmap() const { return false; }
-
-    virtual bool is_hll() const { return false; }
-
     // true if column has null element
     virtual bool has_null() const { return false; }
 
@@ -607,10 +603,6 @@ class IColumn : public COW<IColumn> {
     /// Checks only @sample_ratio ratio of rows.
     virtual double get_ratio_of_default_rows(double sample_ratio = 1.0) const { return 0.0; }
 
-    /// Column is ColumnVector of numbers or ColumnConst of it. Note that Nullable columns are not numeric.
-    /// Implies is_fixed_and_contiguous.
-    virtual bool is_numeric() const { return false; }
-
     // Column is ColumnString/ColumnArray/ColumnMap or other variable length column at every row
     virtual bool is_variable_length() const { return false; }
 
@@ -618,8 +610,6 @@ class IColumn : public COW<IColumn> {
 
     virtual bool is_column_string64() const { return false; }
 
-    virtual bool is_column_decimal() const { return false; }
-
     virtual bool is_column_dictionary() const { return false; }
 
     virtual bool is_column_array() const { return false; }
@@ -752,4 +742,4 @@ struct ColumnPtrWrapper {
 
     ColumnPtrWrapper(vectorized::ColumnPtr col) : column_ptr(std::move(col)) {}
 };
-} // namespace doris
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/columns/column_complex.h b/be/src/vec/columns/column_complex.h
index 14ae940c9d7ba0..f8513d54f57934 100644
--- a/be/src/vec/columns/column_complex.h
+++ b/be/src/vec/columns/column_complex.h
@@ -48,11 +48,6 @@ class ColumnComplexType final : public COWHelper<IColumn, ColumnComplexType<T>>
     using value_type = T;
     using Container = std::vector<value_type>;
 
-    bool is_numeric() const override { return false; }
-
-    bool is_bitmap() const override { return std::is_same_v<T, BitmapValue>; }
-    bool is_hll() const override { return std::is_same_v<T, HyperLogLog>; }
-
     size_t size() const override { return data.size(); }
 
     StringRef get_data_at(size_t n) const override {
diff --git a/be/src/vec/columns/column_const.h b/be/src/vec/columns/column_const.h
index 69177eb2ca377a..80d50e26e16ead 100644
--- a/be/src/vec/columns/column_const.h
+++ b/be/src/vec/columns/column_const.h
@@ -253,7 +253,6 @@ class ColumnConst final : public COWHelper<IColumn, ColumnConst> {
     // ColumnConst is not nullable, but may be concrete nullable.
     bool is_concrete_nullable() const override { return is_column_nullable(*data); }
     bool only_null() const override { return data->is_null_at(0); }
-    bool is_numeric() const override { return data->is_numeric(); }
     StringRef get_raw_data() const override { return data->get_raw_data(); }
 
     /// Not part of the common interface.
diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h
index 946b268436ef6c..a8b1f9573ef185 100644
--- a/be/src/vec/columns/column_decimal.h
+++ b/be/src/vec/columns/column_decimal.h
@@ -105,9 +105,6 @@ class ColumnDecimal final : public COWHelper<IColumn, ColumnDecimal<T>> {
 public:
     std::string get_name() const override { return TypeName<T>::get(); }
 
-    bool is_numeric() const override { return false; }
-    bool is_column_decimal() const override { return true; }
-
     size_t size() const override { return data.size(); }
     size_t byte_size() const override { return data.size() * sizeof(data[0]); }
     size_t allocated_bytes() const override { return data.allocated_bytes(); }
diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h
index 83d5e6af35aeb4..84b3ce0f82aadb 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -328,9 +328,6 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable>, public N
 
     bool is_nullable() const override { return true; }
     bool is_concrete_nullable() const override { return true; }
-    bool is_bitmap() const override { return get_nested_column().is_bitmap(); }
-    bool is_hll() const override { return get_nested_column().is_hll(); }
-    bool is_column_decimal() const override { return get_nested_column().is_column_decimal(); }
     bool is_column_string() const override { return get_nested_column().is_column_string(); }
     bool is_column_array() const override { return get_nested_column().is_column_array(); }
     bool is_column_map() const override { return get_nested_column().is_column_map(); }
diff --git a/be/src/vec/columns/column_vector.h b/be/src/vec/columns/column_vector.h
index 970997a91864cc..8006d76c1f98be 100644
--- a/be/src/vec/columns/column_vector.h
+++ b/be/src/vec/columns/column_vector.h
@@ -152,8 +152,6 @@ class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
     ColumnVector(std::initializer_list<T> il) : data {il} {}
 
 public:
-    bool is_numeric() const override { return IsNumber<T>; }
-
     size_t size() const override { return data.size(); }
 
     StringRef get_data_at(size_t n) const override {
diff --git a/be/src/vec/columns/predicate_column.h b/be/src/vec/columns/predicate_column.h
index 7e15656fe1d204..ab9648c3ae620f 100644
--- a/be/src/vec/columns/predicate_column.h
+++ b/be/src/vec/columns/predicate_column.h
@@ -102,8 +102,6 @@ class PredicateColumnType final : public COWHelper<IColumn, PredicateColumnType<
     using value_type = T;
     using Container = PaddedPODArray<value_type>;
 
-    bool is_numeric() const override { return false; }
-
     size_t size() const override { return data.size(); }
 
     StringRef get_data_at(size_t n) const override {
diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
index 86f7cf36fa8dc7..6c54241ea18a2a 100644
--- a/be/src/vec/data_types/data_type.h
+++ b/be/src/vec/data_types/data_type.h
@@ -289,6 +289,7 @@ struct WhichDataType {
     bool is_aggregate_function() const { return idx == TypeIndex::AggregateFunction; }
     bool is_variant_type() const { return idx == TypeIndex::VARIANT; }
     bool is_simple() const { return is_int() || is_uint() || is_float() || is_string(); }
+    bool is_num_can_compare() const { return is_int_or_uint() || is_float() || is_ip(); }
 };
 
 /// IDataType helpers (alternative for IDataType virtual methods with single point of truth)
diff --git a/be/src/vec/functions/functions_comparison.h b/be/src/vec/functions/functions_comparison.h
index 78a89071c76efd..cb56f176c71eb1 100644
--- a/be/src/vec/functions/functions_comparison.h
+++ b/be/src/vec/functions/functions_comparison.h
@@ -637,8 +637,8 @@ class FunctionComparison : public IFunction {
         WhichDataType which_left {left_type};
         WhichDataType which_right {right_type};
 
-        const bool left_is_num = col_left_untyped->is_numeric();
-        const bool right_is_num = col_right_untyped->is_numeric();
+        const bool left_is_num_can_compare = which_left.is_num_can_compare();
+        const bool right_is_num_can_compare = which_right.is_num_can_compare();
 
         const bool left_is_string = which_left.is_string_or_fixed_string();
         const bool right_is_string = which_right.is_string_or_fixed_string();
@@ -648,7 +648,7 @@ class FunctionComparison : public IFunction {
         //        bool date_and_datetime = (left_type != right_type) && which_left.is_date_or_datetime() &&
         //                                 which_right.is_date_or_datetime();
 
-        if (left_is_num && right_is_num) {
+        if (left_is_num_can_compare && right_is_num_can_compare) {
             if (!(execute_num_left_type<UInt8>(block, result, col_left_untyped,
                                                col_right_untyped) ||
                   execute_num_left_type<UInt16>(block, result, col_left_untyped,

From 0ffb71b558949840bc3d83b073177b0952b82cbb Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Fri, 29 Nov 2024 18:09:48 +0800
Subject: [PATCH 081/399] [improvement](mysql)Support mysql
 COM_RESET_CONNECTION command. (#44747)

### What problem does this PR solve?
Support mysql COM_RESET_CONNECTION command.
Doris server side reset default catalog and return
ctx.getState().setOk()

Python test code:
```
import mysql.connector

connection = mysql.connector.connect(
    host="172.20.32.136",
    port=29030,
    user="root",
    password="",
    database=""
)
print(connection.cmd_reset_connection())
```

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None
---
 .../src/main/java/org/apache/doris/qe/ConnectContext.java  | 5 +++++
 .../main/java/org/apache/doris/qe/ConnectProcessor.java    | 7 +++++++
 .../java/org/apache/doris/qe/MysqlConnectProcessor.java    | 3 +++
 3 files changed, 15 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index a16422ba9e5979..c21c9ee3f86db9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -342,6 +342,11 @@ public String removeLastDBOfCatalog(String catalog) {
         return lastDBOfCatalog.get(catalog);
     }
 
+    // Used by COM_RESET_CONNECTION
+    public void clearLastDBOfCatalog() {
+        lastDBOfCatalog.clear();
+    }
+
     public void setNotEvalNondeterministicFunction(boolean notEvalNondeterministicFunction) {
         this.notEvalNondeterministicFunction = notEvalNondeterministicFunction;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 2340aa37aebb55..fcc6c2362cf276 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -44,6 +44,7 @@
 import org.apache.doris.common.util.SqlUtils;
 import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.InternalCatalog;
 import org.apache.doris.metric.MetricRepo;
 import org.apache.doris.mysql.MysqlChannel;
 import org.apache.doris.mysql.MysqlPacket;
@@ -198,6 +199,12 @@ protected void handleDebug() {
         ctx.getState().setOk();
     }
 
+    protected void handleResetConnection() {
+        ctx.changeDefaultCatalog(InternalCatalog.INTERNAL_CATALOG_NAME);
+        ctx.clearLastDBOfCatalog();
+        ctx.getState().setOk();
+    }
+
     protected void handleStmtReset() {
         ctx.getState().setOk();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
index 376d4740e632a1..da3baced5fad78 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
@@ -264,6 +264,9 @@ private void dispatch() throws IOException {
             case COM_SET_OPTION:
                 handleSetOption();
                 break;
+            case COM_RESET_CONNECTION:
+                handleResetConnection();
+                break;
             default:
                 ctx.getState().setError(ErrorCode.ERR_UNKNOWN_COM_ERROR, "Unsupported command(" + command + ")");
                 LOG.warn("Unsupported command(" + command + ")");

From 851d780c14d9e880547181b826ee57953869944d Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Fri, 29 Nov 2024 19:06:37 +0800
Subject: [PATCH 082/399] [Chore](check) add check for bf not inited and not
 ignored (#44781)

add check for bf not inited and not ignored
---
 be/src/exprs/bloom_filter_func.h | 2 ++
 be/src/exprs/runtime_filter.cpp  | 7 +++++++
 2 files changed, 9 insertions(+)

diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h
index 674a429581643b..ff749420ad286e 100644
--- a/be/src/exprs/bloom_filter_func.h
+++ b/be/src/exprs/bloom_filter_func.h
@@ -233,6 +233,8 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
                                                 uint16_t* offsets, int number,
                                                 bool is_parse_column) = 0;
 
+    bool inited() const { return _inited; }
+
 private:
     void _limit_length() {
         if (_runtime_bloom_filter_min_size > 0) {
diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 19789fbd743cbc..e26452c9ef69e5 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -1283,6 +1283,13 @@ PrimitiveType IRuntimeFilter::column_type() const {
 
 void IRuntimeFilter::signal() {
     DCHECK(is_consumer());
+
+    if (!_wrapper->is_ignored() && _wrapper->is_bloomfilter() &&
+        !_wrapper->get_bloomfilter()->inited()) {
+        throw Exception(ErrorCode::INTERNAL_ERROR, "bf not inited and not ignored, rf: {}",
+                        debug_string());
+    }
+
     COUNTER_SET(_wait_timer, int64_t((MonotonicMillis() - registration_time_) * NANOS_PER_MILLIS));
     _rf_state_atomic.store(RuntimeFilterState::READY);
     if (!_filter_timer.empty()) {

From fed46d2d8467148c87e88ca8bec69280125dd8e3 Mon Sep 17 00:00:00 2001
From: hui lai <laihui@selectdb.com>
Date: Fri, 29 Nov 2024 20:43:43 +0800
Subject: [PATCH 083/399] [improve](routine load) ensure load balance after
 scaling up BE nodes (#44693)

We scaling up from three BE nodes to five BE nodes, but from the
monitoring perspective, only the previous three nodes have written
traffic.

![image](https://github.com/user-attachments/assets/947fe8a9-745d-4c37-93d4-15b0e27f12fc)

This pr aims to ensure load balance after scaling up BE nodes.
---
 .../org/apache/doris/load/routineload/RoutineLoadManager.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java
index 8b0ebab3fbe7d5..97f6aba8c589c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java
@@ -487,6 +487,7 @@ public long getAvailableBeForTask(long jobId, long previousBeId) throws LoadExce
         // check if be has idle slot
         readLock();
         try {
+            updateBeIdToMaxConcurrentTasks();
             Map<Long, Integer> beIdToConcurrentTasks = getBeCurrentTasksNumMap();
             int previousBeIdleTaskNum = 0;
 

From 5f8051ace2947f510cda40afb3ed825f0d0c3236 Mon Sep 17 00:00:00 2001
From: TengJianPing <tengjianping@selectdb.com>
Date: Fri, 29 Nov 2024 20:46:45 +0800
Subject: [PATCH 084/399] [improvemment](profile) use one counter to represent
 MemoryUsage and MemoryUsagePeak (#44645)

---
 .../exec/aggregation_sink_operator.cpp        |  2 -
 .../pipeline/exec/analytic_sink_operator.cpp  |  1 -
 be/src/pipeline/exec/exchange_sink_buffer.cpp |  2 -
 .../pipeline/exec/exchange_sink_operator.cpp  | 10 ----
 be/src/pipeline/exec/hashjoin_build_sink.cpp  |  2 -
 .../pipeline/exec/hashjoin_probe_operator.cpp |  2 -
 .../exec/join/process_hash_table_probe_impl.h |  1 -
 be/src/pipeline/exec/operator.cpp             | 22 ++------
 be/src/pipeline/exec/operator.h               | 14 +----
 be/src/pipeline/exec/scan_operator.cpp        |  1 -
 be/src/pipeline/exec/scan_operator.h          |  1 -
 be/src/pipeline/exec/sort_sink_operator.cpp   |  1 -
 .../exec/spill_sort_sink_operator.cpp         |  1 -
 .../exec/streaming_aggregation_operator.cpp   |  3 -
 be/src/runtime/load_channel.cpp               |  1 -
 be/src/runtime/load_channel.h                 |  1 -
 be/src/util/runtime_profile.cpp               | 52 ++++++++----------
 be/src/util/runtime_profile.h                 | 55 ++++++++++++++++---
 be/src/vec/exec/scan/scanner_context.cpp      | 10 +---
 be/src/vec/exec/scan/scanner_context.h        |  2 -
 be/src/vec/exec/scan/scanner_scheduler.cpp    |  6 --
 be/src/vec/runtime/vdata_stream_recvr.cpp     |  4 --
 22 files changed, 79 insertions(+), 115 deletions(-)

diff --git a/be/src/pipeline/exec/aggregation_sink_operator.cpp b/be/src/pipeline/exec/aggregation_sink_operator.cpp
index 27400fba474eef..44e58535b75b71 100644
--- a/be/src/pipeline/exec/aggregation_sink_operator.cpp
+++ b/be/src/pipeline/exec/aggregation_sink_operator.cpp
@@ -231,7 +231,6 @@ void AggSinkLocalState::_update_memusage_with_serialized_key() {
 
                            COUNTER_SET(_memory_used_counter,
                                        arena_memory_usage + hash_table_memory_usage);
-                           COUNTER_SET(_peak_memory_usage_counter, _memory_used_counter->value());
 
                            COUNTER_SET(_serialize_key_arena_memory_usage, arena_memory_usage);
                            COUNTER_SET(_hash_table_memory_usage, hash_table_memory_usage);
@@ -415,7 +414,6 @@ Status AggSinkLocalState::_merge_without_key(vectorized::Block* block) {
 void AggSinkLocalState::_update_memusage_without_key() {
     int64_t arena_memory_usage = _agg_arena_pool->size();
     COUNTER_SET(_memory_used_counter, arena_memory_usage);
-    COUNTER_SET(_peak_memory_usage_counter, arena_memory_usage);
     COUNTER_SET(_serialize_key_arena_memory_usage, arena_memory_usage);
 }
 
diff --git a/be/src/pipeline/exec/analytic_sink_operator.cpp b/be/src/pipeline/exec/analytic_sink_operator.cpp
index 377aeb6fa12be4..7cc25eef9446d6 100644
--- a/be/src/pipeline/exec/analytic_sink_operator.cpp
+++ b/be/src/pipeline/exec/analytic_sink_operator.cpp
@@ -323,7 +323,6 @@ Status AnalyticSinkOperatorX::sink(doris::RuntimeState* state, vectorized::Block
     }
 
     COUNTER_UPDATE(local_state._memory_used_counter, input_block->allocated_bytes());
-    COUNTER_SET(local_state._peak_memory_usage_counter, local_state._memory_used_counter->value());
 
     //TODO: if need improvement, the is a tips to maintain a free queue,
     //so the memory could reuse, no need to new/delete again;
diff --git a/be/src/pipeline/exec/exchange_sink_buffer.cpp b/be/src/pipeline/exec/exchange_sink_buffer.cpp
index 0f02ffc2b9a4b1..6e6108d13a919f 100644
--- a/be/src/pipeline/exec/exchange_sink_buffer.cpp
+++ b/be/src/pipeline/exec/exchange_sink_buffer.cpp
@@ -159,8 +159,6 @@ Status ExchangeSinkBuffer::add_block(TransmitInfo&& request) {
             RETURN_IF_ERROR(
                     BeExecVersionManager::check_be_exec_version(request.block->be_exec_version()));
             COUNTER_UPDATE(_parent->memory_used_counter(), request.block->ByteSizeLong());
-            COUNTER_SET(_parent->peak_memory_usage_counter(),
-                        _parent->memory_used_counter()->value());
         }
         _instance_to_package_queue[ins_id].emplace(std::move(request));
         _total_queue_size++;
diff --git a/be/src/pipeline/exec/exchange_sink_operator.cpp b/be/src/pipeline/exec/exchange_sink_operator.cpp
index 85f58417197d52..dfa6df392b74ba 100644
--- a/be/src/pipeline/exec/exchange_sink_operator.cpp
+++ b/be/src/pipeline/exec/exchange_sink_operator.cpp
@@ -385,10 +385,6 @@ Status ExchangeSinkOperatorX::sink(RuntimeState* state, vectorized::Block* block
     if (all_receiver_eof) {
         return Status::EndOfFile("all data stream channels EOF");
     }
-    Defer defer([&]() {
-        COUNTER_SET(local_state._peak_memory_usage_counter,
-                    local_state._memory_used_counter->value());
-    });
 
     if (_part_type == TPartitionType::UNPARTITIONED || local_state.channels.size() == 1) {
         // 1. serialize depends on it is not local exchange
@@ -505,8 +501,6 @@ Status ExchangeSinkOperatorX::sink(RuntimeState* state, vectorized::Block* block
         }
         COUNTER_UPDATE(local_state.memory_used_counter(),
                        new_channel_mem_usage - old_channel_mem_usage);
-        COUNTER_SET(local_state.peak_memory_usage_counter(),
-                    local_state.memory_used_counter()->value());
     } else if (_part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED) {
         int64_t old_channel_mem_usage = 0;
         for (const auto& channel : local_state.channels) {
@@ -555,8 +549,6 @@ Status ExchangeSinkOperatorX::sink(RuntimeState* state, vectorized::Block* block
         }
         COUNTER_UPDATE(local_state.memory_used_counter(),
                        new_channel_mem_usage - old_channel_mem_usage);
-        COUNTER_SET(local_state.peak_memory_usage_counter(),
-                    local_state.memory_used_counter()->value());
     } else if (_part_type == TPartitionType::TABLE_SINK_HASH_PARTITIONED) {
         int64_t old_channel_mem_usage = 0;
         for (const auto& channel : local_state.channels) {
@@ -581,8 +573,6 @@ Status ExchangeSinkOperatorX::sink(RuntimeState* state, vectorized::Block* block
         }
         COUNTER_UPDATE(local_state.memory_used_counter(),
                        new_channel_mem_usage - old_channel_mem_usage);
-        COUNTER_SET(local_state.peak_memory_usage_counter(),
-                    local_state.memory_used_counter()->value());
     } else if (_part_type == TPartitionType::TABLE_SINK_RANDOM_PARTITIONED) {
         // Control the number of channels according to the flow, thereby controlling the number of table sink writers.
         // 1. select channel
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index a15dbb8dd03111..0a71b86bed0e86 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -310,7 +310,6 @@ Status HashJoinBuildSinkLocalState::process_build_block(RuntimeState* state,
                                     _build_blocks_memory_usage->value() +
                                             (int64_t)(arg.hash_table->get_byte_size() +
                                                       arg.serialized_keys_size(true)));
-                        COUNTER_SET(_peak_memory_usage_counter, _memory_used_counter->value());
                         return st;
                     }},
             _shared_state->hash_table_variants->method_variant, _shared_state->join_op_variants,
@@ -492,7 +491,6 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
                     std::move(*in_block)));
             int64_t blocks_mem_usage = local_state._build_side_mutable_block.allocated_bytes();
             COUNTER_SET(local_state._memory_used_counter, blocks_mem_usage);
-            COUNTER_SET(local_state._peak_memory_usage_counter, blocks_mem_usage);
             COUNTER_SET(local_state._build_blocks_memory_usage, blocks_mem_usage);
         }
     }
diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
index 426bfcb219dc04..7c663b256832ed 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
@@ -483,8 +483,6 @@ Status HashJoinProbeOperatorX::push(RuntimeState* state, vectorized::Block* inpu
             input_block->swap(local_state._probe_block);
             COUNTER_SET(local_state._memory_used_counter,
                         (int64_t)local_state._probe_block.allocated_bytes());
-            COUNTER_SET(local_state._peak_memory_usage_counter,
-                        local_state._memory_used_counter->value());
         }
     }
     return Status::OK();
diff --git a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
index fc1153b3419d3e..24a9a7f67431da 100644
--- a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
+++ b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
@@ -184,7 +184,6 @@ typename HashTableType::State ProcessHashTableProbe<JoinOpType>::_init_probe_sid
         int64_t arena_memory_usage = hash_table_ctx.serialized_keys_size(false);
         COUNTER_SET(_parent->_probe_arena_memory_usage, arena_memory_usage);
         COUNTER_UPDATE(_parent->_memory_used_counter, arena_memory_usage);
-        COUNTER_SET(_parent->_peak_memory_usage_counter, _parent->_memory_used_counter->value());
     }
 
     return typename HashTableType::State(_parent->_probe_columns);
diff --git a/be/src/pipeline/exec/operator.cpp b/be/src/pipeline/exec/operator.cpp
index 3b5174d87c0f7f..09d0eef0f04340 100644
--- a/be/src/pipeline/exec/operator.cpp
+++ b/be/src/pipeline/exec/operator.cpp
@@ -342,7 +342,6 @@ Status OperatorXBase::get_block_after_projects(RuntimeState* state, vectorized::
         return status;
     }
     status = get_block(state, block, eos);
-    local_state->_peak_memory_usage_counter->set(local_state->_memory_used_counter->value());
     return status;
 }
 
@@ -441,11 +440,7 @@ PipelineXSinkLocalStateBase::PipelineXSinkLocalStateBase(DataSinkOperatorXBase*
 }
 
 PipelineXLocalStateBase::PipelineXLocalStateBase(RuntimeState* state, OperatorXBase* parent)
-        : _num_rows_returned(0),
-          _rows_returned_counter(nullptr),
-          _peak_memory_usage_counter(nullptr),
-          _parent(parent),
-          _state(state) {
+        : _num_rows_returned(0), _rows_returned_counter(nullptr), _parent(parent), _state(state) {
     _query_statistics = std::make_shared<QueryStatistics>();
 }
 
@@ -484,9 +479,8 @@ Status PipelineXLocalState<SharedStateArg>::init(RuntimeState* state, LocalState
     _open_timer = ADD_TIMER_WITH_LEVEL(_runtime_profile, "OpenTime", 1);
     _close_timer = ADD_TIMER_WITH_LEVEL(_runtime_profile, "CloseTime", 1);
     _exec_timer = ADD_TIMER_WITH_LEVEL(_runtime_profile, "ExecTime", 1);
-    _memory_used_counter = ADD_COUNTER_WITH_LEVEL(_runtime_profile, "MemoryUsage", TUnit::BYTES, 1);
-    _peak_memory_usage_counter =
-            _runtime_profile->AddHighWaterMarkCounter("MemoryUsagePeak", TUnit::BYTES, "", 1);
+    _memory_used_counter =
+            _runtime_profile->AddHighWaterMarkCounter("MemoryUsage", TUnit::BYTES, "", 1);
     return Status::OK();
 }
 
@@ -519,9 +513,6 @@ Status PipelineXLocalState<SharedStateArg>::close(RuntimeState* state) {
     if constexpr (!std::is_same_v<SharedStateArg, FakeSharedState>) {
         COUNTER_SET(_wait_for_dependency_timer, _dependency->watcher_elapse_time());
     }
-    if (_peak_memory_usage_counter) {
-        _peak_memory_usage_counter->set(_memory_used_counter->value());
-    }
     _closed = true;
     // Some kinds of source operators has a 1-1 relationship with a sink operator (such as AnalyticOperator).
     // We must ensure AnalyticSinkOperator will not be blocked if AnalyticSourceOperator already closed.
@@ -560,9 +551,7 @@ Status PipelineXSinkLocalState<SharedState>::init(RuntimeState* state, LocalSink
     _close_timer = ADD_TIMER_WITH_LEVEL(_profile, "CloseTime", 1);
     _exec_timer = ADD_TIMER_WITH_LEVEL(_profile, "ExecTime", 1);
     info.parent_profile->add_child(_profile, true, nullptr);
-    _memory_used_counter = ADD_COUNTER_WITH_LEVEL(_profile, "MemoryUsage", TUnit::BYTES, 1);
-    _peak_memory_usage_counter =
-            _profile->AddHighWaterMarkCounter("MemoryUsagePeak", TUnit::BYTES, "", 1);
+    _memory_used_counter = _profile->AddHighWaterMarkCounter("MemoryUsage", TUnit::BYTES, "", 1);
     return Status::OK();
 }
 
@@ -574,9 +563,6 @@ Status PipelineXSinkLocalState<SharedState>::close(RuntimeState* state, Status e
     if constexpr (!std::is_same_v<SharedState, FakeSharedState>) {
         COUNTER_SET(_wait_for_dependency_timer, _dependency->watcher_elapse_time());
     }
-    if (_peak_memory_usage_counter) {
-        _peak_memory_usage_counter->set(_memory_used_counter->value());
-    }
     _closed = true;
     return Status::OK();
 }
diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h
index 6053b1a2f48e87..c84c4e7b43f981 100644
--- a/be/src/pipeline/exec/operator.h
+++ b/be/src/pipeline/exec/operator.h
@@ -165,9 +165,6 @@ class PipelineXLocalStateBase {
 
     RuntimeProfile::Counter* exec_time_counter() { return _exec_timer; }
     RuntimeProfile::Counter* memory_used_counter() { return _memory_used_counter; }
-    RuntimeProfile::HighWaterMarkCounter* peak_memory_usage_counter() {
-        return _peak_memory_usage_counter;
-    }
     OperatorXBase* parent() { return _parent; }
     RuntimeState* state() { return _state; }
     vectorized::VExprContextSPtrs& conjuncts() { return _conjuncts; }
@@ -202,11 +199,10 @@ class PipelineXLocalStateBase {
     RuntimeProfile::Counter* _rows_returned_counter = nullptr;
     RuntimeProfile::Counter* _blocks_returned_counter = nullptr;
     RuntimeProfile::Counter* _wait_for_dependency_timer = nullptr;
-    RuntimeProfile::Counter* _memory_used_counter = nullptr;
+    // Account for current memory and peak memory used by this node
+    RuntimeProfile::HighWaterMarkCounter* _memory_used_counter = nullptr;
     RuntimeProfile::Counter* _projection_timer = nullptr;
     RuntimeProfile::Counter* _exec_timer = nullptr;
-    // Account for peak memory used by this node
-    RuntimeProfile::HighWaterMarkCounter* _peak_memory_usage_counter = nullptr;
     RuntimeProfile::Counter* _init_timer = nullptr;
     RuntimeProfile::Counter* _open_timer = nullptr;
     RuntimeProfile::Counter* _close_timer = nullptr;
@@ -348,9 +344,6 @@ class PipelineXSinkLocalStateBase {
     RuntimeProfile::Counter* rows_input_counter() { return _rows_input_counter; }
     RuntimeProfile::Counter* exec_time_counter() { return _exec_timer; }
     RuntimeProfile::Counter* memory_used_counter() { return _memory_used_counter; }
-    RuntimeProfile::HighWaterMarkCounter* peak_memory_usage_counter() {
-        return _peak_memory_usage_counter;
-    }
     virtual std::vector<Dependency*> dependencies() const { return {nullptr}; }
 
     // override in exchange sink , AsyncWriterSink
@@ -380,8 +373,7 @@ class PipelineXSinkLocalStateBase {
     RuntimeProfile::Counter* _wait_for_dependency_timer = nullptr;
     RuntimeProfile::Counter* _wait_for_finish_dependency_timer = nullptr;
     RuntimeProfile::Counter* _exec_timer = nullptr;
-    RuntimeProfile::Counter* _memory_used_counter = nullptr;
-    RuntimeProfile::HighWaterMarkCounter* _peak_memory_usage_counter = nullptr;
+    RuntimeProfile::HighWaterMarkCounter* _memory_used_counter = nullptr;
 
     std::shared_ptr<QueryStatistics> _query_statistics = nullptr;
 };
diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp
index 21c3103fe5a708..ae4396b22c7eec 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -1037,7 +1037,6 @@ Status ScanLocalState<Derived>::_init_profile() {
     _total_throughput_counter =
             profile()->add_rate_counter("TotalReadThroughput", _rows_read_counter);
     _num_scanners = ADD_COUNTER(_runtime_profile, "NumScanners", TUnit::UNIT);
-    _scanner_peak_memory_usage = _peak_memory_usage_counter;
     //_runtime_profile->AddHighWaterMarkCounter("PeakMemoryUsage", TUnit::BYTES);
 
     // 2. counters for scanners
diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h
index 5d41c800383bd0..4519a3ca283f6f 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -108,7 +108,6 @@ class ScanLocalStateBase : public PipelineXLocalState<>, public RuntimeFilterCon
     // Max num of scanner thread
     RuntimeProfile::Counter* _max_scanner_thread_num = nullptr;
     RuntimeProfile::HighWaterMarkCounter* _peak_running_scanner = nullptr;
-    RuntimeProfile::HighWaterMarkCounter* _scanner_peak_memory_usage = nullptr;
     // time of get block from scanner
     RuntimeProfile::Counter* _scan_timer = nullptr;
     RuntimeProfile::Counter* _scan_cpu_timer = nullptr;
diff --git a/be/src/pipeline/exec/sort_sink_operator.cpp b/be/src/pipeline/exec/sort_sink_operator.cpp
index faec4961af93b7..072f28723a36ea 100644
--- a/be/src/pipeline/exec/sort_sink_operator.cpp
+++ b/be/src/pipeline/exec/sort_sink_operator.cpp
@@ -128,7 +128,6 @@ Status SortSinkOperatorX::sink(doris::RuntimeState* state, vectorized::Block* in
         int64_t data_size = local_state._shared_state->sorter->data_size();
         COUNTER_SET(local_state._sort_blocks_memory_usage, data_size);
         COUNTER_SET(local_state._memory_used_counter, data_size);
-        COUNTER_SET(local_state._peak_memory_usage_counter, data_size);
 
         RETURN_IF_CANCELLED(state);
 
diff --git a/be/src/pipeline/exec/spill_sort_sink_operator.cpp b/be/src/pipeline/exec/spill_sort_sink_operator.cpp
index 267bcc83aad92c..6e6689d4134deb 100644
--- a/be/src/pipeline/exec/spill_sort_sink_operator.cpp
+++ b/be/src/pipeline/exec/spill_sort_sink_operator.cpp
@@ -160,7 +160,6 @@ Status SpillSortSinkOperatorX::sink(doris::RuntimeState* state, vectorized::Bloc
     int64_t data_size = local_state._shared_state->in_mem_shared_state->sorter->data_size();
     COUNTER_SET(local_state._sort_blocks_memory_usage, data_size);
     COUNTER_SET(local_state._memory_used_counter, data_size);
-    COUNTER_SET(local_state._peak_memory_usage_counter, data_size);
 
     if (eos) {
         if (local_state._shared_state->is_spilled) {
diff --git a/be/src/pipeline/exec/streaming_aggregation_operator.cpp b/be/src/pipeline/exec/streaming_aggregation_operator.cpp
index cf5071d62e4737..1c8d2c47bc698a 100644
--- a/be/src/pipeline/exec/streaming_aggregation_operator.cpp
+++ b/be/src/pipeline/exec/streaming_aggregation_operator.cpp
@@ -353,7 +353,6 @@ Status StreamingAggLocalState::_merge_without_key(vectorized::Block* block) {
 void StreamingAggLocalState::_update_memusage_without_key() {
     int64_t arena_memory_usage = _agg_arena_pool->size();
     COUNTER_SET(_memory_used_counter, arena_memory_usage);
-    COUNTER_SET(_peak_memory_usage_counter, arena_memory_usage);
     COUNTER_SET(_serialize_key_arena_memory_usage, arena_memory_usage);
 }
 
@@ -378,8 +377,6 @@ void StreamingAggLocalState::_update_memusage_with_serialized_key() {
 
                            COUNTER_SET(_memory_used_counter,
                                        arena_memory_usage + hash_table_memory_usage);
-                           COUNTER_SET(_peak_memory_usage_counter,
-                                       arena_memory_usage + hash_table_memory_usage);
 
                            COUNTER_SET(_serialize_key_arena_memory_usage, arena_memory_usage);
                            COUNTER_SET(_hash_table_memory_usage, hash_table_memory_usage);
diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp
index 9369c0c833c53c..0cb313747b0373 100644
--- a/be/src/runtime/load_channel.cpp
+++ b/be/src/runtime/load_channel.cpp
@@ -98,7 +98,6 @@ void LoadChannel::_init_profile() {
                                                _load_id.to_string(), _sender_ip, _backend_id),
                                    true, true);
     _add_batch_number_counter = ADD_COUNTER(_self_profile, "NumberBatchAdded", TUnit::UNIT);
-    _peak_memory_usage_counter = ADD_COUNTER(_self_profile, "PeakMemoryUsage", TUnit::BYTES);
     _add_batch_timer = ADD_TIMER(_self_profile, "AddBatchTime");
     _handle_eos_timer = ADD_CHILD_TIMER(_self_profile, "HandleEosTime", "AddBatchTime");
     _add_batch_times = ADD_COUNTER(_self_profile, "AddBatchTimes", TUnit::UNIT);
diff --git a/be/src/runtime/load_channel.h b/be/src/runtime/load_channel.h
index 36a8f363ba9bac..2889bcf256515b 100644
--- a/be/src/runtime/load_channel.h
+++ b/be/src/runtime/load_channel.h
@@ -91,7 +91,6 @@ class LoadChannel {
     std::unique_ptr<RuntimeProfile> _profile;
     RuntimeProfile* _self_profile = nullptr;
     RuntimeProfile::Counter* _add_batch_number_counter = nullptr;
-    RuntimeProfile::Counter* _peak_memory_usage_counter = nullptr;
     RuntimeProfile::Counter* _add_batch_timer = nullptr;
     RuntimeProfile::Counter* _add_batch_times = nullptr;
     RuntimeProfile::Counter* _mgr_add_batch_timer = nullptr;
diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp
index e87301880d2479..45db607a342743 100644
--- a/be/src/util/runtime_profile.cpp
+++ b/be/src/util/runtime_profile.cpp
@@ -363,28 +363,24 @@ const std::string* RuntimeProfile::get_info_string(const std::string& key) {
     return &it->second;
 }
 
-#define ADD_COUNTER_IMPL(NAME, T)                                                                  \
-    RuntimeProfile::T* RuntimeProfile::NAME(const std::string& name, TUnit::type unit,             \
-                                            const std::string& parent_counter_name,                \
-                                            int64_t level) {                                       \
-        DCHECK_EQ(_is_averaged_profile, false);                                                    \
-        std::lock_guard<std::mutex> l(_counter_map_lock);                                          \
-        if (_counter_map.find(name) != _counter_map.end()) {                                       \
-            return reinterpret_cast<T*>(_counter_map[name]);                                       \
-        }                                                                                          \
-        DCHECK(parent_counter_name == ROOT_COUNTER ||                                              \
-               _counter_map.find(parent_counter_name) != _counter_map.end());                      \
-        T* counter = _pool->add(new T(unit, level));                                               \
-        _counter_map[name] = counter;                                                              \
-        std::set<std::string>* child_counters =                                                    \
-                find_or_insert(&_child_counter_map, parent_counter_name, std::set<std::string>()); \
-        child_counters->insert(name);                                                              \
-        return counter;                                                                            \
-    }
-
-//ADD_COUNTER_IMPL(AddCounter, Counter);
-ADD_COUNTER_IMPL(AddHighWaterMarkCounter, HighWaterMarkCounter);
-//ADD_COUNTER_IMPL(AddConcurrentTimerCounter, ConcurrentTimerCounter);
+RuntimeProfile::HighWaterMarkCounter* RuntimeProfile::AddHighWaterMarkCounter(
+        const std::string& name, TUnit::type unit, const std::string& parent_counter_name,
+        int64_t level) {
+    DCHECK_EQ(_is_averaged_profile, false);
+    std::lock_guard<std::mutex> l(_counter_map_lock);
+    if (_counter_map.find(name) != _counter_map.end()) {
+        return reinterpret_cast<RuntimeProfile::HighWaterMarkCounter*>(_counter_map[name]);
+    }
+    DCHECK(parent_counter_name == ROOT_COUNTER ||
+           _counter_map.find(parent_counter_name) != _counter_map.end());
+    RuntimeProfile::HighWaterMarkCounter* counter =
+            _pool->add(new RuntimeProfile::HighWaterMarkCounter(unit, level, parent_counter_name));
+    _counter_map[name] = counter;
+    std::set<std::string>* child_counters =
+            find_or_insert(&_child_counter_map, parent_counter_name, std::set<std::string>());
+    child_counters->insert(name);
+    return counter;
+}
 
 std::shared_ptr<RuntimeProfile::HighWaterMarkCounter> RuntimeProfile::AddSharedHighWaterMarkCounter(
         const std::string& name, TUnit::type unit, const std::string& parent_counter_name) {
@@ -395,7 +391,8 @@ std::shared_ptr<RuntimeProfile::HighWaterMarkCounter> RuntimeProfile::AddSharedH
     }
     DCHECK(parent_counter_name == ROOT_COUNTER ||
            _counter_map.find(parent_counter_name) != _counter_map.end());
-    std::shared_ptr<HighWaterMarkCounter> counter = std::make_shared<HighWaterMarkCounter>(unit);
+    std::shared_ptr<HighWaterMarkCounter> counter =
+            std::make_shared<HighWaterMarkCounter>(unit, 2, parent_counter_name);
     _shared_counter_pool[name] = counter;
 
     DCHECK(_counter_map.find(name) == _counter_map.end())
@@ -697,17 +694,14 @@ void RuntimeProfile::print_child_counters(const std::string& prefix,
                                           const CounterMap& counter_map,
                                           const ChildCounterMap& child_counter_map,
                                           std::ostream* s) {
-    std::ostream& stream = *s;
-    ChildCounterMap::const_iterator itr = child_counter_map.find(counter_name);
+    auto itr = child_counter_map.find(counter_name);
 
     if (itr != child_counter_map.end()) {
         const std::set<std::string>& child_counters = itr->second;
         for (const std::string& child_counter : child_counters) {
-            CounterMap::const_iterator iter = counter_map.find(child_counter);
+            auto iter = counter_map.find(child_counter);
             DCHECK(iter != counter_map.end());
-            stream << prefix << "   - " << iter->first << ": "
-                   << PrettyPrinter::print(iter->second->value(), iter->second->type())
-                   << std::endl;
+            iter->second->pretty_print(s, prefix, iter->first);
             RuntimeProfile::print_child_counters(prefix + "  ", child_counter, counter_map,
                                                  child_counter_map, s);
         }
diff --git a/be/src/util/runtime_profile.h b/be/src/util/runtime_profile.h
index 955d77b72aa51c..6e393ac673a628 100644
--- a/be/src/util/runtime_profile.h
+++ b/be/src/util/runtime_profile.h
@@ -39,6 +39,7 @@
 
 #include "common/compiler_util.h" // IWYU pragma: keep
 #include "util/binary_cast.hpp"
+#include "util/container_util.hpp"
 #include "util/pretty_printer.h"
 #include "util/stopwatch.hpp"
 
@@ -126,6 +127,14 @@ class RuntimeProfile {
             tcounters.push_back(std::move(counter));
         }
 
+        virtual void pretty_print(std::ostream* s, const std::string& prefix,
+                                  const std::string& name) const {
+            std::ostream& stream = *s;
+            stream << prefix << "   - " << name << ": "
+                   << PrettyPrinter::print(_value.load(std::memory_order_relaxed), type())
+                   << std::endl;
+        }
+
         TUnit::type type() const { return _type; }
 
         virtual int64_t level() { return _level; }
@@ -142,15 +151,49 @@ class RuntimeProfile {
     /// as value()) and the current value.
     class HighWaterMarkCounter : public Counter {
     public:
-        HighWaterMarkCounter(TUnit::type unit, int64_t level = 2)
-                : Counter(unit, 0, level), current_value_(0) {}
+        HighWaterMarkCounter(TUnit::type unit, int64_t level, const std::string& parent_name)
+                : Counter(unit, 0, level), current_value_(0), _parent_name(parent_name) {}
 
-        virtual void add(int64_t delta) {
+        void add(int64_t delta) {
             current_value_.fetch_add(delta, std::memory_order_relaxed);
             if (delta > 0) {
                 UpdateMax(current_value_);
             }
         }
+        virtual void update(int64_t delta) override { add(delta); }
+
+        virtual void to_thrift(
+                const std::string& name, std::vector<TCounter>& tcounters,
+                std::map<std::string, std::set<std::string>>& child_counters_map) override {
+            {
+                TCounter counter;
+                counter.name = name;
+                counter.value = this->current_value();
+                counter.type = this->type();
+                counter.__set_level(this->level());
+                tcounters.push_back(std::move(counter));
+            }
+            {
+                TCounter counter;
+                std::string peak_name = name + "Peak";
+                counter.name = peak_name;
+                counter.value = this->value();
+                counter.type = this->type();
+                counter.__set_level(this->level());
+                tcounters.push_back(std::move(counter));
+                child_counters_map[_parent_name].insert(peak_name);
+            }
+        }
+
+        virtual void pretty_print(std::ostream* s, const std::string& prefix,
+                                  const std::string& name) const override {
+            std::ostream& stream = *s;
+            stream << prefix << "   - " << name << ": "
+                   << PrettyPrinter::print(current_value(), type()) << std::endl;
+            stream << prefix << "      - " << name << "Peak: "
+                   << PrettyPrinter::print(_value.load(std::memory_order_relaxed), type())
+                   << std::endl;
+        }
 
         /// Tries to increase the current value by delta. If current_value() + delta
         /// exceeds max, return false and current_value is not changed.
@@ -194,6 +237,8 @@ class RuntimeProfile {
         /// The current value of the counter. _value in the super class represents
         /// the high water mark.
         std::atomic<int64_t> current_value_;
+
+        const std::string _parent_name;
     };
 
     using DerivedCounterFunction = std::function<int64_t()>;
@@ -561,10 +606,6 @@ class RuntimeProfile {
     static void print_child_counters(const std::string& prefix, const std::string& counter_name,
                                      const CounterMap& counter_map,
                                      const ChildCounterMap& child_counter_map, std::ostream* s);
-
-    static std::string print_counter(Counter* counter) {
-        return PrettyPrinter::print(counter->value(), counter->type());
-    }
 };
 
 // Utility class to update the counter at object construction and destruction.
diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp
index d37d26b09f7815..65812cb428a8f8 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -236,7 +236,6 @@ vectorized::BlockUPtr ScannerContext::get_free_block(bool force) {
         _scanner_memory_used_counter->set(_block_memory_usage);
         // A free block is reused, so the memory usage should be decreased
         // The caller of get_free_block will increase the memory usage
-        update_peak_memory_usage(-block->allocated_bytes());
     } else if (_block_memory_usage < _max_bytes_in_queue || force) {
         _newly_create_free_blocks_num->update(1);
         block = vectorized::Block::create_unique(_output_tuple_desc->slots(), 0,
@@ -251,9 +250,7 @@ void ScannerContext::return_free_block(vectorized::BlockUPtr block) {
         _block_memory_usage += block_size_to_reuse;
         _scanner_memory_used_counter->set(_block_memory_usage);
         block->clear_column_data();
-        if (_free_blocks.enqueue(std::move(block))) {
-            update_peak_memory_usage(block_size_to_reuse);
-        }
+        _free_blocks.enqueue(std::move(block));
     }
 }
 
@@ -324,7 +321,6 @@ Status ScannerContext::get_block_from_queue(RuntimeState* state, vectorized::Blo
                 _estimated_block_size = block_size;
             }
             _block_memory_usage -= block_size;
-            update_peak_memory_usage(-current_block->allocated_bytes());
             // consume current block
             block->swap(*current_block);
             return_free_block(std::move(current_block));
@@ -540,8 +536,4 @@ void ScannerContext::update_peak_running_scanner(int num) {
     _local_state->_peak_running_scanner->add(num);
 }
 
-void ScannerContext::update_peak_memory_usage(int64_t usage) {
-    _local_state->_scanner_peak_memory_usage->add(usage);
-}
-
 } // namespace doris::vectorized
diff --git a/be/src/vec/exec/scan/scanner_context.h b/be/src/vec/exec/scan/scanner_context.h
index 82e0a06799940b..d1cf06d56686ac 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -127,8 +127,6 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
 
     // Caller should make sure the pipeline task is still running when calling this function
     void update_peak_running_scanner(int num);
-    // Caller should make sure the pipeline task is still running when calling this function
-    void update_peak_memory_usage(int64_t usage);
 
     // Get next block from blocks queue. Called by ScanNode/ScanOperator
     // Set eos to true if there is no more data to read.
diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp
index 385b581d2a5725..7c5aa8db0a77af 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -283,8 +283,6 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
                     break;
                 }
                 // We got a new created block or a reused block.
-                ctx->update_peak_memory_usage(free_block->allocated_bytes());
-                ctx->update_peak_memory_usage(-free_block->allocated_bytes());
                 status = scanner->get_block_after_projects(state, free_block.get(), &eos);
                 first_read = false;
                 if (!status.ok()) {
@@ -293,7 +291,6 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
                 }
                 // Projection will truncate useless columns, makes block size change.
                 auto free_block_bytes = free_block->allocated_bytes();
-                ctx->update_peak_memory_usage(free_block_bytes);
                 raw_bytes_read += free_block_bytes;
                 if (!scan_task->cached_blocks.empty() &&
                     scan_task->cached_blocks.back().first->rows() + free_block->rows() <=
@@ -301,9 +298,7 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
                     size_t block_size = scan_task->cached_blocks.back().first->allocated_bytes();
                     vectorized::MutableBlock mutable_block(
                             scan_task->cached_blocks.back().first.get());
-                    ctx->update_peak_memory_usage(-mutable_block.allocated_bytes());
                     status = mutable_block.merge(*free_block);
-                    ctx->update_peak_memory_usage(mutable_block.allocated_bytes());
                     if (!status.ok()) {
                         LOG(WARNING) << "Block merge failed: " << status.to_string();
                         break;
@@ -313,7 +308,6 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
                             std::move(mutable_block.mutable_columns()));
 
                     // Return block succeed or not, this free_block is not used by this scan task any more.
-                    ctx->update_peak_memory_usage(-free_block_bytes);
                     // If block can be reused, its memory usage will be added back.
                     ctx->return_free_block(std::move(free_block));
                     ctx->inc_block_usage(scan_task->cached_blocks.back().first->allocated_bytes() -
diff --git a/be/src/vec/runtime/vdata_stream_recvr.cpp b/be/src/vec/runtime/vdata_stream_recvr.cpp
index 81e4e1cd5f037e..c3277b0917e84c 100644
--- a/be/src/vec/runtime/vdata_stream_recvr.cpp
+++ b/be/src/vec/runtime/vdata_stream_recvr.cpp
@@ -199,8 +199,6 @@ Status VDataStreamRecvr::SenderQueue::add_block(std::unique_ptr<PBlock> pblock,
         *done = nullptr;
     }
     _recvr->_parent->memory_used_counter()->update(block_byte_size);
-    _recvr->_parent->peak_memory_usage_counter()->set(
-            _recvr->_parent->memory_used_counter()->value());
     add_blocks_memory_usage(block_byte_size);
     return Status::OK();
 }
@@ -240,8 +238,6 @@ void VDataStreamRecvr::SenderQueue::add_block(Block* block, bool use_move) {
         try_set_dep_ready_without_lock();
         COUNTER_UPDATE(_recvr->_local_bytes_received_counter, block_mem_size);
         _recvr->_parent->memory_used_counter()->update(block_mem_size);
-        _recvr->_parent->peak_memory_usage_counter()->set(
-                _recvr->_parent->memory_used_counter()->value());
         add_blocks_memory_usage(block_mem_size);
     }
 }

From cd0cd55f11d21a9bf25a02c1567e378c1f672e07 Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Fri, 29 Nov 2024 20:52:54 +0800
Subject: [PATCH 085/399] [opt](auto-inc) Allow to miss auto-increment column
 and other value columns in partial update (#44528)

Previously, if a partial update misses auto-increment key columns and
other value columns, the load will fail with error `distributed column
not found, column=xxx`. Considering the uniqueness of the generated
value of auto-increment column, this PR converts the partial update load
to upsert in this situation to make it llegal.

### Release note

Allow to miss auto-increment key column and other value columns in
partial update load
---
 .../plans/commands/insert/InsertUtils.java    | 10 +++++
 .../test_partial_update_auto_inc.out          |  8 ++++
 .../test_partial_update_auto_inc.groovy       | 42 ++++++++++++++++++-
 3 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java
index 6b301c16f73aa2..60e7e5bf805a64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java
@@ -297,6 +297,7 @@ public static Plan normalizePlan(Plan plan, TableIf table, Optional<InsertComman
                                         + " with sync materialized view.");
                             }
                             boolean hasMissingColExceptAutoIncKey = false;
+                            boolean hasMissingAutoIncKey = false;
                             for (Column col : olapTable.getFullSchema()) {
                                 Optional<String> insertCol = unboundLogicalSink.getColNames().stream()
                                         .filter(c -> c.equalsIgnoreCase(col.getName())).findFirst();
@@ -313,9 +314,18 @@ public static Plan normalizePlan(Plan plan, TableIf table, Optional<InsertComman
                                 if (!(col.isAutoInc() && col.isKey()) && !insertCol.isPresent() && col.isVisible()) {
                                     hasMissingColExceptAutoIncKey = true;
                                 }
+                                if (col.isAutoInc() && col.isKey() && !insertCol.isPresent()) {
+                                    hasMissingAutoIncKey = true;
+                                }
                             }
                             if (!hasMissingColExceptAutoIncKey) {
                                 ((UnboundTableSink<? extends Plan>) unboundLogicalSink).setPartialUpdate(false);
+                            } else {
+                                if (hasMissingAutoIncKey) {
+                                    // becuase of the uniqueness of genetaed value of auto-increment column,
+                                    // we convert this load to upsert when is misses auto-increment key column
+                                    ((UnboundTableSink<? extends Plan>) unboundLogicalSink).setPartialUpdate(false);
+                                }
                             }
                         }
                     }
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.out
index 19b192c95facb9..b672eb10570508 100644
--- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.out
+++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.out
@@ -35,3 +35,11 @@ doris9
 3	888	888	30
 4	40	40	40
 
+-- !sql --
+test1	15
+test2	29
+test3	49
+
+-- !sql --
+3
+
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
index 1ec60fbb10e837..d689deeb8afe02 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
@@ -103,6 +103,46 @@ suite("test_partial_update_auto_inc") {
             time 10000
         }
         order_qt_select_6 "select * from test_primary_key_partial_update_auto_inc2"
-        sql """ DROP TABLE IF EXISTS test_primary_key_partial_update_auto_inc2 """
+
+
+        sql """ DROP TABLE IF EXISTS test_primary_key_partial_update_auto_inc3 force; """
+        sql """ create table test_primary_key_partial_update_auto_inc3
+            (
+                `id`                      bigint         not null AUTO_INCREMENT,
+                `project_code`            varchar(20)    not null,
+                `period_num`              int,
+                `c2`     int
+            ) unique KEY(`id`)
+            DISTRIBUTED BY HASH(`id`) BUCKETS auto
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "enable_unique_key_merge_on_write" = "true"
+            );   """
+        sql "set enable_unique_key_partial_update=true;"
+        sql "set enable_insert_strict=false;"
+        sql "sync;"
+        
+        sql "insert into test_primary_key_partial_update_auto_inc3(project_code,period_num) values ('test1',15),('test2',29),('test3',49);"
+        qt_sql "select project_code,period_num from test_primary_key_partial_update_auto_inc3 order by project_code,period_num;"
+        qt_sql "select count(distinct id) from test_primary_key_partial_update_auto_inc3;"
+
+
+        sql """ DROP TABLE IF EXISTS test_primary_key_partial_update_auto_inc4 """
+        sql """ CREATE TABLE test_primary_key_partial_update_auto_inc4 (
+                        `k1` BIGINT NOT NULL AUTO_INCREMENT,
+                        `k2` int,
+                        `c1` int,
+                        `c2` int,
+                        `c3` int)
+                        UNIQUE KEY(`k1`,`k2`) DISTRIBUTED BY HASH(`k1`,`k2`) BUCKETS 1
+                        PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "true"); """
+        sql "set enable_unique_key_partial_update=true;"
+        sql "set enable_insert_strict=false;"
+        sql "sync;"
+
+        test {
+            sql "insert into test_primary_key_partial_update_auto_inc4(c1,c2) values(1,1),(2,2),(3,3)"
+            exception "Partial update should include all key columns, missing: k2"
+        }
     }
 }

From 7c6ece0fd1d47f56f31e4bf82ac86d527fa3f1af Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Fri, 29 Nov 2024 21:30:36 +0800
Subject: [PATCH 086/399] [fix](case) exclude some case when
 `random_add_cluster_keys_for_mow` is enabled (#44775)

---
 .../src/main/groovy/org/apache/doris/regression/Config.groovy | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 5aa6d7ce7e7218..379d177fbc280d 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -1025,8 +1025,10 @@ class Config {
         if (isCKEnabled) {
             excludeDirectorySet.add("unique_with_mow_p0/partial_update")
             excludeDirectorySet.add("unique_with_mow_p0/flexible")
+            excludeDirectorySet.add("fault_injection_p0/partial_update")
+            excludeDirectorySet.add("fault_injection_p0/flexible")
             excludeDirectorySet.add("doc")
-            List<String> excludeCases = ["test_table_properties", "test_default_hll", "test_default_pi", "test_full_compaction", "test_full_compaction_by_table_id", "test_create_table", "txn_insert", "test_update_mow", "test_new_update", "test_update_unique", "test_partial_update_generated_column", "nereids_partial_update_native_insert_stmt", "partial_update", "nereids_update_on_current_timestamp", "update_on_current_timestamp", "test_default_bitmap_empty", "nereids_delete_mow_partial_update", "delete_mow_partial_update", "partial_update_seq_col", "nereids_partial_update_native_insert_stmt_complex", "regression_test_variant_delete_and_update", "test_unique_table_auto_inc_partial_update_correct_stream_load", "test_unique_table_auto_inc", "test_unique_table_auto_inc_partial_update_correct_insert", "test_update_schema_change"]
+            List<String> excludeCases = ["test_table_properties", "test_default_hll", "test_default_pi", "test_full_compaction", "test_full_compaction_by_table_id", "test_create_table", "txn_insert", "test_update_mow", "test_new_update", "test_update_unique", "test_partial_update_generated_column", "nereids_partial_update_native_insert_stmt", "partial_update", "nereids_update_on_current_timestamp", "update_on_current_timestamp", "test_default_bitmap_empty", "nereids_delete_mow_partial_update", "delete_mow_partial_update", "partial_update_seq_col", "nereids_partial_update_native_insert_stmt_complex", "regression_test_variant_delete_and_update", "test_unique_table_auto_inc_partial_update_correct_stream_load", "test_unique_table_auto_inc", "test_unique_table_auto_inc_partial_update_correct_insert", "test_update_schema_change", "test_partial_update_rowset_not_found_fault_injection"]
             for (def excludeCase in excludeCases) {
                 excludeSuiteWildcard.add(excludeCase)
             }

From c34c8899f0bbcd27a4b4e8288195727c618b4112 Mon Sep 17 00:00:00 2001
From: airborne12 <jiangkai@selectdb.com>
Date: Fri, 29 Nov 2024 23:20:57 +0800
Subject: [PATCH 087/399] [feat](test) add invalid params check and add ut case
 for bf (#44778)

add bf ut
---
 be/src/olap/rowset/segment_v2/bloom_filter.h  |  10 +-
 be/test/olap/date_bloom_filter_test.cpp       |   2 +-
 .../rowset/segment_v2/bloom_filter_test.cpp   | 302 ++++++++++++++++++
 3 files changed, 311 insertions(+), 3 deletions(-)
 create mode 100644 be/test/olap/rowset/segment_v2/bloom_filter_test.cpp

diff --git a/be/src/olap/rowset/segment_v2/bloom_filter.h b/be/src/olap/rowset/segment_v2/bloom_filter.h
index 20a903e65c1b3b..a7845d1ca36704 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter.h
+++ b/be/src/olap/rowset/segment_v2/bloom_filter.h
@@ -132,14 +132,20 @@ class BloomFilter {
     // for read
     // use deep copy to acquire the data
     virtual Status init(const char* buf, uint32_t size, HashStrategyPB strategy) {
+        if (size <= 1) {
+            return Status::InvalidArgument("invalid size:{}", size);
+        }
         DCHECK(size > 1);
         if (strategy == HASH_MURMUR3_X64_64) {
             _hash_func = murmur_hash3_x64_64;
         } else {
             return Status::InvalidArgument("invalid strategy:{}", strategy);
         }
-        if (size == 0) {
-            return Status::InvalidArgument("invalid size:{}", size);
+        if (buf == nullptr) {
+            return Status::InvalidArgument("buf is nullptr");
+        }
+        if (((size - 1) & (size - 2)) != 0) {
+            return Status::InvalidArgument("size - 1 must be power of two");
         }
         _data = new char[size];
         memcpy(_data, buf, size);
diff --git a/be/test/olap/date_bloom_filter_test.cpp b/be/test/olap/date_bloom_filter_test.cpp
index 1678f5ee71a9e1..715301419e228f 100644
--- a/be/test/olap/date_bloom_filter_test.cpp
+++ b/be/test/olap/date_bloom_filter_test.cpp
@@ -59,7 +59,7 @@ class DateBloomFilterTest : public ::testing::Test {
 
         construct_column(schema_pb.add_column(), 0, "DATE", "date_column");
         construct_column(schema_pb.add_column(), 1, "DATETIME", "datetime_column");
-
+        schema_pb.set_bf_fpp(0.05);
         _tablet_schema.reset(new TabletSchema);
         _tablet_schema->init_from_pb(schema_pb);
 
diff --git a/be/test/olap/rowset/segment_v2/bloom_filter_test.cpp b/be/test/olap/rowset/segment_v2/bloom_filter_test.cpp
new file mode 100644
index 00000000000000..355482173f28b0
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/bloom_filter_test.cpp
@@ -0,0 +1,302 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include "olap/rowset/segment_v2/bloom_filter.h"
+
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <random>
+#include <unordered_set>
+#include <vector>
+
+using namespace doris;
+using namespace segment_v2;
+
+// Test class for BloomFilter
+class BloomFilterTest : public testing::Test {
+public:
+    void SetUp() override {
+        // Any setup required before each test
+    }
+    void TearDown() override {
+        // Any cleanup required after each test
+    }
+};
+
+// Test the creation of BloomFilter instances
+TEST_F(BloomFilterTest, TestCreateBloomFilter) {
+    // Create a BlockSplitBloomFilter
+    std::unique_ptr<BloomFilter> bf;
+    Status st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf);
+    EXPECT_TRUE(st.ok());
+    EXPECT_NE(bf, nullptr);
+
+    // Create an NGramBloomFilter with a specific size
+    std::unique_ptr<BloomFilter> ngram_bf;
+    size_t bf_size = 1024;
+    st = BloomFilter::create(NGRAM_BLOOM_FILTER, &ngram_bf, bf_size);
+    EXPECT_TRUE(st.ok());
+    EXPECT_NE(ngram_bf, nullptr);
+}
+
+// Test the initialization of the BloomFilter with filter size
+TEST_F(BloomFilterTest, TestInitBloomFilterWithFilterSize) {
+    std::unique_ptr<BloomFilter> bf;
+    Status st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf);
+    EXPECT_TRUE(st.ok());
+
+    // Initialize with a specific filter size
+    uint64_t filter_size = 1024; // Filter size in bytes
+    st = bf->init(filter_size);
+    EXPECT_TRUE(st.ok());
+
+    // Verify the internal size is set correctly
+    EXPECT_EQ(bf->num_bytes(), static_cast<uint32_t>(filter_size));
+}
+
+// Test the initialization of the BloomFilter with expected items and FPP
+TEST_F(BloomFilterTest, TestInitBloomFilter) {
+    std::unique_ptr<BloomFilter> bf;
+    Status st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf);
+    EXPECT_TRUE(st.ok());
+
+    // Initialize with expected number of items and FPP
+    uint64_t n = 1000; // Expected number of items
+    double fpp = 0.01; // Desired false positive probability
+    st = bf->init(n, fpp, HASH_MURMUR3_X64_64);
+    EXPECT_TRUE(st.ok());
+
+    // Verify the internal size is set correctly
+    uint32_t expected_num_bits = BloomFilter::optimal_bit_num(n, fpp);
+    uint32_t expected_num_bytes = expected_num_bits / 8;
+    EXPECT_EQ(bf->num_bytes(), expected_num_bytes);
+}
+
+// Test the num_bytes() function
+TEST_F(BloomFilterTest, TestNumBytesFunction) {
+    std::unique_ptr<BloomFilter> bf;
+    Status st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf);
+    EXPECT_TRUE(st.ok());
+
+    // Initialize with a specific filter size
+    uint64_t filter_size = 2048; // Filter size in bytes
+    st = bf->init(filter_size);
+    EXPECT_TRUE(st.ok());
+
+    // Verify num_bytes() returns the correct value
+    EXPECT_EQ(bf->num_bytes(), static_cast<uint32_t>(filter_size));
+}
+
+// Test the contains() function
+TEST_F(BloomFilterTest, TestContainsFunction) {
+    // Create and initialize two BloomFilters
+    std::unique_ptr<BloomFilter> bf1;
+    Status st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf1);
+    EXPECT_TRUE(st.ok());
+    uint64_t n = 1000;
+    double fpp = 0.01;
+    st = bf1->init(n, fpp, HASH_MURMUR3_X64_64);
+    EXPECT_TRUE(st.ok());
+
+    std::unique_ptr<BloomFilter> bf2;
+    st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf2);
+    EXPECT_TRUE(st.ok());
+    st = bf2->init(n, fpp, HASH_MURMUR3_X64_64);
+    EXPECT_TRUE(st.ok());
+
+    // Add elements to bf1
+    for (int i = 0; i < n; ++i) {
+        std::string elem = "bf1_element_" + std::to_string(i);
+        bf1->add_bytes(elem.c_str(), elem.size());
+    }
+
+    // Add elements to bf2
+    for (int i = 0; i < n; ++i) {
+        std::string elem = "bf2_element_" + std::to_string(i);
+        bf2->add_bytes(elem.c_str(), elem.size());
+    }
+
+    // Test the contains() function
+    // Since the base class implementation always returns true, we expect true
+    bool result = bf1->contains(*bf2);
+    EXPECT_TRUE(result);
+}
+
+// Test adding elements and checking them
+TEST_F(BloomFilterTest, TestAddAndTestElements) {
+    std::unique_ptr<BloomFilter> bf;
+    Status st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf);
+    EXPECT_TRUE(st.ok());
+
+    uint64_t n = 1000;
+    double fpp = 0.01;
+    st = bf->init(n, fpp, HASH_MURMUR3_X64_64);
+    EXPECT_TRUE(st.ok());
+
+    // Add elements to the BloomFilter
+    std::vector<std::string> inserted_elements;
+    for (int i = 0; i < n; ++i) {
+        std::string elem = "element_" + std::to_string(i);
+        bf->add_bytes(elem.c_str(), elem.size());
+        inserted_elements.push_back(elem);
+    }
+
+    // Test that inserted elements are found
+    for (const auto& elem : inserted_elements) {
+        bool exists = bf->test_bytes(elem.c_str(), elem.size());
+        EXPECT_TRUE(exists);
+    }
+
+    // Test that non-inserted elements are not found (allowing for false positives)
+    int false_positives = 0;
+    int test_count = 10000;
+    for (int i = 0; i < test_count; ++i) {
+        std::string elem = "nonexistent_element_" + std::to_string(i);
+        bool exists = bf->test_bytes(elem.c_str(), elem.size());
+        if (exists) {
+            false_positives++;
+        }
+    }
+    double actual_fpp = static_cast<double>(false_positives) / test_count;
+    std::cout << "Expected FPP: " << fpp << ", Actual FPP: " << actual_fpp << std::endl;
+    // Allow some margin in FPP
+    EXPECT_LE(actual_fpp, fpp * 1.2);
+}
+
+// Test the merge function
+TEST_F(BloomFilterTest, TestMergeBloomFilters) {
+    // Create and initialize two BloomFilters
+    std::unique_ptr<BloomFilter> bf1;
+    Status st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf1);
+    EXPECT_TRUE(st.ok());
+    uint64_t n = 1000;
+    double fpp = 0.01;
+    st = bf1->init(n, fpp, HASH_MURMUR3_X64_64);
+    EXPECT_TRUE(st.ok());
+
+    std::unique_ptr<BloomFilter> bf2;
+    st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf2);
+    EXPECT_TRUE(st.ok());
+    st = bf2->init(n, fpp, HASH_MURMUR3_X64_64);
+    EXPECT_TRUE(st.ok());
+
+    // Add elements to bf1
+    std::vector<std::string> elements_bf1;
+    for (int i = 0; i < n; ++i) {
+        std::string elem = "bf1_element_" + std::to_string(i);
+        bf1->add_bytes(elem.c_str(), elem.size());
+        elements_bf1.push_back(elem);
+    }
+
+    // Add elements to bf2
+    std::vector<std::string> elements_bf2;
+    for (int i = 0; i < n; ++i) {
+        std::string elem = "bf2_element_" + std::to_string(i);
+        bf2->add_bytes(elem.c_str(), elem.size());
+        elements_bf2.push_back(elem);
+    }
+
+    // Merge bf2 into bf1
+    st = bf1->merge(bf2.get());
+    EXPECT_TRUE(st.ok());
+
+    // Test that elements from bf1 are found
+    for (const auto& elem : elements_bf1) {
+        bool exists = bf1->test_bytes(elem.c_str(), elem.size());
+        EXPECT_TRUE(exists);
+    }
+
+    // Test that elements from bf2 are found in merged bf1
+    for (const auto& elem : elements_bf2) {
+        bool exists = bf1->test_bytes(elem.c_str(), elem.size());
+        EXPECT_TRUE(exists);
+    }
+}
+
+// Test null value handling
+TEST_F(BloomFilterTest, TestNullValueHandling) {
+    std::unique_ptr<BloomFilter> bf;
+    Status st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf);
+    EXPECT_TRUE(st.ok());
+
+    uint64_t n = 1000;
+    double fpp = 0.01;
+    st = bf->init(n, fpp, HASH_MURMUR3_X64_64);
+    EXPECT_TRUE(st.ok());
+
+    // Initially, has_null should be false
+    EXPECT_FALSE(bf->has_null());
+
+    // Add null value
+    bf->add_bytes(nullptr, 0);
+
+    // Now, has_null should be true
+    EXPECT_TRUE(bf->has_null());
+
+    // Test for null value
+    bool exists = bf->test_bytes(nullptr, 0);
+    EXPECT_TRUE(exists);
+}
+
+// Test the optimal_bit_num function
+TEST_F(BloomFilterTest, TestOptimalBitNum) {
+    uint64_t n = 1000;
+    double fpp = 0.01;
+    uint32_t num_bits = BloomFilter::optimal_bit_num(n, fpp);
+
+    // Expected num_bits should be within a reasonable range
+    EXPECT_GT(num_bits, 0u);
+    EXPECT_LE(num_bits, BloomFilter::MAXIMUM_BYTES * 8);
+
+    // Verify that num_bits is a power of 2
+    EXPECT_EQ(num_bits & (num_bits - 1), 0u);
+}
+
+// Test init function with invalid inputs to cover exception branches
+TEST_F(BloomFilterTest, TestInitWithInvalidInputs) {
+    std::unique_ptr<BloomFilter> bf;
+    Status st = BloomFilter::create(BLOCK_BLOOM_FILTER, &bf);
+    EXPECT_TRUE(st.ok());
+
+    // Test with invalid hash strategy
+    const char* buffer = "test_buffer";
+    uint32_t size = 1024;
+    HashStrategyPB invalid_strategy = static_cast<HashStrategyPB>(-1); // Invalid strategy
+    st = bf->init(buffer, size, invalid_strategy);
+    EXPECT_FALSE(st.ok());
+    EXPECT_EQ(st.code(), TStatusCode::INVALID_ARGUMENT);
+
+    // Test with size zero
+    st = bf->init(buffer, 0, HASH_MURMUR3_X64_64);
+    EXPECT_FALSE(st.ok());
+    EXPECT_EQ(st.code(), TStatusCode::INVALID_ARGUMENT);
+
+    // Test with nullptr buffer
+    st = bf->init(nullptr, size, HASH_MURMUR3_X64_64);
+    EXPECT_EQ(st.code(), TStatusCode::INVALID_ARGUMENT);
+
+    // Test with size less than minimum allowed
+    uint32_t invalid_size = 1; // Less than minimum bytes
+    st = bf->init(buffer, invalid_size, HASH_MURMUR3_X64_64);
+    EXPECT_FALSE(st.ok());
+    EXPECT_EQ(st.code(), TStatusCode::INVALID_ARGUMENT);
+
+    // Test with size not a power of two
+    uint32_t non_power_of_two_size = 1000; // Not a power of two
+    st = bf->init(buffer, non_power_of_two_size, HASH_MURMUR3_X64_64);
+    EXPECT_EQ(st.code(), TStatusCode::INVALID_ARGUMENT);
+}
\ No newline at end of file

From cf40dba7fba7ca258d3ee7b6275ded1a4cc345ea Mon Sep 17 00:00:00 2001
From: Tiewei Fang <fangtiewei@selectdb.com>
Date: Sat, 30 Nov 2024 09:18:53 +0800
Subject: [PATCH 088/399] [fix](outfile) add regression test for outfile
 (#44734)

### What problem does this PR solve?
Problem Summary:

add regression test for this pr: #43929

TODO:
Support to export data of doris NULL type to orc file format.
---
 .../outfile/test_outfile_null_type.out        |  37 ++++++
 .../outfile/test_outfile_null_type.groovy     | 111 ++++++++++++++++++
 2 files changed, 148 insertions(+)
 create mode 100644 regression-test/data/export_p0/outfile/test_outfile_null_type.out
 create mode 100644 regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy

diff --git a/regression-test/data/export_p0/outfile/test_outfile_null_type.out b/regression-test/data/export_p0/outfile/test_outfile_null_type.out
new file mode 100644
index 00000000000000..1777bba72c1b5c
--- /dev/null
+++ b/regression-test/data/export_p0/outfile/test_outfile_null_type.out
@@ -0,0 +1,37 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_export --
+1	ftw-1	19
+2	ftw-2	20
+3	ftw-3	21
+4	ftw-4	22
+5	ftw-5	23
+6	ftw-6	24
+7	ftw-7	25
+8	ftw-8	26
+9	ftw-9	27
+10	\N	\N
+
+-- !select_load_parquet --
+1	ftw-1	19	\N
+10	\N	\N	\N
+2	ftw-2	20	\N
+3	ftw-3	21	\N
+4	ftw-4	22	\N
+5	ftw-5	23	\N
+6	ftw-6	24	\N
+7	ftw-7	25	\N
+8	ftw-8	26	\N
+9	ftw-9	27	\N
+
+-- !select_load_csv --
+1	ftw-1	19	\N
+10	\N	\N	\N
+2	ftw-2	20	\N
+3	ftw-3	21	\N
+4	ftw-4	22	\N
+5	ftw-5	23	\N
+6	ftw-6	24	\N
+7	ftw-7	25	\N
+8	ftw-8	26	\N
+9	ftw-9	27	\N
+
diff --git a/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy b/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
new file mode 100644
index 00000000000000..492b6770195341
--- /dev/null
+++ b/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_outfile_null_type", "p0") {
+    String ak = getS3AK()
+    String sk = getS3SK()
+    String s3_endpoint = getS3Endpoint()
+    String region = getS3Region()
+    String bucket = context.config.otherConfigs.get("s3BucketName");
+
+    def export_table_name = "test_outfile_null_type"
+    def outFilePath = "${bucket}/outfile/null_type/exp_"
+
+    def outfile_to_S3 = { format ->
+        // select ... into outfile ...
+        def res = sql """
+            SELECT *, NULL AS null_col FROM ${export_table_name} t
+            INTO OUTFILE "s3://${outFilePath}"
+            FORMAT AS ${format}
+            PROPERTIES (
+                "s3.endpoint" = "${s3_endpoint}",
+                "s3.region" = "${region}",
+                "s3.secret_key"="${sk}",
+                "s3.access_key" = "${ak}"
+            );
+        """
+
+        return res[0][3]
+    }
+
+    sql """ DROP TABLE IF EXISTS ${export_table_name} """
+    sql """
+        CREATE TABLE `${export_table_name}` (
+            `id` int(11) NULL,
+            `Name` string NULL,
+            `age` int(11) NULL
+        ) ENGINE=OLAP
+        DISTRIBUTED BY HASH(`id`)
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+        """
+
+    StringBuilder sb = new StringBuilder()
+    int i = 1
+    for (; i < 10; i ++) {
+        sb.append("""
+            (${i}, 'ftw-${i}', ${i + 18}),
+        """)
+    }
+    sb.append("""
+            (${i}, NULL, NULL)
+        """)
+    sql """ INSERT INTO ${export_table_name} VALUES
+            ${sb.toString()}
+        """
+    def insert_res = sql "show last insert;"
+    logger.info("insert result: " + insert_res.toString())
+    qt_select_export """ SELECT * FROM ${export_table_name} t ORDER BY id; """
+
+    // parquet file format
+    def format = "parquet"
+    def outfile_url = outfile_to_S3("${format}")
+    order_qt_select_load_parquet """ SELECT * FROM S3 (
+                "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+                "ACCESS_KEY"= "${ak}",
+                "SECRET_KEY" = "${sk}",
+                "format" = "${format}",
+                "region" = "${region}"
+            );
+            """
+
+    // TODO(ftw): orc supports to export data of NULL type.
+    // orc file foramt
+    // format = "orc"
+    // outfile_url = outfile_to_S3("${format}")
+    // qt_select_load_orc """ SELECT * FROM S3 (
+    //             "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+    //             "ACCESS_KEY"= "${ak}",
+    //             "SECRET_KEY" = "${sk}",
+    //             "format" = "${format}",
+    //             "region" = "${region}"
+    //         );
+    //         """
+
+    // csv file foramt
+    format = "csv"
+    outfile_url = outfile_to_S3("${format}")
+    order_qt_select_load_csv """ SELECT * FROM S3 (
+                "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+                "ACCESS_KEY"= "${ak}",
+                "SECRET_KEY" = "${sk}",
+                "format" = "${format}",
+                "region" = "${region}"
+            );
+            """
+}
\ No newline at end of file

From 42a7734007dc3d20713cfb89b3aaf83e6d25608e Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Sat, 30 Nov 2024 14:17:12 +0800
Subject: [PATCH 089/399] [opt](nereids)flattern and/or (#44574)

### What problem does this PR solve?
And/or expressions are represented by binary trees. The depth of the
tree causes stack overflow in recursive program calls. To solve this
problem, this pr will flatten the binary tree when processing and/or,
reducing the number of recursions. At the same time, we also rewrite
some recursive programs into non-recursive forms to avoid stack
overflow.

Issue
---
 .../apache/doris/job/manager/JobManager.java  |   8 +-
 .../java/org/apache/doris/load/ExportMgr.java |  15 ++-
 .../apache/doris/load/loadv2/LoadManager.java |   6 +-
 .../doris/nereids/cost/ExprCostModel.java     |  12 ++
 .../glue/translator/ExpressionTranslator.java |  81 +++++++++++--
 .../nereids/parser/LogicalPlanBuilder.java    |  10 +-
 .../rules/analysis/ExpressionAnalyzer.java    |  64 +++++++++-
 .../rules/analysis/SubqueryToApply.java       |  24 ++--
 .../rules/ExtractCommonFactorRule.java        |  20 +++-
 .../rules/FoldConstantRuleOnFE.java           |  16 +--
 .../PartitionPruneExpressionExtractor.java    |  80 +++++++------
 .../expression/rules/SimplifyNotExprRule.java |   5 +-
 .../TrySimplifyPredicateWithMarkJoinSlot.java |  42 ++++---
 .../rewrite/InferPredicateByReplace.java      |   6 -
 .../stats/ColumnStatsAdjustVisitor.java       |  11 +-
 .../nereids/stats/ExpressionEstimation.java   |  27 ++++-
 .../doris/nereids/stats/FilterEstimation.java |  49 +++++---
 .../doris/nereids/trees/expressions/And.java  |  36 ++++--
 .../trees/expressions/CompoundPredicate.java  |  82 +++++++++++--
 .../nereids/trees/expressions/Expression.java |  18 +--
 .../doris/nereids/trees/expressions/Or.java   |  35 ++++--
 .../visitor/ExpressionVisitor.java            |   2 +-
 .../trees/plans/commands/CancelCommand.java   |   6 +-
 .../plans/commands/CancelExportCommand.java   |   3 +-
 .../plans/commands/CancelLoadCommand.java     |   5 +-
 .../plans/commands/DeleteFromCommand.java     |   4 +-
 .../doris/nereids/util/ExpressionUtils.java   |  42 +++++--
 .../doris/nereids/util/TypeCoercionUtils.java |  23 ----
 .../rules/expression/SimplifyRangeTest.java   |   2 +-
 .../ArrayContainsToArrayOverlapTest.java      |   3 +-
 .../nereids/rules/rewrite/OrToInTest.java     |  19 +--
 .../doris/nereids/sqltest/InferTest.java      |   6 +-
 .../nereids/stats/FilterEstimationTest.java   |   2 +-
 .../UpdateMvByPartitionCommandTest.java       |   4 +-
 .../nereids_hint_tpcds_p0/shape/query10.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query13.out   |   6 +-
 .../nereids_hint_tpcds_p0/shape/query15.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query26.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query28.out   |  12 +-
 .../nereids_hint_tpcds_p0/shape/query34.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query41.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query45.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query46.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query47.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query48.out   |   6 +-
 .../nereids_hint_tpcds_p0/shape/query49.out   |   6 +-
 .../nereids_hint_tpcds_p0/shape/query53.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query57.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query61.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query63.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query68.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query7.out    |   2 +-
 .../nereids_hint_tpcds_p0/shape/query78.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query79.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query85.out   |   6 +-
 .../nereids_hint_tpcds_p0/shape/query88.out   |  16 +--
 .../nereids_hint_tpcds_p0/shape/query89.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query91.out   |   2 +-
 .../data/nereids_hint_tpch_p0/shape/q19.out   |   4 +-
 .../data/nereids_hint_tpch_p0/shape/q7.out    |   2 +-
 .../huge_compound/huge_compound.out           |   4 +
 .../eliminate_not_null/eliminate_not_null.out |   2 +-
 .../eliminate_outer_join.out                  |   2 +-
 .../push_down_expression_in_hash_join.out     |   2 +-
 .../push_down_filter_other_condition.out      |   6 +-
 .../filter_push_down/push_filter_through.out  |  14 +--
 .../extend_infer_equal_predicate.out          |   4 +-
 .../pull_up_predicate_set_op.out              |   2 +-
 .../predicate_infer/infer_predicate.out       |   6 +-
 .../extract_from_disjunction_in_join.out      |  16 +--
 .../push_down_filter_through_window.out       |   2 +-
 .../bs_downgrade_shape/query13.out            |   6 +-
 .../bs_downgrade_shape/query45.out            |   2 +-
 .../bs_downgrade_shape/query61.out            |   2 +-
 .../bs_downgrade_shape/query68.out            |   2 +-
 .../bs_downgrade_shape/query91.out            |   2 +-
 .../eliminate_empty/query10_empty.out         |   2 +-
 .../shape/query10.out                         |   2 +-
 .../shape/query13.out                         |   6 +-
 .../shape/query15.out                         |   2 +-
 .../shape/query26.out                         |   2 +-
 .../shape/query28.out                         |  12 +-
 .../shape/query34.out                         |   2 +-
 .../shape/query35.out                         |   2 +-
 .../shape/query41.out                         |   2 +-
 .../shape/query45.out                         |   2 +-
 .../shape/query46.out                         |   2 +-
 .../shape/query47.out                         |   2 +-
 .../shape/query48.out                         |   6 +-
 .../shape/query49.out                         |   6 +-
 .../shape/query53.out                         |   2 +-
 .../shape/query57.out                         |   2 +-
 .../shape/query61.out                         |   2 +-
 .../shape/query63.out                         |   2 +-
 .../shape/query68.out                         |   2 +-
 .../shape/query7.out                          |   2 +-
 .../shape/query78.out                         |   2 +-
 .../shape/query79.out                         |   2 +-
 .../shape/query85.out                         |   6 +-
 .../shape/query88.out                         |  16 +--
 .../shape/query89.out                         |   2 +-
 .../shape/query91.out                         |   2 +-
 .../noStatsRfPrune/query10.out                |   2 +-
 .../noStatsRfPrune/query13.out                |   6 +-
 .../noStatsRfPrune/query15.out                |   2 +-
 .../noStatsRfPrune/query26.out                |   2 +-
 .../noStatsRfPrune/query28.out                |  12 +-
 .../noStatsRfPrune/query34.out                |   2 +-
 .../noStatsRfPrune/query35.out                |   2 +-
 .../noStatsRfPrune/query41.out                |   2 +-
 .../noStatsRfPrune/query45.out                |   2 +-
 .../noStatsRfPrune/query46.out                |   2 +-
 .../noStatsRfPrune/query47.out                |   2 +-
 .../noStatsRfPrune/query48.out                |   6 +-
 .../noStatsRfPrune/query49.out                |   6 +-
 .../noStatsRfPrune/query53.out                |   2 +-
 .../noStatsRfPrune/query57.out                |   2 +-
 .../noStatsRfPrune/query61.out                |   2 +-
 .../noStatsRfPrune/query63.out                |   2 +-
 .../noStatsRfPrune/query68.out                |   2 +-
 .../noStatsRfPrune/query7.out                 |   2 +-
 .../noStatsRfPrune/query78.out                |   2 +-
 .../noStatsRfPrune/query79.out                |   2 +-
 .../noStatsRfPrune/query85.out                |   6 +-
 .../noStatsRfPrune/query88.out                |  16 +--
 .../noStatsRfPrune/query89.out                |   2 +-
 .../noStatsRfPrune/query91.out                |   2 +-
 .../no_stats_shape/query10.out                |   2 +-
 .../no_stats_shape/query13.out                |   6 +-
 .../no_stats_shape/query15.out                |   2 +-
 .../no_stats_shape/query26.out                |   2 +-
 .../no_stats_shape/query28.out                |  12 +-
 .../no_stats_shape/query34.out                |   2 +-
 .../no_stats_shape/query35.out                |   2 +-
 .../no_stats_shape/query41.out                |   2 +-
 .../no_stats_shape/query45.out                |   2 +-
 .../no_stats_shape/query46.out                |   2 +-
 .../no_stats_shape/query47.out                |   2 +-
 .../no_stats_shape/query48.out                |   6 +-
 .../no_stats_shape/query49.out                |   6 +-
 .../no_stats_shape/query53.out                |   2 +-
 .../no_stats_shape/query57.out                |   2 +-
 .../no_stats_shape/query61.out                |   2 +-
 .../no_stats_shape/query63.out                |   2 +-
 .../no_stats_shape/query68.out                |   2 +-
 .../no_stats_shape/query7.out                 |   2 +-
 .../no_stats_shape/query78.out                |   2 +-
 .../no_stats_shape/query79.out                |   2 +-
 .../no_stats_shape/query85.out                |   6 +-
 .../no_stats_shape/query88.out                |  16 +--
 .../no_stats_shape/query89.out                |   2 +-
 .../no_stats_shape/query91.out                |   2 +-
 .../rf_prune/query10.out                      |   2 +-
 .../rf_prune/query13.out                      |   6 +-
 .../rf_prune/query15.out                      |   2 +-
 .../rf_prune/query26.out                      |   2 +-
 .../rf_prune/query28.out                      |  12 +-
 .../rf_prune/query34.out                      |   2 +-
 .../rf_prune/query35.out                      |   2 +-
 .../rf_prune/query41.out                      |   2 +-
 .../rf_prune/query45.out                      |   2 +-
 .../rf_prune/query46.out                      |   2 +-
 .../rf_prune/query47.out                      |   2 +-
 .../rf_prune/query48.out                      |   6 +-
 .../rf_prune/query49.out                      |   6 +-
 .../rf_prune/query53.out                      |   2 +-
 .../rf_prune/query57.out                      |   2 +-
 .../rf_prune/query61.out                      |   2 +-
 .../rf_prune/query63.out                      |   2 +-
 .../rf_prune/query68.out                      |   2 +-
 .../rf_prune/query7.out                       |   2 +-
 .../rf_prune/query78.out                      |   2 +-
 .../rf_prune/query79.out                      |   2 +-
 .../rf_prune/query85.out                      |   6 +-
 .../rf_prune/query88.out                      |  16 +--
 .../rf_prune/query89.out                      |   2 +-
 .../rf_prune/query91.out                      |   2 +-
 .../shape/query10.out                         |   2 +-
 .../shape/query13.out                         |   6 +-
 .../shape/query15.out                         |   2 +-
 .../shape/query26.out                         |   2 +-
 .../shape/query28.out                         |  12 +-
 .../shape/query34.out                         |   2 +-
 .../shape/query35.out                         |   2 +-
 .../shape/query41.out                         |   2 +-
 .../shape/query45.out                         |   2 +-
 .../shape/query46.out                         |   2 +-
 .../shape/query47.out                         |   2 +-
 .../shape/query48.out                         |   6 +-
 .../shape/query49.out                         |   6 +-
 .../shape/query53.out                         |   2 +-
 .../shape/query57.out                         |   2 +-
 .../shape/query61.out                         |   2 +-
 .../shape/query63.out                         |   2 +-
 .../shape/query68.out                         |   2 +-
 .../shape/query7.out                          |   2 +-
 .../shape/query78.out                         |   2 +-
 .../shape/query79.out                         |   2 +-
 .../shape/query85.out                         |   6 +-
 .../shape/query88.out                         |  16 +--
 .../shape/query89.out                         |   2 +-
 .../shape/query91.out                         |   2 +-
 .../shape/query10.out                         |   2 +-
 .../shape/query13.out                         |   6 +-
 .../shape/query15.out                         |   2 +-
 .../shape/query26.out                         |   2 +-
 .../shape/query28.out                         |  12 +-
 .../shape/query34.out                         |   2 +-
 .../shape/query35.out                         |   2 +-
 .../shape/query41.out                         |   2 +-
 .../shape/query45.out                         |   2 +-
 .../shape/query46.out                         |   2 +-
 .../shape/query47.out                         |   2 +-
 .../shape/query48.out                         |   6 +-
 .../shape/query49.out                         |   6 +-
 .../shape/query53.out                         |  16 +--
 .../shape/query57.out                         |   2 +-
 .../shape/query61.out                         |   2 +-
 .../shape/query63.out                         |  16 +--
 .../shape/query68.out                         |   2 +-
 .../shape/query7.out                          |   2 +-
 .../shape/query78.out                         |   2 +-
 .../shape/query79.out                         |   2 +-
 .../shape/query85.out                         |   6 +-
 .../shape/query88.out                         |  16 +--
 .../shape/query89.out                         |   2 +-
 .../shape/query91.out                         |   2 +-
 .../nostats_rf_prune/q19.out                  |   4 +-
 .../nostats_rf_prune/q7.out                   |   2 +-
 .../rf_prune/q19.out                          |   4 +-
 .../rf_prune/q7.out                           |   2 +-
 .../shape/q19.out                             |   4 +-
 .../nereids_tpch_shape_sf1000_p0/shape/q7.out |   2 +-
 .../shape_no_stats/q19.out                    |   4 +-
 .../shape_no_stats/q7.out                     |   2 +-
 .../hint_tpcds/shape/query78.out              |   2 +-
 .../new_shapes_p0/hint_tpch/shape/q19.out     |   4 +-
 .../data/new_shapes_p0/hint_tpch/shape/q7.out |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query10.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query13.out    |   6 +-
 .../tpcds_sf100/noStatsRfPrune/query15.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query26.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query28.out    |  12 +-
 .../tpcds_sf100/noStatsRfPrune/query34.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query35.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query41.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query45.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query46.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query47.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query48.out    |   6 +-
 .../tpcds_sf100/noStatsRfPrune/query49.out    |   6 +-
 .../tpcds_sf100/noStatsRfPrune/query53.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query57.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query61.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query63.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query68.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query7.out     |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query78.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query79.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query85.out    |   6 +-
 .../tpcds_sf100/noStatsRfPrune/query88.out    |  16 +--
 .../tpcds_sf100/noStatsRfPrune/query89.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query91.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query10.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query13.out    |   6 +-
 .../tpcds_sf100/no_stats_shape/query15.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query26.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query28.out    |  12 +-
 .../tpcds_sf100/no_stats_shape/query34.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query35.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query41.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query45.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query46.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query47.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query48.out    |   6 +-
 .../tpcds_sf100/no_stats_shape/query49.out    |   6 +-
 .../tpcds_sf100/no_stats_shape/query53.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query57.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query61.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query63.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query68.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query7.out     |   2 +-
 .../tpcds_sf100/no_stats_shape/query78.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query79.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query85.out    |   6 +-
 .../tpcds_sf100/no_stats_shape/query88.out    |  16 +--
 .../tpcds_sf100/no_stats_shape/query89.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query91.out    |   2 +-
 .../tpcds_sf100/rf_prune/query10.out          |   2 +-
 .../tpcds_sf100/rf_prune/query13.out          |   6 +-
 .../tpcds_sf100/rf_prune/query15.out          |   2 +-
 .../tpcds_sf100/rf_prune/query26.out          |   2 +-
 .../tpcds_sf100/rf_prune/query28.out          |  12 +-
 .../tpcds_sf100/rf_prune/query34.out          |   2 +-
 .../tpcds_sf100/rf_prune/query35.out          |   2 +-
 .../tpcds_sf100/rf_prune/query41.out          |   2 +-
 .../tpcds_sf100/rf_prune/query45.out          |   2 +-
 .../tpcds_sf100/rf_prune/query46.out          |   2 +-
 .../tpcds_sf100/rf_prune/query47.out          |   2 +-
 .../tpcds_sf100/rf_prune/query48.out          |  16 +--
 .../tpcds_sf100/rf_prune/query49.out          |   6 +-
 .../tpcds_sf100/rf_prune/query53.out          |   2 +-
 .../tpcds_sf100/rf_prune/query57.out          |   2 +-
 .../tpcds_sf100/rf_prune/query61.out          |   2 +-
 .../tpcds_sf100/rf_prune/query63.out          |   2 +-
 .../tpcds_sf100/rf_prune/query68.out          |   2 +-
 .../tpcds_sf100/rf_prune/query7.out           |   2 +-
 .../tpcds_sf100/rf_prune/query78.out          |   2 +-
 .../tpcds_sf100/rf_prune/query79.out          |   2 +-
 .../tpcds_sf100/rf_prune/query85.out          |   6 +-
 .../tpcds_sf100/rf_prune/query88.out          |  16 +--
 .../tpcds_sf100/rf_prune/query89.out          |   2 +-
 .../tpcds_sf100/rf_prune/query91.out          |   2 +-
 .../tpcds_sf100/shape/query10.out             |   2 +-
 .../tpcds_sf100/shape/query13.out             |   6 +-
 .../tpcds_sf100/shape/query15.out             |   2 +-
 .../tpcds_sf100/shape/query26.out             |   2 +-
 .../tpcds_sf100/shape/query28.out             |  12 +-
 .../tpcds_sf100/shape/query34.out             |   2 +-
 .../tpcds_sf100/shape/query35.out             |   2 +-
 .../tpcds_sf100/shape/query41.out             |   2 +-
 .../tpcds_sf100/shape/query45.out             |   2 +-
 .../tpcds_sf100/shape/query46.out             |   2 +-
 .../tpcds_sf100/shape/query47.out             |   2 +-
 .../tpcds_sf100/shape/query48.out             |  16 +--
 .../tpcds_sf100/shape/query49.out             |   6 +-
 .../tpcds_sf100/shape/query53.out             |   2 +-
 .../tpcds_sf100/shape/query57.out             |   2 +-
 .../tpcds_sf100/shape/query61.out             |   2 +-
 .../tpcds_sf100/shape/query63.out             |   2 +-
 .../tpcds_sf100/shape/query68.out             |   2 +-
 .../tpcds_sf100/shape/query7.out              |   2 +-
 .../tpcds_sf100/shape/query78.out             |   2 +-
 .../tpcds_sf100/shape/query79.out             |   2 +-
 .../tpcds_sf100/shape/query85.out             |   6 +-
 .../tpcds_sf100/shape/query88.out             |  16 +--
 .../tpcds_sf100/shape/query89.out             |   2 +-
 .../tpcds_sf100/shape/query91.out             |   2 +-
 .../bs_downgrade_shape/query13.out            |   6 +-
 .../bs_downgrade_shape/query45.out            |   2 +-
 .../bs_downgrade_shape/query61.out            |   2 +-
 .../bs_downgrade_shape/query68.out            |   2 +-
 .../bs_downgrade_shape/query91.out            |   2 +-
 .../eliminate_empty/query10_empty.out         |   2 +-
 .../tpcds_sf1000/shape/query10.out            |   2 +-
 .../tpcds_sf1000/shape/query13.out            |   6 +-
 .../tpcds_sf1000/shape/query15.out            |   2 +-
 .../tpcds_sf1000/shape/query26.out            |   2 +-
 .../tpcds_sf1000/shape/query28.out            |  12 +-
 .../tpcds_sf1000/shape/query34.out            |   2 +-
 .../tpcds_sf1000/shape/query35.out            |   2 +-
 .../tpcds_sf1000/shape/query41.out            |   2 +-
 .../tpcds_sf1000/shape/query45.out            |   2 +-
 .../tpcds_sf1000/shape/query46.out            |   2 +-
 .../tpcds_sf1000/shape/query47.out            |   2 +-
 .../tpcds_sf1000/shape/query48.out            |   6 +-
 .../tpcds_sf1000/shape/query49.out            |   6 +-
 .../tpcds_sf1000/shape/query53.out            |   2 +-
 .../tpcds_sf1000/shape/query57.out            |   2 +-
 .../tpcds_sf1000/shape/query61.out            |   2 +-
 .../tpcds_sf1000/shape/query63.out            |   2 +-
 .../tpcds_sf1000/shape/query68.out            |   2 +-
 .../tpcds_sf1000/shape/query7.out             |   2 +-
 .../tpcds_sf1000/shape/query78.out            |   2 +-
 .../tpcds_sf1000/shape/query79.out            |   2 +-
 .../tpcds_sf1000/shape/query85.out            |   6 +-
 .../tpcds_sf1000/shape/query88.out            |  16 +--
 .../tpcds_sf1000/shape/query89.out            |   2 +-
 .../tpcds_sf1000/shape/query91.out            |   2 +-
 .../tpch_sf1000/nostats_rf_prune/q19.out      |   4 +-
 .../tpch_sf1000/nostats_rf_prune/q7.out       |   2 +-
 .../tpch_sf1000/rf_prune/q19.out              |   4 +-
 .../new_shapes_p0/tpch_sf1000/rf_prune/q7.out |   2 +-
 .../new_shapes_p0/tpch_sf1000/shape/q19.out   |   4 +-
 .../new_shapes_p0/tpch_sf1000/shape/q7.out    |   2 +-
 .../tpch_sf1000/shape_no_stats/q19.out        |   4 +-
 .../tpch_sf1000/shape_no_stats/q7.out         |   2 +-
 .../huge_compound/huge_compound.groovy        |  69 +++++++++++
 .../pull_up_predicate_set_op.groovy           |   2 +-
 .../test_compoundpredicate_explain.groovy     | 111 +++++++++++-------
 380 files changed, 1300 insertions(+), 930 deletions(-)
 create mode 100644 regression-test/data/nereids_p0/huge_compound/huge_compound.out
 create mode 100644 regression-test/suites/nereids_p0/huge_compound/huge_compound.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
index db2ca6eb81d156..2a957775e113b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
@@ -42,7 +42,7 @@
 import org.apache.doris.load.loadv2.JobState;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.nereids.trees.expressions.And;
-import org.apache.doris.nereids.trees.expressions.BinaryOperator;
+import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.collect.Lists;
@@ -528,7 +528,8 @@ private static void addNeedCancelLoadJob(String label, String state,
     /**
      * used for nereids planner
      */
-    public void cancelLoadJob(String dbName, String label, String state, BinaryOperator operator)
+    public void cancelLoadJob(String dbName, String label, String state,
+                              Expression operator)
             throws JobException, AnalysisException, DdlException {
         Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
         // List of load jobs waiting to be cancelled
@@ -582,7 +583,8 @@ public void cancelLoadJob(String dbName, String label, String state, BinaryOpera
     }
 
     private static void addNeedCancelLoadJob(String label, String state,
-                                             BinaryOperator operator, List<InsertJob> loadJobs,
+                                             Expression operator,
+                                             List<InsertJob> loadJobs,
                                              List<InsertJob> matchLoadJobs)
             throws AnalysisException {
         PatternMatcher matcher = PatternMatcherWrapper.createMysqlPattern(label,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index 398cc8f4ac2518..eddd5fb27eeef1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -38,7 +38,7 @@
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.datasource.InternalCatalog;
 import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.nereids.trees.expressions.BinaryOperator;
+import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.scheduler.exception.JobException;
@@ -162,7 +162,9 @@ public void cancelExportJob(CancelExportStmt stmt) throws DdlException, Analysis
         }
     }
 
-    private List<ExportJob> getWaitingCancelJobs(String label, String state, BinaryOperator operator)
+    private List<ExportJob> getWaitingCancelJobs(
+            String label, String state,
+            Expression operator)
             throws AnalysisException {
         Predicate<ExportJob> jobFilter = buildCancelJobFilter(label, state, operator);
         readLock();
@@ -174,7 +176,9 @@ private List<ExportJob> getWaitingCancelJobs(String label, String state, BinaryO
     }
 
     @VisibleForTesting
-    public static Predicate<ExportJob> buildCancelJobFilter(String label, String state, BinaryOperator operator)
+    public static Predicate<ExportJob> buildCancelJobFilter(
+            String label, String state,
+            Expression operator)
             throws AnalysisException {
         PatternMatcher matcher = PatternMatcherWrapper.createMysqlPattern(label,
                 CaseSensibility.LABEL.getCaseSensibility());
@@ -201,7 +205,10 @@ public static Predicate<ExportJob> buildCancelJobFilter(String label, String sta
     /**
      * used for Nereids planner
      */
-    public void cancelExportJob(String label, String state, BinaryOperator operator, String dbName)
+    public void cancelExportJob(
+            String label,
+            String state,
+            Expression operator, String dbName)
             throws DdlException, AnalysisException {
         // List of export jobs waiting to be cancelled
         List<ExportJob> matchExportJobs = getWaitingCancelJobs(label, state, operator);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
index 5c39a27e24f828..eb9b3f0e831a22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
@@ -47,7 +47,7 @@
 import org.apache.doris.load.Load;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.nereids.trees.expressions.And;
-import org.apache.doris.nereids.trees.expressions.BinaryOperator;
+import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.persist.CleanLabelOperationLog;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.OriginStatement;
@@ -248,7 +248,7 @@ public void recordFinishedLoadJob(String label, long transactionId, String dbNam
      * Match need cancel loadJob by stmt.
      **/
     @VisibleForTesting
-    public static void addNeedCancelLoadJob(String label, String state, BinaryOperator operator,
+    public static void addNeedCancelLoadJob(String label, String state, Expression operator,
                                             List<LoadJob> loadJobs, List<LoadJob> matchLoadJobs)
             throws AnalysisException {
         PatternMatcher matcher = PatternMatcherWrapper.createMysqlPattern(label,
@@ -281,7 +281,7 @@ public static void addNeedCancelLoadJob(String label, String state, BinaryOperat
     /**
      * Cancel load job by stmt.
      **/
-    public void cancelLoadJob(String dbName, String label, String state, BinaryOperator operator)
+    public void cancelLoadJob(String dbName, String label, String state, Expression operator)
             throws DdlException, AnalysisException {
         Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
         // List of load jobs waiting to be cancelled
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/ExprCostModel.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/ExprCostModel.java
index 4a8f9ccd029276..43c98969b9b2a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/ExprCostModel.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/ExprCostModel.java
@@ -18,7 +18,9 @@
 package org.apache.doris.nereids.cost;
 
 import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
@@ -70,4 +72,14 @@ public Double visitLiteral(Literal literal, Void context) {
         return 0.0;
     }
 
+    @Override
+    public Double visitAnd(And and, Void context) {
+        return 0.0;
+    }
+
+    @Override
+    public Double visitOr(Or or, Void context) {
+        return 0.0;
+    }
+
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
index 627a983c123b8d..166892f48a9db6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
@@ -104,7 +104,9 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 
+import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Deque;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -322,22 +324,85 @@ public Expr visitNullLiteral(NullLiteral nullLiteral, PlanTranslatorContext cont
         return nullLit;
     }
 
+    private static class Frame {
+        int low;
+        int high;
+        CompoundPredicate.Operator op;
+        boolean processed;
+
+        Frame(int low, int high, CompoundPredicate.Operator op) {
+            this.low = low;
+            this.high = high;
+            this.op = op;
+            this.processed = false;
+        }
+    }
+
+    private Expr toBalancedTree(int low, int high, List<Expr> children,
+            CompoundPredicate.Operator op) {
+        Deque<Frame> stack = new ArrayDeque<>();
+        Deque<Expr> results = new ArrayDeque<>();
+
+        stack.push(new Frame(low, high, op));
+
+        while (!stack.isEmpty()) {
+            Frame currentFrame = stack.peek();
+
+            if (!currentFrame.processed) {
+                int l = currentFrame.low;
+                int h = currentFrame.high;
+                int diff = h - l;
+
+                if (diff == 0) {
+                    results.push(children.get(l));
+                    stack.pop();
+                } else if (diff == 1) {
+                    Expr left = children.get(l);
+                    Expr right = children.get(h);
+                    CompoundPredicate cp = new CompoundPredicate(op, left, right);
+                    results.push(cp);
+                    stack.pop();
+                } else {
+                    int mid = l + (h - l) / 2;
+
+                    currentFrame.processed = true;
+
+                    stack.push(new Frame(mid + 1, h, op));
+                    stack.push(new Frame(l, mid, op));
+                }
+            } else {
+                stack.pop();
+                if (results.size() >= 2) {
+                    Expr right = results.pop();
+                    Expr left = results.pop();
+                    CompoundPredicate cp = new CompoundPredicate(currentFrame.op, left, right);
+                    results.push(cp);
+                }
+            }
+        }
+        return results.pop();
+    }
+
     @Override
     public Expr visitAnd(And and, PlanTranslatorContext context) {
-        org.apache.doris.analysis.CompoundPredicate cp = new org.apache.doris.analysis.CompoundPredicate(
-                org.apache.doris.analysis.CompoundPredicate.Operator.AND,
-                and.child(0).accept(this, context),
-                and.child(1).accept(this, context));
+        List<Expr> children = and.children().stream().map(
+                e -> e.accept(this, context)
+        ).collect(Collectors.toList());
+        CompoundPredicate cp = (CompoundPredicate) toBalancedTree(0, children.size() - 1,
+                children, CompoundPredicate.Operator.AND);
+
         cp.setNullableFromNereids(and.nullable());
         return cp;
     }
 
     @Override
     public Expr visitOr(Or or, PlanTranslatorContext context) {
-        org.apache.doris.analysis.CompoundPredicate cp = new org.apache.doris.analysis.CompoundPredicate(
-                org.apache.doris.analysis.CompoundPredicate.Operator.OR,
-                or.child(0).accept(this, context),
-                or.child(1).accept(this, context));
+        List<Expr> children = or.children().stream().map(
+                e -> e.accept(this, context)
+        ).collect(Collectors.toList());
+        CompoundPredicate cp = (CompoundPredicate) toBalancedTree(0, children.size() - 1,
+                children, CompoundPredicate.Operator.OR);
+
         cp.setNullableFromNereids(or.nullable());
         return cp;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 2db26dc15011f0..e5f4322225de53 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2081,8 +2081,14 @@ public Expression visitLogicalBinary(LogicalBinaryContext ctx) {
             // into expressions.
             Collections.reverse(contexts);
             List<Expression> expressions = contexts.stream().map(this::getExpression).collect(Collectors.toList());
-            // Create a balanced tree.
-            return reduceToExpressionTree(0, expressions.size() - 1, expressions, ctx);
+            if (ctx.operator.getType() == DorisParser.AND) {
+                return new And(expressions);
+            } else if (ctx.operator.getType() == DorisParser.OR) {
+                return new Or(expressions);
+            } else {
+                // Create a balanced tree.
+                return reduceToExpressionTree(0, expressions.size() - 1, expressions, ctx);
+            }
         });
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
index 49789aa66e1ff8..adc68ac6ecac1b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
@@ -39,6 +39,7 @@
 import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
 import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
 import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
 import org.apache.doris.nereids.trees.expressions.BinaryArithmetic;
 import org.apache.doris.nereids.trees.expressions.BitNot;
@@ -46,7 +47,6 @@
 import org.apache.doris.nereids.trees.expressions.CaseWhen;
 import org.apache.doris.nereids.trees.expressions.Cast;
 import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
-import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
 import org.apache.doris.nereids.trees.expressions.Divide;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
@@ -57,6 +57,7 @@
 import org.apache.doris.nereids.trees.expressions.Match;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.nereids.trees.expressions.Placeholder;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -73,6 +74,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.udf.UdfBuilder;
 import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
 import org.apache.doris.nereids.trees.expressions.typecoercion.ImplicitCastInputTypes;
 import org.apache.doris.nereids.trees.plans.Plan;
@@ -81,6 +83,7 @@
 import org.apache.doris.nereids.types.BigIntType;
 import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.TypeCoercionUtils;
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.qe.ConnectContext;
@@ -95,6 +98,7 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.Lists;
 import org.apache.commons.lang3.StringUtils;
 
 import java.util.ArrayList;
@@ -498,11 +502,59 @@ public Expression visitBinaryArithmetic(BinaryArithmetic binaryArithmetic, Expre
     }
 
     @Override
-    public Expression visitCompoundPredicate(CompoundPredicate compoundPredicate, ExpressionRewriteContext context) {
-        Expression left = compoundPredicate.left().accept(this, context);
-        Expression right = compoundPredicate.right().accept(this, context);
-        CompoundPredicate ret = (CompoundPredicate) compoundPredicate.withChildren(left, right);
-        return TypeCoercionUtils.processCompoundPredicate(ret);
+    public Expression visitOr(Or or, ExpressionRewriteContext context) {
+        List<Expression> children = ExpressionUtils.extractDisjunction(or);
+        List<Expression> newChildren = Lists.newArrayListWithCapacity(children.size());
+        boolean hasNewChild = false;
+        for (Expression child : children) {
+            Expression newChild = child.accept(this, context);
+            if (newChild == null) {
+                newChild = child;
+            }
+            if (newChild.getDataType().isNullType()) {
+                newChild = new NullLiteral(BooleanType.INSTANCE);
+            } else {
+                newChild = TypeCoercionUtils.castIfNotSameType(newChild, BooleanType.INSTANCE);
+            }
+
+            if (! child.equals(newChild)) {
+                hasNewChild = true;
+            }
+            newChildren.add(newChild);
+        }
+        if (hasNewChild) {
+            return ExpressionUtils.or(newChildren);
+        } else {
+            return or;
+        }
+    }
+
+    @Override
+    public Expression visitAnd(And and, ExpressionRewriteContext context) {
+        List<Expression> children = ExpressionUtils.extractConjunction(and);
+        List<Expression> newChildren = Lists.newArrayListWithCapacity(children.size());
+        boolean hasNewChild = false;
+        for (Expression child : children) {
+            Expression newChild = child.accept(this, context);
+            if (newChild == null) {
+                newChild = child;
+            }
+            if (newChild.getDataType().isNullType()) {
+                newChild = new NullLiteral(BooleanType.INSTANCE);
+            } else {
+                newChild = TypeCoercionUtils.castIfNotSameType(newChild, BooleanType.INSTANCE);
+            }
+
+            if (! child.equals(newChild)) {
+                hasNewChild = true;
+            }
+            newChildren.add(newChild);
+        }
+        if (hasNewChild) {
+            return ExpressionUtils.and(newChildren);
+        } else {
+            return and;
+        }
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
index 17e7d098cad552..14700b030d68b7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
@@ -27,7 +27,7 @@
 import org.apache.doris.nereids.trees.TreeNode;
 import org.apache.doris.nereids.trees.expressions.Alias;
 import org.apache.doris.nereids.trees.expressions.And;
-import org.apache.doris.nereids.trees.expressions.BinaryOperator;
+import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
 import org.apache.doris.nereids.trees.expressions.Exists;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.InSubquery;
@@ -589,17 +589,19 @@ public Expression visitScalarSubquery(ScalarSubquery scalar, SubqueryContext con
         }
 
         @Override
-        public Expression visitBinaryOperator(BinaryOperator binaryOperator, SubqueryContext context) {
+        public Expression visitCompoundPredicate(CompoundPredicate compound, SubqueryContext context) {
             // update isMarkJoin flag
-            isMarkJoin =
-                    isMarkJoin || ((binaryOperator.left().anyMatch(SubqueryExpr.class::isInstance)
-                            || binaryOperator.right().anyMatch(SubqueryExpr.class::isInstance))
-                            && (binaryOperator instanceof Or));
-
-            Expression left = replace(binaryOperator.left(), context);
-            Expression right = replace(binaryOperator.right(), context);
-
-            return binaryOperator.withChildren(left, right);
+            if (compound instanceof Or) {
+                for (Expression child : compound.children()) {
+                    if (child.anyMatch(SubqueryExpr.class::isInstance)) {
+                        isMarkJoin = true;
+                        break;
+                    }
+                }
+            }
+            return compound.withChildren(
+                    compound.children().stream().map(c -> replace(c, context)).collect(Collectors.toList())
+            );
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ExtractCommonFactorRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ExtractCommonFactorRule.java
index 4032db4aadf550..d79fafa80a71b9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ExtractCommonFactorRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ExtractCommonFactorRule.java
@@ -40,6 +40,7 @@
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Extract common expr for `CompoundPredicate`.
@@ -60,11 +61,24 @@ public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
 
     private static Expression extractCommonFactor(CompoundPredicate originExpr) {
         // fast return
-        if (!(originExpr.left() instanceof CompoundPredicate || originExpr.left() instanceof BooleanLiteral)
-                && !(originExpr.right() instanceof CompoundPredicate || originExpr.right() instanceof BooleanLiteral)) {
+        boolean canExtract = false;
+        Set<Expression> childrenSet = new LinkedHashSet<>();
+        for (Expression child : originExpr.children()) {
+            if ((child instanceof CompoundPredicate || child instanceof BooleanLiteral)) {
+                canExtract = true;
+            }
+            childrenSet.add(child);
+        }
+        if (!canExtract) {
+            if (childrenSet.size() != originExpr.children().size()) {
+                if (childrenSet.size() == 1) {
+                    return childrenSet.iterator().next();
+                } else {
+                    return originExpr.withChildren(childrenSet.stream().collect(Collectors.toList()));
+                }
+            }
             return originExpr;
         }
-
         // flatten same type to a list
         // e.g. ((a and (b or c)) and c) -> [a, (b or c), c]
         List<Expression> flatten = ExpressionUtils.extract(originExpr);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
index 1e01e544a2510d..d1c385ec621062 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
@@ -401,15 +401,15 @@ public Expression visitAnd(And and, ExpressionRewriteContext context) {
                     // x and y
                     return and.withChildren(nonTrueLiteral);
             }
-        } else if (nullCount == 1) {
+        } else if (nullCount < and.children().size()) {
             if (nonTrueLiteral.size() == 1) {
-                // null and true
-                return new NullLiteral(BooleanType.INSTANCE);
+                return nonTrueLiteral.get(0);
+            } else {
+                // null and x
+                return and.withChildren(nonTrueLiteral);
             }
-            // null and x
-            return and.withChildren(nonTrueLiteral);
         } else {
-            // null and null
+            // null and null and null and ...
             return new NullLiteral(BooleanType.INSTANCE);
         }
     }
@@ -442,10 +442,10 @@ public Expression visitOr(Or or, ExpressionRewriteContext context) {
                     // x or y
                     return or.withChildren(nonFalseLiteral);
             }
-        } else if (nullCount == 1) {
+        } else if (nullCount < nonFalseLiteral.size()) {
             if (nonFalseLiteral.size() == 1) {
                 // null or false
-                return new NullLiteral(BooleanType.INSTANCE);
+                return nonFalseLiteral.get(0);
             }
             // null or x
             return or.withChildren(nonFalseLiteral);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruneExpressionExtractor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruneExpressionExtractor.java
index 322016fd45c4a9..90375e0c0e6a36 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruneExpressionExtractor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruneExpressionExtractor.java
@@ -29,7 +29,9 @@
 import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
 
+import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
@@ -100,49 +102,55 @@ public Expression visit(Expression originExpr, Context parentContext) {
     }
 
     @Override
-    public Expression visitAnd(And node, Context parentContext) {
-        // handle left node
-        Context leftContext = new Context();
-        Expression newLeft = node.left().accept(this, leftContext);
-        // handle right node
-        Context rightContext = new Context();
-        Expression newRight = node.right().accept(this, rightContext);
-
-        // if anyone of them is FALSE, the whole expression should be FALSE.
-        if (newLeft == BooleanLiteral.FALSE || newRight == BooleanLiteral.FALSE) {
-            return BooleanLiteral.FALSE;
+    public Expression visitAnd(And and, Context parentContext) {
+        List<Expression> children = and.children();
+        List<Expression> newChildren = Lists.newArrayListWithCapacity(children.size());
+        boolean changed = false;
+        for (Expression child : children) {
+            Context childContext = new Context();
+            Expression newChild = child.accept(this, childContext);
+            // if anyone of them is FALSE, the whole expression should be FALSE.
+            if (newChild == BooleanLiteral.FALSE) {
+                return BooleanLiteral.FALSE;
+            }
+            if (newChild != BooleanLiteral.TRUE && !childContext.containsUnEvaluableExpression) {
+                newChildren.add(newChild);
+                changed |= !newChild.equals(child);
+            } else {
+                changed = true;
+            }
         }
-
-        // If left node contains non-partition slot or is TURE, just discard it.
-        if (newLeft == BooleanLiteral.TRUE || leftContext.containsUnEvaluableExpression) {
-            return rightContext.containsUnEvaluableExpression ? BooleanLiteral.TRUE : newRight;
+        if (newChildren.isEmpty()) {
+            return BooleanLiteral.TRUE;
         }
-
-        // If right node contains non-partition slot or is TURE, just discard it.
-        if (newRight == BooleanLiteral.TRUE || rightContext.containsUnEvaluableExpression) {
-            return newLeft;
+        if (newChildren.size() == 1) {
+            return newChildren.get(0);
+        }
+        if (changed) {
+            if (newChildren.isEmpty()) {
+                return BooleanLiteral.TRUE;
+            } else {
+                return and.withChildren(newChildren);
+            }
+        } else {
+            return and;
         }
-
-        // both does not contains non-partition slot.
-        return new And(newLeft, newRight);
     }
 
     @Override
-    public Expression visitOr(Or node, Context parentContext) {
-        // handle left node
-        Context leftContext = new Context();
-        Expression newLeft = node.left().accept(this, leftContext);
-        // handle right node
-        Context rightContext = new Context();
-        Expression newRight = node.right().accept(this, rightContext);
-
-        // if anyone of them is TRUE or contains non-partition slot, just return TRUE.
-        if (newLeft == BooleanLiteral.TRUE || newRight == BooleanLiteral.TRUE
-                || leftContext.containsUnEvaluableExpression || rightContext.containsUnEvaluableExpression) {
-            return BooleanLiteral.TRUE;
+    public Expression visitOr(Or or, Context parentContext) {
+        List<Expression> children = or.children();
+        List<Expression> newChildren = Lists.newArrayListWithCapacity(children.size());
+        for (Expression child : children) {
+            Context childContext = new Context();
+            Expression newChild = child.accept(this, childContext);
+            // if anyone of them is TRUE or contains non-partition slot, just return TRUE.
+            if (newChild == BooleanLiteral.TRUE || childContext.containsUnEvaluableExpression) {
+                return BooleanLiteral.TRUE;
+            }
+            newChildren.add(newChild);
         }
-
-        return new Or(newLeft, newRight);
+        return or.withChildren(newChildren);
     }
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyNotExprRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyNotExprRule.java
index 484d68f0d7317d..92ea5ddfd5c9ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyNotExprRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyNotExprRule.java
@@ -31,6 +31,7 @@
 import com.google.common.collect.ImmutableList;
 
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * Rewrite rule of NOT expression.
@@ -76,9 +77,7 @@ public static Expression simplify(Not not) {
             }
         } else if (child instanceof CompoundPredicate) {
             CompoundPredicate cp = (CompoundPredicate) child;
-            Not left = new Not(cp.left());
-            Not right = new Not(cp.right());
-            return cp.flip(left, right);
+            return cp.flip(cp.children().stream().map(c -> new Not(c)).collect(Collectors.toList()));
         } else if (child instanceof Not) {
             return child.child(0);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/TrySimplifyPredicateWithMarkJoinSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/TrySimplifyPredicateWithMarkJoinSlot.java
index d4dc6697d80aaa..323a6951688290 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/TrySimplifyPredicateWithMarkJoinSlot.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/TrySimplifyPredicateWithMarkJoinSlot.java
@@ -25,6 +25,10 @@
 import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
 /**
  * TrySimplifyPredicateWithMarkJoinSlot
  */
@@ -56,19 +60,15 @@ public Expression visitAnd(And and, ExpressionRewriteContext context) {
          * we change 'predicate(with mark slot) and predicate(no mark slot)' -> predicate(with mark slot) and true
          * to evaluate the predicate
          */
-        Expression left = and.left();
-        Expression newLeft = left.accept(this, context);
-
-        if (newLeft.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
-            newLeft = BooleanLiteral.TRUE;
+        List<Expression> newChildren = Lists.newArrayListWithCapacity(and.children().size());
+        for (Expression child : and.children()) {
+            Expression newChild = child.accept(this, context);
+            if (newChild.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
+                newChild = BooleanLiteral.TRUE;
+            }
+            newChildren.add(newChild);
         }
-
-        Expression right = and.right();
-        Expression newRight = right.accept(this, context);
-        if (newRight.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
-            newRight = BooleanLiteral.TRUE;
-        }
-        Expression expr = new And(newLeft, newRight);
+        Expression expr = new And(newChildren);
         return expr;
     }
 
@@ -94,19 +94,17 @@ public Expression visitOr(Or or, ExpressionRewriteContext context) {
          * we change 'predicate(with mark slot) or predicate(no mark slot)' -> predicate(with mark slot) or false
          * to evaluate the predicate
          */
-        Expression left = or.left();
-        Expression newLeft = left.accept(this, context);
 
-        if (newLeft.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
-            newLeft = BooleanLiteral.FALSE;
+        List<Expression> newChildren = Lists.newArrayListWithCapacity(or.children().size());
+        for (Expression child : or.children()) {
+            Expression newChild = child.accept(this, context);
+            if (newChild.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
+                newChild = BooleanLiteral.FALSE;
+            }
+            newChildren.add(newChild);
         }
 
-        Expression right = or.right();
-        Expression newRight = right.accept(this, context);
-        if (newRight.getInputSlots().stream().noneMatch(MarkJoinSlotReference.class::isInstance)) {
-            newRight = BooleanLiteral.FALSE;
-        }
-        Expression expr = new Or(newLeft, newRight);
+        Expression expr = new Or(newChildren);
         return expr;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java
index d6f4925c7adeb7..4fc9efc1943ba6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java
@@ -29,7 +29,6 @@
 import org.apache.doris.nereids.trees.expressions.InPredicate;
 import org.apache.doris.nereids.trees.expressions.Like;
 import org.apache.doris.nereids.trees.expressions.Not;
-import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
@@ -81,11 +80,6 @@ public Void visit(Expression expr, Map<Expression, Set<Expression>> context) {
             return null;
         }
 
-        @Override
-        public Void visitOr(Or expr, Map<Expression, Set<Expression>> context) {
-            return null;
-        }
-
         @Override
         public Void visitInPredicate(InPredicate inPredicate, Map<Expression, Set<Expression>> context) {
             if (!validInPredicate(inPredicate)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ColumnStatsAdjustVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ColumnStatsAdjustVisitor.java
index eaf2ce3734408d..d91cb5b4a6883b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ColumnStatsAdjustVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ColumnStatsAdjustVisitor.java
@@ -20,9 +20,8 @@
 import org.apache.doris.analysis.LiteralExpr;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.nereids.trees.expressions.Cast;
-import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
-import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
 import org.apache.doris.nereids.types.DataType;
 import org.apache.doris.nereids.types.coercion.CharacterType;
 import org.apache.doris.statistics.ColumnStatistic;
@@ -57,16 +56,10 @@
  * for other expressions(except cast), we also need to adjust their input column stats.
  *
  */
-public class ColumnStatsAdjustVisitor extends ExpressionVisitor<ColumnStatistic, Statistics> {
+public class ColumnStatsAdjustVisitor extends DefaultExpressionVisitor<ColumnStatistic, Statistics> {
 
     private static final Logger LOG = LogManager.getLogger(ColumnStatsAdjustVisitor.class);
 
-    @Override
-    public ColumnStatistic visit(Expression expr, Statistics context) {
-        expr.children().forEach(child -> child.accept(this, context));
-        return null;
-    }
-
     @Override
     public ColumnStatistic visitCast(Cast cast, Statistics context) {
         ColumnStatistic colStats = context.findColumnStatistics(cast);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
index 2307a6dfba3525..7d1b5439bace23 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
@@ -24,17 +24,18 @@
 import org.apache.doris.nereids.trees.expressions.Add;
 import org.apache.doris.nereids.trees.expressions.AggregateExpression;
 import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.BinaryArithmetic;
 import org.apache.doris.nereids.trees.expressions.CaseWhen;
 import org.apache.doris.nereids.trees.expressions.Cast;
 import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
-import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
 import org.apache.doris.nereids.trees.expressions.Divide;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.IntegralDivide;
 import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference;
 import org.apache.doris.nereids.trees.expressions.Mod;
 import org.apache.doris.nereids.trees.expressions.Multiply;
+import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.Subtract;
 import org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
@@ -451,12 +452,26 @@ public ColumnStatistic visitComparisonPredicate(ComparisonPredicate cp, Statisti
     }
 
     @Override
-    public ColumnStatistic visitCompoundPredicate(CompoundPredicate compoundPredicate, Statistics context) {
-        List<Expression> childExprs = compoundPredicate.children();
-        ColumnStatistic firstChild = childExprs.get(0).accept(this, context);
+    public ColumnStatistic visitOr(Or or, Statistics inputStats) {
+        List<Expression> children = or.children();
+        // TODO: this algorithm is not right, fix it latter
+        ColumnStatistic firstChild = children.get(0).accept(this, inputStats);
         double maxNull = StatsMathUtil.maxNonNaN(firstChild.numNulls, 1);
-        for (int i = 1; i < childExprs.size(); i++) {
-            ColumnStatistic columnStatistic = childExprs.get(i).accept(this, context);
+        for (int i = 1; i < children.size(); i++) {
+            ColumnStatistic columnStatistic = children.get(i).accept(this, inputStats);
+            maxNull = StatsMathUtil.maxNonNaN(maxNull, columnStatistic.numNulls);
+        }
+        return new ColumnStatisticBuilder(firstChild).setNumNulls(maxNull).setNdv(2).build();
+    }
+
+    @Override
+    public ColumnStatistic visitAnd(And and, Statistics inputStats) {
+        List<Expression> children = and.children();
+        // TODO: this algorithm is not right, fix it latter
+        ColumnStatistic firstChild = children.get(0).accept(this, inputStats);
+        double maxNull = StatsMathUtil.maxNonNaN(firstChild.numNulls, 1);
+        for (int i = 1; i < children.size(); i++) {
+            ColumnStatistic columnStatistic = children.get(i).accept(this, inputStats);
             maxNull = StatsMathUtil.maxNonNaN(maxNull, columnStatistic.numNulls);
         }
         return new ColumnStatisticBuilder(firstChild).setNumNulls(maxNull).setNdv(2).build();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index e050ae9fe0feec..afd2bdaf25f43a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -23,7 +23,6 @@
 import org.apache.doris.nereids.stats.FilterEstimation.EstimationContext;
 import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
-import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
 import org.apache.doris.nereids.trees.expressions.EqualPredicate;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.GreaterThan;
@@ -113,22 +112,36 @@ public Statistics visit(Expression expr, EstimationContext context) {
     }
 
     @Override
-    public Statistics visitCompoundPredicate(CompoundPredicate predicate, EstimationContext context) {
-        Expression leftExpr = predicate.child(0);
-        Expression rightExpr = predicate.child(1);
-        Statistics leftStats = leftExpr.accept(this, context);
+    public Statistics visitAnd(And and, EstimationContext context) {
+        List<Expression> children = and.children();
+        Statistics inputStats = context.statistics;
+        Statistics outputStats = inputStats;
+        Preconditions.checkArgument(children.size() > 1, "and expression abnormal: " + and);
+        for (Expression child : children) {
+            outputStats = child.accept(this, new EstimationContext(inputStats));
+            outputStats.normalizeColumnStatistics(inputStats.getRowCount(), true);
+            inputStats = outputStats;
+        }
+        return outputStats;
+    }
+
+    @Override
+    public Statistics visitOr(Or or, EstimationContext context) {
+        List<Expression> children = or.children();
+        Set<Slot> leftInputSlots = Sets.newHashSet(children.get(0).getInputSlots());
+        Statistics leftStats = children.get(0).accept(this, context);
         leftStats.normalizeColumnStatistics(context.statistics.getRowCount(), true);
-        Statistics andStats = rightExpr.accept(this, new EstimationContext(leftStats));
-        if (predicate instanceof And) {
-            andStats.normalizeColumnStatistics(context.statistics.getRowCount(), true);
-            return andStats;
-        } else if (predicate instanceof Or) {
-            Statistics rightStats = rightExpr.accept(this, context);
+        Statistics outputStats = leftStats;
+        Preconditions.checkArgument(children.size() > 1, "and expression abnormal: " + or);
+        for (int i = 1; i < children.size(); i++) {
+            Expression child = children.get(i);
+            Statistics andStats = child.accept(this, new EstimationContext(leftStats));
+            Statistics rightStats = child.accept(this, context);
             rightStats.normalizeColumnStatistics(context.statistics.getRowCount(), true);
             double rowCount = leftStats.getRowCount() + rightStats.getRowCount() - andStats.getRowCount();
             Statistics orStats = context.statistics.withRowCount(rowCount);
-            Set<Slot> leftInputSlots = leftExpr.getInputSlots();
-            Set<Slot> rightInputSlots = rightExpr.getInputSlots();
+
+            Set<Slot> rightInputSlots = child.getInputSlots();
             for (Slot slot : context.keyColumns) {
                 if (leftInputSlots.contains(slot) && rightInputSlots.contains(slot)) {
                     ColumnStatistic leftColStats = leftStats.findColumnStatistics(slot);
@@ -146,13 +159,11 @@ public Statistics visitCompoundPredicate(CompoundPredicate predicate, Estimation
                     orStats.addColumnStats(slot, colBuilder.build());
                 }
             }
-            return orStats;
+            leftStats = orStats;
+            outputStats = orStats;
+            leftInputSlots.addAll(child.getInputSlots());
         }
-        // should not come here
-        Preconditions.checkArgument(false,
-                "unsupported compound operator: %s in %s",
-                predicate.getClass().getName(), predicate.toSql());
-        return context.statistics;
+        return outputStats;
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/And.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/And.java
index 5e76c3afa1a6b3..cc302e76061cfa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/And.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/And.java
@@ -18,9 +18,9 @@
 package org.apache.doris.nereids.trees.expressions;
 
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
 
 import java.util.List;
 
@@ -28,7 +28,6 @@
  * And predicate expression.
  */
 public class And extends CompoundPredicate {
-
     /**
      * Desc: Constructor for CompoundPredicate.
      *
@@ -36,16 +35,18 @@ public class And extends CompoundPredicate {
      * @param right right child of comparison predicate
      */
     public And(Expression left, Expression right) {
-        super(ImmutableList.of(left, right), "AND");
+        super(ExpressionUtils.mergeList(
+                ExpressionUtils.extractConjunction(left),
+                ExpressionUtils.extractConjunction(right)), "AND");
     }
 
-    private And(List<Expression> children) {
+    public And(List<Expression> children) {
         super(children, "AND");
     }
 
     @Override
     public Expression withChildren(List<Expression> children) {
-        Preconditions.checkArgument(children.size() == 2);
+        Preconditions.checkArgument(children.size() >= 2);
         return new And(children);
     }
 
@@ -56,16 +57,35 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public CompoundPredicate flip() {
-        return new Or(left(), right());
+        return new Or(children);
     }
 
     @Override
-    public CompoundPredicate flip(Expression left, Expression right) {
-        return new Or(left, right);
+    public CompoundPredicate flip(List<Expression> children) {
+        return new Or(children);
     }
 
     @Override
     public Class<? extends CompoundPredicate> flipType() {
         return Or.class;
     }
+
+    @Override
+    protected List<Expression> extract() {
+        return ExpressionUtils.extractConjunction(this);
+    }
+
+    @Override
+    public List<Expression> children() {
+        if (flattenChildren.isEmpty()) {
+            for (Expression child : children) {
+                if (child instanceof And) {
+                    flattenChildren.addAll(((And) child).extract());
+                } else {
+                    flattenChildren.add(child);
+                }
+            }
+        }
+        return flattenChildren;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java
index ccc55122125784..d58d1ba8193de5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java
@@ -18,25 +18,32 @@
 package org.apache.doris.nereids.trees.expressions;
 
 import org.apache.doris.nereids.exceptions.UnboundException;
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+import org.apache.doris.nereids.trees.expressions.typecoercion.ExpectsInputTypes;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.types.DataType;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * Compound predicate expression.
- * Such as &&,||,AND,OR.
+ * Such as AND,OR.
  */
-public abstract class CompoundPredicate extends BinaryOperator {
+public abstract class CompoundPredicate extends Expression implements ExpectsInputTypes {
+    protected final List<Expression> flattenChildren = new ArrayList<>();
+    private String symbol;
 
     public CompoundPredicate(List<Expression> children, String symbol) {
-        super(children, symbol);
+        super(children);
+        this.symbol = symbol;
     }
 
     @Override
     public boolean nullable() throws UnboundException {
-        return left().nullable() || right().nullable();
+        return children.stream().anyMatch(ExpressionTrait::nullable);
     }
 
     @Override
@@ -50,8 +57,8 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
     }
 
     @Override
-    public DataType inputType() {
-        return BooleanType.INSTANCE;
+    public List<DataType> expectedInputTypes() {
+        return children.stream().map(c -> BooleanType.INSTANCE).collect(Collectors.toList());
     }
 
     /**
@@ -62,8 +69,69 @@ public DataType inputType() {
     /**
      * Flip logical `and` and `or` operator with new children.
      */
-    public abstract CompoundPredicate flip(Expression left, Expression right);
+    public abstract CompoundPredicate flip(List<Expression> children);
 
     public abstract Class<? extends CompoundPredicate> flipType();
 
+    protected abstract List<Expression> extract();
+
+    @Override
+    public boolean equals(Object o) {
+        if (compareWidthAndDepth) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            List<Expression> thisChildren = this.children();
+            List<Expression> thatChildren = ((CompoundPredicate) o).children();
+            if (thisChildren.size() != thatChildren.size()) {
+                return false;
+            }
+            for (int i = 0; i < thisChildren.size(); i++) {
+                if (!thisChildren.get(i).equals(thatChildren.get(i))) {
+                    return false;
+                }
+            }
+            return true;
+        } else {
+            return super.equals(o);
+        }
+    }
+
+    @Override
+    public String toSql() {
+        StringBuilder sb = new StringBuilder();
+        children().forEach(c -> sb.append(c.toSql()).append(","));
+        sb.deleteCharAt(sb.length() - 1);
+        return symbol + "[" + sb + "]";
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        children().forEach(c -> sb.append(c.toString()).append(","));
+        sb.deleteCharAt(sb.length() - 1);
+        return symbol + "[" + sb + "]";
+    }
+
+    @Override
+    public String shapeInfo() {
+        StringBuilder sb = new StringBuilder();
+        children().forEach(c -> sb.append(c.shapeInfo()).append(","));
+        sb.deleteCharAt(sb.length() - 1);
+        return symbol + "[" + sb + "]";
+    }
+
+    @Override
+    public int arity() {
+        // get flattern children
+        return children().size();
+    }
+
+    @Override
+    public Expression child(int index) {
+        return children().get(index);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
index 6063ad2b1cd6d1..e20290e8b59c41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
@@ -17,7 +17,6 @@
 
 package org.apache.doris.nereids.trees.expressions;
 
-import org.apache.doris.common.Config;
 import org.apache.doris.nereids.analyzer.Unbound;
 import org.apache.doris.nereids.analyzer.UnboundVariable;
 import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -60,12 +59,12 @@ public abstract class Expression extends AbstractTreeNode<Expression> implements
     public static final String DEFAULT_EXPRESSION_NAME = "expression";
     // Mask this expression is generated by rule, should be removed.
     protected Optional<String> exprName = Optional.empty();
+    protected final boolean compareWidthAndDepth;
     private final int depth;
     private final int width;
     // Mark this expression is from predicate infer or something else infer
     private final boolean inferred;
     private final boolean hasUnbound;
-    private final boolean compareWidthAndDepth;
     private final Supplier<Set<Slot>> inputSlots = Suppliers.memoize(
             () -> collect(e -> e instanceof Slot && !(e instanceof ArrayItemSlot)));
     private final int fastChildrenHashCode;
@@ -115,8 +114,6 @@ protected Expression(Expression... children) {
                 this.compareWidthAndDepth = compareWidthAndDepth;
                 this.fastChildrenHashCode = fastChildrenHashCode;
         }
-
-        checkLimit();
         this.inferred = false;
         this.hasUnbound = hasUnbound || this instanceof Unbound;
     }
@@ -170,23 +167,10 @@ protected Expression(List<Expression> children, boolean inferred) {
                 this.compareWidthAndDepth = compareWidthAndDepth && supportCompareWidthAndDepth();
                 this.fastChildrenHashCode = fastChildrenhashCode;
         }
-
-        checkLimit();
         this.inferred = inferred;
         this.hasUnbound = hasUnbound || this instanceof Unbound;
     }
 
-    private void checkLimit() {
-        if (depth > Config.expr_depth_limit) {
-            throw new AnalysisException(String.format("Exceeded the maximum depth of an "
-                    + "expression tree (%s).", Config.expr_depth_limit));
-        }
-        if (width > Config.expr_children_limit) {
-            throw new AnalysisException(String.format("Exceeded the maximum children of an "
-                    + "expression tree (%s).", Config.expr_children_limit));
-        }
-    }
-
     public Alias alias(String alias) {
         return new Alias(this, alias);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Or.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Or.java
index 61249fb91c48ce..cf6c46c3ea4215 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Or.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Or.java
@@ -18,9 +18,9 @@
 package org.apache.doris.nereids.trees.expressions;
 
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
 
 import java.util.List;
 
@@ -36,16 +36,18 @@ public class Or extends CompoundPredicate {
      * @param right right child of comparison predicate
      */
     public Or(Expression left, Expression right) {
-        super(ImmutableList.of(left, right), "OR");
+        super(ExpressionUtils.mergeList(
+                ExpressionUtils.extractDisjunction(left),
+                ExpressionUtils.extractDisjunction(right)), "OR");
     }
 
-    private Or(List<Expression> children) {
+    public Or(List<Expression> children) {
         super(children, "OR");
     }
 
     @Override
     public Expression withChildren(List<Expression> children) {
-        Preconditions.checkArgument(children.size() == 2);
+        Preconditions.checkArgument(children.size() >= 2);
         return new Or(children);
     }
 
@@ -56,16 +58,35 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public CompoundPredicate flip() {
-        return new And(left(), right());
+        return new And(children);
     }
 
     @Override
-    public CompoundPredicate flip(Expression left, Expression right) {
-        return new And(left, right);
+    public CompoundPredicate flip(List<Expression> children) {
+        return new And(children);
     }
 
     @Override
     public Class<? extends CompoundPredicate> flipType() {
         return And.class;
     }
+
+    @Override
+    protected List<Expression> extract() {
+        return ExpressionUtils.extractDisjunction(this);
+    }
+
+    @Override
+    public List<Expression> children() {
+        if (flattenChildren.isEmpty()) {
+            for (Expression child : children) {
+                if (child instanceof Or) {
+                    flattenChildren.addAll(((Or) child).extract());
+                } else {
+                    flattenChildren.add(child);
+                }
+            }
+        }
+        return flattenChildren;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
index ab367a2bf7398e..406d0835610a17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
@@ -334,7 +334,7 @@ public R visitStructLiteral(StructLiteral structLiteral, C context) {
     }
 
     public R visitCompoundPredicate(CompoundPredicate compoundPredicate, C context) {
-        return visitBinaryOperator(compoundPredicate, context);
+        return visit(compoundPredicate, context);
     }
 
     public R visitAnd(And and, C context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelCommand.java
index 88ecf3cba1e6d8..e82191c3648e97 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelCommand.java
@@ -144,18 +144,18 @@ private void binaryCheck(Expression expr, Map<String, String> supportedColumns)
     private void compoundCheck(Expression expr, Map<String, String> supportedColumns) throws AnalysisException {
         // current only support label and state
         if (expr instanceof Not) {
-            throw new AnalysisException("Current not support NOT operator");
+            throw new AnalysisException("not support NOT operator");
         }
         for (int i = 0; i < 2; i++) {
             Expression child = expr.child(i);
             if (child instanceof CompoundPredicate) {
-                throw new AnalysisException("Current not support nested clause");
+                throw new AnalysisException("not support where clause: " + expr.toSql());
             } else if (child instanceof Like) {
                 likeCheck(child, supportedColumns);
             } else if (child instanceof BinaryOperator) {
                 binaryCheck(child, supportedColumns);
             } else {
-                throw new AnalysisException("Only support like/binary predicate");
+                throw new AnalysisException("Only support like/equalTo/And/Or predicate");
             }
         }
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelExportCommand.java
index f7defefb422476..c7dc786a733a85 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelExportCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelExportCommand.java
@@ -20,7 +20,6 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.UserException;
 import org.apache.doris.load.ExportJobState;
-import org.apache.doris.nereids.trees.expressions.BinaryOperator;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
@@ -55,7 +54,7 @@ public CancelExportCommand(String dbName, Expression whereClause) {
     @Override
     public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
         validate(ctx);
-        ctx.getEnv().getExportMgr().cancelExportJob(label, state, (BinaryOperator) whereClause, dbName);
+        ctx.getEnv().getExportMgr().cancelExportJob(label, state, whereClause, dbName);
     }
 
     private void validate(ConnectContext ctx) throws UserException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelLoadCommand.java
index 4d3068ed08f559..80c622a1957a18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelLoadCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelLoadCommand.java
@@ -21,7 +21,6 @@
 import org.apache.doris.common.UserException;
 import org.apache.doris.job.exception.JobException;
 import org.apache.doris.load.ExportJobState;
-import org.apache.doris.nereids.trees.expressions.BinaryOperator;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
@@ -57,9 +56,9 @@ public CancelLoadCommand(String dbName, Expression whereClause) {
     public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
         validate(ctx);
         try {
-            ctx.getEnv().getJobManager().cancelLoadJob(dbName, label, state, (BinaryOperator) whereClause);
+            ctx.getEnv().getJobManager().cancelLoadJob(dbName, label, state, whereClause);
         } catch (JobException e) {
-            ctx.getEnv().getLoadManager().cancelLoadJob(dbName, label, state, (BinaryOperator) whereClause);
+            ctx.getEnv().getLoadManager().cancelLoadJob(dbName, label, state, whereClause);
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
index 34a54ec2651f65..ab2a7d3f7414c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
@@ -334,8 +334,8 @@ private void checkInPredicate(InPredicate in) {
 
     private void checkPredicate(Expression predicate) {
         if (predicate instanceof And) {
-            checkPredicate(((And) predicate).left());
-            checkPredicate(((And) predicate).right());
+            And and = (And) predicate;
+            and.children().forEach(child -> checkPredicate(child));
         } else if (predicate instanceof ComparisonPredicate) {
             checkComparisonPredicate((ComparisonPredicate) predicate);
         } else if (predicate instanceof IsNull) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index bf4d6e084795f1..e3909b87bd3726 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -78,10 +78,12 @@
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Collection;
+import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -131,18 +133,24 @@ public static List<Expression> extract(CompoundPredicate expr) {
 
     private static List<Expression> extract(Class<? extends Expression> type, Expression expr) {
         List<Expression> result = Lists.newArrayList();
-        extract(type, expr, result);
+        Deque<Expression> stack = new ArrayDeque<>();
+        stack.push(expr);
+        while (!stack.isEmpty()) {
+            Expression current = stack.pop();
+            if (type.isInstance(current)) {
+                for (Expression child : current.children()) {
+                    stack.push(child);
+                }
+            } else {
+                result.add(current);
+            }
+        }
+        result = Lists.reverse(result);
         return result;
     }
 
     private static void extract(Class<? extends Expression> type, Expression expr, Collection<Expression> result) {
-        if (type.isInstance(expr)) {
-            CompoundPredicate predicate = (CompoundPredicate) expr;
-            extract(type, predicate.left(), result);
-            extract(type, predicate.right(), result);
-        } else {
-            result.add(expr);
-        }
+        result.addAll(extract(type, expr));
     }
 
     public static Optional<Pair<Slot, Slot>> extractEqualSlot(Expression expr) {
@@ -1002,6 +1010,24 @@ public static Literal analyzeAndFoldToLiteral(ConnectContext ctx, Expression exp
         }
     }
 
+    /**
+     * mergeList
+     */
+    public static List<Expression> mergeList(List<Expression> list1, List<Expression> list2) {
+        ImmutableList.Builder<Expression> builder = ImmutableList.builder();
+        for (Expression expression : list1) {
+            if (expression != null) {
+                builder.add(expression);
+            }
+        }
+        for (Expression expression : list2) {
+            if (expression != null) {
+                builder.add(expression);
+            }
+        }
+        return builder.build();
+    }
+
     private static class UnboundSlotRewriter extends DefaultExpressionRewriter<Void> {
         public static final UnboundSlotRewriter INSTANCE = new UnboundSlotRewriter();
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
index d7f9fc83baf288..603a891d2d2a49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
@@ -32,7 +32,6 @@
 import org.apache.doris.nereids.trees.expressions.CaseWhen;
 import org.apache.doris.nereids.trees.expressions.Cast;
 import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
-import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
 import org.apache.doris.nereids.trees.expressions.Divide;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.InPredicate;
@@ -1192,28 +1191,6 @@ public static Expression processCaseWhen(CaseWhen caseWhen) {
                 .orElseThrow(() -> new AnalysisException("Cannot find common type for case when " + caseWhen));
     }
 
-    /**
-     * process compound predicate type coercion.
-     */
-    public static Expression processCompoundPredicate(CompoundPredicate compoundPredicate) {
-        // check
-        compoundPredicate.checkLegalityBeforeTypeCoercion();
-        ImmutableList.Builder<Expression> newChildren
-                = ImmutableList.builderWithExpectedSize(compoundPredicate.arity());
-        boolean changed = false;
-        for (Expression child : compoundPredicate.children()) {
-            Expression newChild;
-            if (child.getDataType().isNullType()) {
-                newChild = new NullLiteral(BooleanType.INSTANCE);
-            } else {
-                newChild = castIfNotSameType(child, BooleanType.INSTANCE);
-            }
-            changed |= child != newChild;
-            newChildren.add(newChild);
-        }
-        return changed ? compoundPredicate.withChildren(newChildren.build()) : compoundPredicate;
-    }
-
     private static boolean canCompareDate(DataType t1, DataType t2) {
         DataType dateType = t1;
         DataType anotherType = t2;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
index d6a7b9e5585d14..2848965fb06b7a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
@@ -189,7 +189,7 @@ public void testSimplify() {
         assertRewrite("TA + TC = 1 and TA + TC = 3", "(TA + TC) is null and null");
         assertRewriteNotNull("TA + TC in (1) and TA + TC in (3)", "FALSE");
         assertRewrite("TA + TC in (1) and TA + TC in (3)", "(TA + TC) is null and null");
-        assertRewrite("TA + TC in (1) and TA + TC in (1)", "TA + TC = 1");
+        assertRewrite("TA + TC in (1) and TA + TC in (1)", "(TA + TC) IN (1)");
         assertRewriteNotNull("(TA + TC > 3 and TA + TC < 1) and TB < 5", "FALSE");
         assertRewrite("(TA + TC > 3 and TA + TC < 1) and TB < 5", "(TA + TC) is null and null and TB < 5");
         assertRewrite("(TA + TC > 3 and TA + TC < 1) or TB < 5", "((TA + TC) is null and null) OR TB < 5");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ArrayContainsToArrayOverlapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ArrayContainsToArrayOverlapTest.java
index 028d85ce097fe6..ed56bd2a13c3bc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ArrayContainsToArrayOverlapTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ArrayContainsToArrayOverlapTest.java
@@ -89,8 +89,7 @@ void testAndOverlap() {
                 .rewrite()
                 .getPlan();
         Expression expression = plan.child(0).getExpressions().get(0).child(0);
-        Assertions.assertEquals("(array_contains([1], 0) OR "
-                        + "(array_contains([1], 1) AND arrays_overlap([1], [2, 3, 4])))",
+        Assertions.assertEquals("OR[array_contains([1], 0),AND[array_contains([1], 1),arrays_overlap([1], [2, 3, 4])]]",
                 expression.toSql());
     }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
index 5ce7871db144bc..774be68643feb2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
@@ -38,7 +38,7 @@ void test1() {
         String expr = "col1 = 1 or col1 = 2 or col1 = 3 and (col2 = 4)";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("(col1 IN (1, 2, 3) AND (col1 IN (1, 2) OR ((col1 = 3) AND (col2 = 4))))",
+        Assertions.assertEquals("AND[col1 IN (1, 2, 3),OR[col1 IN (1, 2),AND[(col1 = 3),(col2 = 4)]]]",
                 rewritten.toSql());
     }
 
@@ -106,7 +106,8 @@ void test7() {
         String expr = "A = 1 or A = 2 or abs(A)=5 or A in (1, 2, 3) or B = 1 or B = 2 or B in (1, 2, 3) or B+1 in (4, 5, 7)";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("(((A IN (1, 2, 3) OR (abs(A) = 5)) OR B IN (1, 2, 3)) OR (B + 1) IN (4, 5, 7))", rewritten.toSql());
+        Assertions.assertEquals("OR[A IN (1, 2, 3),(abs(A) = 5),B IN (1, 2, 3),(B + 1) IN (4, 5, 7)]",
+                rewritten.toSql());
     }
 
     @Test
@@ -114,7 +115,7 @@ void test8() {
         String expr = "col = 1 or (col = 2 and (col = 3 or col = '4' or col = 5.0))";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("((col = 1) OR ((col = 2) AND col IN ('4', 3, 5.0)))",
+        Assertions.assertEquals("OR[(col = 1),AND[(col = 2),col IN ('4', 3, 5.0)]]",
                 rewritten.toSql());
     }
 
@@ -124,7 +125,7 @@ void testEnsureOrder() {
         String expr = "col1 IN (1, 2) OR col2 IN (1, 2)";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("(col1 IN (1, 2) OR col2 IN (1, 2))",
+        Assertions.assertEquals("OR[col1 IN (1, 2),col2 IN (1, 2)]",
                 rewritten.toSql());
     }
 
@@ -133,7 +134,7 @@ void test9() {
         String expr = "col1=1 and (col2=1 or col2=2)";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("((col1 = 1) AND col2 IN (1, 2))",
+        Assertions.assertEquals("AND[(col1 = 1),col2 IN (1, 2)]",
                 rewritten.toSql());
     }
 
@@ -143,7 +144,7 @@ void test10() {
         String expr = "col1=1 or (col2 = 2 and (col3=4 or col3=5))";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("((col1 = 1) OR ((col2 = 2) AND col3 IN (4, 5)))",
+        Assertions.assertEquals("OR[(col1 = 1),AND[(col2 = 2),col3 IN (4, 5)]]",
                 rewritten.toSql());
     }
 
@@ -153,7 +154,7 @@ void test11() {
         String expr = "(a=1 and b=2 and c=3) or (a=2 and b=2 and c=4)";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("((b = 2) AND ((a IN (1, 2) AND c IN (3, 4)) AND (((a = 1) AND (c = 3)) OR ((a = 2) AND (c = 4)))))",
+        Assertions.assertEquals("AND[(b = 2),a IN (1, 2),c IN (3, 4),OR[AND[(a = 1),(c = 3)],AND[(a = 2),(c = 4)]]]",
                 rewritten.toSql());
     }
 
@@ -183,7 +184,7 @@ void test14() {
         String expr = "(a=1 and f(a)=2) or a=3";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("(((a = 1) AND (f(a) = 2)) OR (a = 3))",
+        Assertions.assertEquals("OR[AND[(a = 1),(f(a) = 2)],(a = 3)]",
                 rewritten.toSql());
     }
 
@@ -193,7 +194,7 @@ void test15() {
         String expr = "x=1 or (a=1 and b=2) or (a=2 and c=3)";
         Expression expression = PARSER.parseExpression(expr);
         Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("(((x = 1) OR ((a = 1) AND (b = 2))) OR ((a = 2) AND (c = 3)))",
+        Assertions.assertEquals("OR[(x = 1),AND[(a = 1),(b = 2)],AND[(a = 2),(c = 3)]]",
                 rewritten.toSql());
     }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
index 494183d196c1ad..613965b1238e27 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
@@ -51,7 +51,7 @@ void testInferNotNullFromFilterAndEliminateOuter2() {
                     innerLogicalJoin(
                         logicalOlapScan(),
                         logicalFilter().when(
-                                f -> f.getPredicate().toString().equals("((id#0 = 4) OR (id#0 > 4))"))
+                                f -> f.getPredicate().toString().equals("OR[(id#0 = 4),(id#0 > 4)]"))
                     )
 
                 );
@@ -69,11 +69,11 @@ void testInferNotNullFromFilterAndEliminateOuter3() {
                         logicalFilter(
                             leftOuterLogicalJoin(
                                 logicalFilter().when(
-                                        f -> f.getPredicate().toString().equals("((id#0 = 4) OR (id#0 > 4))")),
+                                        f -> f.getPredicate().toString().equals("OR[(id#0 = 4),(id#0 > 4)]")),
                                 logicalOlapScan()
                             )
                         ).when(f -> f.getPredicate().toString()
-                                .equals("((id#0 = 4) OR ((id#0 > 4) AND score#3 IS NULL))"))
+                                .equals("OR[(id#0 = 4),AND[(id#0 > 4),score#3 IS NULL]]"))
                 );
     }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
index 9b0fdc3880dd47..2009a98fbc1c8a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
@@ -1214,7 +1214,7 @@ void testMultiAndWithNull() {
 
         FilterEstimation filterEstimation = new FilterEstimation();
         Statistics result = filterEstimation.estimate(allAnd, stats);
-        Assertions.assertEquals(result.getRowCount(), 2109.16, 0.01);
+        Assertions.assertEquals(result.getRowCount(), 2109.0, 10);
     }
 
     /**
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java
index a2909f85a76129..26a1b7cabd3eae 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java
@@ -45,7 +45,7 @@ void testFirstPartWithoutLowerBound() throws AnalysisException {
         RangePartitionItem item1 = new RangePartitionItem(range1);
 
         Set<Expression> predicates = UpdateMvByPartitionCommand.constructPredicates(Sets.newHashSet(item1), "s");
-        Assertions.assertEquals("((s < 1) OR s IS NULL)", predicates.iterator().next().toSql());
+        Assertions.assertEquals("OR[(s < 1),s IS NULL]", predicates.iterator().next().toSql());
 
     }
 
@@ -82,6 +82,6 @@ void testNull() throws AnalysisException {
         listPartitionItem = new ListPartitionItem(ImmutableList.of(v1, v2));
         expr = UpdateMvByPartitionCommand.constructPredicates(Sets.newHashSet(listPartitionItem), "s").iterator()
                 .next();
-        Assertions.assertEquals("(s IS NULL OR s IN (1))", expr.toSql());
+        Assertions.assertEquals("OR[s IS NULL,s IN (1)]", expr.toSql());
     }
 }
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query10.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query10.out
index bdf370f6b1793f..f12195a12a34f7 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query10.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
 ------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF5 ss_customer_sk->[c_customer_sk]
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query13.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query13.out
index 1c8d917a8612c1..5a2ada8884b3a1 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query13.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query13.out
@@ -7,16 +7,16 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('D', 'W')) AND cd_education_status IN ('2 yr Degree', 'Primary')) AND ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
-------------------filter(((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary'))) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
+------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IL', 'TN', 'TX') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('ID', 'OH', 'WY') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('IA', 'MS', 'SC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query15.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query15.out
index 212fd68ca76555..46aab2d9f4aa45 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query15.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query15.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
+----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
 ----------------------PhysicalProject
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query26.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query26.out
index 6687b771fe4cf2..ce6e2a80111885 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query26.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query26.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2002))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query28.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query28.out
index 2fdcc86b103f71..36ec7305d96abb 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query28.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 107.00) AND (store_sales.ss_list_price <= 117.00)) OR ((store_sales.ss_coupon_amt >= 1319.00) AND (store_sales.ss_coupon_amt <= 2319.00))) OR ((store_sales.ss_wholesale_cost >= 60.00) AND (store_sales.ss_wholesale_cost <= 80.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 107.00),(store_sales.ss_list_price <= 117.00)],AND[(store_sales.ss_coupon_amt >= 1319.00),(store_sales.ss_coupon_amt <= 2319.00)],AND[(store_sales.ss_wholesale_cost >= 60.00),(store_sales.ss_wholesale_cost <= 80.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 23.00) AND (store_sales.ss_list_price <= 33.00)) OR ((store_sales.ss_coupon_amt >= 825.00) AND (store_sales.ss_coupon_amt <= 1825.00))) OR ((store_sales.ss_wholesale_cost >= 43.00) AND (store_sales.ss_wholesale_cost <= 63.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 23.00),(store_sales.ss_list_price <= 33.00)],AND[(store_sales.ss_coupon_amt >= 825.00),(store_sales.ss_coupon_amt <= 1825.00)],AND[(store_sales.ss_wholesale_cost >= 43.00),(store_sales.ss_wholesale_cost <= 63.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 74.00) AND (store_sales.ss_list_price <= 84.00)) OR ((store_sales.ss_coupon_amt >= 4381.00) AND (store_sales.ss_coupon_amt <= 5381.00))) OR ((store_sales.ss_wholesale_cost >= 57.00) AND (store_sales.ss_wholesale_cost <= 77.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 74.00),(store_sales.ss_list_price <= 84.00)],AND[(store_sales.ss_coupon_amt >= 4381.00),(store_sales.ss_coupon_amt <= 5381.00)],AND[(store_sales.ss_wholesale_cost >= 57.00),(store_sales.ss_wholesale_cost <= 77.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 3117.00) AND (store_sales.ss_coupon_amt <= 4117.00))) OR ((store_sales.ss_wholesale_cost >= 68.00) AND (store_sales.ss_wholesale_cost <= 88.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 3117.00),(store_sales.ss_coupon_amt <= 4117.00)],AND[(store_sales.ss_wholesale_cost >= 68.00),(store_sales.ss_wholesale_cost <= 88.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 58.00) AND (store_sales.ss_list_price <= 68.00)) OR ((store_sales.ss_coupon_amt >= 9402.00) AND (store_sales.ss_coupon_amt <= 10402.00))) OR ((store_sales.ss_wholesale_cost >= 38.00) AND (store_sales.ss_wholesale_cost <= 58.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 58.00),(store_sales.ss_list_price <= 68.00)],AND[(store_sales.ss_coupon_amt >= 9402.00),(store_sales.ss_coupon_amt <= 10402.00)],AND[(store_sales.ss_wholesale_cost >= 38.00),(store_sales.ss_wholesale_cost <= 58.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 64.00) AND (store_sales.ss_list_price <= 74.00)) OR ((store_sales.ss_coupon_amt >= 5792.00) AND (store_sales.ss_coupon_amt <= 6792.00))) OR ((store_sales.ss_wholesale_cost >= 73.00) AND (store_sales.ss_wholesale_cost <= 93.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 64.00),(store_sales.ss_list_price <= 74.00)],AND[(store_sales.ss_coupon_amt >= 5792.00),(store_sales.ss_coupon_amt <= 6792.00)],AND[(store_sales.ss_wholesale_cost >= 73.00),(store_sales.ss_wholesale_cost <= 93.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query34.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query34.out
index 4b86edab02cf41..12e90cc7312c45 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query34.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query34.out
@@ -25,7 +25,7 @@ PhysicalResultSink
 ------------------------------------filter((store.s_county = 'Williamson County'))
 --------------------------------------PhysicalOlapScan[store]
 ------------------------------PhysicalProject
---------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (2000, 2001, 2002))
+--------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (2000, 2001, 2002))
 ----------------------------------PhysicalOlapScan[date_dim]
 --------------------------PhysicalProject
 ----------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query41.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query41.out
index ea5acccd883550..0bba60d4cdac39 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query41.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke') AND i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound')) AND (((((((item.i_category = 'Women') AND i_color IN ('forest', 'lime')) AND i_units IN ('Pallet', 'Pound')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('navy', 'slate')) AND i_units IN ('Bunch', 'Gross')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('powder', 'sky')) AND i_units IN ('Dozen', 'Lb')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('maroon', 'smoke')) AND i_units IN ('Case', 'Ounce')) AND i_size IN ('economy', 'small')))) OR ((i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna') AND i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'dark')) AND i_units IN ('Tbl', 'Ton')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('frosted', 'plum')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('papaya', 'peach')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('firebrick', 'sienna')) AND i_units IN ('Cup', 'Each')) AND i_size IN ('economy', 'small'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
+------------------------------filter(OR[AND[i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke'),i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound'),OR[AND[(item.i_category = 'Women'),i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce'),i_size IN ('economy', 'small')]]],AND[i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna'),i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each'),i_size IN ('economy', 'small')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query45.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query45.out
index b65fa9047c0c87..dcc2e202e23752 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query45.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ws_bill_customer_sk]
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query46.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query46.out
index 2f09cd1407d04e..441387467eaf87 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query46.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query46.out
@@ -29,7 +29,7 @@ PhysicalResultSink
 ----------------------------------filter(d_dow IN (0, 6) and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query47.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query47.out
index 5554797a7b9dc3..b1694f8470571f 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query47.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2000) OR ((date_dim.d_year = 1999) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2001) AND (date_dim.d_moy = 1))) and d_year IN (1999, 2000, 2001))
+--------------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and d_year IN (1999, 2000, 2001))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query48.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query48.out
index 06ecefed7e0624..6b34ba86efb1b0 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query48.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query48.out
@@ -9,14 +9,14 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('ND', 'NY', 'SD') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('GA', 'KS', 'MD') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('CO', 'MN', 'NC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'Secondary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Advanced Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'Secondary')) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '2 yr Degree'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD'))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query49.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query49.out
index 889d5069ff9751..8b807baf5409e4 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query49.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query53.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query53.out
index 5e987921a5d3f7..cdbe5ee8705ae0 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query53.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query57.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query57.out
index 91234f18164852..f1d71f63e53c41 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query57.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+--------------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query61.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query61.out
index 4e0a1573b53eb0..4f8ab146d22a36 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query61.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query61.out
@@ -34,7 +34,7 @@ PhysicalResultSink
 --------------------------------------filter((item.i_category = 'Home'))
 ----------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+----------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ------------------------------------PhysicalOlapScan[promotion]
 ----------------------------PhysicalProject
 ------------------------------filter((store.s_gmt_offset = -7.00))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query63.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query63.out
index 89a666497689a1..f9d54e03f75154 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query63.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query68.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query68.out
index 084a62e24b6954..9e66e56512ebc7 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query68.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query68.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Fairview', 'Midway'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_vehicle_count = 4)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
 Hint log:
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query7.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query7.out
index 1ccce08635ab65..89f1c0f0b3a615 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query7.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query78.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query78.out
index 52ab844c3f1bbc..e57834b15ff42a 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query78.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query79.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query79.out
index a03517f058c033..62c0b640f35218 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query79.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query79.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter((date_dim.d_dow = 1) and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 7) OR (household_demographics.hd_vehicle_count > -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 7),(household_demographics.hd_vehicle_count > -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query85.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query85.out
index aca2a36b6a8e3a..2826be96b9c8c8 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query85.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query85.out
@@ -20,12 +20,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('IA', 'NC', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('GA', 'WI', 'WV') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('KY', 'OK', 'VA') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV'))
 ------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'D') AND (cd1.cd_education_status = 'Primary')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'College')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'U') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
@@ -39,7 +39,7 @@ PhysicalResultSink
 ----------------------------------------------filter((date_dim.d_year = 1998))
 ------------------------------------------------PhysicalOlapScan[date_dim]
 ------------------------------------PhysicalProject
---------------------------------------filter(((((cd1.cd_marital_status = 'D') AND (cd1.cd_education_status = 'Primary')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'College'))) OR ((cd1.cd_marital_status = 'U') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U'))
+--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U'))
 ----------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[reason]
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out
index 49e27f92b64a61..8f366f3d8c0202 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 --------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 ----------------------------------------PhysicalOlapScan[time_dim]
 --------------------------------PhysicalProject
-----------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((store.s_store_name = 'ese'))
@@ -45,7 +45,7 @@ PhysicalResultSink
 --------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 ----------------------------------------PhysicalOlapScan[time_dim]
 --------------------------------PhysicalProject
-----------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((store.s_store_name = 'ese'))
@@ -66,7 +66,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -87,7 +87,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -108,7 +108,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -129,7 +129,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -150,7 +150,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -171,7 +171,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+----------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query89.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query89.out
index 91ff8765933815..e3f71a702a8ae1 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query89.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('audio', 'history', 'school-uniforms')) OR (i_category IN ('Men', 'Shoes', 'Sports') AND i_class IN ('pants', 'tennis', 'womens'))) and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens'))
+------------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('audio', 'history', 'school-uniforms')],AND[i_category IN ('Men', 'Shoes', 'Sports'),i_class IN ('pants', 'tennis', 'womens')]] and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 2001))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query91.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query91.out
index 95b8e947317d70..3c8761017d349b 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query91.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query91.out
@@ -28,7 +28,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 --------------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------------------PhysicalProject
 --------------------------------------filter((hd_buy_potential like 'Unknown%'))
diff --git a/regression-test/data/nereids_hint_tpch_p0/shape/q19.out b/regression-test/data/nereids_hint_tpch_p0/shape/q19.out
index 940b16c90e40d5..5cabdf9f163c56 100644
--- a/regression-test/data/nereids_hint_tpch_p0/shape/q19.out
+++ b/regression-test/data/nereids_hint_tpch_p0/shape/q19.out
@@ -5,12 +5,12 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15))))
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]])
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem]
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
 Hint log:
diff --git a/regression-test/data/nereids_hint_tpch_p0/shape/q7.out b/regression-test/data/nereids_hint_tpch_p0/shape/q7.out
index 9cffd25181dbe0..d9fb0e0791a203 100644
--- a/regression-test/data/nereids_hint_tpch_p0/shape/q7.out
+++ b/regression-test/data/nereids_hint_tpch_p0/shape/q7.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=((((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE'))))
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=(OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]])
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=()
 ----------------------PhysicalProject
diff --git a/regression-test/data/nereids_p0/huge_compound/huge_compound.out b/regression-test/data/nereids_p0/huge_compound/huge_compound.out
new file mode 100644
index 00000000000000..e0843bea1df999
--- /dev/null
+++ b/regression-test/data/nereids_p0/huge_compound/huge_compound.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !1 --
+72	93
+
diff --git a/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out b/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out
index 8f1bab13986b7d..877492a2eed6ad 100644
--- a/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out
+++ b/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out
@@ -37,7 +37,7 @@ PhysicalResultSink
 
 -- !eliminate_not_null_complex_logic --
 PhysicalResultSink
---filter(( not score IS NULL) and ((t.score > 5) OR (t.id < 10)))
+--filter(( not score IS NULL) and OR[(t.score > 5),(t.id < 10)])
 ----PhysicalOlapScan[t]
 
 -- !eliminate_not_null_date_function --
diff --git a/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out b/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out
index d92655e4e73e0b..48b3db99631679 100644
--- a/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out
+++ b/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out
@@ -128,7 +128,7 @@ PhysicalResultSink
 -- !full_outer_join_compound_conditions --
 PhysicalResultSink
 --PhysicalDistribute[DistributionSpecGather]
-----filter(((t1.score > 5) OR (t2.score > 5)))
+----filter(OR[(t1.score > 5),(t2.score > 5)])
 ------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[t]
 --------PhysicalOlapScan[t]
diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_down_expression_in_hash_join.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_down_expression_in_hash_join.out
index ec63cd6aabdc4f..b4bddac5a5e077 100644
--- a/regression-test/data/nereids_rules_p0/filter_push_down/push_down_expression_in_hash_join.out
+++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_down_expression_in_hash_join.out
@@ -70,7 +70,7 @@ PhysicalResultSink
 
 -- !pushdown_left_outer_join_subquery --
 PhysicalResultSink
---NestedLoopJoin[INNER_JOIN]((cast(id as BIGINT) = (sum(id) - 1)) OR id IS NULL)
+--NestedLoopJoin[INNER_JOIN]OR[(cast(id as BIGINT) = (sum(id) - 1)),id IS NULL]
 ----PhysicalOlapScan[t1]
 ----hashAgg[GLOBAL]
 ------hashAgg[LOCAL]
diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out
index afa4e4cb6ae315..7bc1aded5877cc 100644
--- a/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out
+++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out
@@ -190,7 +190,7 @@ PhysicalResultSink
 
 -- !pushdown_left_outer_join_subquery --
 PhysicalResultSink
---filter(((cast(id as BIGINT) = sum(id)) OR id IS NULL))
+--filter(OR[(cast(id as BIGINT) = sum(id)),id IS NULL])
 ----NestedLoopJoin[LEFT_OUTER_JOIN](id = 1)
 ------PhysicalOlapScan[t1]
 ------hashAgg[GLOBAL]
@@ -199,7 +199,7 @@ PhysicalResultSink
 
 -- !pushdown_left_anti_join_subquery --
 PhysicalResultSink
---NestedLoopJoin[LEFT_ANTI_JOIN](((t1.id = t2.id) OR id IS NULL) OR id IS NULL)(id > 1)
+--NestedLoopJoin[LEFT_ANTI_JOIN]OR[(t1.id = t2.id),id IS NULL,id IS NULL](id > 1)
 ----PhysicalOlapScan[t1]
 ----PhysicalOlapScan[t2]
 
@@ -229,7 +229,7 @@ PhysicalResultSink
 
 -- !pushdown_left_outer_join_subquery_outer --
 PhysicalResultSink
---NestedLoopJoin[INNER_JOIN]((t1.id = t2.id) OR (id IS NULL AND (id > 1)))
+--NestedLoopJoin[INNER_JOIN]OR[(t1.id = t2.id),AND[id IS NULL,(id > 1)]]
 ----PhysicalOlapScan[t1]
 ----PhysicalAssertNumRows
 ------PhysicalOlapScan[t2]
diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
index 906d59fa9016dc..a9f9442ee62559 100644
--- a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
+++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
@@ -59,7 +59,7 @@ PhysicalResultSink
 
 -- !filter_join_inner --
 PhysicalResultSink
---hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=(((t1.id = 1) OR (t2.id = 2)))
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=(OR[(t1.id = 1),(t2.id = 2)])
 ----PhysicalOlapScan[t1]
 ----PhysicalOlapScan[t2]
 
@@ -77,7 +77,7 @@ PhysicalResultSink
 
 -- !filter_join_left --
 PhysicalResultSink
---filter(((t1.id = 1) OR (t2.id = 2)))
+--filter(OR[(t1.id = 1),(t2.id = 2)])
 ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 ------PhysicalOlapScan[t1]
 ------PhysicalOlapScan[t2]
@@ -104,7 +104,7 @@ PhysicalResultSink
 
 -- !filter_join_left --
 PhysicalResultSink
---filter(((t1.id = 1) OR (t2.id = 2)))
+--filter(OR[(t1.id = 1),(t2.id = 2)])
 ----hashJoin[FULL_OUTER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 ------PhysicalOlapScan[t1]
 ------PhysicalOlapScan[t2]
@@ -169,7 +169,7 @@ PhysicalResultSink
 
 -- !filter_mixed_inner_left --
 PhysicalResultSink
---filter((((t1.id = 1) AND (t2.id = 2)) OR (t3.id = 2)))
+--filter(OR[AND[(t1.id = 1),(t2.id = 2)],(t3.id = 2)])
 ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.id = t3.id)) otherCondition=()
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[t1]
@@ -178,7 +178,7 @@ PhysicalResultSink
 
 -- !filter_multi_left --
 PhysicalResultSink
---filter((((t1.id = 1) AND (t2.id > 1)) OR (t3.id < 4)))
+--filter(OR[AND[(t1.id = 1),(t2.id > 1)],(t3.id < 4)])
 ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.id = t3.id)) otherCondition=()
 ------hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[t1]
@@ -236,7 +236,7 @@ PhysicalResultSink
 
 -- !filter_aggregation_group_set --
 PhysicalResultSink
---filter(((t1.id > 10) OR (cast(msg as DOUBLE) = 1.0)))
+--filter(OR[(t1.id > 10),(cast(msg as DOUBLE) = 1.0)])
 ----hashAgg[LOCAL]
 ------PhysicalRepeat
 --------PhysicalOlapScan[t1]
@@ -372,6 +372,6 @@ PhysicalResultSink
 ----PhysicalQuickSort[LOCAL_SORT]
 ------PhysicalWindow
 --------PhysicalQuickSort[LOCAL_SORT]
-----------filter(((t1.msg = '') OR (t1.id = 2)))
+----------filter(OR[(t1.msg = ''),(t1.id = 2)])
 ------------PhysicalOlapScan[t1]
 
diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/extend_infer_equal_predicate.out b/regression-test/data/nereids_rules_p0/infer_predicate/extend_infer_equal_predicate.out
index 79f3ad0b7939ea..ed43d254b5063f 100644
--- a/regression-test/data/nereids_rules_p0/infer_predicate/extend_infer_equal_predicate.out
+++ b/regression-test/data/nereids_rules_p0/infer_predicate/extend_infer_equal_predicate.out
@@ -112,14 +112,14 @@ PhysicalResultSink
 -- !test_or1 --
 PhysicalResultSink
 --hashJoin[INNER_JOIN] hashCondition=((t1.a = t2.a)) otherCondition=()
-----filter(((t1.a < 2) OR (t1.a > 10)))
+----filter(OR[(t1.a < 2),(t1.a > 10)])
 ------PhysicalOlapScan[extend_infer_t3]
 ----PhysicalOlapScan[extend_infer_t4]
 
 -- !test_or2 --
 PhysicalResultSink
 --hashJoin[INNER_JOIN] hashCondition=((t1.a = t2.a)) otherCondition=()
-----filter(((t1.a < 2) OR (t1.a > 10)))
+----filter(OR[(t1.a < 2),(t1.a > 10)])
 ------PhysicalOlapScan[extend_infer_t3]
 ----PhysicalOlapScan[extend_infer_t4]
 
diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.out b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.out
index 3e7889f460b585..db72758ec618fd 100644
--- a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.out
+++ b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.out
@@ -202,7 +202,7 @@ PhysicalResultSink
 ----hashAgg[GLOBAL]
 ------hashAgg[LOCAL]
 --------PhysicalUnion
-----------filter((a IN (1, 2) OR b IN ('2d', '3')))
+----------filter(OR[a IN (1, 2),b IN ('2d', '3')])
 ------------PhysicalOlapScan[test_pull_up_predicate_set_op1]
 ----PhysicalOlapScan[test_pull_up_predicate_set_op3]
 
diff --git a/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out b/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out
index 14817af2ee3200..288c30bb28c1cf 100644
--- a/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out
+++ b/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out
@@ -30,7 +30,7 @@ PhysicalResultSink
 
 -- !infer_predicate_full_outer_join --
 PhysicalResultSink
---filter(((t1.name = 'Test') OR (t2.name = 'Test')))
+--filter(OR[(t1.name = 'Test'),(t2.name = 'Test')])
 ----hashJoin[FULL_OUTER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 ------PhysicalOlapScan[t]
 ------PhysicalOlapScan[t]
@@ -128,8 +128,8 @@ PhysicalResultSink
 
 -- !infer_predicate_complex_and_or_logic --
 PhysicalResultSink
---hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=(((t1.score > 80) OR ((t2.name = 'Dave') AND (t1.id < 50))))
-----filter(((t1.score > 80) OR (t1.id < 50)))
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=(OR[(t1.score > 80),AND[(t2.name = 'Dave'),(t1.id < 50)]])
+----filter(OR[(t1.score > 80),(t1.id < 50)])
 ------PhysicalOlapScan[t]
 ----PhysicalOlapScan[t]
 
diff --git a/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out b/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out
index 898621c7da765c..1c9c25ba337c78 100644
--- a/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out
+++ b/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out
@@ -1,7 +1,7 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !left_semi --
 PhysicalResultSink
---hashJoin[LEFT_SEMI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))))
+--hashJoin[LEFT_SEMI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]])
 ----filter(a IN (1, 2))
 ------PhysicalOlapScan[extract_from_disjunction_in_join_t1]
 ----filter(a IN (8, 9))
@@ -9,7 +9,7 @@ PhysicalResultSink
 
 -- !right_semi --
 PhysicalResultSink
---hashJoin[RIGHT_SEMI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))))
+--hashJoin[RIGHT_SEMI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]])
 ----filter(a IN (1, 2))
 ------PhysicalOlapScan[extract_from_disjunction_in_join_t1]
 ----filter(a IN (8, 9))
@@ -17,35 +17,35 @@ PhysicalResultSink
 
 -- !left --
 PhysicalResultSink
---hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))) and a IN (1, 2))
+--hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (1, 2))
 ----PhysicalOlapScan[extract_from_disjunction_in_join_t1]
 ----filter(a IN (8, 9))
 ------PhysicalOlapScan[extract_from_disjunction_in_join_t2]
 
 -- !right --
 PhysicalResultSink
---hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))) and a IN (8, 9))
+--hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (8, 9))
 ----filter(a IN (1, 2))
 ------PhysicalOlapScan[extract_from_disjunction_in_join_t1]
 ----PhysicalOlapScan[extract_from_disjunction_in_join_t2]
 
 -- !left_anti --
 PhysicalResultSink
---hashJoin[LEFT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))) and a IN (1, 2))
+--hashJoin[LEFT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (1, 2))
 ----PhysicalOlapScan[extract_from_disjunction_in_join_t1]
 ----filter(a IN (8, 9))
 ------PhysicalOlapScan[extract_from_disjunction_in_join_t2]
 
 -- !right_anti --
 PhysicalResultSink
---hashJoin[RIGHT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))) and a IN (8, 9))
+--hashJoin[RIGHT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (8, 9))
 ----filter(a IN (1, 2))
 ------PhysicalOlapScan[extract_from_disjunction_in_join_t1]
 ----PhysicalOlapScan[extract_from_disjunction_in_join_t2]
 
 -- !inner --
 PhysicalResultSink
---hashJoin[INNER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))))
+--hashJoin[INNER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]])
 ----filter(a IN (1, 2))
 ------PhysicalOlapScan[extract_from_disjunction_in_join_t1]
 ----filter(a IN (8, 9))
@@ -53,7 +53,7 @@ PhysicalResultSink
 
 -- !outer --
 PhysicalResultSink
---hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))) and a IN (1, 2))
+--hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (1, 2))
 ----filter((t1.c = 3))
 ------PhysicalOlapScan[extract_from_disjunction_in_join_t1]
 ----filter(a IN (8, 9))
diff --git a/regression-test/data/nereids_rules_p0/push_down_filter/push_down_filter_through_window.out b/regression-test/data/nereids_rules_p0/push_down_filter/push_down_filter_through_window.out
index 9ecb96e1fc0b74..a200972a0fa5a3 100644
--- a/regression-test/data/nereids_rules_p0/push_down_filter/push_down_filter_through_window.out
+++ b/regression-test/data/nereids_rules_p0/push_down_filter/push_down_filter_through_window.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ------PhysicalQuickSort[LOCAL_SORT]
 --------PhysicalPartitionTopN
 ----------PhysicalProject
-------------filter(((t.id > 1) OR (t.value1 > 2)))
+------------filter(OR[(t.id > 1),(t.value1 > 2)])
 --------------PhysicalOlapScan[push_down_multi_column_predicate_through_window_t]
 
 -- !multi_column_or_predicate_push_down_window --
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.out
index 461f54e0e1767c..8138a8e20ed7e5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.out
@@ -7,21 +7,21 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IL', 'TN', 'TX') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('ID', 'OH', 'WY') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('IA', 'MS', 'SC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF3 ss_addr_sk->[ca_address_sk]
+--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY'))
 --------------------PhysicalOlapScan[customer_address] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('D', 'W')) AND cd_education_status IN ('2 yr Degree', 'Primary')) AND ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
 ----------------------------PhysicalProject
-------------------------------filter(((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary'))) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
+------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------PhysicalProject
 --------------------------filter(hd_dep_count IN (1, 3))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.out
index b65fa9047c0c87..dcc2e202e23752 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ws_bill_customer_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.out
index 7453f3ef1a7080..e768a09ec1494d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ------------------------------------filter((item.i_category = 'Home'))
 --------------------------------------PhysicalOlapScan[item]
 ------------------------------PhysicalProject
---------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+--------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ----------------------------------PhysicalOlapScan[promotion]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_gmt_offset = -7.00))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.out
index f4503a600d6272..2f4fbe401f1315 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.out
@@ -33,6 +33,6 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Fairview', 'Midway'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_vehicle_count = 4)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.out
index 624685e9dab961..9d3c77acb23ca8 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.out
@@ -28,7 +28,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 --------------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------------------PhysicalProject
 --------------------------------------filter((hd_buy_potential like 'Unknown%'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/eliminate_empty/query10_empty.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/eliminate_empty/query10_empty.out
index f68d46db52fbb8..78fd7c847c29ed 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/eliminate_empty/query10_empty.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/eliminate_empty/query10_empty.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query10.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query10.out
index f68d46db52fbb8..78fd7c847c29ed 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out
index 461f54e0e1767c..8138a8e20ed7e5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out
@@ -7,21 +7,21 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IL', 'TN', 'TX') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('ID', 'OH', 'WY') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('IA', 'MS', 'SC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF3 ss_addr_sk->[ca_address_sk]
+--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY'))
 --------------------PhysicalOlapScan[customer_address] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('D', 'W')) AND cd_education_status IN ('2 yr Degree', 'Primary')) AND ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
 ----------------------------PhysicalProject
-------------------------------filter(((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary'))) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
+------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------PhysicalProject
 --------------------------filter(hd_dep_count IN (1, 3))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query15.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query15.out
index 2de8417f7ab831..06c1b08293ef85 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query15.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query15.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
+----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
 ----------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query26.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query26.out
index dc5aee071856c1..383242890f9dd4 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query26.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query26.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2002))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query28.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query28.out
index 2fdcc86b103f71..36ec7305d96abb 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query28.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 107.00) AND (store_sales.ss_list_price <= 117.00)) OR ((store_sales.ss_coupon_amt >= 1319.00) AND (store_sales.ss_coupon_amt <= 2319.00))) OR ((store_sales.ss_wholesale_cost >= 60.00) AND (store_sales.ss_wholesale_cost <= 80.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 107.00),(store_sales.ss_list_price <= 117.00)],AND[(store_sales.ss_coupon_amt >= 1319.00),(store_sales.ss_coupon_amt <= 2319.00)],AND[(store_sales.ss_wholesale_cost >= 60.00),(store_sales.ss_wholesale_cost <= 80.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 23.00) AND (store_sales.ss_list_price <= 33.00)) OR ((store_sales.ss_coupon_amt >= 825.00) AND (store_sales.ss_coupon_amt <= 1825.00))) OR ((store_sales.ss_wholesale_cost >= 43.00) AND (store_sales.ss_wholesale_cost <= 63.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 23.00),(store_sales.ss_list_price <= 33.00)],AND[(store_sales.ss_coupon_amt >= 825.00),(store_sales.ss_coupon_amt <= 1825.00)],AND[(store_sales.ss_wholesale_cost >= 43.00),(store_sales.ss_wholesale_cost <= 63.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 74.00) AND (store_sales.ss_list_price <= 84.00)) OR ((store_sales.ss_coupon_amt >= 4381.00) AND (store_sales.ss_coupon_amt <= 5381.00))) OR ((store_sales.ss_wholesale_cost >= 57.00) AND (store_sales.ss_wholesale_cost <= 77.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 74.00),(store_sales.ss_list_price <= 84.00)],AND[(store_sales.ss_coupon_amt >= 4381.00),(store_sales.ss_coupon_amt <= 5381.00)],AND[(store_sales.ss_wholesale_cost >= 57.00),(store_sales.ss_wholesale_cost <= 77.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 3117.00) AND (store_sales.ss_coupon_amt <= 4117.00))) OR ((store_sales.ss_wholesale_cost >= 68.00) AND (store_sales.ss_wholesale_cost <= 88.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 3117.00),(store_sales.ss_coupon_amt <= 4117.00)],AND[(store_sales.ss_wholesale_cost >= 68.00),(store_sales.ss_wholesale_cost <= 88.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 58.00) AND (store_sales.ss_list_price <= 68.00)) OR ((store_sales.ss_coupon_amt >= 9402.00) AND (store_sales.ss_coupon_amt <= 10402.00))) OR ((store_sales.ss_wholesale_cost >= 38.00) AND (store_sales.ss_wholesale_cost <= 58.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 58.00),(store_sales.ss_list_price <= 68.00)],AND[(store_sales.ss_coupon_amt >= 9402.00),(store_sales.ss_coupon_amt <= 10402.00)],AND[(store_sales.ss_wholesale_cost >= 38.00),(store_sales.ss_wholesale_cost <= 58.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 64.00) AND (store_sales.ss_list_price <= 74.00)) OR ((store_sales.ss_coupon_amt >= 5792.00) AND (store_sales.ss_coupon_amt <= 6792.00))) OR ((store_sales.ss_wholesale_cost >= 73.00) AND (store_sales.ss_wholesale_cost <= 93.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 64.00),(store_sales.ss_list_price <= 74.00)],AND[(store_sales.ss_coupon_amt >= 5792.00),(store_sales.ss_coupon_amt <= 6792.00)],AND[(store_sales.ss_wholesale_cost >= 73.00),(store_sales.ss_wholesale_cost <= 93.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query34.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query34.out
index 135b461a144762..ad28c2d46104d0 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query34.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------------filter((store.s_county = 'Williamson County'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (2000, 2001, 2002))
+------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (2000, 2001, 2002))
 --------------------------------PhysicalOlapScan[date_dim]
 ------------------------PhysicalProject
 --------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out
index a69bca459222b5..2f4be8c2912555 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query41.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query41.out
index ea5acccd883550..0bba60d4cdac39 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke') AND i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound')) AND (((((((item.i_category = 'Women') AND i_color IN ('forest', 'lime')) AND i_units IN ('Pallet', 'Pound')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('navy', 'slate')) AND i_units IN ('Bunch', 'Gross')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('powder', 'sky')) AND i_units IN ('Dozen', 'Lb')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('maroon', 'smoke')) AND i_units IN ('Case', 'Ounce')) AND i_size IN ('economy', 'small')))) OR ((i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna') AND i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'dark')) AND i_units IN ('Tbl', 'Ton')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('frosted', 'plum')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('papaya', 'peach')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('firebrick', 'sienna')) AND i_units IN ('Cup', 'Each')) AND i_size IN ('economy', 'small'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
+------------------------------filter(OR[AND[i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke'),i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound'),OR[AND[(item.i_category = 'Women'),i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce'),i_size IN ('economy', 'small')]]],AND[i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna'),i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each'),i_size IN ('economy', 'small')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query45.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query45.out
index b65fa9047c0c87..dcc2e202e23752 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query45.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ws_bill_customer_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query46.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query46.out
index e84100fe65fac8..cb7d73da5838ed 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query46.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query46.out
@@ -33,6 +33,6 @@ PhysicalResultSink
 ----------------------------------filter(d_dow IN (0, 6) and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out
index d51d48c5ab5677..0e9f713243773a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2000) OR ((date_dim.d_year = 1999) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2001) AND (date_dim.d_moy = 1))) and d_year IN (1999, 2000, 2001))
+--------------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and d_year IN (1999, 2000, 2001))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out
index 3fcfef0d8f68ba..6ed75b1e21eca8 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out
@@ -9,14 +9,14 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('ND', 'NY', 'SD') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('GA', 'KS', 'MD') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('CO', 'MN', 'NC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'Secondary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Advanced Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'Secondary')) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '2 yr Degree'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query49.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query49.out
index 889d5069ff9751..8b807baf5409e4 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query49.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query53.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query53.out
index 07d64a35b071ca..d2467a65e93e09 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query53.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out
index c1b802ebfc4498..2e8174812f69ea 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+--------------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query61.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query61.out
index 7453f3ef1a7080..e768a09ec1494d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query61.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query61.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ------------------------------------filter((item.i_category = 'Home'))
 --------------------------------------PhysicalOlapScan[item]
 ------------------------------PhysicalProject
---------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+--------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ----------------------------------PhysicalOlapScan[promotion]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_gmt_offset = -7.00))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query63.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query63.out
index d1807a29db2f0d..bbbb80bc4b68e0 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query63.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query68.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query68.out
index f4503a600d6272..2f4fbe401f1315 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query68.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query68.out
@@ -33,6 +33,6 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Fairview', 'Midway'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_vehicle_count = 4)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query7.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query7.out
index aadb31a78e012f..2d63af9e61b19e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query7.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query78.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query78.out
index 3b2d980a6ac2c7..0663ee2198a5fe 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query78.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query79.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query79.out
index b8b18b2657766b..e3d8f3af326d91 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query79.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query79.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter((date_dim.d_dow = 1) and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 7) OR (household_demographics.hd_vehicle_count > -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 7),(household_demographics.hd_vehicle_count > -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out
index 1781cd24d5b9f5..c734d7945f3a56 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF5 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('IA', 'NC', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('GA', 'WI', 'WV') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('KY', 'OK', 'VA') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV'))
 ------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'D') AND (cd1.cd_education_status = 'Primary')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'College')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'U') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
@@ -37,7 +37,7 @@ PhysicalResultSink
 ----------------------------------------------filter((date_dim.d_year = 1998))
 ------------------------------------------------PhysicalOlapScan[date_dim]
 ------------------------------------PhysicalProject
---------------------------------------filter(((((cd1.cd_marital_status = 'D') AND (cd1.cd_education_status = 'Primary')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'College'))) OR ((cd1.cd_marital_status = 'U') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U'))
+--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U'))
 ----------------------------------------PhysicalOlapScan[customer_demographics]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[web_page]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out
index 3c8f0335f1f415..d016dca48db832 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+----------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query89.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query89.out
index 786de940dad5d8..b8751687a0ff29 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query89.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('audio', 'history', 'school-uniforms')) OR (i_category IN ('Men', 'Shoes', 'Sports') AND i_class IN ('pants', 'tennis', 'womens'))) and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens'))
+------------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('audio', 'history', 'school-uniforms')],AND[i_category IN ('Men', 'Shoes', 'Sports'),i_class IN ('pants', 'tennis', 'womens')]] and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 2001))
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query91.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query91.out
index 624685e9dab961..9d3c77acb23ca8 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query91.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query91.out
@@ -28,7 +28,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 --------------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------------------PhysicalProject
 --------------------------------------filter((hd_buy_potential like 'Unknown%'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query10.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query10.out
index bc11b07f3a3809..5fb40519b131d5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out
index 6df93fe3046d5c..9c7d9382777bee 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('M', 'S')) AND cd_education_status IN ('4 yr Degree', 'College')) AND ((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree'))) and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter(hd_dep_count IN (1, 3))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query15.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query15.out
index 7b0fd9b10fd861..5825559155b8e7 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query15.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query15.out
@@ -10,7 +10,7 @@ PhysicalResultSink
 --------------PhysicalProject
 ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00)))
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)])
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
 --------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query26.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query26.out
index 3762fabaeead07..52f628f8b600a2 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query26.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query26.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 2001))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query28.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query28.out
index dae7cb1c23a402..7a6bdd8868ef00 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query28.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 131.00) AND (store_sales.ss_list_price <= 141.00)) OR ((store_sales.ss_coupon_amt >= 16798.00) AND (store_sales.ss_coupon_amt <= 17798.00))) OR ((store_sales.ss_wholesale_cost >= 25.00) AND (store_sales.ss_wholesale_cost <= 45.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 131.00),(store_sales.ss_list_price <= 141.00)],AND[(store_sales.ss_coupon_amt >= 16798.00),(store_sales.ss_coupon_amt <= 17798.00)],AND[(store_sales.ss_wholesale_cost >= 25.00),(store_sales.ss_wholesale_cost <= 45.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 145.00) AND (store_sales.ss_list_price <= 155.00)) OR ((store_sales.ss_coupon_amt >= 14792.00) AND (store_sales.ss_coupon_amt <= 15792.00))) OR ((store_sales.ss_wholesale_cost >= 46.00) AND (store_sales.ss_wholesale_cost <= 66.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 145.00),(store_sales.ss_list_price <= 155.00)],AND[(store_sales.ss_coupon_amt >= 14792.00),(store_sales.ss_coupon_amt <= 15792.00)],AND[(store_sales.ss_wholesale_cost >= 46.00),(store_sales.ss_wholesale_cost <= 66.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 150.00) AND (store_sales.ss_list_price <= 160.00)) OR ((store_sales.ss_coupon_amt >= 6600.00) AND (store_sales.ss_coupon_amt <= 7600.00))) OR ((store_sales.ss_wholesale_cost >= 9.00) AND (store_sales.ss_wholesale_cost <= 29.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 150.00),(store_sales.ss_list_price <= 160.00)],AND[(store_sales.ss_coupon_amt >= 6600.00),(store_sales.ss_coupon_amt <= 7600.00)],AND[(store_sales.ss_wholesale_cost >= 9.00),(store_sales.ss_wholesale_cost <= 29.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 91.00) AND (store_sales.ss_list_price <= 101.00)) OR ((store_sales.ss_coupon_amt >= 13493.00) AND (store_sales.ss_coupon_amt <= 14493.00))) OR ((store_sales.ss_wholesale_cost >= 36.00) AND (store_sales.ss_wholesale_cost <= 56.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 91.00),(store_sales.ss_list_price <= 101.00)],AND[(store_sales.ss_coupon_amt >= 13493.00),(store_sales.ss_coupon_amt <= 14493.00)],AND[(store_sales.ss_wholesale_cost >= 36.00),(store_sales.ss_wholesale_cost <= 56.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 0.00) AND (store_sales.ss_list_price <= 10.00)) OR ((store_sales.ss_coupon_amt >= 7629.00) AND (store_sales.ss_coupon_amt <= 8629.00))) OR ((store_sales.ss_wholesale_cost >= 6.00) AND (store_sales.ss_wholesale_cost <= 26.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 0.00),(store_sales.ss_list_price <= 10.00)],AND[(store_sales.ss_coupon_amt >= 7629.00),(store_sales.ss_coupon_amt <= 8629.00)],AND[(store_sales.ss_wholesale_cost >= 6.00),(store_sales.ss_wholesale_cost <= 26.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 15257.00) AND (store_sales.ss_coupon_amt <= 16257.00))) OR ((store_sales.ss_wholesale_cost >= 31.00) AND (store_sales.ss_wholesale_cost <= 51.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 15257.00),(store_sales.ss_coupon_amt <= 16257.00)],AND[(store_sales.ss_wholesale_cost >= 31.00),(store_sales.ss_wholesale_cost <= 51.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query34.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query34.out
index dad99a6b6465dd..6c90e3a48d7cba 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query35.out
index 9f3bb8dab0f9e3..c14ce7550a3069 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query41.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query41.out
index c9c1ffff499741..3034a77fe0897a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query45.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query45.out
index c6b4daafdce0df..7270fe9092a53b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query45.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query46.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query46.out
index be021ce25b68ae..48e2eae68f5c61 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query46.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query46.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out
index 0182bbf46eb817..77dcc9357f98b8 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out
index 338a70100eb21b..57618c21f263d3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------------------PhysicalProject
@@ -18,7 +18,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store]
 --------------------PhysicalProject
-----------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
+----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
 ------------------------PhysicalOlapScan[customer_demographics]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query49.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query49.out
index dc56d7f9e81411..0db3fa841189b2 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query49.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query53.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query53.out
index 93289fca48e4ef..89dc632eb527c4 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query53.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out
index 842b18a4eac9e6..96d8f68090e5de 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query61.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query61.out
index f9dcb3c9460ca4..62da8c9cb21a0f 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query61.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query61.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ------------------------------------filter((store.s_gmt_offset = -7.00))
 --------------------------------------PhysicalOlapScan[store]
 ------------------------------PhysicalProject
---------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+--------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ----------------------------------PhysicalOlapScan[promotion]
 --------------------------PhysicalProject
 ----------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 1999))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query63.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query63.out
index 3bd844ce72c196..9653f6c52199aa 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query63.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query68.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query68.out
index 52f4671d103618..0fd8ddd97ecd2c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query68.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query68.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query7.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query7.out
index ff218ca1256a94..f47da720468166 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query7.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query7.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 2001))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query78.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query78.out
index 30527883a49018..a6034ca86ac5c5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query78.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query79.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query79.out
index 77168450e857f5..8f9b721f08ee59 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query79.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query79.out
@@ -25,7 +25,7 @@ PhysicalResultSink
 ------------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
 --------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((household_demographics.hd_dep_count = 5) OR (household_demographics.hd_vehicle_count > 4)))
+--------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)])
 ----------------------------PhysicalOlapScan[household_demographics]
 ------------PhysicalProject
 --------------PhysicalOlapScan[customer]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out
index 590589c61a42a0..9f12763cec5114 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=()
 ----------------------------------------PhysicalProject
@@ -30,7 +30,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[web_page]
 ------------------------------------PhysicalProject
---------------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
+--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ----------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6
 --------------------------------PhysicalProject
 ----------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query88.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query88.out
index dc879d6daa5a4b..a20acf339431ea 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query89.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query89.out
index c709c4df5fe097..2e9294bce91fd2 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query89.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Electronics', 'Jewelry', 'Shoes') AND i_class IN ('athletic', 'portable', 'semi-precious')) OR (i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'maternity', 'rock'))) and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
+------------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 1999))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query91.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query91.out
index aa211a9f494891..6a4b369a5fe1ed 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query91.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query91.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ------------------------------filter((customer_address.ca_gmt_offset = -6.00))
 --------------------------------PhysicalOlapScan[customer_address]
 ------------------------PhysicalProject
---------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((hd_buy_potential like '1001-5000%'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query10.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query10.out
index 36a2c69f6b9516..4fdff8b37961c5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out
index 7f109c5dece1a7..a16b428cbc5159 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('M', 'S')) AND cd_education_status IN ('4 yr Degree', 'College')) AND ((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree'))) and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter(hd_dep_count IN (1, 3))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query15.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query15.out
index 1fba646d62b40a..81b0bae51498c1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query15.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query15.out
@@ -10,7 +10,7 @@ PhysicalResultSink
 --------------PhysicalProject
 ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF1 ca_address_sk->[c_current_addr_sk]
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF1 ca_address_sk->[c_current_addr_sk]
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF0 c_customer_sk->[cs_bill_customer_sk]
 --------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query26.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query26.out
index 4561d649bd82c2..edbed407b77921 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query26.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query26.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 2001))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query28.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query28.out
index dae7cb1c23a402..7a6bdd8868ef00 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query28.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 131.00) AND (store_sales.ss_list_price <= 141.00)) OR ((store_sales.ss_coupon_amt >= 16798.00) AND (store_sales.ss_coupon_amt <= 17798.00))) OR ((store_sales.ss_wholesale_cost >= 25.00) AND (store_sales.ss_wholesale_cost <= 45.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 131.00),(store_sales.ss_list_price <= 141.00)],AND[(store_sales.ss_coupon_amt >= 16798.00),(store_sales.ss_coupon_amt <= 17798.00)],AND[(store_sales.ss_wholesale_cost >= 25.00),(store_sales.ss_wholesale_cost <= 45.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 145.00) AND (store_sales.ss_list_price <= 155.00)) OR ((store_sales.ss_coupon_amt >= 14792.00) AND (store_sales.ss_coupon_amt <= 15792.00))) OR ((store_sales.ss_wholesale_cost >= 46.00) AND (store_sales.ss_wholesale_cost <= 66.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 145.00),(store_sales.ss_list_price <= 155.00)],AND[(store_sales.ss_coupon_amt >= 14792.00),(store_sales.ss_coupon_amt <= 15792.00)],AND[(store_sales.ss_wholesale_cost >= 46.00),(store_sales.ss_wholesale_cost <= 66.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 150.00) AND (store_sales.ss_list_price <= 160.00)) OR ((store_sales.ss_coupon_amt >= 6600.00) AND (store_sales.ss_coupon_amt <= 7600.00))) OR ((store_sales.ss_wholesale_cost >= 9.00) AND (store_sales.ss_wholesale_cost <= 29.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 150.00),(store_sales.ss_list_price <= 160.00)],AND[(store_sales.ss_coupon_amt >= 6600.00),(store_sales.ss_coupon_amt <= 7600.00)],AND[(store_sales.ss_wholesale_cost >= 9.00),(store_sales.ss_wholesale_cost <= 29.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 91.00) AND (store_sales.ss_list_price <= 101.00)) OR ((store_sales.ss_coupon_amt >= 13493.00) AND (store_sales.ss_coupon_amt <= 14493.00))) OR ((store_sales.ss_wholesale_cost >= 36.00) AND (store_sales.ss_wholesale_cost <= 56.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 91.00),(store_sales.ss_list_price <= 101.00)],AND[(store_sales.ss_coupon_amt >= 13493.00),(store_sales.ss_coupon_amt <= 14493.00)],AND[(store_sales.ss_wholesale_cost >= 36.00),(store_sales.ss_wholesale_cost <= 56.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 0.00) AND (store_sales.ss_list_price <= 10.00)) OR ((store_sales.ss_coupon_amt >= 7629.00) AND (store_sales.ss_coupon_amt <= 8629.00))) OR ((store_sales.ss_wholesale_cost >= 6.00) AND (store_sales.ss_wholesale_cost <= 26.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 0.00),(store_sales.ss_list_price <= 10.00)],AND[(store_sales.ss_coupon_amt >= 7629.00),(store_sales.ss_coupon_amt <= 8629.00)],AND[(store_sales.ss_wholesale_cost >= 6.00),(store_sales.ss_wholesale_cost <= 26.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 15257.00) AND (store_sales.ss_coupon_amt <= 16257.00))) OR ((store_sales.ss_wholesale_cost >= 31.00) AND (store_sales.ss_wholesale_cost <= 51.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 15257.00),(store_sales.ss_coupon_amt <= 16257.00)],AND[(store_sales.ss_wholesale_cost >= 31.00),(store_sales.ss_wholesale_cost <= 51.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query34.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query34.out
index 25429183c8fe4a..9a53073479fb8c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query35.out
index f5f9508f9b6cfb..b414e22cb15150 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query41.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query41.out
index c9c1ffff499741..3034a77fe0897a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query45.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query45.out
index 038ece83d7c044..68d1ef7855a7fd 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query45.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query46.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query46.out
index 71f3746264834f..802c3969afc6e1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query46.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query46.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out
index b6f046eb01fe81..29aa8e6ae22e6c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out
index ed7ed3b4bdf669..fa9006ce24f673 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF0 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
@@ -18,7 +18,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store]
 --------------------PhysicalProject
-----------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
+----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
 ------------------------PhysicalOlapScan[customer_demographics]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query49.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query49.out
index dc56d7f9e81411..0db3fa841189b2 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query49.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query53.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query53.out
index 81e6091b663bf7..04920e65ac6894 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query53.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out
index e640ec24d7b3e3..88777bc1ff548d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query61.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query61.out
index f9dcb3c9460ca4..62da8c9cb21a0f 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query61.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query61.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ------------------------------------filter((store.s_gmt_offset = -7.00))
 --------------------------------------PhysicalOlapScan[store]
 ------------------------------PhysicalProject
---------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+--------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ----------------------------------PhysicalOlapScan[promotion]
 --------------------------PhysicalProject
 ----------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 1999))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query63.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query63.out
index 2cd49bd8c40786..d4fb4990da98b8 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query63.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query68.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query68.out
index 04dab40ade6bb2..20e70268ebb862 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query68.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query68.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query7.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query7.out
index 12c9bd7d3a0b25..2b6615e0b93b84 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query7.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query7.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 2001))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query78.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query78.out
index 30527883a49018..a6034ca86ac5c5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query78.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query79.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query79.out
index 5d935352c491c6..1a19308d991441 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query79.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query79.out
@@ -25,7 +25,7 @@ PhysicalResultSink
 ------------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
 --------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((household_demographics.hd_dep_count = 5) OR (household_demographics.hd_vehicle_count > 4)))
+--------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)])
 ----------------------------PhysicalOlapScan[household_demographics]
 ------------PhysicalProject
 --------------PhysicalOlapScan[customer]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out
index 08e59c1e36507e..424102e34021a1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF2 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------------------PhysicalProject
@@ -30,7 +30,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[web_page]
 ------------------------------------PhysicalProject
---------------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
+--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ----------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6
 --------------------------------PhysicalProject
 ----------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query88.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query88.out
index dc879d6daa5a4b..a20acf339431ea 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query89.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query89.out
index 661b47d0ea980a..e4d2ae3435f174 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query89.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Electronics', 'Jewelry', 'Shoes') AND i_class IN ('athletic', 'portable', 'semi-precious')) OR (i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'maternity', 'rock'))) and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
+------------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 1999))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query91.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query91.out
index c19b43be01a9df..a2e5d4a7660114 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query91.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query91.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ------------------------------filter((customer_address.ca_gmt_offset = -6.00))
 --------------------------------PhysicalOlapScan[customer_address]
 ------------------------PhysicalProject
---------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((hd_buy_potential like '1001-5000%'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query10.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query10.out
index 6027b8b2684b4b..4dfc2de4cf3fe5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out
index e92ddac6d97742..4b72ea24b2c114 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out
@@ -7,16 +7,16 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('M', 'S')) AND cd_education_status IN ('4 yr Degree', 'College')) AND ((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
-------------------filter(((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree'))) and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
+------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query15.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query15.out
index 685f61ffed3387..f5813cde38de7d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query15.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query15.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
+----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
 ----------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query26.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query26.out
index 2ee30b92ac642f..e2b0cdc5351213 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query26.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query26.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query28.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query28.out
index dae7cb1c23a402..7a6bdd8868ef00 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query28.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 131.00) AND (store_sales.ss_list_price <= 141.00)) OR ((store_sales.ss_coupon_amt >= 16798.00) AND (store_sales.ss_coupon_amt <= 17798.00))) OR ((store_sales.ss_wholesale_cost >= 25.00) AND (store_sales.ss_wholesale_cost <= 45.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 131.00),(store_sales.ss_list_price <= 141.00)],AND[(store_sales.ss_coupon_amt >= 16798.00),(store_sales.ss_coupon_amt <= 17798.00)],AND[(store_sales.ss_wholesale_cost >= 25.00),(store_sales.ss_wholesale_cost <= 45.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 145.00) AND (store_sales.ss_list_price <= 155.00)) OR ((store_sales.ss_coupon_amt >= 14792.00) AND (store_sales.ss_coupon_amt <= 15792.00))) OR ((store_sales.ss_wholesale_cost >= 46.00) AND (store_sales.ss_wholesale_cost <= 66.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 145.00),(store_sales.ss_list_price <= 155.00)],AND[(store_sales.ss_coupon_amt >= 14792.00),(store_sales.ss_coupon_amt <= 15792.00)],AND[(store_sales.ss_wholesale_cost >= 46.00),(store_sales.ss_wholesale_cost <= 66.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 150.00) AND (store_sales.ss_list_price <= 160.00)) OR ((store_sales.ss_coupon_amt >= 6600.00) AND (store_sales.ss_coupon_amt <= 7600.00))) OR ((store_sales.ss_wholesale_cost >= 9.00) AND (store_sales.ss_wholesale_cost <= 29.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 150.00),(store_sales.ss_list_price <= 160.00)],AND[(store_sales.ss_coupon_amt >= 6600.00),(store_sales.ss_coupon_amt <= 7600.00)],AND[(store_sales.ss_wholesale_cost >= 9.00),(store_sales.ss_wholesale_cost <= 29.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 91.00) AND (store_sales.ss_list_price <= 101.00)) OR ((store_sales.ss_coupon_amt >= 13493.00) AND (store_sales.ss_coupon_amt <= 14493.00))) OR ((store_sales.ss_wholesale_cost >= 36.00) AND (store_sales.ss_wholesale_cost <= 56.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 91.00),(store_sales.ss_list_price <= 101.00)],AND[(store_sales.ss_coupon_amt >= 13493.00),(store_sales.ss_coupon_amt <= 14493.00)],AND[(store_sales.ss_wholesale_cost >= 36.00),(store_sales.ss_wholesale_cost <= 56.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 0.00) AND (store_sales.ss_list_price <= 10.00)) OR ((store_sales.ss_coupon_amt >= 7629.00) AND (store_sales.ss_coupon_amt <= 8629.00))) OR ((store_sales.ss_wholesale_cost >= 6.00) AND (store_sales.ss_wholesale_cost <= 26.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 0.00),(store_sales.ss_list_price <= 10.00)],AND[(store_sales.ss_coupon_amt >= 7629.00),(store_sales.ss_coupon_amt <= 8629.00)],AND[(store_sales.ss_wholesale_cost >= 6.00),(store_sales.ss_wholesale_cost <= 26.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 15257.00) AND (store_sales.ss_coupon_amt <= 16257.00))) OR ((store_sales.ss_wholesale_cost >= 31.00) AND (store_sales.ss_wholesale_cost <= 51.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 15257.00),(store_sales.ss_coupon_amt <= 16257.00)],AND[(store_sales.ss_wholesale_cost >= 31.00),(store_sales.ss_wholesale_cost <= 51.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query34.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query34.out
index bb286ee190e816..489abcd4c2e33c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query34.out
@@ -21,7 +21,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out
index 49d318314dee3a..dea7b62c38003d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
 ------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF5 ss_customer_sk->[c_customer_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query41.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query41.out
index c9c1ffff499741..3034a77fe0897a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query45.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query45.out
index 6f437edc5cb1a9..baafdf90cdef7a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query45.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ws_bill_customer_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query46.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query46.out
index 23864cbc2a5e14..63a6814934ecb4 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query46.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query46.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query47.out
index e115c8751281e0..048f93392595ec 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query47.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+--------------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out
index 476aa9dd8793af..a0f0e2107fcda4 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out
@@ -9,14 +9,14 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query49.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query49.out
index e34962e0847b0d..7d0f6b1ce22d0f 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query49.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query53.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query53.out
index 93289fca48e4ef..89dc632eb527c4 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query53.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out
index 4a7de98eb56b50..4f23fac89cf958 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+--------------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query61.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query61.out
index d034c5b43231d7..faf30604b86926 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query61.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query61.out
@@ -29,7 +29,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF6 ss_promo_sk->[p_promo_sk]
 ----------------------------------PhysicalProject
-------------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+------------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 --------------------------------------PhysicalOlapScan[promotion] apply RFs: RF6
 ----------------------------------PhysicalProject
 ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF5 s_store_sk->[ss_store_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query63.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query63.out
index 3bd844ce72c196..9653f6c52199aa 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query63.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query68.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query68.out
index 8975dc78ab76e5..aa07d1b2a42d9b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query68.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query68.out
@@ -33,6 +33,6 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query7.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query7.out
index 911daeee06f6a4..ad074d92dcc43c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query7.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query78.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query78.out
index 30527883a49018..a6034ca86ac5c5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query78.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query79.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query79.out
index 0ceac85291e12d..f57418546e7fb9 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query79.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query79.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter((date_dim.d_dow = 1) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 5) OR (household_demographics.hd_vehicle_count > 4)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out
index d08017427d5c94..91ce7e962e317d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
 --------------------------------PhysicalProject
-----------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
+----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk]
+----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk]
 ------------------------------------PhysicalProject
 --------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX'))
 ----------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF3
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query88.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query88.out
index bca8b0d069014b..09c630cef443ef 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query89.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query89.out
index c709c4df5fe097..2e9294bce91fd2 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query89.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Electronics', 'Jewelry', 'Shoes') AND i_class IN ('athletic', 'portable', 'semi-precious')) OR (i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'maternity', 'rock'))) and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
+------------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 1999))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query91.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query91.out
index e2a4ed89af442d..6593bb83f07994 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query91.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query91.out
@@ -28,7 +28,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 --------------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------------------PhysicalProject
 --------------------------------------filter((hd_buy_potential like '1001-5000%'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out
index 6027b8b2684b4b..4dfc2de4cf3fe5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out
index 7528312c6dae6b..e8a4d92c4f7bf1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out
@@ -7,16 +7,16 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('M', 'S')) AND cd_education_status IN ('4 yr Degree', 'College')) AND ((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
-------------------filter(((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree'))) and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
+------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query15.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query15.out
index 685f61ffed3387..f5813cde38de7d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query15.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query15.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
+----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
 ----------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query26.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query26.out
index 2ee30b92ac642f..e2b0cdc5351213 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query26.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query26.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query28.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query28.out
index dae7cb1c23a402..7a6bdd8868ef00 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query28.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 131.00) AND (store_sales.ss_list_price <= 141.00)) OR ((store_sales.ss_coupon_amt >= 16798.00) AND (store_sales.ss_coupon_amt <= 17798.00))) OR ((store_sales.ss_wholesale_cost >= 25.00) AND (store_sales.ss_wholesale_cost <= 45.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 131.00),(store_sales.ss_list_price <= 141.00)],AND[(store_sales.ss_coupon_amt >= 16798.00),(store_sales.ss_coupon_amt <= 17798.00)],AND[(store_sales.ss_wholesale_cost >= 25.00),(store_sales.ss_wholesale_cost <= 45.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 145.00) AND (store_sales.ss_list_price <= 155.00)) OR ((store_sales.ss_coupon_amt >= 14792.00) AND (store_sales.ss_coupon_amt <= 15792.00))) OR ((store_sales.ss_wholesale_cost >= 46.00) AND (store_sales.ss_wholesale_cost <= 66.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 145.00),(store_sales.ss_list_price <= 155.00)],AND[(store_sales.ss_coupon_amt >= 14792.00),(store_sales.ss_coupon_amt <= 15792.00)],AND[(store_sales.ss_wholesale_cost >= 46.00),(store_sales.ss_wholesale_cost <= 66.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 150.00) AND (store_sales.ss_list_price <= 160.00)) OR ((store_sales.ss_coupon_amt >= 6600.00) AND (store_sales.ss_coupon_amt <= 7600.00))) OR ((store_sales.ss_wholesale_cost >= 9.00) AND (store_sales.ss_wholesale_cost <= 29.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 150.00),(store_sales.ss_list_price <= 160.00)],AND[(store_sales.ss_coupon_amt >= 6600.00),(store_sales.ss_coupon_amt <= 7600.00)],AND[(store_sales.ss_wholesale_cost >= 9.00),(store_sales.ss_wholesale_cost <= 29.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 91.00) AND (store_sales.ss_list_price <= 101.00)) OR ((store_sales.ss_coupon_amt >= 13493.00) AND (store_sales.ss_coupon_amt <= 14493.00))) OR ((store_sales.ss_wholesale_cost >= 36.00) AND (store_sales.ss_wholesale_cost <= 56.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 91.00),(store_sales.ss_list_price <= 101.00)],AND[(store_sales.ss_coupon_amt >= 13493.00),(store_sales.ss_coupon_amt <= 14493.00)],AND[(store_sales.ss_wholesale_cost >= 36.00),(store_sales.ss_wholesale_cost <= 56.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 0.00) AND (store_sales.ss_list_price <= 10.00)) OR ((store_sales.ss_coupon_amt >= 7629.00) AND (store_sales.ss_coupon_amt <= 8629.00))) OR ((store_sales.ss_wholesale_cost >= 6.00) AND (store_sales.ss_wholesale_cost <= 26.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 0.00),(store_sales.ss_list_price <= 10.00)],AND[(store_sales.ss_coupon_amt >= 7629.00),(store_sales.ss_coupon_amt <= 8629.00)],AND[(store_sales.ss_wholesale_cost >= 6.00),(store_sales.ss_wholesale_cost <= 26.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 15257.00) AND (store_sales.ss_coupon_amt <= 16257.00))) OR ((store_sales.ss_wholesale_cost >= 31.00) AND (store_sales.ss_wholesale_cost <= 51.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 15257.00),(store_sales.ss_coupon_amt <= 16257.00)],AND[(store_sales.ss_wholesale_cost >= 31.00),(store_sales.ss_wholesale_cost <= 51.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query34.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query34.out
index bb286ee190e816..489abcd4c2e33c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query34.out
@@ -21,7 +21,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out
index dcc86d24029059..9728d7d30707cd 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
 ------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF5 ss_customer_sk->[c_customer_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query41.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query41.out
index c9c1ffff499741..3034a77fe0897a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query45.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query45.out
index 6f437edc5cb1a9..baafdf90cdef7a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query45.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ws_bill_customer_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query46.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query46.out
index 8279b1c53b5c39..136c96ba2856c9 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query46.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query46.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out
index 7a668b06c433e5..ecc4777c22f664 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+--------------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out
index c81868cacca7a6..f8e2288f356010 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out
@@ -9,14 +9,14 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query49.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query49.out
index e34962e0847b0d..7d0f6b1ce22d0f 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query49.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query53.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query53.out
index 81e6091b663bf7..04920e65ac6894 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query53.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out
index a0157318139d88..2cab4f33e1358d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+--------------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query61.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query61.out
index d034c5b43231d7..faf30604b86926 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query61.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query61.out
@@ -29,7 +29,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF6 ss_promo_sk->[p_promo_sk]
 ----------------------------------PhysicalProject
-------------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+------------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 --------------------------------------PhysicalOlapScan[promotion] apply RFs: RF6
 ----------------------------------PhysicalProject
 ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF5 s_store_sk->[ss_store_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query63.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query63.out
index 2cd49bd8c40786..d4fb4990da98b8 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query63.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query68.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query68.out
index 8975dc78ab76e5..aa07d1b2a42d9b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query68.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query68.out
@@ -33,6 +33,6 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query7.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query7.out
index 516105a1c2b73f..e3c9dc2bfd0e67 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query7.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query78.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query78.out
index 30527883a49018..a6034ca86ac5c5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query78.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query79.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query79.out
index 4f8ac0ce4d7dea..974fb1da39b734 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query79.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query79.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter((date_dim.d_dow = 1) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 5) OR (household_demographics.hd_vehicle_count > 4)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out
index 8e543215ca9ce5..6818ffdb89889a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
 --------------------------------PhysicalProject
-----------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
+----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk]
+----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk]
 ------------------------------------PhysicalProject
 --------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX'))
 ----------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF3
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query88.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query88.out
index bca8b0d069014b..09c630cef443ef 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query89.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query89.out
index 661b47d0ea980a..e4d2ae3435f174 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query89.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Electronics', 'Jewelry', 'Shoes') AND i_class IN ('athletic', 'portable', 'semi-precious')) OR (i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'maternity', 'rock'))) and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
+------------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 1999))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query91.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query91.out
index 6f3970cbb29d50..6af6c014d0c298 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query91.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query91.out
@@ -28,7 +28,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 --------------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------------------PhysicalProject
 --------------------------------------filter((hd_buy_potential like '1001-5000%'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query10.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query10.out
index e1cace0be0ccec..43aef07e0be2c0 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query10.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query13.out
index 169daff229f9e5..862b1f5d64b5f6 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('MI', 'OK', 'TX') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('NC', 'OH', 'WA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('FL', 'GA', 'MT') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('MI', 'OK', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('NC', 'OH', 'WA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('FL', 'GA', 'MT'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('D', 'W')) AND cd_education_status IN ('College', 'Primary')) AND ((((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Secondary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('College', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Secondary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary'))) and cd_education_status IN ('College', 'Primary', 'Secondary') and cd_marital_status IN ('D', 'U', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')]] and cd_education_status IN ('College', 'Primary', 'Secondary') and cd_marital_status IN ('D', 'U', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter(hd_dep_count IN (1, 3))
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query15.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query15.out
index 31789e71f5b27a..828af0129d7187 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query15.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query15.out
@@ -10,7 +10,7 @@ PhysicalResultSink
 --------------PhysicalProject
 ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF1 ca_address_sk->[c_current_addr_sk]
+--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF1 ca_address_sk->[c_current_addr_sk]
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF0 c_customer_sk->[cs_bill_customer_sk]
 --------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query26.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query26.out
index 68a8944b8ce179..2b5d23c4aac26d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query26.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query26.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 1998))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query28.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query28.out
index eb43c81cde41c2..fd2a30be06e564 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query28.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 73.00) AND (store_sales.ss_list_price <= 83.00)) OR ((store_sales.ss_coupon_amt >= 7826.00) AND (store_sales.ss_coupon_amt <= 8826.00))) OR ((store_sales.ss_wholesale_cost >= 70.00) AND (store_sales.ss_wholesale_cost <= 90.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 73.00),(store_sales.ss_list_price <= 83.00)],AND[(store_sales.ss_coupon_amt >= 7826.00),(store_sales.ss_coupon_amt <= 8826.00)],AND[(store_sales.ss_wholesale_cost >= 70.00),(store_sales.ss_wholesale_cost <= 90.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 152.00) AND (store_sales.ss_list_price <= 162.00)) OR ((store_sales.ss_coupon_amt >= 2196.00) AND (store_sales.ss_coupon_amt <= 3196.00))) OR ((store_sales.ss_wholesale_cost >= 56.00) AND (store_sales.ss_wholesale_cost <= 76.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 152.00),(store_sales.ss_list_price <= 162.00)],AND[(store_sales.ss_coupon_amt >= 2196.00),(store_sales.ss_coupon_amt <= 3196.00)],AND[(store_sales.ss_wholesale_cost >= 56.00),(store_sales.ss_wholesale_cost <= 76.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 53.00) AND (store_sales.ss_list_price <= 63.00)) OR ((store_sales.ss_coupon_amt >= 3430.00) AND (store_sales.ss_coupon_amt <= 4430.00))) OR ((store_sales.ss_wholesale_cost >= 13.00) AND (store_sales.ss_wholesale_cost <= 33.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 53.00),(store_sales.ss_list_price <= 63.00)],AND[(store_sales.ss_coupon_amt >= 3430.00),(store_sales.ss_coupon_amt <= 4430.00)],AND[(store_sales.ss_wholesale_cost >= 13.00),(store_sales.ss_wholesale_cost <= 33.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 182.00) AND (store_sales.ss_list_price <= 192.00)) OR ((store_sales.ss_coupon_amt >= 3262.00) AND (store_sales.ss_coupon_amt <= 4262.00))) OR ((store_sales.ss_wholesale_cost >= 20.00) AND (store_sales.ss_wholesale_cost <= 40.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 182.00),(store_sales.ss_list_price <= 192.00)],AND[(store_sales.ss_coupon_amt >= 3262.00),(store_sales.ss_coupon_amt <= 4262.00)],AND[(store_sales.ss_wholesale_cost >= 20.00),(store_sales.ss_wholesale_cost <= 40.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 85.00) AND (store_sales.ss_list_price <= 95.00)) OR ((store_sales.ss_coupon_amt >= 3310.00) AND (store_sales.ss_coupon_amt <= 4310.00))) OR ((store_sales.ss_wholesale_cost >= 37.00) AND (store_sales.ss_wholesale_cost <= 57.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 85.00),(store_sales.ss_list_price <= 95.00)],AND[(store_sales.ss_coupon_amt >= 3310.00),(store_sales.ss_coupon_amt <= 4310.00)],AND[(store_sales.ss_wholesale_cost >= 37.00),(store_sales.ss_wholesale_cost <= 57.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 180.00) AND (store_sales.ss_list_price <= 190.00)) OR ((store_sales.ss_coupon_amt >= 12592.00) AND (store_sales.ss_coupon_amt <= 13592.00))) OR ((store_sales.ss_wholesale_cost >= 22.00) AND (store_sales.ss_wholesale_cost <= 42.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 180.00),(store_sales.ss_list_price <= 190.00)],AND[(store_sales.ss_coupon_amt >= 12592.00),(store_sales.ss_coupon_amt <= 13592.00)],AND[(store_sales.ss_wholesale_cost >= 22.00),(store_sales.ss_wholesale_cost <= 42.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query34.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query34.out
index a883135293b760..5aa611041037f9 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (2000, 2001, 2002))
+----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Arthur County', 'Halifax County', 'Lunenburg County', 'Oglethorpe County', 'Perry County', 'Salem County', 'Sumner County', 'Terrell County'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query35.out
index b14e619cc70b34..beedc2a19350ab 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query35.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query41.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query41.out
index 16a3a620b77c8e..c99d2b2031c4ea 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query41.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('black', 'chocolate', 'cornflower', 'firebrick', 'frosted', 'magenta', 'rose', 'slate') AND i_units IN ('Box', 'Bundle', 'Carton', 'Dram', 'Gross', 'Lb', 'Oz', 'Pound')) AND (((((((item.i_category = 'Women') AND i_color IN ('frosted', 'rose')) AND i_units IN ('Gross', 'Lb')) AND i_size IN ('large', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('black', 'chocolate')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('economy', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('magenta', 'slate')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('cornflower', 'firebrick')) AND i_units IN ('Oz', 'Pound')) AND i_size IN ('large', 'medium')))) OR ((i_color IN ('almond', 'aquamarine', 'cyan', 'lavender', 'maroon', 'papaya', 'purple', 'steel') AND i_units IN ('Bunch', 'Case', 'Cup', 'Each', 'Gram', 'N/A', 'Pallet', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('almond', 'steel')) AND i_units IN ('Case', 'Tsp')) AND i_size IN ('large', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'purple')) AND i_units IN ('Bunch', 'Gram')) AND i_size IN ('economy', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('lavender', 'papaya')) AND i_units IN ('Cup', 'Pallet')) AND i_size IN ('N/A', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('cyan', 'maroon')) AND i_units IN ('Each', 'N/A')) AND i_size IN ('large', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'large', 'medium', 'petite', 'small'))
+------------------------------filter(OR[AND[i_color IN ('black', 'chocolate', 'cornflower', 'firebrick', 'frosted', 'magenta', 'rose', 'slate'),i_units IN ('Box', 'Bundle', 'Carton', 'Dram', 'Gross', 'Lb', 'Oz', 'Pound'),OR[AND[(item.i_category = 'Women'),i_color IN ('frosted', 'rose'),i_units IN ('Gross', 'Lb'),i_size IN ('large', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('black', 'chocolate'),i_units IN ('Box', 'Dram'),i_size IN ('economy', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('magenta', 'slate'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'small')],AND[(item.i_category = 'Men'),i_color IN ('cornflower', 'firebrick'),i_units IN ('Oz', 'Pound'),i_size IN ('large', 'medium')]]],AND[i_color IN ('almond', 'aquamarine', 'cyan', 'lavender', 'maroon', 'papaya', 'purple', 'steel'),i_units IN ('Bunch', 'Case', 'Cup', 'Each', 'Gram', 'N/A', 'Pallet', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('almond', 'steel'),i_units IN ('Case', 'Tsp'),i_size IN ('large', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'purple'),i_units IN ('Bunch', 'Gram'),i_size IN ('economy', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('lavender', 'papaya'),i_units IN ('Cup', 'Pallet'),i_size IN ('N/A', 'small')],AND[(item.i_category = 'Men'),i_color IN ('cyan', 'maroon'),i_units IN ('Each', 'N/A'),i_size IN ('large', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'large', 'medium', 'petite', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query45.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query45.out
index 98618affdbd763..9f0c50ae0a648a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query45.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk]
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query46.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query46.out
index 2831d1a7f8f00a..effe4533d7debc 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query46.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query46.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Fairview', 'Farmington', 'Five Forks', 'Oakland', 'Winchester'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 0) OR (household_demographics.hd_vehicle_count = 1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count = 1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query47.out
index a3205932929569..52c9d9bc883f66 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query47.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query47.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query48.out
index cb48bf987d7340..2cac05386db12c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('GA', 'MI', 'NH') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('KY', 'SD', 'TX') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('FL', 'NY', 'OH') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('GA', 'MI', 'NH'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('KY', 'SD', 'TX'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('FL', 'NY', 'OH'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF0 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
@@ -18,7 +18,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store]
 --------------------PhysicalProject
-----------------------filter(((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary'))) and cd_education_status IN ('College', 'Primary', 'Unknown') and cd_marital_status IN ('D', 'M', 'W'))
+----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')]] and cd_education_status IN ('College', 'Primary', 'Unknown') and cd_marital_status IN ('D', 'M', 'W'))
 ------------------------PhysicalOlapScan[customer_demographics]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('FL', 'GA', 'KY', 'MI', 'NH', 'NY', 'OH', 'SD', 'TX'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query49.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query49.out
index 398dcef280ccea..04db2001d08cf7 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query49.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query53.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query53.out
index 0ef41f5f16c373..382c6a18e33765 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query53.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query53.out
@@ -14,19 +14,19 @@ PhysicalResultSink
 ----------------------PhysicalDistribute[DistributionSpecHash]
 ------------------------hashAgg[LOCAL]
 --------------------------PhysicalProject
-----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk]
 ------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+--------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk]
 ----------------------------------PhysicalProject
-------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[ss_item_sk]
+------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
-------------------------------------------PhysicalOlapScan[item]
+----------------------------------------filter(d_month_seq IN (1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223))
+------------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------------PhysicalProject
-------------------------------------filter(d_month_seq IN (1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223))
---------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalOlapScan[store]
 ------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[store]
+--------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query57.out
index e640ec24d7b3e3..88777bc1ff548d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query61.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query61.out
index a45d788af7c5a8..98825fe6cf8c4a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query61.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query61.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF5 RF6 RF7 RF8 RF10
 --------------------------------------PhysicalProject
-----------------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+----------------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ------------------------------------------PhysicalOlapScan[promotion]
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[customer] apply RFs: RF9
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query63.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query63.out
index 5d47dbe07f57cb..1dfce555f674d1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query63.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query63.out
@@ -14,19 +14,19 @@ PhysicalResultSink
 ----------------------PhysicalDistribute[DistributionSpecHash]
 ------------------------hashAgg[LOCAL]
 --------------------------PhysicalProject
-----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
+----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk]
 ------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+--------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk]
 ----------------------------------PhysicalProject
-------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[ss_item_sk]
+------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
-------------------------------------------PhysicalOlapScan[item]
+----------------------------------------filter(d_month_seq IN (1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222))
+------------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------------PhysicalProject
-------------------------------------filter(d_month_seq IN (1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222))
---------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalOlapScan[store]
 ------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[store]
+--------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query68.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query68.out
index a96932c173fb69..2339b9d070f674 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query68.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query68.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Bethel', 'Pleasant Hill'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 4) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query7.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query7.out
index 9ebde668ccdb31..82a5672ef981f2 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query7.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query7.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 2001))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query78.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query78.out
index 8662cb5a1ad007..81f28190a1b678 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query78.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query79.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query79.out
index 8684d60ad233dd..0239a35e7676cd 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query79.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query79.out
@@ -25,7 +25,7 @@ PhysicalResultSink
 ------------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
 --------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((household_demographics.hd_dep_count = 0) OR (household_demographics.hd_vehicle_count > 3)))
+--------------------------filter(OR[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count > 3)])
 ----------------------------PhysicalOlapScan[household_demographics]
 ------------PhysicalProject
 --------------PhysicalOlapScan[customer]
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query85.out
index 74754368bc4f59..106cf30dfcdd28 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('CA', 'TX', 'VA') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('AR', 'MO', 'NE') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IA', 'MS', 'WA') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('CA', 'TX', 'VA'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('AR', 'MO', 'NE'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'WA'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'College')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'D') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF2 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------------------PhysicalProject
@@ -30,7 +30,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[web_page]
 ------------------------------------PhysicalProject
---------------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'College'))) OR ((cd1.cd_marital_status = 'D') AND (cd1.cd_education_status = 'Secondary'))) and cd_education_status IN ('4 yr Degree', 'College', 'Secondary') and cd_marital_status IN ('D', 'M', 'S'))
+--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Secondary')]] and cd_education_status IN ('4 yr Degree', 'College', 'Secondary') and cd_marital_status IN ('D', 'M', 'S'))
 ----------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6
 --------------------------------PhysicalProject
 ----------------------------------filter(cd_education_status IN ('4 yr Degree', 'College', 'Secondary') and cd_marital_status IN ('D', 'M', 'S'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query88.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query88.out
index fa1d26b12e7fa8..3d5752485f6b0a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = 2) AND (household_demographics.hd_vehicle_count <= 4)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (2, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = 2) AND (household_demographics.hd_vehicle_count <= 4)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (2, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(((((household_demographics.hd_dep_count = 2) AND (household_demographics.hd_vehicle_count <= 4)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (2, 3, 4))
+----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 2) AND (household_demographics.hd_vehicle_count <= 4)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (2, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = 2) AND (household_demographics.hd_vehicle_count <= 4)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (2, 3, 4))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = 2) AND (household_demographics.hd_vehicle_count <= 4)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (2, 3, 4))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = 2) AND (household_demographics.hd_vehicle_count <= 4)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (2, 3, 4))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = 2) AND (household_demographics.hd_vehicle_count <= 4)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (2, 3, 4))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query89.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query89.out
index a57dfec0fc83fc..a24231e1dd660d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query89.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Books', 'Home', 'Music') AND i_class IN ('classical', 'fiction', 'glassware')) OR (i_category IN ('Jewelry', 'Sports', 'Women') AND i_class IN ('baseball', 'dresses', 'semi-precious'))) and i_category IN ('Books', 'Home', 'Jewelry', 'Music', 'Sports', 'Women') and i_class IN ('baseball', 'classical', 'dresses', 'fiction', 'glassware', 'semi-precious'))
+------------------------------------------filter(OR[AND[i_category IN ('Books', 'Home', 'Music'),i_class IN ('classical', 'fiction', 'glassware')],AND[i_category IN ('Jewelry', 'Sports', 'Women'),i_class IN ('baseball', 'dresses', 'semi-precious')]] and i_category IN ('Books', 'Home', 'Jewelry', 'Music', 'Sports', 'Women') and i_class IN ('baseball', 'classical', 'dresses', 'fiction', 'glassware', 'semi-precious'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 2000))
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query91.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query91.out
index 48587871410146..15a09077f4ccd5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query91.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query91.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ------------------------------filter((customer_address.ca_gmt_offset = -7.00))
 --------------------------------PhysicalOlapScan[customer_address]
 ------------------------PhysicalProject
---------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((hd_buy_potential like 'Unknown%'))
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out
index bf86e68a7f9837..ad82594176a219 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q7.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q7.out
index 9c2ccaeb016726..e953eb4cbcee33 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q7.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 --------------------------------PhysicalOlapScan[supplier] apply RFs: RF4
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[customer] apply RFs: RF3
-------------------NestedLoopJoin[INNER_JOIN](((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')))
+------------------NestedLoopJoin[INNER_JOIN]OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]]
 --------------------PhysicalProject
 ----------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ------------------------PhysicalOlapScan[nation]
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out
index bf86e68a7f9837..ad82594176a219 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q7.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q7.out
index f1424cc0f730c4..957b17a7402749 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q7.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q7.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=((((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')))) build RFs:RF4 c_custkey->[o_custkey]
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=(OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]]) build RFs:RF4 c_custkey->[o_custkey]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF3 l_orderkey->[o_orderkey]
 ----------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out
index bf86e68a7f9837..ad82594176a219 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q7.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q7.out
index f1424cc0f730c4..957b17a7402749 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q7.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q7.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=((((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')))) build RFs:RF4 c_custkey->[o_custkey]
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=(OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]]) build RFs:RF4 c_custkey->[o_custkey]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF3 l_orderkey->[o_orderkey]
 ----------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out
index bf86e68a7f9837..ad82594176a219 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q7.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q7.out
index 9c2ccaeb016726..e953eb4cbcee33 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q7.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 --------------------------------PhysicalOlapScan[supplier] apply RFs: RF4
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[customer] apply RFs: RF3
-------------------NestedLoopJoin[INNER_JOIN](((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')))
+------------------NestedLoopJoin[INNER_JOIN]OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]]
 --------------------PhysicalProject
 ----------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ------------------------PhysicalOlapScan[nation]
diff --git a/regression-test/data/new_shapes_p0/hint_tpcds/shape/query78.out b/regression-test/data/new_shapes_p0/hint_tpcds/shape/query78.out
index 52ab844c3f1bbc..e57834b15ff42a 100644
--- a/regression-test/data/new_shapes_p0/hint_tpcds/shape/query78.out
+++ b/regression-test/data/new_shapes_p0/hint_tpcds/shape/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/new_shapes_p0/hint_tpch/shape/q19.out b/regression-test/data/new_shapes_p0/hint_tpch/shape/q19.out
index 940b16c90e40d5..5cabdf9f163c56 100644
--- a/regression-test/data/new_shapes_p0/hint_tpch/shape/q19.out
+++ b/regression-test/data/new_shapes_p0/hint_tpch/shape/q19.out
@@ -5,12 +5,12 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15))))
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]])
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem]
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
 Hint log:
diff --git a/regression-test/data/new_shapes_p0/hint_tpch/shape/q7.out b/regression-test/data/new_shapes_p0/hint_tpch/shape/q7.out
index 74cca15a749172..919b8547bb8a69 100644
--- a/regression-test/data/new_shapes_p0/hint_tpch/shape/q7.out
+++ b/regression-test/data/new_shapes_p0/hint_tpch/shape/q7.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=((((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE'))))
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=(OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]])
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=()
 ----------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query10.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query10.out
index bc11b07f3a3809..5fb40519b131d5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query10.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.out
index 6df93fe3046d5c..9c7d9382777bee 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('M', 'S')) AND cd_education_status IN ('4 yr Degree', 'College')) AND ((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree'))) and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter(hd_dep_count IN (1, 3))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query15.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query15.out
index 7b0fd9b10fd861..5825559155b8e7 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query15.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query15.out
@@ -10,7 +10,7 @@ PhysicalResultSink
 --------------PhysicalProject
 ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00)))
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)])
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
 --------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query26.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query26.out
index 3762fabaeead07..52f628f8b600a2 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query26.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query26.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 2001))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query28.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query28.out
index dae7cb1c23a402..7a6bdd8868ef00 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query28.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 131.00) AND (store_sales.ss_list_price <= 141.00)) OR ((store_sales.ss_coupon_amt >= 16798.00) AND (store_sales.ss_coupon_amt <= 17798.00))) OR ((store_sales.ss_wholesale_cost >= 25.00) AND (store_sales.ss_wholesale_cost <= 45.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 131.00),(store_sales.ss_list_price <= 141.00)],AND[(store_sales.ss_coupon_amt >= 16798.00),(store_sales.ss_coupon_amt <= 17798.00)],AND[(store_sales.ss_wholesale_cost >= 25.00),(store_sales.ss_wholesale_cost <= 45.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 145.00) AND (store_sales.ss_list_price <= 155.00)) OR ((store_sales.ss_coupon_amt >= 14792.00) AND (store_sales.ss_coupon_amt <= 15792.00))) OR ((store_sales.ss_wholesale_cost >= 46.00) AND (store_sales.ss_wholesale_cost <= 66.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 145.00),(store_sales.ss_list_price <= 155.00)],AND[(store_sales.ss_coupon_amt >= 14792.00),(store_sales.ss_coupon_amt <= 15792.00)],AND[(store_sales.ss_wholesale_cost >= 46.00),(store_sales.ss_wholesale_cost <= 66.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 150.00) AND (store_sales.ss_list_price <= 160.00)) OR ((store_sales.ss_coupon_amt >= 6600.00) AND (store_sales.ss_coupon_amt <= 7600.00))) OR ((store_sales.ss_wholesale_cost >= 9.00) AND (store_sales.ss_wholesale_cost <= 29.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 150.00),(store_sales.ss_list_price <= 160.00)],AND[(store_sales.ss_coupon_amt >= 6600.00),(store_sales.ss_coupon_amt <= 7600.00)],AND[(store_sales.ss_wholesale_cost >= 9.00),(store_sales.ss_wholesale_cost <= 29.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 91.00) AND (store_sales.ss_list_price <= 101.00)) OR ((store_sales.ss_coupon_amt >= 13493.00) AND (store_sales.ss_coupon_amt <= 14493.00))) OR ((store_sales.ss_wholesale_cost >= 36.00) AND (store_sales.ss_wholesale_cost <= 56.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 91.00),(store_sales.ss_list_price <= 101.00)],AND[(store_sales.ss_coupon_amt >= 13493.00),(store_sales.ss_coupon_amt <= 14493.00)],AND[(store_sales.ss_wholesale_cost >= 36.00),(store_sales.ss_wholesale_cost <= 56.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 0.00) AND (store_sales.ss_list_price <= 10.00)) OR ((store_sales.ss_coupon_amt >= 7629.00) AND (store_sales.ss_coupon_amt <= 8629.00))) OR ((store_sales.ss_wholesale_cost >= 6.00) AND (store_sales.ss_wholesale_cost <= 26.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 0.00),(store_sales.ss_list_price <= 10.00)],AND[(store_sales.ss_coupon_amt >= 7629.00),(store_sales.ss_coupon_amt <= 8629.00)],AND[(store_sales.ss_wholesale_cost >= 6.00),(store_sales.ss_wholesale_cost <= 26.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 15257.00) AND (store_sales.ss_coupon_amt <= 16257.00))) OR ((store_sales.ss_wholesale_cost >= 31.00) AND (store_sales.ss_wholesale_cost <= 51.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 15257.00),(store_sales.ss_coupon_amt <= 16257.00)],AND[(store_sales.ss_wholesale_cost >= 31.00),(store_sales.ss_wholesale_cost <= 51.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.out
index dad99a6b6465dd..6c90e3a48d7cba 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.out
index a3f752e2ce0e3d..6def6ef536b340 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.out
index c9c1ffff499741..3034a77fe0897a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query45.out
index c6b4daafdce0df..7270fe9092a53b 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query45.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.out
index 495863c5dc9ef9..37d045d1ebc4a1 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query47.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query47.out
index 0182bbf46eb817..77dcc9357f98b8 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query47.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query47.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.out
index 338a70100eb21b..57618c21f263d3 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------------------PhysicalProject
@@ -18,7 +18,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store]
 --------------------PhysicalProject
-----------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
+----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
 ------------------------PhysicalOlapScan[customer_demographics]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query49.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query49.out
index dc56d7f9e81411..0db3fa841189b2 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query49.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query53.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query53.out
index 93289fca48e4ef..89dc632eb527c4 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query53.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.out
index 842b18a4eac9e6..96d8f68090e5de 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query61.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query61.out
index f9dcb3c9460ca4..62da8c9cb21a0f 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query61.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query61.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ------------------------------------filter((store.s_gmt_offset = -7.00))
 --------------------------------------PhysicalOlapScan[store]
 ------------------------------PhysicalProject
---------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+--------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ----------------------------------PhysicalOlapScan[promotion]
 --------------------------PhysicalProject
 ----------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 1999))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query63.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query63.out
index 3bd844ce72c196..9653f6c52199aa 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query63.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.out
index 046c5866f5196c..d645b44a1efaa7 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query7.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query7.out
index ff218ca1256a94..f47da720468166 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query7.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query7.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 2001))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query78.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query78.out
index 30527883a49018..a6034ca86ac5c5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query78.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query79.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query79.out
index 77168450e857f5..8f9b721f08ee59 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query79.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query79.out
@@ -25,7 +25,7 @@ PhysicalResultSink
 ------------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
 --------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((household_demographics.hd_dep_count = 5) OR (household_demographics.hd_vehicle_count > 4)))
+--------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)])
 ----------------------------PhysicalOlapScan[household_demographics]
 ------------PhysicalProject
 --------------PhysicalOlapScan[customer]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.out
index 590589c61a42a0..9f12763cec5114 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=()
 ----------------------------------------PhysicalProject
@@ -30,7 +30,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[web_page]
 ------------------------------------PhysicalProject
---------------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
+--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ----------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6
 --------------------------------PhysicalProject
 ----------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.out
index dc879d6daa5a4b..a20acf339431ea 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query89.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query89.out
index c709c4df5fe097..2e9294bce91fd2 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query89.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Electronics', 'Jewelry', 'Shoes') AND i_class IN ('athletic', 'portable', 'semi-precious')) OR (i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'maternity', 'rock'))) and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
+------------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 1999))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query91.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query91.out
index aa211a9f494891..6a4b369a5fe1ed 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query91.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query91.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ------------------------------filter((customer_address.ca_gmt_offset = -6.00))
 --------------------------------PhysicalOlapScan[customer_address]
 ------------------------PhysicalProject
---------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((hd_buy_potential like '1001-5000%'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query10.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query10.out
index 36a2c69f6b9516..4fdff8b37961c5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query10.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[c_current_addr_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.out
index 7f109c5dece1a7..a16b428cbc5159 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('M', 'S')) AND cd_education_status IN ('4 yr Degree', 'College')) AND ((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree'))) and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter(hd_dep_count IN (1, 3))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query15.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query15.out
index 1fba646d62b40a..81b0bae51498c1 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query15.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query15.out
@@ -10,7 +10,7 @@ PhysicalResultSink
 --------------PhysicalProject
 ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[cs_sold_date_sk]
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF1 ca_address_sk->[c_current_addr_sk]
+--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF1 ca_address_sk->[c_current_addr_sk]
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF0 c_customer_sk->[cs_bill_customer_sk]
 --------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query26.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query26.out
index 4561d649bd82c2..edbed407b77921 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query26.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query26.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 2001))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query28.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query28.out
index dae7cb1c23a402..7a6bdd8868ef00 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query28.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 131.00) AND (store_sales.ss_list_price <= 141.00)) OR ((store_sales.ss_coupon_amt >= 16798.00) AND (store_sales.ss_coupon_amt <= 17798.00))) OR ((store_sales.ss_wholesale_cost >= 25.00) AND (store_sales.ss_wholesale_cost <= 45.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 131.00),(store_sales.ss_list_price <= 141.00)],AND[(store_sales.ss_coupon_amt >= 16798.00),(store_sales.ss_coupon_amt <= 17798.00)],AND[(store_sales.ss_wholesale_cost >= 25.00),(store_sales.ss_wholesale_cost <= 45.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 145.00) AND (store_sales.ss_list_price <= 155.00)) OR ((store_sales.ss_coupon_amt >= 14792.00) AND (store_sales.ss_coupon_amt <= 15792.00))) OR ((store_sales.ss_wholesale_cost >= 46.00) AND (store_sales.ss_wholesale_cost <= 66.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 145.00),(store_sales.ss_list_price <= 155.00)],AND[(store_sales.ss_coupon_amt >= 14792.00),(store_sales.ss_coupon_amt <= 15792.00)],AND[(store_sales.ss_wholesale_cost >= 46.00),(store_sales.ss_wholesale_cost <= 66.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 150.00) AND (store_sales.ss_list_price <= 160.00)) OR ((store_sales.ss_coupon_amt >= 6600.00) AND (store_sales.ss_coupon_amt <= 7600.00))) OR ((store_sales.ss_wholesale_cost >= 9.00) AND (store_sales.ss_wholesale_cost <= 29.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 150.00),(store_sales.ss_list_price <= 160.00)],AND[(store_sales.ss_coupon_amt >= 6600.00),(store_sales.ss_coupon_amt <= 7600.00)],AND[(store_sales.ss_wholesale_cost >= 9.00),(store_sales.ss_wholesale_cost <= 29.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 91.00) AND (store_sales.ss_list_price <= 101.00)) OR ((store_sales.ss_coupon_amt >= 13493.00) AND (store_sales.ss_coupon_amt <= 14493.00))) OR ((store_sales.ss_wholesale_cost >= 36.00) AND (store_sales.ss_wholesale_cost <= 56.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 91.00),(store_sales.ss_list_price <= 101.00)],AND[(store_sales.ss_coupon_amt >= 13493.00),(store_sales.ss_coupon_amt <= 14493.00)],AND[(store_sales.ss_wholesale_cost >= 36.00),(store_sales.ss_wholesale_cost <= 56.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 0.00) AND (store_sales.ss_list_price <= 10.00)) OR ((store_sales.ss_coupon_amt >= 7629.00) AND (store_sales.ss_coupon_amt <= 8629.00))) OR ((store_sales.ss_wholesale_cost >= 6.00) AND (store_sales.ss_wholesale_cost <= 26.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 0.00),(store_sales.ss_list_price <= 10.00)],AND[(store_sales.ss_coupon_amt >= 7629.00),(store_sales.ss_coupon_amt <= 8629.00)],AND[(store_sales.ss_wholesale_cost >= 6.00),(store_sales.ss_wholesale_cost <= 26.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 15257.00) AND (store_sales.ss_coupon_amt <= 16257.00))) OR ((store_sales.ss_wholesale_cost >= 31.00) AND (store_sales.ss_wholesale_cost <= 51.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 15257.00),(store_sales.ss_coupon_amt <= 16257.00)],AND[(store_sales.ss_wholesale_cost >= 31.00),(store_sales.ss_wholesale_cost <= 51.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.out
index 25429183c8fe4a..9a53073479fb8c 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.out
index 14b71f576d97e0..83807f4b912bfe 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.out
index c9c1ffff499741..3034a77fe0897a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query45.out
index 038ece83d7c044..68d1ef7855a7fd 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query45.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.out
index ae7ea5a5a310fe..4f754d410d4b6d 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query47.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query47.out
index b6f046eb01fe81..29aa8e6ae22e6c 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query47.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query47.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.out
index ed7ed3b4bdf669..fa9006ce24f673 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF0 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
@@ -18,7 +18,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store]
 --------------------PhysicalProject
-----------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
+----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
 ------------------------PhysicalOlapScan[customer_demographics]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query49.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query49.out
index dc56d7f9e81411..0db3fa841189b2 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query49.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query53.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query53.out
index 81e6091b663bf7..04920e65ac6894 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query53.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.out
index e640ec24d7b3e3..88777bc1ff548d 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.out
@@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[item]
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query61.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query61.out
index f9dcb3c9460ca4..62da8c9cb21a0f 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query61.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query61.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ------------------------------------filter((store.s_gmt_offset = -7.00))
 --------------------------------------PhysicalOlapScan[store]
 ------------------------------PhysicalProject
---------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+--------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ----------------------------------PhysicalOlapScan[promotion]
 --------------------------PhysicalProject
 ----------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 1999))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query63.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query63.out
index 2cd49bd8c40786..d4fb4990da98b8 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query63.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.out
index 70945beb33b729..82e85b4fc4697a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.out
@@ -27,7 +27,7 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query7.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query7.out
index 12c9bd7d3a0b25..2b6615e0b93b84 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query7.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query7.out
@@ -26,6 +26,6 @@ PhysicalResultSink
 ------------------------filter((date_dim.d_year = 2001))
 --------------------------PhysicalOlapScan[date_dim]
 ------------------PhysicalProject
---------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+--------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 ----------------------PhysicalOlapScan[promotion]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query78.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query78.out
index 30527883a49018..a6034ca86ac5c5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query78.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query79.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query79.out
index 5d935352c491c6..1a19308d991441 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query79.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query79.out
@@ -25,7 +25,7 @@ PhysicalResultSink
 ------------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
 --------------------------------PhysicalOlapScan[store]
 ------------------------PhysicalProject
---------------------------filter(((household_demographics.hd_dep_count = 5) OR (household_demographics.hd_vehicle_count > 4)))
+--------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)])
 ----------------------------PhysicalOlapScan[household_demographics]
 ------------PhysicalProject
 --------------PhysicalOlapScan[customer]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.out
index 08e59c1e36507e..424102e34021a1 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF2 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------------------PhysicalProject
@@ -30,7 +30,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[web_page]
 ------------------------------------PhysicalProject
---------------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
+--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ----------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6
 --------------------------------PhysicalProject
 ----------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.out
index dc879d6daa5a4b..a20acf339431ea 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query89.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query89.out
index 661b47d0ea980a..e4d2ae3435f174 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query89.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Electronics', 'Jewelry', 'Shoes') AND i_class IN ('athletic', 'portable', 'semi-precious')) OR (i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'maternity', 'rock'))) and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
+------------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 1999))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query91.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query91.out
index c19b43be01a9df..a2e5d4a7660114 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query91.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query91.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ------------------------------filter((customer_address.ca_gmt_offset = -6.00))
 --------------------------------PhysicalOlapScan[customer_address]
 ------------------------PhysicalProject
---------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((hd_buy_potential like '1001-5000%'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query10.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query10.out
index 6027b8b2684b4b..4dfc2de4cf3fe5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query10.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query13.out
index e92ddac6d97742..4b72ea24b2c114 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query13.out
@@ -7,16 +7,16 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('M', 'S')) AND cd_education_status IN ('4 yr Degree', 'College')) AND ((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
-------------------filter(((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree'))) and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
+------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query15.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query15.out
index f15477f65e12fb..c070b7d34c00d4 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query15.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query15.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00)))
+----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)])
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
 ----------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query26.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query26.out
index c1cc05ec8ee03f..37ede3b355320f 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query26.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query26.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query28.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query28.out
index dae7cb1c23a402..7a6bdd8868ef00 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query28.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 131.00) AND (store_sales.ss_list_price <= 141.00)) OR ((store_sales.ss_coupon_amt >= 16798.00) AND (store_sales.ss_coupon_amt <= 17798.00))) OR ((store_sales.ss_wholesale_cost >= 25.00) AND (store_sales.ss_wholesale_cost <= 45.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 131.00),(store_sales.ss_list_price <= 141.00)],AND[(store_sales.ss_coupon_amt >= 16798.00),(store_sales.ss_coupon_amt <= 17798.00)],AND[(store_sales.ss_wholesale_cost >= 25.00),(store_sales.ss_wholesale_cost <= 45.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 145.00) AND (store_sales.ss_list_price <= 155.00)) OR ((store_sales.ss_coupon_amt >= 14792.00) AND (store_sales.ss_coupon_amt <= 15792.00))) OR ((store_sales.ss_wholesale_cost >= 46.00) AND (store_sales.ss_wholesale_cost <= 66.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 145.00),(store_sales.ss_list_price <= 155.00)],AND[(store_sales.ss_coupon_amt >= 14792.00),(store_sales.ss_coupon_amt <= 15792.00)],AND[(store_sales.ss_wholesale_cost >= 46.00),(store_sales.ss_wholesale_cost <= 66.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 150.00) AND (store_sales.ss_list_price <= 160.00)) OR ((store_sales.ss_coupon_amt >= 6600.00) AND (store_sales.ss_coupon_amt <= 7600.00))) OR ((store_sales.ss_wholesale_cost >= 9.00) AND (store_sales.ss_wholesale_cost <= 29.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 150.00),(store_sales.ss_list_price <= 160.00)],AND[(store_sales.ss_coupon_amt >= 6600.00),(store_sales.ss_coupon_amt <= 7600.00)],AND[(store_sales.ss_wholesale_cost >= 9.00),(store_sales.ss_wholesale_cost <= 29.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 91.00) AND (store_sales.ss_list_price <= 101.00)) OR ((store_sales.ss_coupon_amt >= 13493.00) AND (store_sales.ss_coupon_amt <= 14493.00))) OR ((store_sales.ss_wholesale_cost >= 36.00) AND (store_sales.ss_wholesale_cost <= 56.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 91.00),(store_sales.ss_list_price <= 101.00)],AND[(store_sales.ss_coupon_amt >= 13493.00),(store_sales.ss_coupon_amt <= 14493.00)],AND[(store_sales.ss_wholesale_cost >= 36.00),(store_sales.ss_wholesale_cost <= 56.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 0.00) AND (store_sales.ss_list_price <= 10.00)) OR ((store_sales.ss_coupon_amt >= 7629.00) AND (store_sales.ss_coupon_amt <= 8629.00))) OR ((store_sales.ss_wholesale_cost >= 6.00) AND (store_sales.ss_wholesale_cost <= 26.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 0.00),(store_sales.ss_list_price <= 10.00)],AND[(store_sales.ss_coupon_amt >= 7629.00),(store_sales.ss_coupon_amt <= 8629.00)],AND[(store_sales.ss_wholesale_cost >= 6.00),(store_sales.ss_wholesale_cost <= 26.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 15257.00) AND (store_sales.ss_coupon_amt <= 16257.00))) OR ((store_sales.ss_wholesale_cost >= 31.00) AND (store_sales.ss_wholesale_cost <= 51.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 15257.00),(store_sales.ss_coupon_amt <= 16257.00)],AND[(store_sales.ss_wholesale_cost >= 31.00),(store_sales.ss_wholesale_cost <= 51.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query34.out
index bb286ee190e816..489abcd4c2e33c 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query34.out
@@ -21,7 +21,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query35.out
index 4da981f140aa0e..1e865046f6cf27 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query35.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
 ------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF5 ss_customer_sk->[c_customer_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query41.out
index c9c1ffff499741..3034a77fe0897a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query45.out
index 83f6b9ca5df1f0..40b25ae51ad929 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query45.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=()
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query46.out
index 2895fa954ff4eb..75c9af2b354fae 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query46.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query46.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query47.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query47.out
index e115c8751281e0..048f93392595ec 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query47.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+--------------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out
index 476aa9dd8793af..3f16a98d674c76 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out
@@ -7,23 +7,23 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
+--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
-----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
-------------------------PhysicalOlapScan[customer_address]
+----------------------filter((date_dim.d_year = 1999))
+------------------------PhysicalOlapScan[date_dim]
 ----------------PhysicalProject
-------------------filter((date_dim.d_year = 1999))
---------------------PhysicalOlapScan[date_dim]
+------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
+--------------------PhysicalOlapScan[customer_address]
 ------------PhysicalProject
 --------------PhysicalOlapScan[store]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query49.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query49.out
index e34962e0847b0d..7d0f6b1ce22d0f 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query49.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query53.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query53.out
index 93289fca48e4ef..89dc632eb527c4 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query53.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query57.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query57.out
index 4a7de98eb56b50..4f23fac89cf958 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query57.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+--------------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query61.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query61.out
index d034c5b43231d7..faf30604b86926 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query61.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query61.out
@@ -29,7 +29,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF6 ss_promo_sk->[p_promo_sk]
 ----------------------------------PhysicalProject
-------------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+------------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 --------------------------------------PhysicalOlapScan[promotion] apply RFs: RF6
 ----------------------------------PhysicalProject
 ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF5 s_store_sk->[ss_store_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query63.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query63.out
index 3bd844ce72c196..9653f6c52199aa 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query63.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query68.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query68.out
index 8975dc78ab76e5..aa07d1b2a42d9b 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query68.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query68.out
@@ -33,6 +33,6 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query7.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query7.out
index 3d8133e9159fea..18425373b08452 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query7.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query78.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query78.out
index 30527883a49018..a6034ca86ac5c5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query78.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query79.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query79.out
index 0ceac85291e12d..f57418546e7fb9 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query79.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query79.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter((date_dim.d_dow = 1) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 5) OR (household_demographics.hd_vehicle_count > 4)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out
index c428ce03bde80e..d9d74a4dd87f7e 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
 --------------------------------PhysicalProject
-----------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
+----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk]
+----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query88.out
index bca8b0d069014b..09c630cef443ef 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query89.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query89.out
index c709c4df5fe097..2e9294bce91fd2 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query89.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Electronics', 'Jewelry', 'Shoes') AND i_class IN ('athletic', 'portable', 'semi-precious')) OR (i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'maternity', 'rock'))) and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
+------------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 1999))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query91.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query91.out
index e2a4ed89af442d..6593bb83f07994 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query91.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query91.out
@@ -28,7 +28,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 --------------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------------------PhysicalProject
 --------------------------------------filter((hd_buy_potential like '1001-5000%'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query10.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query10.out
index 6027b8b2684b4b..4dfc2de4cf3fe5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query10.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query13.out
index 7528312c6dae6b..e8a4d92c4f7bf1 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query13.out
@@ -7,16 +7,16 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('M', 'S')) AND cd_education_status IN ('4 yr Degree', 'College')) AND ((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
-------------------filter(((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '4 yr Degree'))) and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
+------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query15.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query15.out
index b80017b7ebd0f9..b93c82158d296a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query15.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query15.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
+----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
 ----------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query26.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query26.out
index d5fbe1d1762fb9..0c3a3f432aeb48 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query26.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query26.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query28.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query28.out
index dae7cb1c23a402..7a6bdd8868ef00 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query28.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 131.00) AND (store_sales.ss_list_price <= 141.00)) OR ((store_sales.ss_coupon_amt >= 16798.00) AND (store_sales.ss_coupon_amt <= 17798.00))) OR ((store_sales.ss_wholesale_cost >= 25.00) AND (store_sales.ss_wholesale_cost <= 45.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 131.00),(store_sales.ss_list_price <= 141.00)],AND[(store_sales.ss_coupon_amt >= 16798.00),(store_sales.ss_coupon_amt <= 17798.00)],AND[(store_sales.ss_wholesale_cost >= 25.00),(store_sales.ss_wholesale_cost <= 45.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 145.00) AND (store_sales.ss_list_price <= 155.00)) OR ((store_sales.ss_coupon_amt >= 14792.00) AND (store_sales.ss_coupon_amt <= 15792.00))) OR ((store_sales.ss_wholesale_cost >= 46.00) AND (store_sales.ss_wholesale_cost <= 66.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 145.00),(store_sales.ss_list_price <= 155.00)],AND[(store_sales.ss_coupon_amt >= 14792.00),(store_sales.ss_coupon_amt <= 15792.00)],AND[(store_sales.ss_wholesale_cost >= 46.00),(store_sales.ss_wholesale_cost <= 66.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 150.00) AND (store_sales.ss_list_price <= 160.00)) OR ((store_sales.ss_coupon_amt >= 6600.00) AND (store_sales.ss_coupon_amt <= 7600.00))) OR ((store_sales.ss_wholesale_cost >= 9.00) AND (store_sales.ss_wholesale_cost <= 29.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 150.00),(store_sales.ss_list_price <= 160.00)],AND[(store_sales.ss_coupon_amt >= 6600.00),(store_sales.ss_coupon_amt <= 7600.00)],AND[(store_sales.ss_wholesale_cost >= 9.00),(store_sales.ss_wholesale_cost <= 29.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 91.00) AND (store_sales.ss_list_price <= 101.00)) OR ((store_sales.ss_coupon_amt >= 13493.00) AND (store_sales.ss_coupon_amt <= 14493.00))) OR ((store_sales.ss_wholesale_cost >= 36.00) AND (store_sales.ss_wholesale_cost <= 56.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 91.00),(store_sales.ss_list_price <= 101.00)],AND[(store_sales.ss_coupon_amt >= 13493.00),(store_sales.ss_coupon_amt <= 14493.00)],AND[(store_sales.ss_wholesale_cost >= 36.00),(store_sales.ss_wholesale_cost <= 56.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 0.00) AND (store_sales.ss_list_price <= 10.00)) OR ((store_sales.ss_coupon_amt >= 7629.00) AND (store_sales.ss_coupon_amt <= 8629.00))) OR ((store_sales.ss_wholesale_cost >= 6.00) AND (store_sales.ss_wholesale_cost <= 26.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 0.00),(store_sales.ss_list_price <= 10.00)],AND[(store_sales.ss_coupon_amt >= 7629.00),(store_sales.ss_coupon_amt <= 8629.00)],AND[(store_sales.ss_wholesale_cost >= 6.00),(store_sales.ss_wholesale_cost <= 26.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 15257.00) AND (store_sales.ss_coupon_amt <= 16257.00))) OR ((store_sales.ss_wholesale_cost >= 31.00) AND (store_sales.ss_wholesale_cost <= 51.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 15257.00),(store_sales.ss_coupon_amt <= 16257.00)],AND[(store_sales.ss_wholesale_cost >= 31.00),(store_sales.ss_wholesale_cost <= 51.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query34.out
index bb286ee190e816..489abcd4c2e33c 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query34.out
@@ -21,7 +21,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (1998, 1999, 2000))
+----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query35.out
index 93fdd630b3352d..dc926eb4b522f0 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query35.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=()
 ------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF5 ss_customer_sk->[c_customer_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query41.out
index c9c1ffff499741..3034a77fe0897a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet') AND i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'gainsboro')) AND i_units IN ('Dozen', 'Ounce')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('chiffon', 'violet')) AND i_units IN ('Pound', 'Ton')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('blue', 'chartreuse')) AND i_units IN ('Each', 'Oz')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('dodger', 'tan')) AND i_units IN ('Bunch', 'Tsp')) AND i_size IN ('economy', 'medium')))) OR ((i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato') AND i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown')) AND (((((((item.i_category = 'Women') AND i_color IN ('blanched', 'tomato')) AND i_units IN ('Case', 'Tbl')) AND i_size IN ('economy', 'medium')) OR ((((item.i_category = 'Women') AND i_color IN ('almond', 'lime')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'small'))) OR ((((item.i_category = 'Men') AND i_color IN ('peru', 'saddle')) AND i_units IN ('Gram', 'Pallet')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('indian', 'spring')) AND i_units IN ('Carton', 'Unknown')) AND i_size IN ('economy', 'medium'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query45.out
index e05c3f0537a669..6bbc52ecb1a343 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query45.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ws_bill_customer_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query46.out
index 0a6a5e279d3f80..c1ebbb551053cb 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query46.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query46.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query47.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query47.out
index 7a668b06c433e5..ecc4777c22f664 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query47.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+--------------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out
index c81868cacca7a6..1bf363b1426199 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out
@@ -7,23 +7,23 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
+--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
-----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
-------------------------PhysicalOlapScan[customer_address]
+----------------------filter((date_dim.d_year = 1999))
+------------------------PhysicalOlapScan[date_dim]
 ----------------PhysicalProject
-------------------filter((date_dim.d_year = 1999))
---------------------PhysicalOlapScan[date_dim]
+------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI'))
+--------------------PhysicalOlapScan[customer_address]
 ------------PhysicalProject
 --------------PhysicalOlapScan[store]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query49.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query49.out
index e34962e0847b0d..7d0f6b1ce22d0f 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query49.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query53.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query53.out
index 81e6091b663bf7..04920e65ac6894 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query53.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query57.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query57.out
index a0157318139d88..2cab4f33e1358d 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query57.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1))) and d_year IN (1998, 1999, 2000))
+--------------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query61.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query61.out
index d034c5b43231d7..faf30604b86926 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query61.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query61.out
@@ -29,7 +29,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF6 ss_promo_sk->[p_promo_sk]
 ----------------------------------PhysicalProject
-------------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+------------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 --------------------------------------PhysicalOlapScan[promotion] apply RFs: RF6
 ----------------------------------PhysicalProject
 ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF5 s_store_sk->[ss_store_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query63.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query63.out
index 2cd49bd8c40786..d4fb4990da98b8 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query63.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query68.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query68.out
index 8975dc78ab76e5..aa07d1b2a42d9b 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query68.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query68.out
@@ -33,6 +33,6 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query7.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query7.out
index 03f95005ce9cbc..49acfd90e56654 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query7.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query78.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query78.out
index 30527883a49018..a6034ca86ac5c5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query78.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query79.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query79.out
index 4f8ac0ce4d7dea..974fb1da39b734 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query79.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query79.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter((date_dim.d_dow = 1) and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 5) OR (household_demographics.hd_vehicle_count > 4)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out
index c69ca3b6c0ed48..332573e1b96303 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
 --------------------------------PhysicalProject
-----------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
+----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk]
+----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query88.out
index bca8b0d069014b..09c630cef443ef 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+----------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(((((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1)) OR ((household_demographics.hd_dep_count = 4) AND (household_demographics.hd_vehicle_count <= 6))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 3, 4))
+--------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query89.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query89.out
index 661b47d0ea980a..e4d2ae3435f174 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query89.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Electronics', 'Jewelry', 'Shoes') AND i_class IN ('athletic', 'portable', 'semi-precious')) OR (i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'maternity', 'rock'))) and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
+------------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 1999))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query91.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query91.out
index 6f3970cbb29d50..6af6c014d0c298 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query91.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query91.out
@@ -28,7 +28,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 --------------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------------------PhysicalProject
 --------------------------------------filter((hd_buy_potential like '1001-5000%'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.out
index 461f54e0e1767c..8138a8e20ed7e5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.out
@@ -7,21 +7,21 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IL', 'TN', 'TX') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('ID', 'OH', 'WY') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('IA', 'MS', 'SC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF3 ss_addr_sk->[ca_address_sk]
+--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY'))
 --------------------PhysicalOlapScan[customer_address] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('D', 'W')) AND cd_education_status IN ('2 yr Degree', 'Primary')) AND ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
 ----------------------------PhysicalProject
-------------------------------filter(((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary'))) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
+------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------PhysicalProject
 --------------------------filter(hd_dep_count IN (1, 3))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.out
index 6ac3b85090aceb..72acf1dc6b71ff 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ws_bill_customer_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query61.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query61.out
index 7453f3ef1a7080..e768a09ec1494d 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query61.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query61.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ------------------------------------filter((item.i_category = 'Home'))
 --------------------------------------PhysicalOlapScan[item]
 ------------------------------PhysicalProject
---------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+--------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ----------------------------------PhysicalOlapScan[promotion]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_gmt_offset = -7.00))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query68.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query68.out
index f4503a600d6272..2f4fbe401f1315 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query68.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query68.out
@@ -33,6 +33,6 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Fairview', 'Midway'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_vehicle_count = 4)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query91.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query91.out
index 624685e9dab961..9d3c77acb23ca8 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query91.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query91.out
@@ -28,7 +28,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 --------------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------------------PhysicalProject
 --------------------------------------filter((hd_buy_potential like 'Unknown%'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/eliminate_empty/query10_empty.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/eliminate_empty/query10_empty.out
index f68d46db52fbb8..78fd7c847c29ed 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/eliminate_empty/query10_empty.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/eliminate_empty/query10_empty.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query10.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query10.out
index f68d46db52fbb8..78fd7c847c29ed 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query10.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query10.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query13.out
index 461f54e0e1767c..8138a8e20ed7e5 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query13.out
@@ -7,21 +7,21 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IL', 'TN', 'TX') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('ID', 'OH', 'WY') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('IA', 'MS', 'SC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF3 ss_addr_sk->[ca_address_sk]
+--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY'))
 --------------------PhysicalOlapScan[customer_address] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=((((((household_demographics.hd_dep_count = 1) AND cd_marital_status IN ('D', 'W')) AND cd_education_status IN ('2 yr Degree', 'Primary')) AND ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
 ----------------------------PhysicalProject
-------------------------------filter(((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary'))) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree'))) and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
+------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------PhysicalProject
 --------------------------filter(hd_dep_count IN (1, 3))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query15.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query15.out
index 6f522756f18098..fe0fd80c2f8b05 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query15.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query15.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR ca_state IN ('CA', 'GA', 'WA')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
+----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
 ----------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query26.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query26.out
index dc5aee071856c1..383242890f9dd4 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query26.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query26.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2002))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query28.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query28.out
index 2fdcc86b103f71..36ec7305d96abb 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query28.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query28.out
@@ -17,41 +17,41 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 107.00) AND (store_sales.ss_list_price <= 117.00)) OR ((store_sales.ss_coupon_amt >= 1319.00) AND (store_sales.ss_coupon_amt <= 2319.00))) OR ((store_sales.ss_wholesale_cost >= 60.00) AND (store_sales.ss_wholesale_cost <= 80.00))) and (store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0))
+----------------------------------filter((store_sales.ss_quantity <= 5) and (store_sales.ss_quantity >= 0) and OR[AND[(store_sales.ss_list_price >= 107.00),(store_sales.ss_list_price <= 117.00)],AND[(store_sales.ss_coupon_amt >= 1319.00),(store_sales.ss_coupon_amt <= 2319.00)],AND[(store_sales.ss_wholesale_cost >= 60.00),(store_sales.ss_wholesale_cost <= 80.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 ------------------------PhysicalLimit[LOCAL]
 --------------------------hashAgg[GLOBAL]
 ----------------------------PhysicalDistribute[DistributionSpecGather]
 ------------------------------hashAgg[LOCAL]
 --------------------------------PhysicalProject
-----------------------------------filter(((((store_sales.ss_list_price >= 23.00) AND (store_sales.ss_list_price <= 33.00)) OR ((store_sales.ss_coupon_amt >= 825.00) AND (store_sales.ss_coupon_amt <= 1825.00))) OR ((store_sales.ss_wholesale_cost >= 43.00) AND (store_sales.ss_wholesale_cost <= 63.00))) and (store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6))
+----------------------------------filter((store_sales.ss_quantity <= 10) and (store_sales.ss_quantity >= 6) and OR[AND[(store_sales.ss_list_price >= 23.00),(store_sales.ss_list_price <= 33.00)],AND[(store_sales.ss_coupon_amt >= 825.00),(store_sales.ss_coupon_amt <= 1825.00)],AND[(store_sales.ss_wholesale_cost >= 43.00),(store_sales.ss_wholesale_cost <= 63.00)]])
 ------------------------------------PhysicalOlapScan[store_sales]
 --------------------PhysicalLimit[LOCAL]
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute[DistributionSpecGather]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_list_price >= 74.00) AND (store_sales.ss_list_price <= 84.00)) OR ((store_sales.ss_coupon_amt >= 4381.00) AND (store_sales.ss_coupon_amt <= 5381.00))) OR ((store_sales.ss_wholesale_cost >= 57.00) AND (store_sales.ss_wholesale_cost <= 77.00))) and (store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11))
+------------------------------filter((store_sales.ss_quantity <= 15) and (store_sales.ss_quantity >= 11) and OR[AND[(store_sales.ss_list_price >= 74.00),(store_sales.ss_list_price <= 84.00)],AND[(store_sales.ss_coupon_amt >= 4381.00),(store_sales.ss_coupon_amt <= 5381.00)],AND[(store_sales.ss_wholesale_cost >= 57.00),(store_sales.ss_wholesale_cost <= 77.00)]])
 --------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalLimit[LOCAL]
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute[DistributionSpecGather]
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_list_price >= 89.00) AND (store_sales.ss_list_price <= 99.00)) OR ((store_sales.ss_coupon_amt >= 3117.00) AND (store_sales.ss_coupon_amt <= 4117.00))) OR ((store_sales.ss_wholesale_cost >= 68.00) AND (store_sales.ss_wholesale_cost <= 88.00))) and (store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16))
+--------------------------filter((store_sales.ss_quantity <= 20) and (store_sales.ss_quantity >= 16) and OR[AND[(store_sales.ss_list_price >= 89.00),(store_sales.ss_list_price <= 99.00)],AND[(store_sales.ss_coupon_amt >= 3117.00),(store_sales.ss_coupon_amt <= 4117.00)],AND[(store_sales.ss_wholesale_cost >= 68.00),(store_sales.ss_wholesale_cost <= 88.00)]])
 ----------------------------PhysicalOlapScan[store_sales]
 ------------PhysicalLimit[LOCAL]
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute[DistributionSpecGather]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------filter(((((store_sales.ss_list_price >= 58.00) AND (store_sales.ss_list_price <= 68.00)) OR ((store_sales.ss_coupon_amt >= 9402.00) AND (store_sales.ss_coupon_amt <= 10402.00))) OR ((store_sales.ss_wholesale_cost >= 38.00) AND (store_sales.ss_wholesale_cost <= 58.00))) and (store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21))
+----------------------filter((store_sales.ss_quantity <= 25) and (store_sales.ss_quantity >= 21) and OR[AND[(store_sales.ss_list_price >= 58.00),(store_sales.ss_list_price <= 68.00)],AND[(store_sales.ss_coupon_amt >= 9402.00),(store_sales.ss_coupon_amt <= 10402.00)],AND[(store_sales.ss_wholesale_cost >= 38.00),(store_sales.ss_wholesale_cost <= 58.00)]])
 ------------------------PhysicalOlapScan[store_sales]
 --------PhysicalLimit[LOCAL]
 ----------hashAgg[GLOBAL]
 ------------PhysicalDistribute[DistributionSpecGather]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter(((((store_sales.ss_list_price >= 64.00) AND (store_sales.ss_list_price <= 74.00)) OR ((store_sales.ss_coupon_amt >= 5792.00) AND (store_sales.ss_coupon_amt <= 6792.00))) OR ((store_sales.ss_wholesale_cost >= 73.00) AND (store_sales.ss_wholesale_cost <= 93.00))) and (store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26))
+------------------filter((store_sales.ss_quantity <= 30) and (store_sales.ss_quantity >= 26) and OR[AND[(store_sales.ss_list_price >= 64.00),(store_sales.ss_list_price <= 74.00)],AND[(store_sales.ss_coupon_amt >= 5792.00),(store_sales.ss_coupon_amt <= 6792.00)],AND[(store_sales.ss_wholesale_cost >= 73.00),(store_sales.ss_wholesale_cost <= 93.00)]])
 --------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query34.out
index 135b461a144762..ad28c2d46104d0 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query34.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------------filter((store.s_county = 'Williamson County'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter((((date_dim.d_dom >= 1) AND (date_dim.d_dom <= 3)) OR ((date_dim.d_dom >= 25) AND (date_dim.d_dom <= 28))) and d_year IN (2000, 2001, 2002))
+------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (2000, 2001, 2002))
 --------------------------------PhysicalOlapScan[date_dim]
 ------------------------PhysicalProject
 --------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query35.out
index a7a8d54bd3e135..9012700621a358 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query35.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query35.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalDistribute[DistributionSpecHash]
 --------------hashAgg[LOCAL]
 ----------------PhysicalProject
-------------------filter((ifnull($c$1, FALSE) OR ifnull($c$2, FALSE)))
+------------------filter(OR[ifnull($c$1, FALSE),ifnull($c$2, FALSE)])
 --------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
 ----------------------PhysicalProject
 ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query41.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query41.out
index ea5acccd883550..0bba60d4cdac39 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query41.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter((((i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke') AND i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound')) AND (((((((item.i_category = 'Women') AND i_color IN ('forest', 'lime')) AND i_units IN ('Pallet', 'Pound')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('navy', 'slate')) AND i_units IN ('Bunch', 'Gross')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('powder', 'sky')) AND i_units IN ('Dozen', 'Lb')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('maroon', 'smoke')) AND i_units IN ('Case', 'Ounce')) AND i_size IN ('economy', 'small')))) OR ((i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna') AND i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton')) AND (((((((item.i_category = 'Women') AND i_color IN ('aquamarine', 'dark')) AND i_units IN ('Tbl', 'Ton')) AND i_size IN ('economy', 'small')) OR ((((item.i_category = 'Women') AND i_color IN ('frosted', 'plum')) AND i_units IN ('Box', 'Dram')) AND i_size IN ('extra large', 'petite'))) OR ((((item.i_category = 'Men') AND i_color IN ('papaya', 'peach')) AND i_units IN ('Bundle', 'Carton')) AND i_size IN ('N/A', 'large'))) OR ((((item.i_category = 'Men') AND i_color IN ('firebrick', 'sienna')) AND i_units IN ('Cup', 'Each')) AND i_size IN ('economy', 'small'))))) and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
+------------------------------filter(OR[AND[i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke'),i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound'),OR[AND[(item.i_category = 'Women'),i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce'),i_size IN ('economy', 'small')]]],AND[i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna'),i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each'),i_size IN ('economy', 'small')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query45.out
index 6ac3b85090aceb..72acf1dc6b71ff 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query45.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query45.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1))
+----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1])
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ws_bill_customer_sk]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query46.out
index b80a9affecc0d6..e7aa0e014dcdd9 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query46.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query46.out
@@ -29,7 +29,7 @@ PhysicalResultSink
 ----------------------------------filter(d_dow IN (0, 6) and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = 0)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer_address]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query47.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query47.out
index d51d48c5ab5677..0e9f713243773a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query47.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query47.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2000) OR ((date_dim.d_year = 1999) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2001) AND (date_dim.d_moy = 1))) and d_year IN (1999, 2000, 2001))
+--------------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and d_year IN (1999, 2000, 2001))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out
index 3fcfef0d8f68ba..6ed75b1e21eca8 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out
@@ -9,14 +9,14 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('ND', 'NY', 'SD') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('GA', 'KS', 'MD') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('CO', 'MN', 'NC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'Secondary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Advanced Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 ------------------------PhysicalProject
---------------------------filter(((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'Secondary')) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '2 yr Degree'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S'))
+--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S'))
 ----------------------------PhysicalOlapScan[customer_demographics]
 --------------------PhysicalProject
 ----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query49.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query49.out
index 889d5069ff9751..8b807baf5409e4 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query49.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query49.out
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -48,7 +48,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
@@ -80,7 +80,7 @@ PhysicalResultSink
 --------------------------PhysicalDistribute[DistributionSpecHash]
 ----------------------------hashAgg[LOCAL]
 ------------------------------PhysicalProject
---------------------------------filter(((return_rank <= 10) OR (currency_rank <= 10)))
+--------------------------------filter(OR[(return_rank <= 10),(currency_rank <= 10)])
 ----------------------------------PhysicalWindow
 ------------------------------------PhysicalQuickSort[LOCAL_SORT]
 --------------------------------------PhysicalWindow
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query53.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query53.out
index 07d64a35b071ca..d2467a65e93e09 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query53.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query53.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query57.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query57.out
index c1b802ebfc4498..2e8174812f69ea 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query57.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query57.out
@@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------PhysicalProject
---------------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1))) and d_year IN (2000, 2001, 2002))
+--------------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002))
 ----------------------------------------PhysicalOlapScan[date_dim]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[call_center]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query61.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query61.out
index 7453f3ef1a7080..e768a09ec1494d 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query61.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query61.out
@@ -33,7 +33,7 @@ PhysicalResultSink
 ------------------------------------filter((item.i_category = 'Home'))
 --------------------------------------PhysicalOlapScan[item]
 ------------------------------PhysicalProject
---------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y')))
+--------------------------------filter(OR[(promotion.p_channel_dmail = 'Y'),(promotion.p_channel_email = 'Y'),(promotion.p_channel_tv = 'Y')])
 ----------------------------------PhysicalOlapScan[promotion]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_gmt_offset = -7.00))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query63.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query63.out
index d1807a29db2f0d..bbbb80bc4b68e0 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query63.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query63.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------------PhysicalProject
-----------------------------------------filter((((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('personal', 'portable', 'reference', 'self-help')) AND i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')) OR ((i_category IN ('Men', 'Music', 'Women') AND i_class IN ('accessories', 'classical', 'fragrances', 'pants')) AND i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1'))) and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
+----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help'))
 ------------------------------------------PhysicalOlapScan[item]
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query68.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query68.out
index f4503a600d6272..2f4fbe401f1315 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query68.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query68.out
@@ -33,6 +33,6 @@ PhysicalResultSink
 ----------------------------------filter(s_city IN ('Fairview', 'Midway'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_vehicle_count = 4)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)])
 --------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query7.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query7.out
index aadb31a78e012f..2d63af9e61b19e 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query7.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 ----------------------------filter((date_dim.d_year = 2001))
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalProject
-------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N')))
+------------------------filter(OR[(promotion.p_channel_email = 'N'),(promotion.p_channel_event = 'N')])
 --------------------------PhysicalOlapScan[promotion]
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[item]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query78.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query78.out
index 3b2d980a6ac2c7..0663ee2198a5fe 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query78.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query78.out
@@ -5,7 +5,7 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------filter(((coalesce(ws_qty, 0) > 0) OR (coalesce(cs_qty, 0) > 0)))
+----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)])
 ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=()
 --------------PhysicalProject
 ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=()
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query79.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query79.out
index b8b18b2657766b..e3d8f3af326d91 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query79.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query79.out
@@ -22,7 +22,7 @@ PhysicalResultSink
 ----------------------------------filter((date_dim.d_dow = 1) and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((household_demographics.hd_dep_count = 7) OR (household_demographics.hd_vehicle_count > -1)))
+------------------------------filter(OR[(household_demographics.hd_dep_count = 7),(household_demographics.hd_vehicle_count > -1)])
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query85.out
index 1781cd24d5b9f5..c734d7945f3a56 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF5 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('IA', 'NC', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('GA', 'WI', 'WV') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('KY', 'OK', 'VA') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV'))
 ------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'D') AND (cd1.cd_education_status = 'Primary')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'College')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'U') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
@@ -37,7 +37,7 @@ PhysicalResultSink
 ----------------------------------------------filter((date_dim.d_year = 1998))
 ------------------------------------------------PhysicalOlapScan[date_dim]
 ------------------------------------PhysicalProject
---------------------------------------filter(((((cd1.cd_marital_status = 'D') AND (cd1.cd_education_status = 'Primary')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'College'))) OR ((cd1.cd_marital_status = 'U') AND (cd1.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U'))
+--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U'))
 ----------------------------------------PhysicalOlapScan[customer_demographics]
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[web_page]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out
index 3c8f0335f1f415..d016dca48db832 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+----------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+----------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(((((household_demographics.hd_dep_count = 0) AND (household_demographics.hd_vehicle_count <= 2)) OR ((household_demographics.hd_dep_count = -1) AND (household_demographics.hd_vehicle_count <= 1))) OR ((household_demographics.hd_dep_count = 3) AND (household_demographics.hd_vehicle_count <= 5))) and hd_dep_count IN (-1, 0, 3))
+--------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query89.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query89.out
index 786de940dad5d8..b8751687a0ff29 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query89.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query89.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((i_category IN ('Books', 'Children', 'Electronics') AND i_class IN ('audio', 'history', 'school-uniforms')) OR (i_category IN ('Men', 'Shoes', 'Sports') AND i_class IN ('pants', 'tennis', 'womens'))) and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens'))
+------------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('audio', 'history', 'school-uniforms')],AND[i_category IN ('Men', 'Shoes', 'Sports'),i_class IN ('pants', 'tennis', 'womens')]] and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens'))
 --------------------------------------------PhysicalOlapScan[item]
 ------------------------------------PhysicalProject
 --------------------------------------filter((date_dim.d_year = 2001))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query91.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query91.out
index 624685e9dab961..9d3c77acb23ca8 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query91.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query91.out
@@ -28,7 +28,7 @@ PhysicalResultSink
 ----------------------------------------PhysicalProject
 ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1
 ----------------------------------------PhysicalProject
-------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree'))) and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
+------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W'))
 --------------------------------------------PhysicalOlapScan[customer_demographics]
 ------------------------------------PhysicalProject
 --------------------------------------filter((hd_buy_potential like 'Unknown%'))
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out
index bf86e68a7f9837..ad82594176a219 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.out
index f434b57fc9f511..b98149f8668a1c 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 --------------------------------PhysicalOlapScan[supplier] apply RFs: RF4
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[customer] apply RFs: RF3
-------------------NestedLoopJoin[INNER_JOIN](((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')))
+------------------NestedLoopJoin[INNER_JOIN]OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]]
 --------------------PhysicalProject
 ----------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ------------------------PhysicalOlapScan[nation]
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q19.out
index bf86e68a7f9837..ad82594176a219 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q19.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q7.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q7.out
index f1424cc0f730c4..957b17a7402749 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q7.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q7.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=((((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')))) build RFs:RF4 c_custkey->[o_custkey]
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=(OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]]) build RFs:RF4 c_custkey->[o_custkey]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF3 l_orderkey->[o_orderkey]
 ----------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q19.out
index bf86e68a7f9837..ad82594176a219 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q19.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q7.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q7.out
index f1424cc0f730c4..957b17a7402749 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q7.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q7.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------PhysicalDistribute[DistributionSpecHash]
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=((((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')))) build RFs:RF4 c_custkey->[o_custkey]
+----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=(OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]]) build RFs:RF4 c_custkey->[o_custkey]
 ------------------PhysicalProject
 --------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF3 l_orderkey->[o_orderkey]
 ----------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out
index bf86e68a7f9837..ad82594176a219 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND (part.p_size <= 5)) OR (((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND (part.p_size <= 10))) OR (((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND (part.p_size <= 15))) and (part.p_size >= 1) and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.out
index f434b57fc9f511..b98149f8668a1c 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 --------------------------------PhysicalOlapScan[supplier] apply RFs: RF4
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[customer] apply RFs: RF3
-------------------NestedLoopJoin[INNER_JOIN](((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')))
+------------------NestedLoopJoin[INNER_JOIN]OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]]
 --------------------PhysicalProject
 ----------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ------------------------PhysicalOlapScan[nation]
diff --git a/regression-test/suites/nereids_p0/huge_compound/huge_compound.groovy b/regression-test/suites/nereids_p0/huge_compound/huge_compound.groovy
new file mode 100644
index 00000000000000..3eb38185aef06e
--- /dev/null
+++ b/regression-test/suites/nereids_p0/huge_compound/huge_compound.groovy
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("huge_compound") {
+    sql """
+        drop table if exists tyt_old_order;
+        CREATE TABLE `tyt_old_order` (
+        `n` varchar(65533) NULL,
+        `c` varchar(65533) NULL
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`n`)
+        DISTRIBUTED BY HASH(`n`) BUCKETS AUTO
+        PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1"
+        );
+        insert into tyt_old_order values("72", "93"), ("62", "xx")
+    """
+
+     qt_1 """
+         SELECT * 
+         FROM tyt_old_order 
+         where(n='72'AND c='93')OR(n='04'AND c='85')OR(n='43'AND c='37')OR(n='92'AND c='09')OR(n='85'AND c='69')OR(n='16'AND c='88')OR(n='87'AND c='22')OR(n='38'AND c='44')OR(n='65'AND c='22')OR(n='83'AND c='12')OR(n='14'AND c='44')OR(n='64'AND c='33')OR(n='44'AND c='58')OR(n='37'AND c='27')OR(n='59'AND c='36')OR(n='15'AND c='66')OR(n='44'AND c='66')OR(n='59'AND c='23')OR(n='08'AND c='87')OR(n='03'AND c='09')OR(n='85'AND c='26')OR(n='61'AND c='08')OR(n='23'AND c='13')OR(n='96'AND c='97')OR(n='15'AND c='44')OR(n='33'AND c='33')OR(n='11'AND c='87')OR(n='49'AND c='55')OR(n='70'AND c='29')OR(n='47'AND c='85')OR(n='14'AND c='18')OR(n='77'AND c='17')OR(n='66'AND c='07')OR(n='59'AND c='16')OR(n='48'AND c='11')OR(n='66'AND c='85')OR(n='46'AND c='65')OR(n='26'AND c='68')OR(n='63'AND c='22')OR(n='24'AND c='96')OR(n='17'AND c='00')OR(n='35'AND c='00')OR(n='06'AND c='26')OR(n='55'AND c='65')OR(n='16'AND c='17')OR(n='89'AND c='19')OR(n='57'AND c='30')OR(n='87'AND c='99')OR(n='27'AND c='89')OR(n='72'AND c='91')OR(n='47'AND c='51')OR(n='39'AND c='29')OR(n='87'AND c='45')OR(n='47'AND c='85')OR(n='99'AND c='40')OR(n='04'AND c='67')OR(n='65'AND c='50')OR(n='23'AND c='13')OR(n='54'AND c='44')OR(n='33'AND c='45')OR(n='48'AND c='86')OR(n='82'AND c='33')OR(n='15'AND c='85')OR(n='11'AND c='85')OR(n='33'AND c='93')OR(n='74'AND c='88')OR(n='07'AND c='58')OR(n='34'AND c='35')OR(n='90'AND c='18')OR(n='60'AND c='34')OR(n='78'AND c='77')OR(n='59'AND c='15')OR(n='89'AND c='79') 
+            OR(n='69'AND c='69')OR(n='91'AND c='84')OR(n='28'AND c='76')OR(n='71'AND c='77')OR(n='12'AND c='65')OR(n='08'AND c='36')OR(n='23'AND c='16')OR(n='51'AND c='11')OR(n='48'AND c='19')OR(n='58'AND c='16')OR(n='77'AND c='15')OR(n='80'AND c='15')OR(n='75'AND c='15')OR(n='78'AND c='15')OR(n='61'AND c='15')OR(n='59'AND c='15')OR(n='07'AND c='15')OR(n='18'AND c='15')OR(n='59'AND c='15')OR(n='94'AND c='15')OR(n='55'AND c='15')OR(n='71'AND c='15')OR(n='11'AND c='65')OR(n='42'AND c='48')OR(n='91'AND c='00')OR(n='74'AND c='77')OR(n='20'AND c='80')OR(n='95'AND c='65')OR(n='12'AND c='00')OR(n='26'AND c='77')OR(n='10'AND c='48')OR(n='65'AND c='80')OR(n='20'AND c='48')OR(n='86'AND c='00')OR(n='22'AND c='77')OR(n='16'AND c='65')OR(n='56'AND c='80')OR(n='95'AND c='74')OR(n='61'AND c='68')OR(n='08'AND c='65')OR(n='05'AND c='48')OR(n='90'AND c='00')OR(n='96'AND c='77')OR(n='77'AND c='80')OR(n='46'AND c='65')OR(n='11'AND c='48')OR(n='79'AND c='00')OR(n='54'AND c='77')OR(n='25'AND c='80')OR(n='12'AND c='65')OR(n='39'AND c='48')OR(n='39'AND c='00')OR(n='24'AND c='77')OR(n='78'AND c='80')OR(n='21'AND c='02')OR(n='52'AND c='68')OR(n='12'AND c='66')OR(n='39'AND c='82')OR(n='36'AND c='19')OR(n='08'AND c='19')OR(n='29'AND c='19')OR(n='81'AND c='19')OR(n='25'AND c='19')OR(n='96'AND c='08')OR(n='98'AND c='19')OR(n='40'AND c='19')OR(n='03'AND c='19')OR(n='17'AND c='19')OR(n='97'AND c='19')OR(n='91'AND c='19')OR(n='21'AND c='19')OR(n='46'AND c='19')OR(n='69'AND c='19')OR(n='75'AND c='19')OR(n='68'AND c='19')OR(n='64'AND c='19')OR(n='31'AND c='19') 
+            OR(n='89'AND c='19')OR(n='99'AND c='19')OR(n='60'AND c='19')OR(n='89'AND c='48')OR(n='40'AND c='00')OR(n='04'AND c='77')OR(n='40'AND c='65')OR(n='97'AND c='80')OR(n='59'AND c='48')OR(n='49'AND c='00')OR(n='38'AND c='77')OR(n='89'AND c='65')OR(n='25'AND c='80')OR(n='23'AND c='48')OR(n='88'AND c='00')OR(n='25'AND c='77')OR(n='82'AND c='65')OR(n='94'AND c='80')OR(n='26'AND c='65')OR(n='58'AND c='48')OR(n='95'AND c='00')OR(n='85'AND c='77')OR(n='46'AND c='80')OR(n='65'AND c='65')OR(n='40'AND c='48')OR(n='14'AND c='00')OR(n='93'AND c='77')OR(n='90'AND c='80')OR(n='51'AND c='65')OR(n='40'AND c='48')OR(n='74'AND c='00')OR(n='24'AND c='77')OR(n='56'AND c='80')OR(n='01'AND c='00')OR(n='79'AND c='77')OR(n='24'AND c='80')OR(n='48'AND c='65')OR(n='29'AND c='48')OR(n='09'AND c='00')OR(n='16'AND c='77')OR(n='61'AND c='80')OR(n='64'AND c='23')OR(n='52'AND c='65')OR(n='86'AND c='48')OR(n='62'AND c='00')OR(n='73'AND c='77')OR(n='36'AND c='80')OR(n='11'AND c='65')OR(n='13'AND c='48')OR(n='53'AND c='00')OR(n='39'AND c='77')OR(n='06'AND c='80')OR(n='21'AND c='76')OR(n='53'AND c='88')OR(n='37'AND c='53')OR(n='50'AND c='00')OR(n='42'AND c='00')OR(n='03'AND c='15')OR(n='82'AND c='71')OR(n='15'AND c='77')OR(n='79'AND c='18')OR(n='09'AND c='96')OR(n='15'AND c='98')OR(n='87'AND c='33')OR(n='04'AND c='80')OR(n='64'AND c='80')OR(n='88'AND c='05')OR(n='75'AND c='66')OR(n='85'AND c='80')OR(n='18'AND c='80')OR(n='12'AND c='58')OR(n='28'AND c='60')OR(n='61'AND c='55')OR(n='78'AND c='05')OR(n='39'AND c='77')OR(n='78'AND c='77')OR(n='02'AND c='65')OR(n='29'AND c='11')OR(n='49'AND c='87')OR(n='49'AND c='19')OR(n='43'AND c='19')OR(n='25'AND c='36')OR(n='90'AND c='84')OR(n='52'AND c='87')OR(n='46'AND c='77') 
+            OR(n='99'AND c='89')OR(n='83'AND c='53')OR(n='73'AND c='69')OR(n='38'AND c='18')OR(n='06'AND c='02')OR(n='21'AND c='51')OR(n='27'AND c='88')OR(n='77'AND c='72')OR(n='28'AND c='92')OR(n='90'AND c='55')OR(n='75'AND c='05')OR(n='57'AND c='99')OR(n='45'AND c='07')OR(n='92'AND c='99')OR(n='51'AND c='55')OR(n='29'AND c='77')OR(n='82'AND c='30')OR(n='83'AND c='40')OR(n='79'AND c='89')OR(n='27'AND c='99')OR(n='25'AND c='06')OR(n='71'AND c='51')OR(n='57'AND c='39')OR(n='96'AND c='18')OR(n='18'AND c='02')OR(n='87'AND c='69')OR(n='53'AND c='01')OR(n='14'AND c='50')OR(n='19'AND c='33')OR(n='12'AND c='33')OR(n='99'AND c='86')OR(n='78'AND c='99')OR(n='98'AND c='66')OR(n='94'AND c='52')OR(n='96'AND c='50')OR(n='36'AND c='16')OR(n='58'AND c='18')OR(n='02'AND c='96')OR(n='84'AND c='11')OR(n='94'AND c='73')OR(n='97'AND c='08')OR(n='19'AND c='33')OR(n='41'AND c='06')OR(n='05'AND c='58')OR(n='12'AND c='30')OR(n='58'AND c='22')OR(n='41'AND c='66')OR(n='20'AND c='71')OR(n='41'AND c='71')OR(n='95'AND c='44')OR(n='90'AND c='58')OR(n='96'AND c='06')OR(n='18'AND c='00')OR(n='35'AND c='38')OR(n='45'AND c='02')OR(n='04'AND c='00')OR(n='87'AND c='99')OR(n='51'AND c='38')OR(n='42'AND c='61')OR(n='09'AND c='14')OR(n='48'AND c='70')OR(n='70'AND c='04')OR(n='39'AND c='97')OR(n='72'AND c='32')OR(n='46'AND c='72')OR(n='19'AND c='13')OR(n='98'AND c='37')OR(n='34'AND c='18')OR(n='64'AND c='71')OR(n='73'AND c='87')OR(n='82'AND c='19')OR(n='14'AND c='43')OR(n='36'AND c='22')OR(n='06'AND c='71')OR(n='08'AND c='71')OR(n='61'AND c='71')OR(n='71'AND c='88')OR(n='74'AND c='71') 
+            OR(n='64'AND c='89')OR(n='13'AND c='71')OR(n='77'AND c='35')OR(n='66'AND c='11')OR(n='59'AND c='85')OR(n='29'AND c='66')OR(n='85'AND c='50')OR(n='18'AND c='66')OR(n='10'AND c='92')OR(n='76'AND c='66')OR(n='31'AND c='99')OR(n='86'AND c='91')OR(n='92'AND c='88')OR(n='40'AND c='18')OR(n='18'AND c='77')OR(n='72'AND c='63')OR(n='29'AND c='77')OR(n='24'AND c='87')OR(n='03'AND c='51')OR(n='81'AND c='01')OR(n='50'AND c='72')OR(n='25'AND c='33')OR(n='99'AND c='68')OR(n='51'AND c='13')OR(n='59'AND c='88')OR(n='31'AND c='97')OR(n='71'AND c='10')OR(n='15'AND c='91')OR(n='31'AND c='88')OR(n='92'AND c='00')OR(n='11'AND c='75')OR(n='06'AND c='99')OR(n='46'AND c='32')OR(n='13'AND c='09')OR(n='18'AND c='33')OR(n='88'AND c='58')OR(n='90'AND c='77')OR(n='22'AND c='10')OR(n='37'AND c='19')OR(n='52'AND c='10')OR(n='30'AND c='22')OR(n='98'AND c='66')OR(n='65'AND c='66')OR(n='22'AND c='98')OR(n='99'AND c='77')OR(n='84'AND c='32')OR(n='73'AND c='03')OR(n='99'AND c='58')OR(n='63'AND c='77')OR(n='50'AND c='99')OR(n='23'AND c='35')OR(n='40'AND c='88')OR(n='76'AND c='88')OR(n='10'AND c='88')OR(n='37'AND c='86')OR(n='94'AND c='08')OR(n='84'AND c='30')OR(n='95'AND c='88')OR(n='81'AND c='25')OR(n='34'AND c='25')OR(n='58'AND c='66')OR(n='40'AND c='21')OR(n='02'AND c='88')OR(n='44'AND c='53')OR(n='42'AND c='99')OR(n='56'AND c='16')OR(n='54'AND c='37')OR(n='89'AND c='97')OR(n='49'AND c='22')OR(n='97'AND c='81')OR(n='56'AND c='85')OR(n='13'AND c='93')OR(n='06'AND c='93')OR(n='40'AND c='44')OR(n='84'AND c='09')OR(n='67'AND c='83')OR(n='56'AND c='88')OR(n='72'AND c='58')OR(n='04'AND c='68') 
+            OR(n='96'AND c='68')OR(n='74'AND c='39')OR(n='16'AND c='14')OR(n='28'AND c='00')OR(n='83'AND c='60')OR(n='61'AND c='88')OR(n='13'AND c='70')OR(n='79'AND c='36')OR(n='26'AND c='91')OR(n='41'AND c='91')OR(n='98'AND c='91')OR(n='29'AND c='50')OR(n='02'AND c='45')OR(n='51'AND c='77')OR(n='53'AND c='05')OR(n='02'AND c='14')OR(n='88'AND c='50')OR(n='93'AND c='90')OR(n='32'AND c='09')OR(n='53'AND c='56')OR(n='27'AND c='16')OR(n='30'AND c='68')OR(n='62'AND c='26')OR(n='33'AND c='26')OR(n='04'AND c='74')OR(n='79'AND c='08')OR(n='47'AND c='28')OR(n='93'AND c='44')OR(n='28'AND c='34')OR(n='89'AND c='79')OR(n='63'AND c='88')OR(n='51'AND c='33')OR(n='72'AND c='66')OR(n='22'AND c='79')OR(n='97'AND c='82')OR(n='45'AND c='61')OR(n='02'AND c='33')OR(n='83'AND c='08')OR(n='46'AND c='11')OR(n='09'AND c='33')OR(n='94'AND c='72')OR(n='82'AND c='09')OR(n='15'AND c='92')OR(n='54'AND c='50')OR(n='17'AND c='17')OR(n='89'AND c='26')OR(n='79'AND c='76')OR(n='08'AND c='76')OR(n='46'AND c='94')OR(n='73'AND c='78')OR(n='61'AND c='16')OR(n='42'AND c='43')OR(n='79'AND c='97')OR(n='43'AND c='02')OR(n='50'AND c='15')OR(n='26'AND c='21')OR(n='60'AND c='56')OR(n='08'AND c='51')OR(n='46'AND c='21')OR(n='29'AND c='37')OR(n='03'AND c='58')OR(n='84'AND c='79')OR(n='51'AND c='09')OR(n='15'AND c='09')OR(n='68'AND c='77')OR(n='45'AND c='77')OR(n='65'AND c='17')OR(n='76'AND c='16')OR(n='07'AND c='10')OR(n='13'AND c='56')OR(n='75'AND c='93')OR(n='17'AND c='51')OR(n='86'AND c='68')OR(n='73'AND c='68')OR(n='24'AND c='99')OR(n='88'AND c='99')OR(n='15'AND c='33')OR(n='24'AND c='95')OR(n='36'AND c='83') 
+            OR(n='12'AND c='83')OR(n='32'AND c='83')OR(n='62'AND c='83')OR(n='35'AND c='56')OR(n='01'AND c='67')OR(n='37'AND c='85')OR(n='89'AND c='08')OR(n='57'AND c='39')OR(n='95'AND c='18')OR(n='31'AND c='08')OR(n='06'AND c='08')OR(n='69'AND c='79')OR(n='69'AND c='70')OR(n='43'AND c='06')OR(n='55'AND c='53')OR(n='75'AND c='91')OR(n='26'AND c='66')OR(n='14'AND c='22')OR(n='23'AND c='38')OR(n='07'AND c='18')OR(n='22'AND c='29')OR(n='39'AND c='53')OR(n='60'AND c='28')OR(n='12'AND c='12')OR(n='45'AND c='03')OR(n='54'AND c='60')OR(n='53'AND c='98')OR(n='11'AND c='78')OR(n='43'AND c='53')OR(n='80'AND c='98')OR(n='04'AND c='69')OR(n='75'AND c='68')OR(n='91'AND c='26')OR(n='55'AND c='93')OR(n='12'AND c='88')OR(n='10'AND c='03')OR(n='27'AND c='93')OR(n='96'AND c='89')OR(n='83'AND c='66')OR(n='09'AND c='23')OR(n='27'AND c='68')OR(n='71'AND c='33')OR(n='21'AND c='42')OR(n='49'AND c='56')OR(n='71'AND c='81')OR(n='83'AND c='82')OR(n='91'AND c='82')OR(n='82'AND c='92')OR(n='09'AND c='50')OR(n='99'AND c='45')OR(n='52'AND c='45')OR(n='47'AND c='70')OR(n='98'AND c='88')OR(n='01'AND c='35')OR(n='81'AND c='85')OR(n='02'AND c='53')OR(n='13'AND c='91')OR(n='75'AND c='88')OR(n='30'AND c='46')OR(n='56'AND c='90')OR(n='64'AND c='54')OR(n='37'AND c='95')OR(n='38'AND c='88')OR(n='89'AND c='05')OR(n='73'AND c='88')OR(n='60'AND c='51')OR(n='48'AND c='60')OR(n='25'AND c='62')OR(n='61'AND c='88')OR(n='74'AND c='19')OR(n='24'AND c='62')OR(n='44'AND c='68')OR(n='15'AND c='60')OR(n='89'AND c='69')OR(n='06'AND c='57')OR(n='83'AND c='49')OR(n='66'AND c='99')OR(n='31'AND c='11')OR(n='68'AND c='96') 
+            OR(n='49'AND c='48')OR(n='59'AND c='95')OR(n='30'AND c='05')OR(n='63'AND c='20')OR(n='68'AND c='85')OR(n='75'AND c='99')OR(n='46'AND c='29')OR(n='93'AND c='99')OR(n='48'AND c='99')OR(n='24'AND c='88')OR(n='47'AND c='15')OR(n='52'AND c='99')OR(n='17'AND c='91')OR(n='12'AND c='53')OR(n='38'AND c='16')OR(n='82'AND c='82')OR(n='79'AND c='11')OR(n='99'AND c='01')OR(n='40'AND c='68')OR(n='33'AND c='79')OR(n='32'AND c='07')OR(n='39'AND c='26')OR(n='31'AND c='81')OR(n='16'AND c='66')OR(n='75'AND c='59')OR(n='76'AND c='77')OR(n='22'AND c='82')OR(n='49'AND c='08')OR(n='09'AND c='18')OR(n='79'AND c='87')OR(n='88'AND c='32')OR(n='14'AND c='16')OR(n='47'AND c='37')OR(n='24'AND c='33')OR(n='03'AND c='53')OR(n='96'AND c='88')OR(n='66'AND c='88')OR(n='82'AND c='19')OR(n='29'AND c='26')OR(n='33'AND c='48')OR(n='75'AND c='19')OR(n='76'AND c='39')OR(n='39'AND c='11')OR(n='41'AND c='22')OR(n='96'AND c='76')OR(n='63'AND c='33')OR(n='13'AND c='00')OR(n='08'AND c='28')OR(n='56'AND c='20')OR(n='28'AND c='20')OR(n='54'AND c='88')OR(n='53'AND c='78')OR(n='95'AND c='79')OR(n='08'AND c='75')OR(n='44'AND c='31')OR(n='45'AND c='47')OR(n='47'AND c='96')OR(n='84'AND c='33')OR(n='73'AND c='55')OR(n='74'AND c='77')OR(n='66'AND c='19')OR(n='20'AND c='19')OR(n='42'AND c='19')OR(n='16'AND c='19')OR(n='10'AND c='19')OR(n='37'AND c='65')OR(n='75'AND c='77')OR(n='64'AND c='48')OR(n='19'AND c='00')OR(n='93'AND c='80')OR(n='83'AND c='65')OR(n='45'AND c='77')OR(n='91'AND c='48')OR(n='81'AND c='00')OR(n='74'AND c='80') 
+            OR(n='88'AND c='68')OR(n='21'AND c='34')OR(n='17'AND c='17')OR(n='40'AND c='79')OR(n='90'AND c='71')OR(n='80'AND c='22')OR(n='27'AND c='07')OR(n='12'AND c='88')OR(n='24'AND c='85')OR(n='16'AND c='38')OR(n='67'AND c='54')OR(n='06'AND c='20')OR(n='97'AND c='20')OR(n='41'AND c='66')OR(n='99'AND c='51')OR(n='88'AND c='55')OR(n='44'AND c='70')OR(n='23'AND c='91')OR(n='94'AND c='78')OR(n='64'AND c='00')OR(n='66'AND c='83')OR(n='83'AND c='27')OR(n='06'AND c='91')OR(n='28'AND c='23')OR(n='89'AND c='20')OR(n='79'AND c='13')OR(n='25'AND c='60')OR(n='92'AND c='28')OR(n='28'AND c='66')OR(n='84'AND c='76')OR(n='25'AND c='76')OR(n='59'AND c='66')OR(n='18'AND c='43')OR(n='70'AND c='58')OR(n='11'AND c='77')OR(n='22'AND c='83')OR(n='28'AND c='00')OR(n='05'AND c='00')OR(n='16'AND c='76')OR(n='74'AND c='98')OR(n='41'AND c='06')OR(n='99'AND c='06')OR(n='58'AND c='76')OR(n='24'AND c='22')OR(n='85'AND c='86')OR(n='30'AND c='67')OR(n='02'AND c='76')OR(n='40'AND c='00')OR(n='96'AND c='43')OR(n='51'AND c='39')OR(n='73'AND c='77')OR(n='74'AND c='75')OR(n='85'AND c='85')OR(n='97'AND c='93')OR(n='28'AND c='80')OR(n='29'AND c='80')OR(n='84'AND c='41')OR(n='05'AND c='86')OR(n='50'AND c='86')OR(n='89'AND c='00')OR(n='64'AND c='48')OR(n='85'AND c='65')OR(n='33'AND c='77')OR(n='85'AND c='80')OR(n='70'AND c='99')OR(n='63'AND c='69')OR(n='16'AND c='86')OR(n='42'AND c='86')OR(n='77'AND c='48')OR(n='10'AND c='65')OR(n='95'AND c='77')OR(n='81'AND c='00')OR(n='94'AND c='08')OR(n='09'AND c='44')OR(n='70'AND c='22')OR(n='32'AND c='51')OR(n='16'AND c='51')OR(n='47'AND c='99')OR(n='30'AND c='99')OR(n='32'AND c='99')
+            OR(n='20'AND c='86')OR(n='36'AND c='86')OR(n='50'AND c='86')OR(n='77'AND c='86')OR(n='62'AND c='86')OR(n='92'AND c='86')OR(n='43'AND c='86')OR(n='40'AND c='86')OR(n='26'AND c='22')OR(n='99'AND c='35')OR(n='06'AND c='48')OR(n='77'AND c='65')OR(n='50'AND c='77')OR(n='14'AND c='00')OR(n='85'AND c='95')OR(n='21'AND c='48')OR(n='05'AND c='65')OR(n='19'AND c='77')OR(n='03'AND c='00')OR(n='65'AND c='19')OR(n='30'AND c='48')OR(n='93'AND c='65')OR(n='48'AND c='77')OR(n='74'AND c='00')OR(n='23'AND c='68')OR(n='73'AND c='22')OR(n='10'AND c='00')OR(n='55'AND c='48')OR(n='01'AND c='65')OR(n='17'AND c='77')OR(n='16'AND c='80')OR(n='44'AND c='88')OR(n='85'AND c='00')OR(n='30'AND c='48')OR(n='03'AND c='65')OR(n='91'AND c='77')OR(n='94'AND c='80')OR(n='05'AND c='00')OR(n='52'AND c='48')OR(n='00'AND c='65')OR(n='77'AND c='77')OR(n='77'AND c='80')OR(n='12'AND c='48')OR(n='39'AND c='65')OR(n='60'AND c='77')OR(n='53'AND c='00')OR(n='06'AND c='80')OR(n='22'AND c='97')OR(n='47'AND c='15')OR(n='06'AND c='15')OR(n='58'AND c='48')OR(n='99'AND c='65')OR(n='96'AND c='77')OR(n='40'AND c='00')OR(n='31'AND c='80')OR(n='45'AND c='00')OR(n='67'AND c='48')OR(n='63'AND c='65')OR(n='91'AND c='77')OR(n='41'AND c='80')OR(n='81'AND c='99')OR(n='87'AND c='99')OR(n='67'AND c='99')OR(n='46'AND c='00')OR(n='69'AND c='48')OR(n='62'AND c='65')OR(n='80'AND c='77')OR(n='92'AND c='80')OR(n='44'AND c='00')OR(n='07'AND c='48')OR(n='74'AND c='65')OR(n='92'AND c='77')OR(n='79'AND c='80')OR(n='72'AND c='12')OR(n='42'AND c='00')OR(n='23'AND c='48')OR(n='23'AND c='65')OR(n='88'AND c='77') 
+            OR(n='41'AND c='80')OR(n='26'AND c='71')OR(n='95'AND c='85')OR(n='59'AND c='15')OR(n='63'AND c='70')OR(n='02'AND c='10')OR(n='26'AND c='33')OR(n='94'AND c='16')OR(n='99'AND c='60')OR(n='83'AND c='38')OR(n='40'AND c='85')OR(n='56'AND c='73')OR(n='58'AND c='85')OR(n='16'AND c='67')OR(n='36'AND c='36')OR(n='86'AND c='05')OR(n='73'AND c='82')OR(n='22'AND c='44')OR(n='04'AND c='86')OR(n='74'AND c='95')OR(n='83'AND c='56')OR(n='29'AND c='07')OR(n='74'AND c='71')OR(n='19'AND c='00')OR(n='79'AND c='22')OR(n='07'AND c='91')OR(n='48'AND c='00')OR(n='88'AND c='78')OR(n='05'AND c='88')OR(n='81'AND c='52')OR(n='20'AND c='52')OR(n='30'AND c='51')OR(n='77'AND c='32')OR(n='62'AND c='81')OR(n='78'AND c='97')OR(n='83'AND c='88')OR(n='12'AND c='96')OR(n='58'AND c='13')OR(n='69'AND c='10')OR(n='90'AND c='13')OR(n='60'AND c='13')OR(n='28'AND c='99')OR(n='83'AND c='45')OR(n='67'AND c='01')OR(n='05'AND c='88')OR(n='55'AND c='30')OR(n='74'AND c='40')OR(n='77'AND c='05')OR(n='65'AND c='44')OR(n='62'AND c='93')OR(n='70'AND c='88')OR(n='03'AND c='71')OR(n='35'AND c='86')OR(n='03'AND c='33')OR(n='64'AND c='62')OR(n='86'AND c='55')OR(n='52'AND c='68')OR(n='98'AND c='22')OR(n='81'AND c='96')OR(n='73'AND c='73')OR(n='26'AND c='33')OR(n='77'AND c='99')OR(n='61'AND c='30')OR(n='28'AND c='92')OR(n='28'AND c='66')OR(n='17'AND c='16')OR(n='27'AND c='62')OR(n='43'AND c='22')OR(n='97'AND c='00')OR(n='03'AND c='48')OR(n='65'AND c='65')OR(n='87'AND c='77')OR(n='93'AND c='80')OR(n='75'AND c='62')OR(n='29'AND c='19')OR(n='47'AND c='56')OR(n='42'AND c='33')OR(n='24'AND c='38')OR(n='08'AND c='52')OR(n='19'AND c='77')OR(n='30'AND c='26')OR(n='86'AND c='44')OR(n='94'AND c='95')OR(n='41'AND c='23')OR(n='44'AND c='51')OR(n='10'AND c='66')OR(n='59'AND c='54')OR(n='91'AND c='31')OR(n='95'AND c='99') 
+            OR(n='73'AND c='20')OR(n='57'AND c='22')OR(n='81'AND c='88')OR(n='95'AND c='00')OR(n='96'AND c='88')OR(n='80'AND c='05')OR(n='43'AND c='22')OR(n='70'AND c='99')OR(n='99'AND c='78')OR(n='47'AND c='78')OR(n='85'AND c='77')OR(n='77'AND c='62')OR(n='64'AND c='67')OR(n='40'AND c='97')OR(n='48'AND c='68')OR(n='42'AND c='40')OR(n='54'AND c='02')OR(n='47'AND c='22')OR(n='77'AND c='62')OR(n='96'AND c='02')OR(n='65'AND c='87')OR(n='94'AND c='95')OR(n='67'AND c='52')OR(n='91'AND c='41')OR(n='87'AND c='06')OR(n='55'AND c='09')OR(n='20'AND c='89')OR(n='23'AND c='19')OR(n='59'AND c='27')OR(n='78'AND c='57')OR(n='67'AND c='97')OR(n='36'AND c='33')OR(n='71'AND c='27')OR(n='60'AND c='85')OR(n='06'AND c='69')OR(n='46'AND c='12')OR(n='71'AND c='00')OR(n='33'AND c='87')OR(n='26'AND c='31')OR(n='68'AND c='43')OR(n='47'AND c='41')OR(n='25'AND c='11')OR(n='67'AND c='56')OR(n='62'AND c='21')OR(n='92'AND c='88')OR(n='35'AND c='62')OR(n='34'AND c='99')OR(n='39'AND c='09')OR(n='42'AND c='22')OR(n='67'AND c='88')OR(n='79'AND c='65')OR(n='72'AND c='77')OR(n='21'AND c='38')OR(n='81'AND c='99')OR(n='06'AND c='40')OR(n='06'AND c='00')OR(n='99'AND c='99')OR(n='11'AND c='88')OR(n='38'AND c='55')OR(n='02'AND c='55')OR(n='44'AND c='99')OR(n='47'AND c='09')OR(n='02'AND c='33')OR(n='67'AND c='85')OR(n='00'AND c='53')OR(n='16'AND c='96')OR(n='97'AND c='20')OR(n='40'AND c='56')OR(n='59'AND c='73')OR(n='22'AND c='77')OR(n='88'AND c='55')OR(n='06'AND c='25')OR(n='17'AND c='19')OR(n='51'AND c='03')OR(n='30'AND c='22')OR(n='15'AND c='08')OR(n='95'AND c='00')OR(n='25'AND c='88')OR(n='54'AND c='88')OR(n='81'AND c='09')OR(n='06'AND c='33')OR(n='51'AND c='32')OR(n='79'AND c='32')OR(n='13'AND c='32')OR(n='54'AND c='39')OR(n='55'AND c='99')OR(n='47'AND c='30') 
+            OR(n='03'AND c='66')OR(n='47'AND c='45')OR(n='21'AND c='08')OR(n='01'AND c='18')OR(n='71'AND c='65')OR(n='37'AND c='65')OR(n='66'AND c='99')OR(n='93'AND c='73')OR(n='46'AND c='33')OR(n='45'AND c='18')OR(n='10'AND c='27')OR(n='25'AND c='66')OR(n='84'AND c='71')OR(n='91'AND c='71')OR(n='02'AND c='71')OR(n='81'AND c='33')OR(n='49'AND c='44')OR(n='80'AND c='55')OR(n='22'AND c='05')OR(n='75'AND c='62')OR(n='98'AND c='67')OR(n='75'AND c='18')OR(n='02'AND c='00')OR(n='83'AND c='80')OR(n='36'AND c='00')OR(n='50'AND c='99')OR(n='36'AND c='37')OR(n='41'AND c='37')OR(n='83'AND c='00')OR(n='80'AND c='48')OR(n='71'AND c='77')OR(n='57'AND c='65')OR(n='77'AND c='80')OR(n='96'AND c='99')OR(n='70'AND c='00')OR(n='37'AND c='48')OR(n='09'AND c='77')OR(n='47'AND c='65')OR(n='23'AND c='80')OR(n='10'AND c='77')OR(n='19'AND c='88')OR(n='56'AND c='88')OR(n='11'AND c='93')OR(n='41'AND c='15')OR(n='00'AND c='15')OR(n='70'AND c='19')OR(n='35'AND c='15')OR(n='26'AND c='15')OR(n='56'AND c='00')OR(n='53'AND c='48')OR(n='18'AND c='77')OR(n='73'AND c='65')OR(n='82'AND c='80')OR(n='67'AND c='19')OR(n='69'AND c='48')OR(n='07'AND c='77')OR(n='44'AND c='00')OR(n='54'AND c='65')OR(n='20'AND c='80')OR(n='62'AND c='19')OR(n='91'AND c='19')OR(n='13'AND c='48')OR(n='05'AND c='77')OR(n='20'AND c='00')OR(n='54'AND c='65')OR(n='43'AND c='80')OR(n='53'AND c='80')OR(n='53'AND c='48')OR(n='02'AND c='77')OR(n='01'AND c='00')OR(n='82'AND c='65')OR(n='22'AND c='80')OR(n='23'AND c='36')OR(n='60'AND c='53')OR(n='68'AND c='53')OR(n='77'AND c='83')OR(n='87'AND c='35')OR(n='51'AND c='32')OR(n='44'AND c='76')OR(n='44'AND c='16')OR(n='06'AND c='89')OR(n='17'AND c='57')OR(n='61'AND c='57')OR(n='06'AND c='18')OR(n='18'AND c='89') 
+            OR(n='17'AND c='18')OR(n='85'AND c='20')OR(n='98'AND c='15')OR(n='22'AND c='27')OR(n='00'AND c='27')OR(n='70'AND c='68')OR(n='23'AND c='08')OR(n='43'AND c='99')OR(n='43'AND c='88')OR(n='30'AND c='44')OR(n='69'AND c='78')OR(n='85'AND c='22')OR(n='00'AND c='62')OR(n='39'AND c='96')OR(n='25'AND c='77')OR(n='92'AND c='29')OR(n='77'AND c='77')OR(n='30'AND c='26')OR(n='16'AND c='88')OR(n='68'AND c='95')OR(n='21'AND c='10')OR(n='52'AND c='18')OR(n='07'AND c='19')OR(n='95'AND c='68')OR(n='01'AND c='37')OR(n='08'AND c='66')OR(n='69'AND c='23')OR(n='68'AND c='38')OR(n='60'AND c='25')OR(n='05'AND c='90')OR(n='02'AND c='87')OR(n='81'AND c='41')OR(n='33'AND c='57')OR(n='65'AND c='52')OR(n='03'AND c='28')OR(n='56'AND c='82')OR(n='49'AND c='79')OR(n='84'AND c='94')OR(n='86'AND c='56')OR(n='58'AND c='08')OR(n='42'AND c='50')OR(n='51'AND c='00')OR(n='55'AND c='55')OR(n='27'AND c='69')OR(n='23'AND c='69')OR(n='47'AND c='82')OR(n='82'AND c='18')OR(n='43'AND c='25')OR(n='84'AND c='87')OR(n='04'AND c='31')OR(n='20'AND c='54')OR(n='04'AND c='76')OR(n='31'AND c='99')OR(n='01'AND c='77')OR(n='30'AND c='77')OR(n='01'AND c='88')OR(n='33'AND c='97')OR(n='59'AND c='43')OR(n='49'AND c='77')OR(n='20'AND c='99')OR(n='99'AND c='82')OR(n='92'AND c='88')OR(n='54'AND c='87')OR(n='71'AND c='76')OR(n='02'AND c='22')OR(n='99'AND c='89')OR(n='18'AND c='84')OR(n='52'AND c='73')OR(n='05'AND c='26')OR(n='01'AND c='88')OR(n='48'AND c='78')OR(n='80'AND c='78')OR(n='29'AND c='86')OR(n='88'AND c='11')OR(n='60'AND c='22')OR(n='59'AND c='88')OR(n='33'AND c='99')OR(n='26'AND c='50')OR(n='62'AND c='59')OR(n='76'AND c='23')OR(n='17'AND c='81')OR(n='44'AND c='99')OR(n='57'AND c='00')OR(n='57'AND c='07')OR(n='22'AND c='78')OR(n='36'AND c='52')OR(n='08'AND c='12')OR(n='87'AND c='34')OR(n='94'AND c='77')OR(n='72'AND c='33')OR(n='56'AND c='89')OR(n='36'AND c='37')OR(n='38'AND c='99')OR(n='02'AND c='01')OR(n='89'AND c='97')OR(n='25'AND c='91') 
+            OR(n='79'AND c='67')OR(n='61'AND c='77')OR(n='50'AND c='75')OR(n='05'AND c='07')OR(n='15'AND c='86')OR(n='60'AND c='11')OR(n='54'AND c='19')OR(n='19'AND c='14')OR(n='29'AND c='92') 
+            OR(n='26'AND c='06')OR(n='63'AND c='78')OR(n='24'AND c='67')OR(n='56'AND c='58')OR(n='33'AND c='58')OR(n='79'AND c='53')OR(n='89'AND c='44')OR(n='52'AND c='14')OR(n='28'AND c='94')OR(n='32'AND c='76')OR(n='99'AND c='87')OR(n='73'AND c='87')OR(n='44'AND c='43')OR(n='33'AND c='33')OR(n='72'AND c='14')OR(n='16'AND c='29')OR(n='26'AND c='86')OR(n='24'AND c='99')OR(n='68'AND c='26')OR(n='99'AND c='24')OR(n='32'AND c='09')OR(n='22'AND c='34')OR(n='36'AND c='93')OR(n='78'AND c='57')OR(n='40'AND c='55')OR(n='81'AND c='01')OR(n='39'AND c='19')OR(n='39'AND c='88')OR(n='52'AND c='11')OR(n='61'AND c='58')OR(n='84'AND c='32')OR(n='42'AND c='83')OR(n='48'AND c='00')OR(n='63'AND c='51')OR(n='28'AND c='18')OR(n='18'AND c='60')OR(n='64'AND c='05')OR(n='50'AND c='18')OR(n='10'AND c='88')OR(n='68'AND c='88')OR(n='14'AND c='08')OR(n='62'AND c='88')OR(n='95'AND c='26')OR(n='78'AND c='76')OR(n='73'AND c='91')OR(n='74'AND c='89')OR(n='60'AND c='55')OR(n='46'AND c='90')OR(n='69'AND c='89')OR(n='52'AND c='98')OR(n='38'AND c='65')OR(n='99'AND c='75')OR(n='53'AND c='88')OR(n='35'AND c='77')OR(n='84'AND c='50')OR(n='50'AND c='77')OR(n='67'AND c='06')OR(n='28'AND c='49')OR(n='43'AND c='55')OR(n='79'AND c='31')OR(n='06'AND c='88')OR(n='41'AND c='93')OR(n='20'AND c='11')OR(n='23'AND c='50')OR(n='08'AND c='55')OR(n='15'AND c='00')OR(n='91'AND c='46')OR(n='03'AND c='88')OR(n='81'AND c='48')OR(n='09'AND c='11')OR(n='00'AND c='88')OR(n='49'AND c='00')OR(n='76'AND c='32')OR(n='58'AND c='06')OR(n='18'AND c='99')OR(n='76'AND c='39')OR(n='14'AND c='33')OR(n='66'AND c='05')OR(n='28'AND c='99')OR(n='20'AND c='90')OR(n='07'AND c='11')OR(n='45'AND c='19')OR(n='44'AND c='19')OR(n='51'AND c='80') 
+            OR(n='64'AND c='81')OR(n='36'AND c='19')OR(n='35'AND c='67')OR(n='35'AND c='19')OR(n='21'AND c='11')OR(n='07'AND c='19')OR(n='54'AND c='19')OR(n='42'AND c='55')OR(n='77'AND c='96')OR(n='39'AND c='33')OR(n='04'AND c='01')OR(n='45'AND c='50')OR(n='69'AND c='88')OR(n='87'AND c='53')OR(n='96'AND c='98')OR(n='28'AND c='11')OR(n='55'AND c='71')OR(n='78'AND c='11')OR(n='50'AND c='77')OR(n='47'AND c='11')OR(n='49'AND c='48')OR(n='56'AND c='77')OR(n='09'AND c='77')OR(n='17'AND c='65')OR(n='51'AND c='80')OR(n='30'AND c='22')OR(n='37'AND c='44')OR(n='88'AND c='62')OR(n='17'AND c='88')OR(n='12'AND c='13')OR(n='24'AND c='49')OR(n='36'AND c='11')OR(n='00'AND c='55')OR(n='61'AND c='04')OR(n='88'AND c='32')OR(n='82'AND c='58')OR(n='42'AND c='60')OR(n='76'AND c='99')OR(n='41'AND c='39')OR(n='91'AND c='57')OR(n='28'AND c='18')OR(n='86'AND c='71')OR(n='18'AND c='71')OR(n='26'AND c='98')OR(n='53'AND c='27')OR(n='05'AND c='67')OR(n='32'AND c='82')OR(n='33'AND c='69')OR(n='27'AND c='68')OR(n='64'AND c='89')OR(n='48'AND c='56')OR(n='80'AND c='91')OR(n='02'AND c='39')OR(n='45'AND c='26')OR(n='67'AND c='88')OR(n='52'AND c='56')OR(n='22'AND c='12')OR(n='31'AND c='93')OR(n='36'AND c='50')OR(n='50'AND c='72')OR(n='34'AND c='44')OR(n='10'AND c='26')OR(n='98'AND c='40')OR(n='16'AND c='88')OR(n='89'AND c='66')OR(n='84'AND c='76')OR(n='55'AND c='32')OR(n='06'AND c='89')OR(n='94'AND c='33')OR(n='40'AND c='76')OR(n='01'AND c='00')OR(n='53'AND c='44')OR(n='36'AND c='78')OR(n='96'AND c='89')OR(n='18'AND c='77')OR(n='03'AND c='66')OR(n='61'AND c='83')OR(n='21'AND c='83')OR(n='56'AND c='32')OR(n='58'AND c='74')OR(n='71'AND c='98')OR(n='66'AND c='26')OR(n='75'AND c='30')OR(n='14'AND c='21')OR(n='58'AND c='81')OR(n='39'AND c='07')OR(n='26'AND c='88')OR(n='94'AND c='40')OR(n='02'AND c='29')OR(n='41'AND c='37')OR(n='20'AND c='92')OR(n='49'AND c='27')OR(n='29'AND c='89')OR(n='39'AND c='98')OR(n='40'AND c='98') 
+            OR(n='95'AND c='33')OR(n='58'AND c='98')OR(n='85'AND c='88')OR(n='61'AND c='40')OR(n='85'AND c='77')OR(n='90'AND c='77')OR(n='52'AND c='77')OR(n='54'AND c='66')OR(n='06'AND c='92')OR(n='89'AND c='14')OR(n='37'AND c='69')OR(n='49'AND c='30')OR(n='64'AND c='20')OR(n='77'AND c='40')OR(n='22'AND c='98')OR(n='21'AND c='88')OR(n='98'AND c='66')OR(n='08'AND c='86')OR(n='65'AND c='18')OR(n='08'AND c='08')OR(n='29'AND c='92')OR(n='33'AND c='77')OR(n='12'AND c='89')OR(n='76'AND c='31')OR(n='12'AND c='31')OR(n='13'AND c='44')OR(n='04'AND c='17')OR(n='79'AND c='88')OR(n='87'AND c='83')OR(n='40'AND c='22')OR(n='74'AND c='19')OR(n='55'AND c='88')OR(n='19'AND c='96')OR(n='45'AND c='88')OR(n='55'AND c='88')OR(n='32'AND c='45')OR(n='10'AND c='37')OR(n='16'AND c='12')OR(n='02'AND c='28')OR(n='24'AND c='37')OR(n='34'AND c='56')OR(n='35'AND c='05')OR(n='48'AND c='50')OR(n='80'AND c='51')OR(n='86'AND c='92')OR(n='73'AND c='72')OR(n='37'AND c='37')OR(n='20'AND c='12')OR(n='24'AND c='28')OR(n='53'AND c='37')OR(n='75'AND c='56')OR(n='65'AND c='05')OR(n='52'AND c='50')OR(n='66'AND c='51')OR(n='81'AND c='92')OR(n='58'AND c='72')OR(n='40'AND c='77')OR(n='93'AND c='77')OR(n='05'AND c='60')OR(n='37'AND c='33')OR(n='06'AND c='71')OR(n='40'AND c='68')OR(n='54'AND c='15')OR(n='03'AND c='11')OR(n='15'AND c='77')OR(n='70'AND c='62')OR(n='42'AND c='28')OR(n='57'AND c='11')OR(n='46'AND c='33')OR(n='83'AND c='17')OR(n='12'AND c='81')OR(n='69'AND c='11')OR(n='76'AND c='05')OR(n='57'AND c='19')OR(n='75'AND c='19')OR(n='06'AND c='19')OR(n='09'AND c='77')OR(n='20'AND c='48')OR(n='45'AND c='80')OR(n='38'AND c='65')OR(n='13'AND c='19')OR(n='42'AND c='19')OR(n='55'AND c='11')OR(n='94'AND c='80')OR(n='59'AND c='77')OR(n='50'AND c='48')OR(n='39'AND c='80')OR(n='24'AND c='65')OR(n='21'AND c='77')OR(n='27'AND c='48')OR(n='92'AND c='80')OR(n='11'AND c='65')OR(n='59'AND c='19')OR(n='18'AND c='19') 
+            OR(n='26'AND c='19')OR(n='15'AND c='48')OR(n='21'AND c='80')OR(n='54'AND c='77')OR(n='74'AND c='44')OR(n='00'AND c='48')OR(n='97'AND c='80')OR(n='72'AND c='77')OR(n='67'AND c='65')OR(n='33'AND c='48')OR(n='03'AND c='80')OR(n='37'AND c='77')OR(n='24'AND c='65')OR(n='05'AND c='15')OR(n='81'AND c='22')OR(n='47'AND c='71')OR(n='14'AND c='78')OR(n='11'AND c='99')OR(n='74'AND c='86')OR(n='37'AND c='95')OR(n='18'AND c='56')OR(n='30'AND c='90')OR(n='09'AND c='03')OR(n='55'AND c='53')OR(n='17'AND c='99')OR(n='83'AND c='81')OR(n='47'AND c='52')OR(n='99'AND c='97')OR(n='79'AND c='34')OR(n='02'AND c='22')OR(n='87'AND c='19')OR(n='03'AND c='98')OR(n='32'AND c='61')OR(n='36'AND c='71')OR(n='85'AND c='71')OR(n='54'AND c='71')OR(n='06'AND c='22')OR(n='43'AND c='30')OR(n='90'AND c='22')OR(n='15'AND c='05')OR(n='37'AND c='56')OR(n='24'AND c='00')OR(n='88'AND c='27')OR(n='37'AND c='27')OR(n='70'AND c='85')OR(n='04'AND c='85')OR(n='31'AND c='85')OR(n='25'AND c='85')OR(n='91'AND c='51')OR(n='57'AND c='85')OR(n='25'AND c='77')OR(n='67'AND c='87')OR(n='30'AND c='55')OR(n='46'AND c='48')OR(n='47'AND c='77')OR(n='34'AND c='15')OR(n='08'AND c='55')OR(n='05'AND c='77')OR(n='04'AND c='35')OR(n='64'AND c='55')OR(n='16'AND c='35')OR(n='84'AND c='73')OR(n='90'AND c='17')OR(n='68'AND c='08')OR(n='32'AND c='98')OR(n='82'AND c='27')OR(n='82'AND c='78')OR(n='65'AND c='66')OR(n='35'AND c='30')OR(n='60'AND c='45')OR(n='56'AND c='94')OR(n='42'AND c='02')OR(n='84'AND c='14')OR(n='01'AND c='80')OR(n='15'AND c='97')OR(n='18'AND c='85')OR(n='12'AND c='78')OR(n='50'AND c='85')OR(n='10'AND c='18')OR(n='11'AND c='96')OR(n='62'AND c='18')OR(n='49'AND c='85')OR(n='41'AND c='00')OR(n='33'AND c='82')OR(n='46'AND c='73')OR(n='95'AND c='71')OR(n='48'AND c='88')OR(n='82'AND c='73')OR(n='23'AND c='36')OR(n='43'AND c='67')OR(n='36'AND c='88')OR(n='20'AND c='78')OR(n='99'AND c='43')OR(n='66'AND c='68') 
+            OR(n='82'AND c='09')OR(n='37'AND c='58')OR(n='24'AND c='13')OR(n='03'AND c='52')OR(n='47'AND c='22')OR(n='03'AND c='37')OR(n='46'AND c='76')OR(n='00'AND c='77')OR(n='04'AND c='19')OR(n='63'AND c='08')OR(n='15'AND c='90')OR(n='52'AND c='45')OR(n='18'AND c='05')OR(n='48'AND c='28')OR(n='84'AND c='00')OR(n='22'AND c='19')OR(n='32'AND c='89')OR(n='97'AND c='44')OR(n='74'AND c='89')OR(n='36'AND c='89')OR(n='21'AND c='55')OR(n='88'AND c='97')OR(n='34'AND c='86')OR(n='39'AND c='15')OR(n='00'AND c='19')OR(n='10'AND c='55')OR(n='21'AND c='70')OR(n='96'AND c='06')OR(n='88'AND c='58')OR(n='10'AND c='81')OR(n='39'AND c='99')OR(n='10'AND c='68')OR(n='85'AND c='28')OR(n='08'AND c='65')OR(n='97'AND c='65')OR(n='15'AND c='33')OR(n='85'AND c='55')OR(n='98'AND c='00')OR(n='67'AND c='78')OR(n='15'AND c='08')OR(n='94'AND c='96')OR(n='79'AND c='81')OR(n='36'AND c='10')OR(n='50'AND c='53')OR(n='61'AND c='98')OR(n='74'AND c='56')OR(n='38'AND c='44')OR(n='39'AND c='99')OR(n='78'AND c='92')OR(n='79'AND c='33')OR(n='07'AND c='75')OR(n='05'AND c='38')OR(n='04'AND c='31')OR(n='72'AND c='88')OR(n='22'AND c='36')OR(n='27'AND c='99')OR(n='59'AND c='99')OR(n='44'AND c='87')OR(n='42'AND c='03')OR(n='52'AND c='87')OR(n='42'AND c='91')OR(n='38'AND c='02')OR(n='19'AND c='91')OR(n='43'AND c='04')OR(n='30'AND c='83')OR(n='36'AND c='91')OR(n='09'AND c='95')OR(n='44'AND c='45')OR(n='32'AND c='05')OR(n='06'AND c='38')OR(n='40'AND c='37')OR(n='59'AND c='38')OR(n='39'AND c='37')OR(n='05'AND c='38')OR(n='09'AND c='38')OR(n='94'AND c='37')OR(n='88'AND c='38')OR(n='60'AND c='29')OR(n='52'AND c='38')OR(n='87'AND c='38')OR(n='56'AND c='37')OR(n='50'AND c='38')OR(n='30'AND c='37')OR(n='38'AND c='38')OR(n='24'AND c='29')OR(n='17'AND c='38')OR(n='91'AND c='26')OR(n='80'AND c='38')OR(n='34'AND c='29') 
+            OR(n='68'AND c='38')OR(n='91'AND c='29')OR(n='23'AND c='29')OR(n='20'AND c='38')OR(n='83'AND c='29')OR(n='77'AND c='38')OR(n='61'AND c='38')OR(n='55'AND c='37')OR(n='39'AND c='38')OR(n='19'AND c='37')OR(n='41'AND c='38')OR(n='02'AND c='37')OR(n='40'AND c='38')OR(n='12'AND c='29')OR(n='01'AND c='38')OR(n='59'AND c='38')OR(n='76'AND c='37')OR(n='81'AND c='38')OR(n='54'AND c='37')OR(n='42'AND c='07')OR(n='38'AND c='03')OR(n='99'AND c='72')OR(n='79'AND c='19')OR(n='27'AND c='19')OR(n='31'AND c='19')OR(n='81'AND c='19')OR(n='66'AND c='19')OR(n='80'AND c='82')OR(n='69'AND c='27')OR(n='09'AND c='06')OR(n='97'AND c='87')OR(n='15'AND c='15')OR(n='06'AND c='15')OR(n='27'AND c='15')OR(n='04'AND c='15')OR(n='64'AND c='06')OR(n='14'AND c='15')OR(n='64'AND c='76')OR(n='39'AND c='59')OR(n='07'AND c='40')OR(n='51'AND c='68')OR(n='39'AND c='00')OR(n='35'AND c='33')OR(n='89'AND c='19')OR(n='16'AND c='00')OR(n='44'AND c='58')OR(n='76'AND c='18')OR(n='11'AND c='66')OR(n='27'AND c='87')OR(n='38'AND c='38')OR(n='57'AND c='55')OR(n='88'AND c='23')OR(n='24'AND c='29')OR(n='60'AND c='75')OR(n='81'AND c='27')OR(n='21'AND c='34')OR(n='11'AND c='96')OR(n='53'AND c='33')OR(n='26'AND c='89')OR(n='41'AND c='98')OR(n='59'AND c='98')OR(n='31'AND c='08')OR(n='29'AND c='70')OR(n='86'AND c='22')OR(n='18'AND c='96')OR(n='78'AND c='19')OR(n='32'AND c='81')OR(n='05'AND c='93')OR(n='59'AND c='65')OR(n='51'AND c='76')OR(n='87'AND c='15')OR(n='86'AND c='88')OR(n='18'AND c='15')OR(n='91'AND c='15')OR(n='43'AND c='48')OR(n='02'AND c='65')OR(n='21'AND c='14')OR(n='63'AND c='85')OR(n='50'AND c='48')OR(n='54'AND c='65')OR(n='84'AND c='15')OR(n='89'AND c='78')OR(n='95'AND c='60')OR(n='10'AND c='78')OR(n='42'AND c='87')OR(n='02'AND c='50')OR(n='82'AND c='78')OR(n='72'AND c='85')OR(n='24'AND c='77') 
+            OR(n='43'AND c='98')OR(n='57'AND c='00')OR(n='23'AND c='00')OR(n='64'AND c='23')OR(n='15'AND c='83')OR(n='60'AND c='66')OR(n='40'AND c='88')OR(n='60'AND c='78')OR(n='92'AND c='51')OR(n='08'AND c='77')OR(n='13'AND c='99')OR(n='20'AND c='00')OR(n='23'AND c='89')OR(n='32'AND c='44')OR(n='71'AND c='67')OR(n='95'AND c='67')OR(n='28'AND c='31')OR(n='56'AND c='35')OR(n='02'AND c='36')OR(n='59'AND c='93')OR(n='31'AND c='92')OR(n='70'AND c='09')OR(n='37'AND c='44')OR(n='42'AND c='20')OR(n='16'AND c='13')OR(n='91'AND c='82')OR(n='20'AND c='37')OR(n='39'AND c='51')OR(n='04'AND c='85')OR(n='20'AND c='48')OR(n='65'AND c='11')OR(n='71'AND c='02')OR(n='82'AND c='52')OR(n='55'AND c='24')OR(n='74'AND c='55')OR(n='79'AND c='80')OR(n='31'AND c='33')OR(n='09'AND c='99')OR(n='89'AND c='98')OR(n='79'AND c='98')OR(n='54'AND c='37')OR(n='43'AND c='61')OR(n='13'AND c='53')OR(n='95'AND c='90')OR(n='17'AND c='70')OR(n='94'AND c='33')OR(n='25'AND c='26')OR(n='22'AND c='21')OR(n='06'AND c='03')OR(n='64'AND c='30')OR(n='37'AND c='00')OR(n='61'AND c='45')OR(n='80'AND c='38')OR(n='52'AND c='34')OR(n='84'AND c='29')OR(n='35'AND c='99')OR(n='30'AND c='99')OR(n='60'AND c='99')OR(n='38'AND c='89')OR(n='50'AND c='99')OR(n='16'AND c='18')OR(n='99'AND c='99')OR(n='14'AND c='88')OR(n='55'AND c='56')OR(n='11'AND c='82')OR(n='13'AND c='82')OR(n='74'AND c='78')OR(n='47'AND c='37')OR(n='19'AND c='48')OR(n='83'AND c='65')OR(n='09'AND c='99')OR(n='63'AND c='99')OR(n='64'AND c='99')OR(n='27'AND c='99')OR(n='59'AND c='99')OR(n='02'AND c='48')OR(n='29'AND c='65')OR(n='68'AND c='22')OR(n='34'AND c='48')OR(n='62'AND c='65')OR(n='52'AND c='48')OR(n='90'AND c='65')OR(n='22'AND c='48')OR(n='49'AND c='65')OR(n='97'AND c='48')OR(n='94'AND c='65')OR(n='67'AND c='91')OR(n='15'AND c='97')OR(n='96'AND c='94') 
+            OR(n='66'AND c='15')OR(n='58'AND c='15')OR(n='27'AND c='11')OR(n='17'AND c='88')OR(n='81'AND c='27')OR(n='31'AND c='16')OR(n='00'AND c='19')OR(n='93'AND c='48')OR(n='43'AND c='65')OR(n='72'AND c='97')OR(n='09'AND c='48')OR(n='38'AND c='65')OR(n='52'AND c='11')OR(n='89'AND c='89')OR(n='24'AND c='53')OR(n='14'AND c='18')OR(n='72'AND c='80')OR(n='52'AND c='28')OR(n='01'AND c='26')OR(n='97'AND c='66')OR(n='34'AND c='00')OR(n='29'AND c='88')OR(n='19'AND c='90')OR(n='73'AND c='65')OR(n='67'AND c='71')OR(n='50'AND c='19')OR(n='92'AND c='67')OR(n='60'AND c='48')OR(n='36'AND c='65')OR(n='01'AND c='41')OR(n='19'AND c='78')OR(n='39'AND c='55')OR(n='26'AND c='66')OR(n='18'AND c='78')OR(n='30'AND c='71')OR(n='62'AND c='54')OR(n='41'AND c='66')OR(n='45'AND c='83')OR(n='62'AND c='72')OR(n='08'AND c='06')OR(n='09'AND c='88')OR(n='23'AND c='19')OR(n='90'AND c='46')OR(n='63'AND c='15')OR(n='70'AND c='89')OR(n='60'AND c='50')OR(n='68'AND c='28')OR(n='66'AND c='99')OR(n='56'AND c='93')OR(n='18'AND c='71')OR(n='94'AND c='29')OR(n='90'AND c='54')OR(n='94'AND c='71')OR(n='47'AND c='29')OR(n='96'AND c='71')OR(n='11'AND c='71')OR(n='87'AND c='37')OR(n='03'AND c='71') 
+            OR(n='97'AND c='71')OR(n='64'AND c='71')OR(n='10'AND c='66')OR(n='24'AND c='71')OR(n='25'AND c='71')OR(n='69'AND c='71')OR(n='57'AND c='22')OR(n='12'AND c='71')OR(n='27'AND c='71')OR(n='41'AND c='71')OR(n='12'AND c='71')OR(n='15'AND c='29')OR(n='46'AND c='71')OR(n='00'AND c='85')OR(n='96'AND c='71')OR(n='42'AND c='82')OR(n='05'AND c='71')OR(n='45'AND c='71')OR(n='19'AND c='37')OR(n='55'AND c='71')OR(n='32'AND c='29')OR(n='70'AND c='71')OR(n='56'AND c='71')OR(n='38'AND c='71')OR(n='47'AND c='71')OR(n='24'AND c='37')OR(n='30'AND c='71')OR(n='86'AND c='28')OR(n='28'AND c='58')OR(n='09'AND c='33')OR(n='55'AND c='12')OR(n='37'AND c='32')OR(n='48'AND c='29')OR(n='08'AND c='38')OR(n='42'AND c='11')OR(n='65'AND c='57')OR(n='18'AND c='57')OR(n='42'AND c='08')OR(n='35'AND c='11')OR(n='19'AND c='00')OR(n='84'AND c='52')OR(n='20'AND c='59')OR(n='84'AND c='65')OR(n='01'AND c='59')OR(n='87'AND c='65')OR(n='98'AND c='65')OR(n='87'AND c='59')OR(n='58'AND c='24')OR(n='10'AND c='99')OR(n='50'AND c='57')OR(n='92'AND c='88')OR(n='69'AND c='20')OR(n='47'AND c='44')OR(n='71'AND c='44')OR(n='28'AND c='18')OR(n='12'AND c='18')OR(n='24'AND c='88')OR(n='84'AND c='16')OR(n='19'AND c='37')OR(n='01'AND c='34')OR(n='07'AND c='73')OR(n='48'AND c='02')OR(n='64'AND c='73')OR(n='23'AND c='73')OR(n='95'AND c='05')OR(n='42'AND c='05')OR(n='67'AND c='12')OR(n='02'AND c='13')OR(n='49'AND c='87')OR(n='23'AND c='53')OR(n='97'AND c='24')OR(n='47'AND c='98')OR(n='96'AND c='10')OR(n='17'AND c='78')OR(n='02'AND c='99')OR(n='21'AND c='69')OR(n='76'AND c='33')OR(n='41'AND c='18')OR(n='97'AND c='32')OR(n='40'AND c='97')OR(n='48'AND c='95')OR(n='26'AND c='95')OR(n='28'AND c='88')OR(n='51'AND c='08')OR(n='13'AND c='37')OR(n='43'AND c='83')OR(n='54'AND c='20')OR(n='98'AND c='34')OR(n='32'AND c='93')OR(n='63'AND c='96')OR(n='91'AND c='16')OR(n='24'AND c='16')OR(n='68'AND c='10')OR(n='76'AND c='88')OR(n='62'AND c='68')OR(n='49'AND c='76')OR(n='10'AND c='76')OR(n='95'AND c='62')OR(n='42'AND c='91')OR(n='98'AND c='22')OR(n='55'AND c='19')OR(n='45'AND c='66')OR(n='19'AND c='56')OR(n='54'AND c='98')OR(n='86'AND c='76')OR(n='92'AND c='63')OR(n='89'AND c='11')OR(n='63'AND c='90')OR(n='85'AND c='40')OR(n='01'AND c='67')OR(n='88'AND c='19')OR(n='26'AND c='11')OR(n='62'AND c='79')OR(n='74'AND c='81') 
+            OR(n='47'AND c='55')OR(n='70'AND c='05')OR(n='82'AND c='20')OR(n='10'AND c='47')OR(n='29'AND c='59')OR(n='93'AND c='89')OR(n='06'AND c='88')OR(n='34'AND c='30')OR(n='23'AND c='55')OR(n='57'AND c='57')OR(n='43'AND c='86')OR(n='06'AND c='86')OR(n='28'AND c='86')OR(n='44'AND c='86')OR(n='20'AND c='86')OR(n='96'AND c='48')OR(n='19'AND c='48')OR(n='06'AND c='44')OR(n='82'AND c='68')OR(n='74'AND c='23')OR(n='60'AND c='86')OR(n='05'AND c='10')OR(n='61'AND c='93')OR(n='22'AND c='29')OR(n='22'AND c='92')OR(n='63'AND c='20')OR(n='47'AND c='19')OR(n='16'AND c='07')OR(n='02'AND c='18')OR(n='13'AND c='77')OR(n='35'AND c='11')OR(n='76'AND c='17')OR(n='87'AND c='11')OR(n='93'AND c='90')OR(n='14'AND c='71')OR(n='35'AND c='30')OR(n='37'AND c='44')OR(n='33'AND c='61')OR(n='05'AND c='78')OR(n='25'AND c='88')OR(n='55'AND c='18')OR(n='67'AND c='66')OR(n='55'AND c='68')OR(n='92'AND c='78')OR(n='30'AND c='15')OR(n='14'AND c='15')OR(n='60'AND c='15')OR(n='18'AND c='15')OR(n='64'AND c='89')OR(n='56'AND c='50')OR(n='34'AND c='86')OR(n='47'AND c='22')OR(n='42'AND c='44')OR(n='05'AND c='56')OR(n='67'AND c='82')OR(n='90'AND c='55')OR(n='58'AND c='55')OR(n='01'AND c='55')OR(n='18'AND c='41')OR(n='66'AND c='37')OR(n='89'AND c='97')OR(n='38'AND c='66')OR(n='16'AND c='15')OR(n='29'AND c='15')OR(n='29'AND c='15')OR(n='26'AND c='15')OR(n='04'AND c='19')OR(n='96'AND c='96')OR(n='56'AND c='66')OR(n='08'AND c='48')OR(n='13'AND c='48')OR(n='88'AND c='48')OR(n='67'AND c='69')OR(n='49'AND c='26')OR(n='14'AND c='64')OR(n='39'AND c='48')OR(n='18'AND c='53')OR(n='34'AND c='80')OR(n='43'AND c='17')OR(n='22'AND c='26')OR(n='03'AND c='65')OR(n='50'AND c='85')OR(n='98'AND c='11')OR(n='65'AND c='90')OR(n='54'AND c='97')OR(n='13'AND c='77')OR(n='59'AND c='82')OR(n='82'AND c='98')OR(n='98'AND c='60')OR(n='22'AND c='26')OR(n='83'AND c='11')OR(n='51'AND c='57')OR(n='82'AND c='33')OR(n='08'AND c='32')OR(n='94'AND c='13')OR(n='89'AND c='45')OR(n='32'AND c='75')OR(n='40'AND c='32')OR(n='82'AND c='67')OR(n='13'AND c='98')OR(n='61'AND c='30')OR(n='50'AND c='00')OR(n='78'AND c='60')OR(n='35'AND c='77')OR(n='03'AND c='26')OR(n='84'AND c='45')OR(n='13'AND c='19')OR(n='27'AND c='45')OR(n='01'AND c='19')OR(n='66'AND c='45')OR(n='33'AND c='45')OR(n='51'AND c='19')OR(n='70'AND c='45')OR(n='65'AND c='18')OR(n='68'AND c='19')OR(n='69'AND c='19')OR(n='21'AND c='23')OR(n='95'AND c='45')OR(n='78'AND c='19')OR(n='37'AND c='45')OR(n='16'AND c='50')OR(n='75'AND c='19')OR(n='54'AND c='23')OR(n='56'AND c='22')OR(n='88'AND c='28')OR(n='21'AND c='53')OR(n='32'AND c='23')OR(n='97'AND c='71') 
+            OR(n='76'AND c='69')OR(n='73'AND c='92')OR(n='50'AND c='65')OR(n='34'AND c='88')OR(n='91'AND c='99')OR(n='51'AND c='90')OR(n='97'AND c='93')OR(n='84'AND c='48')OR(n='50'AND c='88')OR(n='65'AND c='35')OR(n='87'AND c='68')OR(n='19'AND c='16')OR(n='99'AND c='71')OR(n='98'AND c='00')OR(n='73'AND c='52')OR(n='66'AND c='16')OR(n='32'AND c='10')OR(n='71'AND c='44')OR(n='01'AND c='55')OR(n='60'AND c='00')OR(n='10'AND c='16')OR(n='03'AND c='88')OR(n='04'AND c='52')OR(n='59'AND c='57')OR(n='00'AND c='99')OR(n='63'AND c='23')OR(n='78'AND c='07')OR(n='90'AND c='87')OR(n='95'AND c='96')OR(n='52'AND c='18')OR(n='61'AND c='99')OR(n='65'AND c='66')OR(n='66'AND c='87')OR(n='24'AND c='80')OR(n='88'AND c='77')OR(n='05'AND c='01')OR(n='35'AND c='11')OR(n='59'AND c='80')OR(n='04'AND c='62')OR(n='53'AND c='88')OR(n='78'AND c='07')OR(n='94'AND c='59')OR(n='12'AND c='79')OR(n='00'AND c='88')OR(n='50'AND c='85')OR(n='66'AND c='55')OR(n='15'AND c='18')OR(n='89'AND c='28')OR(n='77'AND c='69')OR(n='70'AND c='87')OR(n='48'AND c='26')OR(n='55'AND c='03')OR(n='87'AND c='67')OR(n='05'AND c='82')OR(n='64'AND c='15')OR(n='05'AND c='49')OR(n='80'AND c='66')OR(n='60'AND c='78')OR(n='93'AND c='02')OR(n='25'AND c='91')OR(n='65'AND c='00')OR(n='80'AND c='91')OR(n='45'AND c='30')OR(n='68'AND c='98')OR(n='40'AND c='88')OR(n='37'AND c='19')OR(n='24'AND c='62')OR(n='71'AND c='48')OR(n='27'AND c='78')OR(n='56'AND c='21')OR(n='49'AND c='00')OR(n='05'AND c='77')OR(n='41'AND c='57')OR(n='67'AND c='99')OR(n='15'AND c='41')OR(n='68'AND c='70')OR(n='98'AND c='49')OR(n='90'AND c='03')OR(n='92'AND c='41')OR(n='59'AND c='00')OR(n='16'AND c='89')OR(n='63'AND c='76')OR(n='23'AND c='90')OR(n='76'AND c='91')OR(n='41'AND c='06')OR(n='08'AND c='44')OR(n='11'AND c='44')OR(n='46'AND c='91')OR(n='17'AND c='91')OR(n='17'AND c='91')OR(n='74'AND c='72')OR(n='05'AND c='72')OR(n='03'AND c='52')OR(n='52'AND c='81')OR(n='42'AND c='91')OR(n='17'AND c='91')OR(n='54'AND c='60')OR(n='02'AND c='67')OR(n='72'AND c='32')OR(n='71'AND c='91')OR(n='34'AND c='91')OR(n='34'AND c='88')OR(n='42'AND c='60')OR(n='58'AND c='21')OR(n='16'AND c='70')OR(n='18'AND c='38')OR(n='84'AND c='05')OR(n='10'AND c='11')OR(n='85'AND c='85')OR(n='90'AND c='09')OR(n='06'AND c='45')OR(n='01'AND c='21')OR(n='38'AND c='76')OR(n='92'AND c='02')OR(n='01'AND c='66')OR(n='42'AND c='26')OR(n='17'AND c='58')OR(n='07'AND c='12')OR(n='31'AND c='85')OR(n='34'AND c='87')OR(n='24'AND c='19')OR(n='94'AND c='19')OR(n='44'AND c='55')OR(n='97'AND c='47')OR(n='93'AND c='55')OR(n='78'AND c='62')OR(n='36'AND c='95')OR(n='25'AND c='89')OR(n='38'AND c='27')OR(n='72'AND c='32')OR(n='42'AND c='77')OR(n='41'AND c='09')OR(n='25'AND c='27')OR(n='20'AND c='99')OR(n='66'AND c='33')OR(n='13'AND c='65')OR(n='68'AND c='85')OR(n='79'AND c='87')OR(n='01'AND c='08')OR(n='20'AND c='19')OR(n='34'AND c='19')OR(n='24'AND c='19')OR(n='61'AND c='19')OR(n='71'AND c='19')OR(n='86'AND c='19')OR(n='93'AND c='19')OR(n='65'AND c='78')OR(n='25'AND c='55')OR(n='71'AND c='16')OR(n='10'AND c='68')OR(n='44'AND c='98')OR(n='35'AND c='86')OR(n='54'AND c='48')OR(n='11'AND c='82')OR(n='11'AND c='68')OR(n='71'AND c='98')OR(n='44'AND c='55')OR(n='06'AND c='16')OR(n='18'AND c='86')OR(n='31'AND c='48')OR(n='47'AND c='55')OR(n='18'AND c='16')OR(n='89'AND c='98')OR(n='19'AND c='68')OR(n='31'AND c='86')OR(n='65'AND c='48')OR(n='15'AND c='55')OR(n='14'AND c='16')OR(n='39'AND c='98')OR(n='64'AND c='68')OR(n='39'AND c='86')OR(n='34'AND c='48')OR(n='09'AND c='19')OR(n='60'AND c='19')OR(n='94'AND c='19')OR(n='20'AND c='55')OR(n='59'AND c='98')OR(n='82'AND c='16')OR(n='63'AND c='68')OR(n='75'AND c='86')OR(n='41'AND c='48')OR(n='59'AND c='39')OR(n='47'AND c='74')OR(n='60'AND c='87')OR(n='51'AND c='99')OR(n='56'AND c='00')OR(n='35'AND c='32')OR(n='25'AND c='19')OR(n='16'AND c='19')OR(n='20'AND c='38')OR(n='02'AND c='86')OR(n='18'AND c='82')OR(n='16'AND c='68')OR(n='99'AND c='11')OR(n='64'AND c='22')OR(n='84'AND c='66')OR(n='74'AND c='81')OR(n='18'AND c='88')OR(n='69'AND c='19')OR(n='00'AND c='19')OR(n='74'AND c='19')OR(n='05'AND c='88')OR(n='55'AND c='19')OR(n='63'AND c='19')OR(n='45'AND c='19')OR(n='71'AND c='99')OR(n='08'AND c='19')OR(n='82'AND c='19')OR(n='21'AND c='88')OR(n='17'AND c='19')OR(n='33'AND c='19')OR(n='54'AND c='77')OR(n='11'AND c='06')OR(n='32'AND c='77')OR(n='14'AND c='76')OR(n='68'AND c='84')OR(n='63'AND c='60')OR(n='22'AND c='11')OR(n='02'AND c='68') 
+            OR(n='37'AND c='86')OR(n='97'AND c='56')OR(n='14'AND c='78')OR(n='54'AND c='53')OR(n='37'AND c='88')OR(n='60'AND c='01')OR(n='85'AND c='66')OR(n='03'AND c='11')OR(n='41'AND c='66')OR(n='53'AND c='26')OR(n='86'AND c='92')OR(n='23'AND c='82')OR(n='20'AND c='45')OR(n='40'AND c='68')OR(n='64'AND c='28')OR(n='46'AND c='66')OR(n='59'AND c='73')OR(n='74'AND c='68')OR(n='15'AND c='55')OR(n='82'AND c='92')OR(n='58'AND c='70')OR(n='37'AND c='88')OR(n='57'AND c='84')OR(n='89'AND c='89')OR(n='14'AND c='58')OR(n='86'AND c='08')OR(n='23'AND c='98')OR(n='90'AND c='22')OR(n='56'AND c='88')OR(n='93'AND c='12')OR(n='03'AND c='85')OR(n='17'AND c='44')OR(n='38'AND c='89')OR(n='44'AND c='88')OR(n='89'AND c='36')OR(n='52'AND c='21')OR(n='58'AND c='15')OR(n='07'AND c='98')OR(n='41'AND c='70')OR(n='01'AND c='99')OR(n='38'AND c='45')OR(n='00'AND c='02')OR(n='32'AND c='03')OR(n='45'AND c='89')OR(n='49'AND c='86')OR(n='26'AND c='78')OR(n='72'AND c='39')OR(n='81'AND c='91')OR(n='04'AND c='78')OR(n='53'AND c='88')OR(n='01'AND c='48')OR(n='21'AND c='62')OR(n='94'AND c='08')OR(n='06'AND c='70')OR(n='37'AND c='70')OR(n='31'AND c='06')OR(n='87'AND c='32')OR(n='02'AND c='99')OR(n='84'AND c='15')OR(n='83'AND c='15')OR(n='87'AND c='66')OR(n='17'AND c='69')OR(n='62'AND c='85')OR(n='70'AND c='43')OR(n='61'AND c='22')OR(n='17'AND c='82')OR(n='04'AND c='77')OR(n='35'AND c='61')OR(n='42'AND c='08')OR(n='46'AND c='32')OR(n='20'AND c='22')OR(n='17'AND c='13')OR(n='47'AND c='85')OR(n='11'AND c='43')OR(n='63'AND c='75')OR(n='54'AND c='60')OR(n='01'AND c='97')OR(n='71'AND c='45')OR(n='68'AND c='99')OR(n='70'AND c='88')OR(n='64'AND c='19')OR(n='99'AND c='19')OR(n='78'AND c='19')OR(n='41'AND c='19')OR(n='67'AND c='48')OR(n='21'AND c='04')OR(n='49'AND c='15')OR(n='28'AND c='19')OR(n='91'AND c='48')OR(n='56'AND c='04')OR(n='36'AND c='15')OR(n='61'AND c='19')OR(n='45'AND c='48')OR(n='08'AND c='04')OR(n='10'AND c='15')OR(n='93'AND c='19')OR(n='74'AND c='48')OR(n='23'AND c='04')OR(n='80'AND c='15')OR(n='13'AND c='19')OR(n='35'AND c='48')OR(n='36'AND c='04')OR(n='51'AND c='15')OR(n='20'AND c='82')OR(n='32'AND c='19')OR(n='34'AND c='48')OR(n='11'AND c='04')OR(n='20'AND c='15')OR(n='40'AND c='82')OR(n='89'AND c='19')OR(n='51'AND c='69')OR(n='91'AND c='69')OR(n='72'AND c='17')OR(n='20'AND c='98')OR(n='76'AND c='80')OR(n='20'AND c='91')OR(n='99'AND c='69')OR(n='13'AND c='15')OR(n='88'AND c='67')OR(n='53'AND c='98')OR(n='88'AND c='08')OR(n='48'AND c='58')OR(n='94'AND c='70')OR(n='57'AND c='40')OR(n='32'AND c='40')OR(n='92'AND c='40')OR(n='22'AND c='16')OR(n='38'AND c='75')OR(n='32'AND c='99')OR(n='10'AND c='16')OR(n='86'AND c='75')OR(n='19'AND c='99')OR(n='52'AND c='40')OR(n='11'AND c='65')OR(n='53'AND c='65')OR(n='04'AND c='99')OR(n='70'AND c='44')OR(n='37'AND c='87')OR(n='27'AND c='87')OR(n='28'AND c='30')OR(n='34'AND c='34')OR(n='69'AND c='00')OR(n='56'AND c='18')OR(n='28'AND c='34')OR(n='14'AND c='98')OR(n='51'AND c='02')OR(n='39'AND c='51')OR(n='66'AND c='19')OR(n='70'AND c='69')OR(n='64'AND c='78')OR(n='10'AND c='45')OR(n='15'AND c='80')OR(n='78'AND c='78')OR(n='51'AND c='30')OR(n='11'AND c='90')OR(n='87'AND c='20')OR(n='97'AND c='28')OR(n='78'AND c='69')OR(n='58'AND c='88')OR(n='74'AND c='79')OR(n='14'AND c='99')OR(n='63'AND c='99')OR(n='41'AND c='93')OR(n='83'AND c='72')OR(n='52'AND c='77')OR(n='45'AND c='66')OR(n='75'AND c='31')OR(n='81'AND c='27')OR(n='24'AND c='10')OR(n='88'AND c='26')OR(n='25'AND c='68')OR(n='60'AND c='60')OR(n='18'AND c='65')OR(n='55'AND c='16')OR(n='32'AND c='88')OR(n='36'AND c='39')OR(n='27'AND c='01')OR(n='07'AND c='67')OR(n='42'AND c='66')OR(n='03'AND c='65')OR(n='12'AND c='11')OR(n='42'AND c='91')OR(n='77'AND c='13')OR(n='99'AND c='38') 
+            OR(n='33'AND c='93')OR(n='46'AND c='92')OR(n='77'AND c='53')OR(n='20'AND c='00')OR(n='58'AND c='97')OR(n='04'AND c='09')OR(n='42'AND c='51')OR(n='93'AND c='56')OR(n='58'AND c='06')OR(n='38'AND c='08')OR(n='46'AND c='12')OR(n='23'AND c='77')OR(n='17'AND c='33')OR(n='28'AND c='90')OR(n='49'AND c='16')OR(n='47'AND c='21')OR(n='04'AND c='25')OR(n='09'AND c='16')OR(n='38'AND c='95')OR(n='13'AND c='88')OR(n='67'AND c='88')OR(n='31'AND c='30')OR(n='58'AND c='23')OR(n='16'AND c='78')OR(n='94'AND c='99')OR(n='88'AND c='99')OR(n='89'AND c='39')OR(n='42'AND c='87')OR(n='42'AND c='99')OR(n='06'AND c='39')OR(n='26'AND c='20')OR(n='49'AND c='56')OR(n='22'AND c='79')OR(n='21'AND c='05')OR(n='08'AND c='09')OR(n='15'AND c='50')OR(n='99'AND c='60')OR(n='90'AND c='22')OR(n='64'AND c='34')OR(n='35'AND c='27')OR(n='95'AND c='91')OR(n='22'AND c='83')OR(n='23'AND c='45')OR(n='79'AND c='07')OR(n='30'AND c='69')OR(n='59'AND c='11')OR(n='46'AND c='44')OR(n='05'AND c='18')OR(n='42'AND c='81')OR(n='95'AND c='43')OR(n='64'AND c='30')OR(n='69'AND c='25')OR(n='43'AND c='61')OR(n='38'AND c='88')OR(n='98'AND c='91')OR(n='37'AND c='22')OR(n='13'AND c='06')OR(n='02'AND c='02')OR(n='39'AND c='50')OR(n='20'AND c='10')OR(n='69'AND c='26')OR(n='10'AND c='88')OR(n='25'AND c='19')OR(n='60'AND c='66')OR(n='59'AND c='11')OR(n='24'AND c='88')OR(n='89'AND c='99')OR(n='99'AND c='12')OR(n='16'AND c='08')OR(n='73'AND c='99')OR(n='34'AND c='88')OR(n='61'AND c='18')OR(n='37'AND c='50')OR(n='53'AND c='79')OR(n='30'AND c='30')OR(n='87'AND c='37')OR(n='12'AND c='26')OR(n='93'AND c='03')OR(n='25'AND c='88')OR(n='64'AND c='83')OR(n='26'AND c='13')OR(n='83'AND c='00')OR(n='37'AND c='70')OR(n='46'AND c='09')OR(n='05'AND c='10')OR(n='88'AND c='08')OR(n='45'AND c='11')OR(n='73'AND c='11')OR(n='14'AND c='66')OR(n='08'AND c='47')OR(n='50'AND c='44')OR(n='71'AND c='36')OR(n='47'AND c='88')OR(n='72'AND c='59')OR(n='00'AND c='67')OR(n='54'AND c='18')OR(n='64'AND c='58')OR(n='05'AND c='51')OR(n='77'AND c='39')OR(n='88'AND c='66')OR(n='62'AND c='77')OR(n='32'AND c='88')OR(n='19'AND c='38')OR(n='32'AND c='85')OR(n='97'AND c='17')OR(n='06'AND c='01')OR(n='53'AND c='03')OR(n='27'AND c='28')OR(n='55'AND c='51')OR(n='20'AND c='77')OR(n='41'AND c='97')OR(n='54'AND c='77')OR(n='18'AND c='23')OR(n='09'AND c='75')OR(n='63'AND c='89')OR(n='59'AND c='97')OR(n='01'AND c='66')OR(n='47'AND c='27')OR(n='32'AND c='50')OR(n='97'AND c='39')OR(n='44'AND c='98')OR(n='06'AND c='96')OR(n='32'AND c='79')OR(n='95'AND c='00')OR(n='52'AND c='57')OR(n='22'AND c='23')OR(n='00'AND c='19')OR(n='55'AND c='19')OR(n='65'AND c='19')OR(n='93'AND c='19')OR(n='94'AND c='19')OR(n='54'AND c='19')OR(n='91'AND c='20')OR(n='92'AND c='19')OR(n='12'AND c='19') 
+            OR(n='49'AND c='19')OR(n='16'AND c='69')OR(n='97'AND c='29')OR(n='48'AND c='19')OR(n='89'AND c='19')OR(n='75'AND c='19')OR(n='97'AND c='19')OR(n='07'AND c='19')OR(n='67'AND c='19')OR(n='62'AND c='19')OR(n='73'AND c='19')OR(n='70'AND c='19')OR(n='66'AND c='48')OR(n='13'AND c='04')OR(n='13'AND c='15')OR(n='43'AND c='19')OR(n='93'AND c='82')OR(n='10'AND c='62')OR(n='69'AND c='30')OR(n='23'AND c='15')OR(n='33'AND c='89')OR(n='29'AND c='73')OR(n='35'AND c='30')OR(n='16'AND c='77')OR(n='28'AND c='19')OR(n='05'AND c='19')OR(n='68'AND c='19')OR(n='23'AND c='19')OR(n='33'AND c='19')OR(n='77'AND c='19')OR(n='79'AND c='19')OR(n='80'AND c='19')OR(n='22'AND c='19')OR(n='69'AND c='19')OR(n='23'AND c='19')OR(n='29'AND c='19')OR(n='69'AND c='19')OR(n='02'AND c='19')OR(n='72'AND c='19')OR(n='42'AND c='19')OR(n='95'AND c='65')OR(n='53'AND c='77')OR(n='65'AND c='77')OR(n='75'AND c='77')OR(n='67'AND c='15')OR(n='29'AND c='77')OR(n='98'AND c='77')OR(n='78'AND c='77')OR(n='85'AND c='77')OR(n='94'AND c='77')OR(n='58'AND c='77')OR(n='90'AND c='77')OR(n='88'AND c='77')OR(n='83'AND c='77')OR(n='28'AND c='77')OR(n='99'AND c='77')OR(n='37'AND c='77')OR(n='14'AND c='30')OR(n='34'AND c='91')OR(n='20'AND c='99')OR(n='98'AND c='54')OR(n='54'AND c='98')OR(n='88'AND c='98')OR(n='29'AND c='98')OR(n='31'AND c='98')OR(n='37'AND c='98')OR(n='25'AND c='60')OR(n='69'AND c='66')OR(n='98'AND c='89')OR(n='75'AND c='20')OR(n='83'AND c='60')OR(n='61'AND c='48')OR(n='00'AND c='04')OR(n='33'AND c='15')OR(n='36'AND c='19')OR(n='79'AND c='65')OR(n='51'AND c='99')OR(n='70'AND c='34')OR(n='97'AND c='88')OR(n='89'AND c='39')OR(n='73'AND c='11')OR(n='72'AND c='39')OR(n='06'AND c='22')OR(n='48'AND c='39')OR(n='59'AND c='81')OR(n='85'AND c='02')OR(n='40'AND c='39')OR(n='01'AND c='73')OR(n='44'AND c='57')OR(n='01'AND c='33')OR(n='51'AND c='36')OR(n='45'AND c='11')OR(n='31'AND c='88')OR(n='58'AND c='11')OR(n='71'AND c='11')OR(n='77'AND c='52')OR(n='88'AND c='33')OR(n='66'AND c='86')OR(n='14'AND c='40')OR(n='23'AND c='33')OR(n='79'AND c='19')OR(n='15'AND c='05')OR(n='23'AND c='76')OR(n='05'AND c='53')OR(n='69'AND c='76')OR(n='86'AND c='85')OR(n='17'AND c='44')OR(n='32'AND c='77')OR(n='17'AND c='95')OR(n='83'AND c='62')OR(n='59'AND c='28')OR(n='52'AND c='26')OR(n='48'AND c='26')OR(n='65'AND c='77')OR(n='62'AND c='19')OR(n='27'AND c='90')OR(n='10'AND c='88')OR(n='82'AND c='65')OR(n='38'AND c='44')OR(n='65'AND c='91')OR(n='98'AND c='12')OR(n='84'AND c='91')OR(n='94'AND c='85')OR(n='27'AND c='86')OR(n='18'AND c='78')OR(n='90'AND c='83')OR(n='85'AND c='80')OR(n='93'AND c='05') 
+            OR(n='97'AND c='58')OR(n='60'AND c='71')OR(n='96'AND c='88')OR(n='31'AND c='86')OR(n='13'AND c='91')OR(n='33'AND c='91')OR(n='24'AND c='23')OR(n='74'AND c='80')OR(n='52'AND c='56')OR(n='16'AND c='85')OR(n='22'AND c='97')OR(n='74'AND c='30')OR(n='11'AND c='55')OR(n='68'AND c='36')OR(n='61'AND c='79')OR(n='86'AND c='03')OR(n='81'AND c='76')OR(n='77'AND c='82')OR(n='49'AND c='65')OR(n='68'AND c='76')OR(n='70'AND c='82')OR(n='13'AND c='18')OR(n='14'AND c='06')OR(n='52'AND c='88')OR(n='12'AND c='37')OR(n='57'AND c='11')OR(n='61'AND c='08')OR(n='04'AND c='44')OR(n='43'AND c='98')OR(n='40'AND c='00')OR(n='93'AND c='76')OR(n='62'AND c='82')OR(n='57'AND c='83')OR(n='57'AND c='19')OR(n='00'AND c='82')OR(n='28'AND c='48')OR(n='57'AND c='04')OR(n='06'AND c='19')OR(n='13'AND c='15')OR(n='81'AND c='88')OR(n='26'AND c='48')OR(n='45'AND c='04')OR(n='98'AND c='19')OR(n='08'AND c='15')OR(n='70'AND c='86')OR(n='84'AND c='77')OR(n='57'AND c='34')OR(n='99'AND c='90')OR(n='54'AND c='65')OR(n='19'AND c='36')OR(n='70'AND c='11')OR(n='68'AND c='11')OR(n='52'AND c='11')OR(n='11'AND c='11')OR(n='48'AND c='11')OR(n='33'AND c='11')OR(n='60'AND c='26')OR(n='36'AND c='40')OR(n='99'AND c='01')OR(n='52'AND c='99')OR(n='74'AND c='38')OR(n='45'AND c='52')OR(n='53'AND c='83')OR(n='74'AND c='96')OR(n='11'AND c='94')OR(n='43'AND c='22')OR(n='46'AND c='88')OR(n='01'AND c='93')OR(n='87'AND c='52')OR(n='82'AND c='98')OR(n='70'AND c='58')OR(n='43'AND c='57')OR(n='43'AND c='61')OR(n='30'AND c='68')OR(n='82'AND c='89')OR(n='55'AND c='06')OR(n='16'AND c='89')OR(n='57'AND c='89')OR(n='93'AND c='66')OR(n='77'AND c='22')OR(n='99'AND c='88')OR(n='00'AND c='50')OR(n='23'AND c='99')OR(n='17'AND c='22')OR(n='22'AND c='56')OR(n='88'AND c='27')OR(n='47'AND c='26')OR(n='81'AND c='91')OR(n='32'AND c='69')OR(n='68'AND c='58')OR(n='74'AND c='68')OR(n='38'AND c='60')OR(n='36'AND c='51')OR(n='85'AND c='22')OR(n='62'AND c='86')OR(n='54'AND c='82')OR(n='32'AND c='99')OR(n='38'AND c='67')OR(n='65'AND c='29')OR(n='35'AND c='00')OR(n='62'AND c='35')OR(n='24'AND c='73')OR(n='40'AND c='67')OR(n='38'AND c='01')OR(n='92'AND c='15')OR(n='46'AND c='78')OR(n='65'AND c='18')OR(n='31'AND c='44')OR(n='74'AND c='00')OR(n='91'AND c='86')OR(n='23'AND c='78')OR(n='92'AND c='33')OR(n='62'AND c='26')OR(n='08'AND c='58')OR(n='12'AND c='55')OR(n='97'AND c='85')OR(n='21'AND c='88')OR(n='07'AND c='33')OR(n='44'AND c='72')OR(n='32'AND c='29')OR(n='87'AND c='18')OR(n='97'AND c='40')OR(n='42'AND c='98')OR(n='72'AND c='83')OR(n='37'AND c='50')OR(n='33'AND c='63')OR(n='67'AND c='18')OR(n='55'AND c='11')OR(n='67'AND c='59')OR(n='84'AND c='88')OR(n='16'AND c='58')OR(n='92'AND c='55')OR(n='99'AND c='20')OR(n='31'AND c='15')OR(n='56'AND c='83') 
+            OR(n='85'AND c='77')OR(n='70'AND c='89')OR(n='78'AND c='91')OR(n='21'AND c='43')OR(n='20'AND c='72')OR(n='27'AND c='86')OR(n='84'AND c='33')OR(n='46'AND c='30')OR(n='34'AND c='36')OR(n='40'AND c='67')OR(n='48'AND c='25')OR(n='53'AND c='18')OR(n='30'AND c='96')OR(n='16'AND c='95')OR(n='66'AND c='69')OR(n='02'AND c='43')OR(n='50'AND c='11')OR(n='92'AND c='30')OR(n='14'AND c='35')OR(n='34'AND c='45')OR(n='92'AND c='45')OR(n='84'AND c='37')OR(n='81'AND c='29')OR(n='96'AND c='19')OR(n='15'AND c='97')OR(n='89'AND c='32')OR(n='28'AND c='45')OR(n='89'AND c='71')OR(n='98'AND c='45')OR(n='11'AND c='22')OR(n='83'AND c='29')OR(n='71'AND c='19')OR(n='77'AND c='97')OR(n='19'AND c='45')OR(n='26'AND c='71')OR(n='36'AND c='45')OR(n='61'AND c='37')OR(n='67'AND c='29')OR(n='73'AND c='19')OR(n='70'AND c='97')OR(n='24'AND c='32')OR(n='99'AND c='45')OR(n='61'AND c='71')OR(n='50'AND c='11')OR(n='48'AND c='76')OR(n='22'AND c='81')OR(n='62'AND c='15')OR(n='67'AND c='77')OR(n='69'AND c='50')OR(n='97'AND c='34')OR(n='93'AND c='34')OR(n='04'AND c='55')OR(n='67'AND c='88')OR(n='98'AND c='93')OR(n='93'AND c='82')OR(n='63'AND c='77')OR(n='95'AND c='11')OR(n='91'AND c='25')OR(n='59'AND c='08')OR(n='48'AND c='97')OR(n='06'AND c='91')OR(n='02'AND c='22')OR(n='10'AND c='69')OR(n='32'AND c='28')OR(n='30'AND c='18')OR(n='70'AND c='68')OR(n='43'AND c='86')OR(n='76'AND c='95')OR(n='65'AND c='77')OR(n='64'AND c='29')OR(n='66'AND c='69')OR(n='13'AND c='93')OR(n='08'AND c='50')OR(n='65'AND c='66')OR(n='35'AND c='98')OR(n='46'AND c='49')OR(n='85'AND c='22')OR(n='90'AND c='77')OR(n='03'AND c='66')OR(n='33'AND c='99')OR(n='60'AND c='44')OR(n='71'AND c='86')OR(n='11'AND c='86')OR(n='48'AND c='86')OR(n='28'AND c='86')OR(n='40'AND c='86')OR(n='70'AND c='86')OR(n='99'AND c='82')OR(n='83'AND c='59')OR(n='85'AND c='88')OR(n='55'AND c='74')OR(n='97'AND c='96')OR(n='39'AND c='56')OR(n='59'AND c='48')OR(n='42'AND c='04')OR(n='96'AND c='15')OR(n='34'AND c='19')OR(n='26'AND c='15')OR(n='74'AND c='15')OR(n='76'AND c='25')OR(n='24'AND c='58')OR(n='67'AND c='81')OR(n='25'AND c='13')OR(n='98'AND c='75')OR(n='71'AND c='21')OR(n='81'AND c='25')OR(n='85'AND c='99')OR(n='34'AND c='99')OR(n='48'AND c='99')OR(n='53'AND c='79')OR(n='70'AND c='86')OR(n='02'AND c='86')OR(n='13'AND c='86')OR(n='25'AND c='11')OR(n='85'AND c='18')OR(n='50'AND c='78')OR(n='04'AND c='11')OR(n='54'AND c='86')OR(n='00'AND c='86')OR(n='28'AND c='86')OR(n='40'AND c='99')OR(n='97'AND c='99')OR(n='84'AND c='99')OR(n='47'AND c='48')OR(n='41'AND c='04')OR(n='56'AND c='15')OR(n='97'AND c='19')OR(n='91'AND c='48')OR(n='00'AND c='04')OR(n='97'AND c='15')OR(n='05'AND c='19')OR(n='47'AND c='48')OR(n='25'AND c='04')OR(n='72'AND c='15')OR(n='93'AND c='19')OR(n='68'AND c='48')OR(n='41'AND c='04')OR(n='17'AND c='15')OR(n='17'AND c='19')OR(n='70'AND c='48')OR(n='96'AND c='04')OR(n='65'AND c='15')OR(n='50'AND c='19')OR(n='31'AND c='48')OR(n='79'AND c='04')OR(n='01'AND c='15')OR(n='03'AND c='19')OR(n='12'AND c='48')OR(n='14'AND c='04')OR(n='70'AND c='15')OR(n='51'AND c='19')OR(n='30'AND c='48')OR(n='49'AND c='04')OR(n='41'AND c='15')OR(n='32'AND c='19')OR(n='34'AND c='48')OR(n='20'AND c='04')OR(n='13'AND c='15')OR(n='05'AND c='19')OR(n='29'AND c='48')OR(n='21'AND c='04')OR(n='48'AND c='15')OR(n='18'AND c='19')OR(n='17'AND c='99')OR(n='53'AND c='22')OR(n='96'AND c='17')OR(n='98'AND c='79')OR(n='86'AND c='25')OR(n='67'AND c='98')OR(n='46'AND c='12')OR(n='84'AND c='55')OR(n='67'AND c='83')OR(n='86'AND c='28')OR(n='65'AND c='36')OR(n='00'AND c='07')OR(n='78'AND c='60')OR(n='58'AND c='89')OR(n='60'AND c='25')OR(n='82'AND c='29')OR(n='19'AND c='77')OR(n='90'AND c='66')OR(n='82'AND c='95')OR(n='91'AND c='80')OR(n='59'AND c='11')OR(n='66'AND c='38')OR(n='93'AND c='78')OR(n='71'AND c='72')OR(n='95'AND c='32')OR(n='36'AND c='32')OR(n='09'AND c='32')OR(n='39'AND c='15')OR(n='61'AND c='48')OR(n='71'AND c='11')OR(n='30'AND c='22')OR(n='51'AND c='26')OR(n='82'AND c='37')OR(n='35'AND c='79')OR(n='43'AND c='44')OR(n='61'AND c='56')OR(n='39'AND c='08')OR(n='83'AND c='56')OR(n='68'AND c='99')OR(n='51'AND c='99')OR(n='31'AND c='78')OR(n='94'AND c='99')OR(n='01'AND c='99')OR(n='31'AND c='80')OR(n='78'AND c='78')OR(n='16'AND c='99')OR(n='65'AND c='55')OR(n='49'AND c='11')OR(n='35'AND c='94')OR(n='78'AND c='99')OR(n='49'AND c='49')OR(n='58'AND c='27')OR(n='75'AND c='66')OR(n='97'AND c='99')OR(n='55'AND c='23')OR(n='02'AND c='77')OR(n='44'AND c='98')OR(n='58'AND c='18')OR(n='04'AND c='11')OR(n='83'AND c='10')OR(n='49'AND c='93')OR(n='22'AND c='99')OR(n='65'AND c='30')OR(n='01'AND c='87')OR(n='03'AND c='22')OR(n='73'AND c='06')OR(n='41'AND c='76')OR(n='13'AND c='26')OR(n='92'AND c='69')OR(n='88'AND c='37')OR(n='62'AND c='06')OR(n='85'AND c='00')OR(n='28'AND c='77')OR(n='06'AND c='06')OR(n='47'AND c='26') 
+            OR(n='16'AND c='98')OR(n='33'AND c='83')OR(n='14'AND c='56')OR(n='57'AND c='71')OR(n='01'AND c='71')OR(n='12'AND c='45')OR(n='50'AND c='68')OR(n='06'AND c='88')OR(n='07'AND c='68')OR(n='34'AND c='28')OR(n='44'AND c='60')OR(n='48'AND c='96')OR(n='42'AND c='24')OR(n='93'AND c='82')OR(n='33'AND c='67')OR(n='59'AND c='96')OR(n='62'AND c='30')OR(n='03'AND c='18')OR(n='37'AND c='11')OR(n='73'AND c='33')OR(n='84'AND c='18')OR(n='58'AND c='66')OR(n='17'AND c='02')OR(n='92'AND c='56')OR(n='39'AND c='38')OR(n='32'AND c='44')OR(n='62'AND c='77')OR(n='70'AND c='68')OR(n='50'AND c='26')OR(n='17'AND c='48')OR(n='91'AND c='48')OR(n='41'AND c='66')OR(n='11'AND c='60')OR(n='45'AND c='66')OR(n='89'AND c='71')OR(n='97'AND c='75')OR(n='19'AND c='88')OR(n='56'AND c='70')OR(n='28'AND c='08')OR(n='55'AND c='55')OR(n='48'AND c='88')OR(n='92'AND c='82')OR(n='41'AND c='30')OR(n='98'AND c='30')OR(n='27'AND c='66')OR(n='68'AND c='09')OR(n='02'AND c='82')OR(n='65'AND c='88')OR(n='76'AND c='03')OR(n='15'AND c='20')OR(n='39'AND c='56')OR(n='29'AND c='56')OR(n='55'AND c='51')OR(n='20'AND c='22')OR(n='27'AND c='00')OR(n='66'AND c='37')OR(n='75'AND c='27')OR(n='79'AND c='35')OR(n='21'AND c='54')OR(n='13'AND c='98')OR(n='21'AND c='83')OR(n='17'AND c='11')OR(n='95'AND c='26')OR(n='76'AND c='91')OR(n='74'AND c='68')OR(n='28'AND c='71')OR(n='83'AND c='56')OR(n='04'AND c='99')OR(n='17'AND c='60')OR(n='18'AND c='75')OR(n='56'AND c='94')OR(n='00'AND c='73')OR(n='42'AND c='88')OR(n='40'AND c='90')OR(n='63'AND c='87')OR(n='25'AND c='56')OR(n='77'AND c='03')OR(n='08'AND c='96')OR(n='91'AND c='66')OR(n='06'AND c='72')OR(n='94'AND c='86')OR(n='80'AND c='77')OR(n='34'AND c='07')OR(n='75'AND c='72')OR(n='05'AND c='50')OR(n='32'AND c='91')OR(n='94'AND c='71')OR(n='24'AND c='91')OR(n='95'AND c='06')OR(n='94'AND c='91')OR(n='71'AND c='86')OR(n='88'AND c='96')OR(n='36'AND c='88')OR(n='37'AND c='22')OR(n='65'AND c='66')OR(n='04'AND c='71')OR(n='41'AND c='27')OR(n='47'AND c='22')OR(n='32'AND c='97')OR(n='20'AND c='36')OR(n='63'AND c='66')OR(n='43'AND c='20')OR(n='98'AND c='97')OR(n='06'AND c='86')OR(n='45'AND c='58')OR(n='42'AND c='95')OR(n='20'AND c='65')OR(n='44'AND c='77')OR(n='76'AND c='88')OR(n='33'AND c='68')OR(n='48'AND c='83')OR(n='47'AND c='99') 
+            """
+}
diff --git a/regression-test/suites/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.groovy b/regression-test/suites/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.groovy
index 14e3733adab177..1ed15dbfc17883 100644
--- a/regression-test/suites/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.groovy
+++ b/regression-test/suites/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_pull_up_predicate_set_op") {
+suite("pull_up_predicate_set_op") {
     sql "set enable_nereids_planner=true"
     sql "set enable_fallback_to_original_planner=false"
     sql """SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"""
diff --git a/regression-test/suites/query_p0/explain/test_compoundpredicate_explain.groovy b/regression-test/suites/query_p0/explain/test_compoundpredicate_explain.groovy
index fccdd1b2e2fefa..26d91edbb12dea 100644
--- a/regression-test/suites/query_p0/explain/test_compoundpredicate_explain.groovy
+++ b/regression-test/suites/query_p0/explain/test_compoundpredicate_explain.groovy
@@ -23,49 +23,74 @@ suite("test_compoundpredicate_explain") {
 
     sql """INSERT INTO test_compoundpredicate_explain (k1, k2) VALUES (500, 450), (1100, 400), (300, 600), (700, 650), (800, 800), (1500, 300);"""
 
-    def testQueries = [
-        "select * from test_compoundpredicate_explain where k1 > 500 and k2 < 700 or k1 < 3000",
-        "select * from test_compoundpredicate_explain where k1 > 500 or k2 < 700 and k1 < 3000",
-        "select * from test_compoundpredicate_explain where not (k1 > 500 and k2 < 700) or k1 < 3000",
-        "select * from test_compoundpredicate_explain where k1 > 500 and (k2 < 700 or k1 < 3000)",
-        "select * from test_compoundpredicate_explain where not (k1 > 500 or k2 < 700) and k1 < 3000",
-        "select * from test_compoundpredicate_explain where (k1 > 500 and not k2 < 700) or k1 < 3000",
-        "select * from test_compoundpredicate_explain where (k1 > 500 and k2 < 700) and (k1 < 3000 or k2 > 400)",
-        "select * from test_compoundpredicate_explain where not (k1 > 500 or (k2 < 700 and k1 < 3000))",
-        "select * from test_compoundpredicate_explain where k1 > 500 or not (k2 < 700 and k1 < 3000)",
-        "select * from test_compoundpredicate_explain where k1 < 1000 and (k2 < 700 or k1 > 500) and not (k2 > 300)",
-        "select * from test_compoundpredicate_explain where not ((k1 > 500 and k2 < 700) or k1 < 3000)",
-        "select * from test_compoundpredicate_explain where k1 > 500 and not (k2 < 700 or k1 < 3000)",
-        "select * from test_compoundpredicate_explain where (k1 > 500 or k2 < 700) and (k1 < 3000 and k2 > 200)",
-        "select * from test_compoundpredicate_explain where (k1 > 500 and k2 < 700) or not (k1 < 3000 and k2 > 200)"
-    ]
-
-    testQueries.each { query ->
-        def explainResult1 = sql "explain all plan ${query}"
-        def explainResult2 = sql "explain ${query}"
-
-        def predicates2Line = explainResult2.find { line ->
-            line[0].toString().trim().startsWith("PREDICATES:")
-        }
-
-        if (predicates2Line != null) {
-            def predicates2 = predicates2Line[0].split("PREDICATES:").last().trim()
-
-            predicates2 = predicates2?.replaceAll(/\[\#(\d+)\]/) { match, group1 -> "#" + group1 }
-
-            def isMatch = explainResult1.any { line ->
-                line.toString().contains(predicates2)
-            }
-
-            log.info("Testing query: " + query)
-            log.info("Standardized Predicates from PREDICATES: " + predicates2)
-            log.info("Match found in OPTIMIZED PLAN: " + isMatch)
-
-            assert isMatch : "Predicates are not equal for query: ${query}"
-        } else {
-            logger.error("PREDICATES: not found in explain result for query: ${query}")
-            assert false : "PREDICATES: not found in explain result"
-        }
+    explain {
+        sql "select * from test_compoundpredicate_explain where k1 > 500 and k2 < 700 or k1 < 3000"
+        contains "(((k1[#0] > 500) AND (k2[#1] < 700)) OR (k1[#0] < 3000))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where k1 > 500 or k2 < 700 and k1 < 3000"
+        contains "((k1[#0] > 500) OR ((k2[#1] < 700) AND (k1[#0] < 3000)))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where not (k1 > 500 and k2 < 700) or k1 < 3000"
+        contains "((k1[#0] < 3000) OR (k2[#1] >= 700))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where k1 > 500 and (k2 < 700 or k1 < 3000)"
+        contains "PREDICATES: ((k1[#0] > 500) AND ((k2[#1] < 700) OR (k1[#0] < 3000)))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where not (k1 > 500 or k2 < 700) and k1 < 3000"
+        contains "PREDICATES: ((k1[#0] <= 500) AND (k2[#1] >= 700))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where (k1 > 500 and not k2 < 700) or k1 < 3000"
+        contains "PREDICATES: (((k1[#0] > 500) AND (k2[#1] >= 700)) OR (k1[#0] < 3000))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where (k1 > 500 and k2 < 700) and (k1 < 3000 or k2 > 400)"
+        contains "PREDICATES: (((k1[#0] > 500) AND (k2[#1] < 700)) AND ((k1[#0] < 3000) OR (k2[#1] > 400)))"
+    }
+
+    explain {
+        sql  "select * from test_compoundpredicate_explain where not (k1 > 500 or (k2 < 700 and k1 < 3000))"
+        contains "PREDICATES: ((k1[#0] <= 500) AND ((k2[#1] >= 700) OR (k1[#0] >= 3000)))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where k1 > 500 or not (k2 < 700 and k1 < 3000)"
+        contains "PREDICATES: ((k1[#0] > 500) OR (k2[#1] >= 700))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where k1 < 1000 and (k2 < 700 or k1 > 500) and not (k2 > 300)"
+        contains "PREDICATES: (((k1[#0] < 1000) AND ((k2[#1] < 700) OR (k1[#0] > 500))) AND (k2[#1] <= 300))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where not ((k1 > 500 and k2 < 700) or k1 < 3000)"
+        contains "PREDICATES: (((k1[#0] <= 500) OR (k2[#1] >= 700)) AND (k1[#0] >= 3000))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where k1 > 500 and not (k2 < 700 or k1 < 3000)"
+        contains "PREDICATES: ((k1[#0] >= 3000) AND (k2[#1] >= 700))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where (k1 > 500 or k2 < 700) and (k1 < 3000 and k2 > 200)"
+        contains "PREDICATES: ((((k1[#0] > 500) OR (k2[#1] < 700)) AND (k1[#0] < 3000)) AND (k2[#1] > 200))"
+    }
+
+    explain {
+        sql "select * from test_compoundpredicate_explain where (k1 > 500 and k2 < 700) or not (k1 < 3000 and k2 > 200)"
+        contains "PREDICATES: ((((k1[#0] > 500) AND (k2[#1] < 700)) OR (k1[#0] >= 3000)) OR (k2[#1] <= 200))"
     }
 
     sql "drop table if exists test_compoundpredicate_explain"

From ea2a21dc7519142ffcb4af893c88e1dfd4215556 Mon Sep 17 00:00:00 2001
From: htyoung <hty551020@hotmail.com>
Date: Sat, 30 Nov 2024 22:48:56 +0800
Subject: [PATCH 090/399] [fix](cache) fix same sql return wrong result when
 switch database with `use db` and enable sql cache (#44782)

### What problem does this PR solve?

It will return wrong result when running same sql in different db with
`enable_sql_cache=true`

for example, the `db1` and `db2` has the same table `tbl` but the data
are not equals,
if execute the below sql in `db1` and cache the result, then execute it
in `db2`, it will return the wrong result
```sql
select count(*) from tbl
```
Co-authored-by: tongyang.han <tongyang.han@jiduauto.com>
---
 .../doris/common/NereidsSqlCacheManager.java  | 49 +++++++-----
 .../cache/parse_sql_from_sql_cache.groovy     | 74 +++++++++++++++++++
 2 files changed, 105 insertions(+), 18 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java
index 1317fdaefc766e..cd32b52034a5d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java
@@ -74,9 +74,11 @@
 import java.util.Optional;
 import java.util.Set;
 
-/** NereidsSqlCacheManager */
+/**
+ * NereidsSqlCacheManager
+ */
 public class NereidsSqlCacheManager {
-    // key: <user>:<sql>
+    // key: <ctl.db>:<user>:<sql>
     // value: SqlCacheContext
     private volatile Cache<String, SqlCacheContext> sqlCaches;
 
@@ -110,7 +112,7 @@ private static Cache<String, SqlCacheContext> buildSqlCaches(int sqlCacheNum, lo
                 // auto evict cache when jvm memory too low
                 .softValues();
         if (sqlCacheNum > 0) {
-            cacheBuilder = cacheBuilder.maximumSize(sqlCacheNum);
+            cacheBuilder.maximumSize(sqlCacheNum);
         }
         if (expireAfterAccessSeconds > 0) {
             cacheBuilder = cacheBuilder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSeconds));
@@ -119,7 +121,9 @@ private static Cache<String, SqlCacheContext> buildSqlCaches(int sqlCacheNum, lo
         return cacheBuilder.build();
     }
 
-    /** tryAddFeCache */
+    /**
+     * tryAddFeCache
+     */
     public void tryAddFeSqlCache(ConnectContext connectContext, String sql) {
         Optional<SqlCacheContext> sqlCacheContextOpt = connectContext.getStatementContext().getSqlCacheContext();
         if (!sqlCacheContextOpt.isPresent()) {
@@ -127,17 +131,18 @@ public void tryAddFeSqlCache(ConnectContext connectContext, String sql) {
         }
 
         SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get();
-        UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
         String key = sqlCacheContext.getCacheKeyType() == CacheKeyType.SQL
-                ? currentUserIdentity.toString() + ":" + normalizeSql(sql.trim())
-                : currentUserIdentity.toString() + ":" + DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5());
+                ? generateCacheKey(connectContext, normalizeSql(sql))
+                : generateCacheKey(connectContext, DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5()));
         if (sqlCaches.getIfPresent(key) == null && sqlCacheContext.getOrComputeCacheKeyMd5() != null
                 && sqlCacheContext.getResultSetInFe().isPresent()) {
             sqlCaches.put(key, sqlCacheContext);
         }
     }
 
-    /** tryAddBeCache */
+    /**
+     * tryAddBeCache
+     */
     public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyzer analyzer) {
         Optional<SqlCacheContext> sqlCacheContextOpt = connectContext.getStatementContext().getSqlCacheContext();
         if (!sqlCacheContextOpt.isPresent()) {
@@ -147,10 +152,9 @@ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyz
             return;
         }
         SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get();
-        UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
         String key = sqlCacheContext.getCacheKeyType() == CacheKeyType.SQL
-                ? currentUserIdentity.toString() + ":" + normalizeSql(sql.trim())
-                : currentUserIdentity.toString() + ":" + DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5());
+                ? generateCacheKey(connectContext, normalizeSql(sql))
+                : generateCacheKey(connectContext, DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5()));
         if (sqlCaches.getIfPresent(key) == null && sqlCacheContext.getOrComputeCacheKeyMd5() != null) {
             SqlCache cache = (SqlCache) analyzer.getCache();
             sqlCacheContext.setSumOfPartitionNum(cache.getSumOfPartitionNum());
@@ -167,23 +171,23 @@ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyz
         }
     }
 
-    /** tryParseSql */
+    /**
+     * tryParseSql
+     */
     public Optional<LogicalSqlCache> tryParseSql(ConnectContext connectContext, String sql) {
-        UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
-        String key = currentUserIdentity + ":" + normalizeSql(sql.trim());
+        String key = generateCacheKey(connectContext, normalizeSql(sql.trim()));
         SqlCacheContext sqlCacheContext = sqlCaches.getIfPresent(key);
         if (sqlCacheContext == null) {
             return Optional.empty();
         }
 
         // LOG.info("Total size: " + GraphLayout.parseInstance(sqlCacheContext).totalSize());
-
+        UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
         List<Variable> currentVariables = resolveUserVariables(sqlCacheContext);
         if (usedVariablesChanged(currentVariables, sqlCacheContext)) {
             String md5 = DebugUtil.printId(
                     sqlCacheContext.doComputeCacheKeyMd5(Utils.fastToImmutableSet(currentVariables)));
-
-            String md5CacheKey = currentUserIdentity + ":" + md5;
+            String md5CacheKey = generateCacheKey(connectContext, md5);
             SqlCacheContext sqlCacheContextWithVariable = sqlCaches.getIfPresent(md5CacheKey);
 
             // already exist cache in the fe, but the variable is different to this query,
@@ -203,6 +207,15 @@ public Optional<LogicalSqlCache> tryParseSql(ConnectContext connectContext, Stri
         }
     }
 
+    private String generateCacheKey(ConnectContext connectContext, String sqlOrMd5) {
+        CatalogIf<?> currentCatalog = connectContext.getCurrentCatalog();
+        String currentCatalogName = currentCatalog != null ? currentCatalog.getName() : "";
+        String currentDatabase = connectContext.getDatabase();
+        String currentDatabaseName = currentDatabase != null ? currentDatabase : "";
+        return currentCatalogName + "." + currentDatabaseName + ":" + connectContext.getCurrentUserIdentity().toString()
+                + ":" + sqlOrMd5;
+    }
+
     private String normalizeSql(String sql) {
         return NereidsParser.removeCommentAndTrimBlank(sql);
     }
@@ -402,7 +415,7 @@ private boolean usedVariablesChanged(List<Variable> currentVariables, SqlCacheCo
             Variable cachedVariable = cachedUsedVariables.get(i);
             if (!Objects.equals(currentVariable, cachedVariable)
                     || cachedVariable.getRealExpression().anyMatch(
-                            expr -> !((ExpressionTrait) expr).isDeterministic())) {
+                        expr -> !((ExpressionTrait) expr).isDeterministic())) {
                 return true;
             }
         }
diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
index 54ab702888888d..3635936e8be083 100644
--- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
+++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
@@ -827,6 +827,80 @@ suite("parse_sql_from_sql_cache") {
                 def result = sql "select FROM_UNIXTIME(UNIX_TIMESTAMP(), 'yyyy-MM-dd HH:mm:ss')"
                 assertNotEquals("yyyy-MM-dd HH:mm:ss", result[0][0])
             }
+        }),
+        extraThread("test_same_sql_with_different_db", {
+            def dbName1 = "test_db1"
+            def dbName2 = "test_db2"
+            def tableName = "test_cache_table"
+
+            sql "CREATE DATABASE IF NOT EXISTS ${dbName1}"
+            sql "DROP TABLE IF EXISTS ${dbName1}.${tableName}"
+            sql """
+                CREATE TABLE IF NOT EXISTS ${dbName1}.${tableName} (
+                  `k1` date NOT NULL COMMENT "",
+                  `k2` int(11) NOT NULL COMMENT ""
+                ) ENGINE=OLAP
+                DUPLICATE KEY(`k1`, `k2`)
+                COMMENT "OLAP"
+                PARTITION BY RANGE(`k1`)
+                (PARTITION p202411 VALUES [('2024-11-01'), ('2024-12-01')))
+                DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
+                PROPERTIES (
+                "replication_allocation" = "tag.location.default: 1",
+                "in_memory" = "false",
+                "storage_format" = "V2"
+                )
+            """
+            sql "CREATE DATABASE IF NOT EXISTS ${dbName2}"
+            sql "DROP TABLE IF EXISTS ${dbName2}.${tableName}"
+            sql """
+                CREATE TABLE IF NOT EXISTS ${dbName2}.${tableName} (
+                  `k1` date NOT NULL COMMENT "",
+                  `k2` int(11) NOT NULL COMMENT ""
+                ) ENGINE=OLAP
+                DUPLICATE KEY(`k1`, `k2`)
+                COMMENT "OLAP"
+                PARTITION BY RANGE(`k1`)
+                (PARTITION p202411 VALUES [('2024-11-01'), ('2024-12-01')))
+                DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
+                PROPERTIES (
+                "replication_allocation" = "tag.location.default: 1",
+                "in_memory" = "false",
+                "storage_format" = "V2"
+                )
+            """
+
+            sql """
+                INSERT INTO ${dbName1}.${tableName} VALUES 
+                        ("2024-11-29",0),
+                        ("2024-11-30",0)
+            """
+            // after partition changed 10s, the sql cache can be used
+            sleep(10000)
+            sql """
+                INSERT INTO ${dbName2}.${tableName} VALUES 
+                        ("2024-11-29",0)
+            """
+            // after partition changed 10s, the sql cache can be used
+            sleep(10000)
+
+            sql "set enable_sql_cache=true"
+            sql "use ${dbName1}"
+            List<List<Object>> result1 = sql """
+                SELECT COUNT(*) FROM ${tableName}
+            """
+            assertEquals(result1[0][0],2)
+
+            sql "use ${dbName2}"
+            List<List<Object>> result2 = sql """
+                SELECT COUNT(*) FROM ${tableName}
+            """
+            assertEquals(result2[0][0],1)
+
+            sql "DROP TABLE IF EXISTS ${dbName1}.${tableName}"
+            sql "DROP TABLE IF EXISTS ${dbName2}.${tableName}"
+            sql "DROP DATABASE IF EXISTS ${dbName1}"
+            sql "DROP DATABASE IF EXISTS ${dbName2}"
         })
     ).get()
 }

From 97adac19f8c02600e1306a39e61b8237bf5f856b Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Sun, 1 Dec 2024 20:17:20 +0800
Subject: [PATCH 091/399] [fix](maxcompute)add mc catalog  read partition table
 partition prune (#44508)

### What problem does this PR solve?

Based on the `requiredPartitions` API interface of the maxcompute API,
we implemented partition pruning when reading partitioned tables to
avoid queries that scan the entire table.
---
 .../doris/datasource/ExternalTable.java       |   9 +-
 .../datasource/hive/HMSExternalTable.java     |   4 +-
 .../datasource/hive/source/HiveScanNode.java  |   2 +-
 .../maxcompute/MaxComputeExternalTable.java   |  33 +-
 .../maxcompute/source/MaxComputeScanNode.java |  50 +++-
 .../translator/PhysicalPlanTranslator.java    |   3 +-
 .../expression/rules/PartitionPruner.java     |   2 +-
 .../rules/rewrite/PruneFileScanPartition.java |   4 +-
 .../doris/planner/SingleNodePlanner.java      |   5 +-
 .../test_max_compute_partition_prune.out      | 125 ++++++++
 .../test_max_compute_partition_prune.groovy   | 282 ++++++++++++++++++
 11 files changed, 497 insertions(+), 22 deletions(-)
 create mode 100644 regression-test/data/external_table_p2/maxcompute/test_max_compute_partition_prune.out
 create mode 100644 regression-test/suites/external_table_p2/maxcompute/test_max_compute_partition_prune.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index d82959954f2607..6c72d0a67d4732 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -377,7 +377,7 @@ public TableIndexes getTableIndexes() {
      * @return
      */
     public SelectedPartitions initSelectedPartitions(Optional<MvccSnapshot> snapshot) {
-        if (!supportPartitionPruned()) {
+        if (!supportInternalPartitionPruned()) {
             return SelectedPartitions.NOT_PRUNED;
         }
         if (CollectionUtils.isEmpty(this.getPartitionColumns(snapshot))) {
@@ -394,7 +394,7 @@ public SelectedPartitions initSelectedPartitions(Optional<MvccSnapshot> snapshot
      * @param snapshot if not support mvcc, ignore this
      * @return partitionName ==> PartitionItem
      */
-    public Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
+    protected Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
         return Collections.emptyMap();
     }
 
@@ -410,11 +410,12 @@ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
     }
 
     /**
-     * Does it support partition cpruned， If so, this method needs to be overridden in subclasses
+     * Does it support Internal partition pruned, If so, this method needs to be overridden in subclasses
+     * Internal partition pruned : Implement partition pruning logic without relying on external APIs.
      *
      * @return
      */
-    public boolean supportPartitionPruned() {
+    public boolean supportInternalPartitionPruned() {
         return false;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 6d65f8bcdbccb7..134ad362fa1eed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -301,12 +301,12 @@ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
     }
 
     @Override
-    public boolean supportPartitionPruned() {
+    public boolean supportInternalPartitionPruned() {
         return getDlaType() == DLAType.HIVE;
     }
 
     @Override
-    public Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
+    protected Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
         return getNameToPartitionItems();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index 3a2a4d3eb5c6ae..99d3cd1cd21622 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -82,7 +82,7 @@ public class HiveScanNode extends FileQueryScanNode {
 
     // will only be set in Nereids, for lagency planner, it should be null
     @Setter
-    private SelectedPartitions selectedPartitions = null;
+    protected SelectedPartitions selectedPartitions = null;
 
     private boolean partitionInit = false;
     private final AtomicReference<UserException> batchException = new AtomicReference<>(null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
index dc3232f79f5f71..0f748f59e927bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
@@ -21,6 +21,7 @@
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.PartitionItem;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.StructField;
 import org.apache.doris.catalog.StructType;
@@ -28,6 +29,7 @@
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.SchemaCacheValue;
 import org.apache.doris.datasource.TablePartitionValues;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
 import org.apache.doris.thrift.TMCTable;
 import org.apache.doris.thrift.TTableDescriptor;
 import org.apache.doris.thrift.TTableType;
@@ -50,6 +52,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
@@ -71,6 +74,15 @@ protected synchronized void makeSureInitialized() {
         }
     }
 
+    @Override
+    public boolean supportInternalPartitionPruned() {
+        return true;
+    }
+
+    @Override
+    public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
+        return getPartitionColumns();
+    }
 
     public List<Column> getPartitionColumns() {
         makeSureInitialized();
@@ -79,7 +91,24 @@ public List<Column> getPartitionColumns() {
                 .orElse(Collections.emptyList());
     }
 
-    public TablePartitionValues getPartitionValues() {
+    @Override
+    protected Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
+        if (getPartitionColumns().isEmpty()) {
+            return Collections.emptyMap();
+        }
+
+        TablePartitionValues tablePartitionValues = getPartitionValues();
+        Map<Long, PartitionItem> idToPartitionItem = tablePartitionValues.getIdToPartitionItem();
+        Map<Long, String> idToNameMap = tablePartitionValues.getPartitionIdToNameMap();
+
+        Map<String, PartitionItem> nameToPartitionItem = Maps.newHashMapWithExpectedSize(idToPartitionItem.size());
+        for (Entry<Long, PartitionItem> entry : idToPartitionItem.entrySet()) {
+            nameToPartitionItem.put(idToNameMap.get(entry.getKey()), entry.getValue());
+        }
+        return nameToPartitionItem;
+    }
+
+    private TablePartitionValues getPartitionValues() {
         makeSureInitialized();
         Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
         if (!schemaCacheValue.isPresent()) {
@@ -110,6 +139,8 @@ private TablePartitionValues loadPartitionValues(MaxComputeSchemaCacheValue sche
 
     /**
      * parse all values from partitionPath to a single list.
+     * In MaxCompute : Support special characters : _$#.!@
+     * Ref : MaxCompute Error Code: ODPS-0130071  Invalid partition value.
      *
      * @param partitionColumns partitionColumns can contain the part1,part2,part3...
      * @param partitionPath partitionPath format is like the 'part1=123/part2=abc/part3=1bc'
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index e0b84b0860e551..e177e9d8b7c88c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -40,6 +40,7 @@
 import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable;
 import org.apache.doris.datasource.maxcompute.source.MaxComputeSplit.SplitType;
 import org.apache.doris.datasource.property.constants.MCProperties;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
 import org.apache.doris.nereids.util.DateUtils;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.spi.Split;
@@ -50,6 +51,7 @@
 import org.apache.doris.thrift.TTableFormatFileDesc;
 
 import com.aliyun.odps.OdpsType;
+import com.aliyun.odps.PartitionSpec;
 import com.aliyun.odps.table.TableIdentifier;
 import com.aliyun.odps.table.configuration.ArrowOptions;
 import com.aliyun.odps.table.configuration.ArrowOptions.TimestampUnit;
@@ -60,6 +62,7 @@
 import com.aliyun.odps.table.read.split.impl.IndexedInputSplit;
 import com.google.common.collect.Maps;
 import jline.internal.Log;
+import lombok.Setter;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -86,14 +89,28 @@ public class MaxComputeScanNode extends FileQueryScanNode {
     private static final LocationPath ROW_OFFSET_PATH = new LocationPath("/row_offset", Maps.newHashMap());
     private static final LocationPath BYTE_SIZE_PATH = new LocationPath("/byte_size", Maps.newHashMap());
 
+    @Setter
+    private SelectedPartitions selectedPartitions = null;
+
+    // For new planner
+    public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc,
+            SelectedPartitions selectedPartitions, boolean needCheckColumnPriv) {
+        this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE,
+                selectedPartitions, needCheckColumnPriv);
+    }
+
+    // For old planner
     public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
-        this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE, needCheckColumnPriv);
+        this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE,
+                SelectedPartitions.NOT_PRUNED, needCheckColumnPriv);
     }
 
-    public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName,
-                              StatisticalType statisticalType, boolean needCheckColumnPriv) {
+    private MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName,
+            StatisticalType statisticalType, SelectedPartitions selectedPartitions,
+            boolean needCheckColumnPriv) {
         super(id, desc, planNodeName, statisticalType, needCheckColumnPriv);
         table = (MaxComputeExternalTable) desc.getTable();
+        this.selectedPartitions = selectedPartitions;
     }
 
     @Override
@@ -117,10 +134,27 @@ private void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeS
         rangeDesc.setSize(maxComputeSplit.getLength());
     }
 
-    void createTableBatchReadSession() throws UserException {
+    // Return false if no need to read any partition data.
+    // Return true if need to read partition data.
+    boolean createTableBatchReadSession() throws UserException {
         List<String> requiredPartitionColumns = new ArrayList<>();
         List<String> orderedRequiredDataColumns = new ArrayList<>();
 
+        List<PartitionSpec> requiredPartitionSpecs = new ArrayList<>();
+        //if requiredPartitionSpecs is empty, get all partition data.
+        if (!table.getPartitionColumns().isEmpty() && selectedPartitions != SelectedPartitions.NOT_PRUNED) {
+            this.totalPartitionNum = selectedPartitions.totalPartitionNum;
+            this.selectedPartitionNum = selectedPartitions.selectedPartitions.size();
+
+            if (selectedPartitions.selectedPartitions.isEmpty()) {
+                //no need read any partition data.
+                return false;
+            }
+            selectedPartitions.selectedPartitions.forEach(
+                    (key, value) -> requiredPartitionSpecs.add(new PartitionSpec(key))
+            );
+        }
+
         Set<String> requiredSlots =
                 desc.getSlots().stream().map(e -> e.getColumn().getName()).collect(Collectors.toSet());
 
@@ -150,6 +184,7 @@ void createTableBatchReadSession() throws UserException {
                             .withSettings(mcCatalog.getSettings())
                             .withSplitOptions(mcCatalog.getSplitOption())
                             .requiredPartitionColumns(requiredPartitionColumns)
+                            .requiredPartitions(requiredPartitionSpecs)
                             .requiredDataColumns(orderedRequiredDataColumns)
                             .withArrowOptions(
                                     ArrowOptions.newBuilder()
@@ -162,7 +197,7 @@ void createTableBatchReadSession() throws UserException {
         } catch (java.io.IOException e) {
             throw new RuntimeException(e);
         }
-
+        return true;
     }
 
     @Override
@@ -430,7 +465,10 @@ public List<Split> getSplits() throws UserException {
         if (desc.getSlots().isEmpty() || odpsTable.getFileNum() <= 0) {
             return result;
         }
-        createTableBatchReadSession();
+
+        if (!createTableBatchReadSession()) {
+            return result;
+        }
 
         try {
             String scanSessionSerialize =  serializeSession(tableBatchReadSession);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 56ae65ec722941..c474e6bd56e1d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -579,7 +579,8 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla
         } else if (table instanceof TrinoConnectorExternalTable) {
             scanNode = new TrinoConnectorScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
         } else if (table instanceof MaxComputeExternalTable) {
-            scanNode = new MaxComputeScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
+            scanNode = new MaxComputeScanNode(context.nextPlanNodeId(), tupleDescriptor,
+                    fileScan.getSelectedPartitions(), false);
         } else if (table instanceof LakeSoulExternalTable) {
             scanNode = new LakeSoulScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
         } else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
index fac1a7f82d2cfb..ed783aa3d5a9b6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
@@ -55,7 +55,7 @@ public class PartitionPruner extends DefaultExpressionRewriter<Void> {
     /** Different type of table may have different partition prune behavior. */
     public enum PartitionTableType {
         OLAP,
-        HIVE
+        EXTERNAL
     }
 
     private PartitionPruner(List<OnePartitionEvaluator> partitions, Expression partitionPredicate) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
index 4bbb0a8aa76270..ba8b270d1f397d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
@@ -59,7 +59,7 @@ public Rule build() {
                     ExternalTable tbl = scan.getTable();
 
                     SelectedPartitions selectedPartitions;
-                    if (tbl.supportPartitionPruned()) {
+                    if (tbl.supportInternalPartitionPruned()) {
                         selectedPartitions = pruneExternalPartitions(tbl, filter, scan, ctx.cascadesContext);
                     } else {
                         // set isPruned so that it won't go pass the partition prune again
@@ -91,7 +91,7 @@ private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable,
 
         Map<String, PartitionItem> nameToPartitionItem = scan.getSelectedPartitions().selectedPartitions;
         List<String> prunedPartitions = new ArrayList<>(PartitionPruner.prune(
-                partitionSlots, filter.getPredicate(), nameToPartitionItem, ctx, PartitionTableType.HIVE));
+                partitionSlots, filter.getPredicate(), nameToPartitionItem, ctx, PartitionTableType.EXTERNAL));
 
         for (String name : prunedPartitions) {
             selectedPartitionItems.put(name, nameToPartitionItem.get(name));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index d94ad0a2552240..4091640066c1d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -76,7 +76,6 @@
 import org.apache.doris.datasource.trinoconnector.source.TrinoConnectorScanNode;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException;
-import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.thrift.TPushAggOp;
 
 import com.google.common.base.Preconditions;
@@ -1993,9 +1992,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s
                 scanNode = new TrinoConnectorScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
                 break;
             case MAX_COMPUTE_EXTERNAL_TABLE:
-                // TODO: support max compute scan node
-                scanNode = new MaxComputeScanNode(ctx.getNextNodeId(), tblRef.getDesc(), "MCScanNode",
-                        StatisticalType.MAX_COMPUTE_SCAN_NODE, true);
+                scanNode = new MaxComputeScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
                 break;
             case ES_EXTERNAL_TABLE:
                 scanNode = new EsScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
diff --git a/regression-test/data/external_table_p2/maxcompute/test_max_compute_partition_prune.out b/regression-test/data/external_table_p2/maxcompute/test_max_compute_partition_prune.out
new file mode 100644
index 00000000000000..8f443829c4c394
--- /dev/null
+++ b/regression-test/data/external_table_p2/maxcompute/test_max_compute_partition_prune.out
@@ -0,0 +1,125 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
diff --git a/regression-test/suites/external_table_p2/maxcompute/test_max_compute_partition_prune.groovy b/regression-test/suites/external_table_p2/maxcompute/test_max_compute_partition_prune.groovy
new file mode 100644
index 00000000000000..e34569117a167f
--- /dev/null
+++ b/regression-test/suites/external_table_p2/maxcompute/test_max_compute_partition_prune.groovy
@@ -0,0 +1,282 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+
+/*
+CREATE TABLE one_partition_tb (
+    id INT,
+    name string
+)
+PARTITIONED BY (part1 INT);
+INSERT INTO one_partition_tb PARTITION (part1=2024) VALUES (1, 'Alice');
+INSERT INTO one_partition_tb PARTITION (part1=2024) VALUES (2, 'Bob');
+INSERT INTO one_partition_tb PARTITION (part1=2024) VALUES (3, 'Charlie');
+INSERT INTO one_partition_tb PARTITION (part1=2025) VALUES (4, 'David');
+INSERT INTO one_partition_tb PARTITION (part1=2025) VALUES (5, 'Eva');
+CREATE TABLE two_partition_tb (
+    id INT,
+    name string
+)
+PARTITIONED BY (part1 STRING, part2 int);
+INSERT INTO two_partition_tb PARTITION (part1='US', part2=1) VALUES (1, 'Alice');
+INSERT INTO two_partition_tb PARTITION (part1='US', part2=1) VALUES (2, 'Bob');
+INSERT INTO two_partition_tb PARTITION (part1='US', part2=1) VALUES (3, 'Charlie');
+INSERT INTO two_partition_tb PARTITION (part1='US', part2=2) VALUES (4, 'David');
+INSERT INTO two_partition_tb PARTITION (part1='US', part2=2) VALUES (5, 'Eva');
+INSERT INTO two_partition_tb PARTITION (part1='EU', part2=1) VALUES (6, 'Frank');
+INSERT INTO two_partition_tb PARTITION (part1='EU', part2=1) VALUES (7, 'Grace');
+INSERT INTO two_partition_tb PARTITION (part1='EU', part2=2) VALUES (8, 'Hannah');
+INSERT INTO two_partition_tb PARTITION (part1='EU', part2=2) VALUES (9, 'Ivy');
+INSERT INTO two_partition_tb PARTITION (part1='EU', part2=2) VALUES (10, 'Jack');
+CREATE TABLE three_partition_tb (
+    id INT,
+    name string
+)
+PARTITIONED BY (part1 STRING, part2 INT, part3 STRING);
+INSERT INTO three_partition_tb PARTITION (part1='US', part2=2024, part3='Q1') VALUES (1, 'Alice');
+INSERT INTO three_partition_tb PARTITION (part1='US', part2=2024, part3='Q1') VALUES (2, 'Bob');
+INSERT INTO three_partition_tb PARTITION (part1='US', part2=2024, part3='Q1') VALUES (3, 'Charlie');
+INSERT INTO three_partition_tb PARTITION (part1='US', part2=2024, part3='Q2') VALUES (4, 'David');
+INSERT INTO three_partition_tb PARTITION (part1='US', part2=2024, part3='Q2') VALUES (5, 'Eva');
+INSERT INTO three_partition_tb PARTITION (part1='US', part2=2025, part3='Q1') VALUES (6, 'Frank');
+INSERT INTO three_partition_tb PARTITION (part1='US', part2=2025, part3='Q2') VALUES (7, 'Grace');
+INSERT INTO three_partition_tb PARTITION (part1='EU', part2=2024, part3='Q1') VALUES (8, 'Hannah');
+INSERT INTO three_partition_tb PARTITION (part1='EU', part2=2024, part3='Q1') VALUES (9, 'Ivy');
+INSERT INTO three_partition_tb PARTITION (part1='EU', part2=2025, part3='Q2') VALUES (10, 'Jack');
+INSERT INTO three_partition_tb PARTITION (part1='EU', part2=2025, part3='Q2') VALUES (11, 'Leo');
+INSERT INTO three_partition_tb PARTITION (part1='EU', part2=2025, part3='Q3') VALUES (12, 'Mia');
+INSERT INTO three_partition_tb PARTITION (part1='AS', part2=2025, part3='Q1') VALUES (13, 'Nina');
+INSERT INTO three_partition_tb PARTITION (part1='AS', part2=2025, part3='Q2') VALUES (14, 'Oscar');
+INSERT INTO three_partition_tb PARTITION (part1='AS', part2=2025, part3='Q3') VALUES (15, 'Paul');
+select * from one_partition_tb;
+select * from two_partition_tb;
+select * from three_partition_tb;
+show partitions one_partition_tb;
+show partitions two_partition_tb;
+show partitions three_partition_tb;
+*/
+
+suite("test_max_compute_partition_prune", "p2,external,maxcompute,external_remote,external_remote_maxcompute") {
+
+
+    def one_partition_1_1 = """SELECT * FROM one_partition_tb WHERE part1 = 2024 ORDER BY id;"""
+    def one_partition_2_1 = """SELECT * FROM one_partition_tb WHERE part1 = 2025 ORDER BY id;"""
+    def one_partition_3_all = """SELECT * FROM one_partition_tb ORDER BY id;"""
+    def one_partition_4_all = """SELECT * FROM one_partition_tb WHERE id = 5 ORDER BY id;"""
+    def one_partition_5_1 = """SELECT * FROM one_partition_tb WHERE part1 = 2024 AND id >= 3 ORDER BY id;"""
+
+    def two_partition_1_1 = """SELECT * FROM two_partition_tb WHERE part1 = 'US' AND part2 = 1 ORDER BY id;"""
+    def two_partition_2_1 = """SELECT * FROM two_partition_tb WHERE part1 = 'EU' AND part2 = 2 ORDER BY id;"""
+    def two_partition_3_2 = """SELECT * FROM two_partition_tb WHERE part1 = 'US' ORDER BY id;"""
+    def two_partition_4_all = """SELECT * FROM two_partition_tb ORDER BY id;"""
+    def two_partition_5_1 = """SELECT * FROM two_partition_tb WHERE part1 = 'US' AND part2 = 2 AND id > 5 ORDER BY id;"""
+    def two_partition_6_1 = """SELECT * FROM two_partition_tb WHERE part1 = 'EU' AND part2 = 2 ORDER BY id;"""
+
+    def three_partition_1_1 = """SELECT * FROM three_partition_tb WHERE part1 = 'US' AND part2 = 2024 AND part3 = 'Q1' ORDER BY id;"""
+    def three_partition_2_1 = """SELECT * FROM three_partition_tb WHERE part1 = 'EU' AND part2 = 2025 AND part3 = 'Q2' ORDER BY id;"""
+    def three_partition_3_3 = """SELECT * FROM three_partition_tb WHERE part1 = 'AS' AND part2 = 2025 ORDER BY id;"""
+    def three_partition_4_2 = """SELECT * FROM three_partition_tb WHERE part1 = 'US' AND part3 = 'Q1' ORDER BY id;"""
+    def three_partition_5_all = """SELECT * FROM three_partition_tb ORDER BY id;"""
+    def three_partition_6_1 = """SELECT * FROM three_partition_tb WHERE part1 = 'EU' AND part2 = 2024 AND part3 = 'Q1' ORDER BY id;"""
+    def three_partition_7_7 = """SELECT * FROM three_partition_tb WHERE part2 = 2025 ORDER BY id;"""
+    def three_partition_8_2 = """SELECT * FROM three_partition_tb WHERE part1 = 'US' AND part3 = 'Q2' AND id BETWEEN 6 AND 10 ORDER BY id;"""
+
+
+    String enabled = context.config.otherConfigs.get("enableMaxComputeTest")
+    if (enabled != null && enabled.equalsIgnoreCase("true")) {
+        String ak = context.config.otherConfigs.get("ak")
+        String sk = context.config.otherConfigs.get("sk");
+        String mc_db = "mc_datalake"
+        String mc_catalog_name = "test_max_compute_partition_prune"
+
+        sql """drop catalog if exists ${mc_catalog_name};"""
+        sql """
+            create catalog if not exists ${mc_catalog_name} properties (
+                "type" = "max_compute",
+                "mc.default.project" = "${mc_db}",
+                "mc.access_key" = "${ak}",
+                "mc.secret_key" = "${sk}",
+                "mc.endpoint" = "http://service.cn-beijing-vpc.maxcompute.aliyun-inc.com/api"
+            );
+        """
+        sql """ switch ${mc_catalog_name} """
+        sql """ use ${mc_db}"""
+
+        qt_one_partition_1_1 one_partition_1_1
+        explain {
+            sql("${one_partition_1_1}")
+            contains "partition=1/2"
+        }
+
+        qt_one_partition_2_1 one_partition_2_1
+        explain {
+            sql("${one_partition_2_1}")
+            contains "partition=1/2"
+        }
+
+        qt_one_partition_3_all one_partition_3_all
+        explain {
+            sql("${one_partition_3_all}")
+            contains "partition=2/2"
+        }
+
+        qt_one_partition_4_all one_partition_4_all
+        explain {
+            sql("${one_partition_4_all}")
+            contains "partition=2/2"
+        }
+
+        qt_one_partition_5_1 one_partition_5_1
+        explain {
+            sql("${one_partition_5_1}")
+            contains "partition=1/2"
+        }
+
+
+        qt_two_partition_1_1 two_partition_1_1
+        explain {
+            sql("${two_partition_1_1}")
+            contains "partition=1/4"
+        }
+
+        qt_two_partition_2_1 two_partition_2_1
+        explain {
+            sql("${two_partition_2_1}")
+            contains "partition=1/4"
+        }
+
+        qt_two_partition_3_2 two_partition_3_2
+        explain {
+            sql("${two_partition_3_2}")
+            contains "partition=2/4"
+        }
+
+        qt_two_partition_4_all two_partition_4_all
+        explain {
+            sql("${two_partition_4_all}")
+            contains "partition=4/4"
+        }
+
+        qt_two_partition_5_1 two_partition_5_1
+        explain {
+            sql("${two_partition_5_1}")
+            contains "partition=1/4"
+        }
+
+        qt_two_partition_6_1 two_partition_6_1
+        explain {
+            sql("${two_partition_6_1}")
+            contains "partition=1/4"
+        }
+
+
+
+        qt_three_partition_1_1 three_partition_1_1
+        explain {
+            sql("${three_partition_1_1}")
+            contains "partition=1/10"
+        }
+
+        qt_three_partition_2_1 three_partition_2_1
+        explain {
+            sql("${three_partition_2_1}")
+            contains "partition=1/10"
+        }
+
+        qt_three_partition_3_3 three_partition_3_3
+        explain {
+            sql("${three_partition_3_3}")
+            contains "partition=3/10"
+        }
+
+        qt_three_partition_4_2 three_partition_4_2
+        explain {
+            sql("${three_partition_4_2}")
+            contains "partition=2/10"
+        }
+
+        qt_three_partition_5_all three_partition_5_all
+        explain {
+            sql("${three_partition_5_all}")
+            contains "partition=10/10"
+        }
+
+        qt_three_partition_6_1 three_partition_6_1
+        explain {
+            sql("${three_partition_6_1}")
+            contains "partition=1/10"
+        }
+
+        qt_three_partition_7_7 three_partition_7_7
+        explain {
+            sql("${three_partition_7_7}")
+            contains "partition=7/10"
+        }
+
+        qt_three_partition_8_2 three_partition_8_2
+        explain {
+            sql("${three_partition_8_2}")
+            contains "partition=2/10"
+        }
+
+
+        // 0 partitions
+        def one_partition_6_0 = """SELECT * FROM one_partition_tb WHERE part1 = 2023 ORDER BY id;"""
+        qt_one_partition_6_0 one_partition_6_0
+        explain {
+            sql("${one_partition_6_0}")
+            contains "partition=0/2"
+        }
+
+        def two_partition_7_0 = """SELECT * FROM two_partition_tb WHERE part1 = 'CN' AND part2 = 1 ORDER BY id;"""
+        qt_two_partition_7_0 two_partition_7_0
+        explain {
+            sql("${two_partition_7_0}")
+            contains "partition=0/4"
+        }
+
+        def two_partition_8_0 = """SELECT * FROM two_partition_tb WHERE part1 = 'US' AND part2 = 3 ORDER BY id;"""
+        qt_two_partition_8_0 two_partition_8_0
+        explain {
+            sql("${two_partition_8_0}")
+            contains "partition=0/4"
+        }
+
+        def three_partition_9_0 = """SELECT * FROM three_partition_tb WHERE part1 = 'US' AND part2 = 2023 AND part3 = 'Q1' ORDER BY id;"""
+        qt_three_partition_9_0 three_partition_9_0
+        explain {
+            sql("${three_partition_9_0}")
+            contains "partition=0/10"
+        }
+
+        def three_partition_10_0 = """SELECT * FROM three_partition_tb WHERE part1 = 'EU' AND part2 = 2024 AND part3 = 'Q4' ORDER BY id;"""
+        qt_three_partition_10_0 three_partition_10_0
+        explain {
+            sql("${three_partition_10_0}")
+            contains "partition=0/10"
+        }
+
+        def three_partition_11_0 = """SELECT * FROM three_partition_tb WHERE part1 = 'AS' AND part2 = 2025 AND part3 = 'Q4' ORDER BY id;"""
+        qt_three_partition_11_0 three_partition_11_0
+        explain {
+            sql("${three_partition_11_0}")
+            contains "partition=0/10"
+        }
+
+    }
+}
\ No newline at end of file

From 9ca8048d4d3e60f431b8c46212bdc2d29d844f0b Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchi@selectdb.com>
Date: Sun, 1 Dec 2024 22:18:42 +0800
Subject: [PATCH 092/399] [fix](iceberg)Bring field_id with parquet files And
 fix map type's key optional (#44470)

### What problem does this PR solve?

1. Column IDs are required to be stored as [field
IDs](http://github.com/apache/parquet-format/blob/40699d05bd24181de6b1457babbee2c16dce3803/src/main/thrift/parquet.thrift#L459)
on the parquet schema.
ref: https://iceberg.apache.org/spec/?h=field+id#parquet
So, we should add field ids.
2. For `MapType`, its key is always required.
---
 .../table/iceberg/arrow_schema_util.cpp       | 134 ++++++++
 .../format/table/iceberg/arrow_schema_util.h  |  45 +++
 .../vec/exec/format/table/iceberg/types.cpp   |   3 +-
 be/src/vec/exec/format/table/iceberg/types.h  |   4 +
 be/src/vec/runtime/vparquet_transformer.cpp   |  51 +--
 be/src/vec/runtime/vparquet_transformer.h     |   5 +-
 .../iceberg/viceberg_partition_writer.cpp     |   2 +-
 .../table/iceberg/arrow_schema_util_test.cpp  | 304 ++++++++++++++++++
 .../table/iceberg/schema_parser_test.cpp      |  24 ++
 9 files changed, 547 insertions(+), 25 deletions(-)
 create mode 100644 be/src/vec/exec/format/table/iceberg/arrow_schema_util.cpp
 create mode 100644 be/src/vec/exec/format/table/iceberg/arrow_schema_util.h
 create mode 100644 be/test/vec/exec/format/table/iceberg/arrow_schema_util_test.cpp

diff --git a/be/src/vec/exec/format/table/iceberg/arrow_schema_util.cpp b/be/src/vec/exec/format/table/iceberg/arrow_schema_util.cpp
new file mode 100644
index 00000000000000..35a4d51b7f1959
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/arrow_schema_util.cpp
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg/arrow_schema_util.h"
+
+#include <arrow/type.h>
+#include <arrow/util/key_value_metadata.h>
+
+namespace doris {
+namespace iceberg {
+
+const char* ArrowSchemaUtil::PARQUET_FIELD_ID = "PARQUET:field_id";
+const char* ArrowSchemaUtil::ORIGINAL_TYPE = "originalType";
+const char* ArrowSchemaUtil::MAP_TYPE_VALUE = "mapType";
+
+Status ArrowSchemaUtil::convert(const Schema* schema, const std::string& timezone,
+                                std::vector<std::shared_ptr<arrow::Field>>& fields) {
+    for (const auto& column : schema->columns()) {
+        std::shared_ptr<arrow::Field> arrow_field;
+        RETURN_IF_ERROR(convert_to(column, &arrow_field, timezone));
+        fields.push_back(arrow_field);
+    }
+    return Status::OK();
+}
+
+Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field,
+                                   std::shared_ptr<arrow::Field>* arrow_field,
+                                   const std::string& timezone) {
+    std::shared_ptr<arrow::DataType> arrow_type;
+    std::unordered_map<std::string, std::string> metadata;
+    metadata[PARQUET_FIELD_ID] = std::to_string(field.field_id());
+
+    switch (field.field_type()->type_id()) {
+    case iceberg::TypeID::BOOLEAN:
+        arrow_type = arrow::boolean();
+        break;
+
+    case iceberg::TypeID::INTEGER:
+        arrow_type = arrow::int32();
+        break;
+
+    case iceberg::TypeID::LONG:
+        arrow_type = arrow::int64();
+        break;
+
+    case iceberg::TypeID::FLOAT:
+        arrow_type = arrow::float32();
+        break;
+
+    case iceberg::TypeID::DOUBLE:
+        arrow_type = arrow::float64();
+        break;
+
+    case iceberg::TypeID::DATE:
+        arrow_type = arrow::date32();
+        break;
+
+    case iceberg::TypeID::TIMESTAMP: {
+        arrow_type = std::make_shared<arrow::TimestampType>(arrow::TimeUnit::MICRO, timezone);
+        break;
+    }
+
+    case iceberg::TypeID::BINARY:
+    case iceberg::TypeID::STRING:
+    case iceberg::TypeID::UUID:
+    case iceberg::TypeID::FIXED:
+        arrow_type = arrow::utf8();
+        break;
+
+    case iceberg::TypeID::DECIMAL: {
+        auto dt = dynamic_cast<DecimalType*>(field.field_type());
+        arrow_type = arrow::decimal(dt->get_precision(), dt->get_scale());
+        break;
+    }
+
+    case iceberg::TypeID::STRUCT: {
+        std::vector<std::shared_ptr<arrow::Field>> element_fields;
+        StructType* st = field.field_type()->as_struct_type();
+        for (const auto& column : st->fields()) {
+            std::shared_ptr<arrow::Field> element_field;
+            RETURN_IF_ERROR(convert_to(column, &element_field, timezone));
+            element_fields.push_back(element_field);
+        }
+        arrow_type = arrow::struct_(element_fields);
+        break;
+    }
+
+    case iceberg::TypeID::LIST: {
+        std::shared_ptr<arrow::Field> item_field;
+        ListType* list_type = field.field_type()->as_list_type();
+        RETURN_IF_ERROR(convert_to(list_type->element_field(), &item_field, timezone));
+        arrow_type = arrow::list(item_field);
+        break;
+    }
+
+    case iceberg::TypeID::MAP: {
+        std::shared_ptr<arrow::Field> key_field;
+        std::shared_ptr<arrow::Field> value_field;
+        MapType* map_type = field.field_type()->as_map_type();
+        RETURN_IF_ERROR(convert_to(map_type->key_field(), &key_field, timezone));
+        RETURN_IF_ERROR(convert_to(map_type->value_field(), &value_field, timezone));
+        metadata[ORIGINAL_TYPE] = MAP_TYPE_VALUE;
+        arrow_type = std::make_shared<arrow::MapType>(key_field, value_field);
+        break;
+    }
+
+    case iceberg::TypeID::TIME:
+    default:
+        return Status::InternalError("Unsupported field type:" + field.field_type()->to_string());
+    }
+
+    std::shared_ptr<arrow::KeyValueMetadata> schema_metadata =
+            std::make_shared<arrow::KeyValueMetadata>(metadata);
+    *arrow_field =
+            arrow::field(field.field_name(), arrow_type, field.is_optional(), schema_metadata);
+    return Status::OK();
+}
+
+} // namespace iceberg
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/exec/format/table/iceberg/arrow_schema_util.h b/be/src/vec/exec/format/table/iceberg/arrow_schema_util.h
new file mode 100644
index 00000000000000..20b7dbc627cc44
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/arrow_schema_util.h
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <arrow/type.h>
+
+#include <shared_mutex>
+
+#include "vec/exec/format/table/iceberg/schema.h"
+
+namespace doris {
+namespace iceberg {
+
+class ArrowSchemaUtil {
+public:
+    static Status convert(const Schema* schema, const std::string& timezone,
+                          std::vector<std::shared_ptr<arrow::Field>>& fields);
+
+private:
+    static const char* PARQUET_FIELD_ID;
+    static const char* ORIGINAL_TYPE;
+    static const char* MAP_TYPE_VALUE;
+
+    static Status convert_to(const iceberg::NestedField& field,
+                             std::shared_ptr<arrow::Field>* arrow_field,
+                             const std::string& timezone);
+};
+
+} // namespace iceberg
+} // namespace doris
diff --git a/be/src/vec/exec/format/table/iceberg/types.cpp b/be/src/vec/exec/format/table/iceberg/types.cpp
index b56a231979ace1..bf643655ab8810 100644
--- a/be/src/vec/exec/format/table/iceberg/types.cpp
+++ b/be/src/vec/exec/format/table/iceberg/types.cpp
@@ -25,8 +25,9 @@ namespace iceberg {
 std::unique_ptr<MapType> MapType::of_optional(int key_id, int value_id,
                                               std::unique_ptr<Type> key_type,
                                               std::unique_ptr<Type> value_type) {
+    // key is always required
     auto key_field =
-            std::make_unique<NestedField>(true, key_id, "key", std::move(key_type), std::nullopt);
+            std::make_unique<NestedField>(false, key_id, "key", std::move(key_type), std::nullopt);
     auto value_field = std::make_unique<NestedField>(true, value_id, "value", std::move(value_type),
                                                      std::nullopt);
     return std::unique_ptr<MapType>(new MapType(std::move(key_field), std::move(value_field)));
diff --git a/be/src/vec/exec/format/table/iceberg/types.h b/be/src/vec/exec/format/table/iceberg/types.h
index f5262b36f55cd3..91a2f705df0d0b 100644
--- a/be/src/vec/exec/format/table/iceberg/types.h
+++ b/be/src/vec/exec/format/table/iceberg/types.h
@@ -265,6 +265,10 @@ class DecimalType : public PrimitiveType {
         ss << "decimal(" << precision << ", " << scale << ")";
         return ss.str();
     }
+
+    int get_precision() const { return precision; }
+
+    int get_scale() const { return scale; }
 };
 
 class BinaryType : public PrimitiveType {
diff --git a/be/src/vec/runtime/vparquet_transformer.cpp b/be/src/vec/runtime/vparquet_transformer.cpp
index ab288537313973..5409dc0abf3466 100644
--- a/be/src/vec/runtime/vparquet_transformer.cpp
+++ b/be/src/vec/runtime/vparquet_transformer.cpp
@@ -65,6 +65,7 @@
 #include "vec/core/types.h"
 #include "vec/data_types/data_type_decimal.h"
 #include "vec/data_types/data_type_nullable.h"
+#include "vec/exec/format/table/iceberg/arrow_schema_util.h"
 #include "vec/exprs/vexpr.h"
 #include "vec/exprs/vexpr_context.h"
 #include "vec/functions/function_helpers.h"
@@ -202,21 +203,20 @@ void ParquetBuildHelper::build_version(parquet::WriterProperties::Builder& build
     }
 }
 
-VParquetTransformer::VParquetTransformer(RuntimeState* state, doris::io::FileWriter* file_writer,
-                                         const VExprContextSPtrs& output_vexpr_ctxs,
-                                         std::vector<std::string> column_names,
-                                         TParquetCompressionType::type compression_type,
-                                         bool parquet_disable_dictionary,
-                                         TParquetVersion::type parquet_version,
-                                         bool output_object_data,
-                                         const std::string* iceberg_schema_json)
+VParquetTransformer::VParquetTransformer(
+        RuntimeState* state, doris::io::FileWriter* file_writer,
+        const VExprContextSPtrs& output_vexpr_ctxs, std::vector<std::string> column_names,
+        TParquetCompressionType::type compression_type, bool parquet_disable_dictionary,
+        TParquetVersion::type parquet_version, bool output_object_data,
+        const std::string* iceberg_schema_json, const iceberg::Schema* iceberg_schema)
         : VFileFormatTransformer(state, output_vexpr_ctxs, output_object_data),
           _column_names(std::move(column_names)),
           _parquet_schemas(nullptr),
           _compression_type(compression_type),
           _parquet_disable_dictionary(parquet_disable_dictionary),
           _parquet_version(parquet_version),
-          _iceberg_schema_json(iceberg_schema_json) {
+          _iceberg_schema_json(iceberg_schema_json),
+          _iceberg_schema(iceberg_schema) {
     _outstream = std::shared_ptr<ParquetOutputStream>(new ParquetOutputStream(file_writer));
 }
 
@@ -234,6 +234,7 @@ VParquetTransformer::VParquetTransformer(RuntimeState* state, doris::io::FileWri
           _parquet_disable_dictionary(parquet_disable_dictionary),
           _parquet_version(parquet_version),
           _iceberg_schema_json(iceberg_schema_json) {
+    _iceberg_schema = nullptr;
     _outstream = std::shared_ptr<ParquetOutputStream>(new ParquetOutputStream(file_writer));
 }
 
@@ -265,21 +266,27 @@ Status VParquetTransformer::_parse_properties() {
 
 Status VParquetTransformer::_parse_schema() {
     std::vector<std::shared_ptr<arrow::Field>> fields;
-    for (size_t i = 0; i < _output_vexpr_ctxs.size(); i++) {
-        std::shared_ptr<arrow::DataType> type;
-        RETURN_IF_ERROR(convert_to_arrow_type(_output_vexpr_ctxs[i]->root()->type(), &type,
-                                              _state->timezone()));
-        if (_parquet_schemas != nullptr) {
-            std::shared_ptr<arrow::Field> field =
-                    arrow::field(_parquet_schemas->operator[](i).schema_column_name, type,
-                                 _output_vexpr_ctxs[i]->root()->is_nullable());
-            fields.emplace_back(field);
-        } else {
-            std::shared_ptr<arrow::Field> field = arrow::field(
-                    _column_names[i], type, _output_vexpr_ctxs[i]->root()->is_nullable());
-            fields.emplace_back(field);
+    if (_iceberg_schema != nullptr) {
+        RETURN_IF_ERROR(
+                iceberg::ArrowSchemaUtil::convert(_iceberg_schema, _state->timezone(), fields));
+    } else {
+        for (size_t i = 0; i < _output_vexpr_ctxs.size(); i++) {
+            std::shared_ptr<arrow::DataType> type;
+            RETURN_IF_ERROR(convert_to_arrow_type(_output_vexpr_ctxs[i]->root()->type(), &type,
+                                                  _state->timezone()));
+            if (_parquet_schemas != nullptr) {
+                std::shared_ptr<arrow::Field> field =
+                        arrow::field(_parquet_schemas->operator[](i).schema_column_name, type,
+                                     _output_vexpr_ctxs[i]->root()->is_nullable());
+                fields.emplace_back(field);
+            } else {
+                std::shared_ptr<arrow::Field> field = arrow::field(
+                        _column_names[i], type, _output_vexpr_ctxs[i]->root()->is_nullable());
+                fields.emplace_back(field);
+            }
         }
     }
+
     if (_iceberg_schema_json != nullptr) {
         std::shared_ptr<arrow::KeyValueMetadata> schema_metadata =
                 arrow::KeyValueMetadata::Make({"iceberg.schema"}, {*_iceberg_schema_json});
diff --git a/be/src/vec/runtime/vparquet_transformer.h b/be/src/vec/runtime/vparquet_transformer.h
index 9fdbb271373212..ecc4a8ddeac4bc 100644
--- a/be/src/vec/runtime/vparquet_transformer.h
+++ b/be/src/vec/runtime/vparquet_transformer.h
@@ -27,6 +27,7 @@
 #include <parquet/types.h>
 #include <stdint.h>
 
+#include "vec/exec/format/table/iceberg/schema.h"
 #include "vfile_format_transformer.h"
 
 namespace doris {
@@ -95,7 +96,8 @@ class VParquetTransformer final : public VFileFormatTransformer {
                         std::vector<std::string> column_names,
                         TParquetCompressionType::type compression_type,
                         bool parquet_disable_dictionary, TParquetVersion::type parquet_version,
-                        bool output_object_data, const std::string* iceberg_schema_json = nullptr);
+                        bool output_object_data, const std::string* iceberg_schema_json = nullptr,
+                        const iceberg::Schema* iceberg_schema = nullptr);
 
     VParquetTransformer(RuntimeState* state, doris::io::FileWriter* file_writer,
                         const VExprContextSPtrs& output_vexpr_ctxs,
@@ -132,6 +134,7 @@ class VParquetTransformer final : public VFileFormatTransformer {
     const TParquetVersion::type _parquet_version;
     const std::string* _iceberg_schema_json;
     uint64_t _write_size = 0;
+    const iceberg::Schema* _iceberg_schema;
 };
 
 } // namespace doris::vectorized
diff --git a/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp
index 924adf68145a7a..23ee389a8b7d10 100644
--- a/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp
+++ b/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp
@@ -84,7 +84,7 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil
         _file_format_transformer.reset(new VParquetTransformer(
                 state, _file_writer.get(), _write_output_expr_ctxs, _write_column_names,
                 parquet_compression_type, parquet_disable_dictionary, TParquetVersion::PARQUET_1_0,
-                false, _iceberg_schema_json));
+                false, _iceberg_schema_json, &_schema));
         return _file_format_transformer->open();
     }
     case TFileFormatType::FORMAT_ORC: {
diff --git a/be/test/vec/exec/format/table/iceberg/arrow_schema_util_test.cpp b/be/test/vec/exec/format/table/iceberg/arrow_schema_util_test.cpp
new file mode 100644
index 00000000000000..b5f61c9d2e3cdf
--- /dev/null
+++ b/be/test/vec/exec/format/table/iceberg/arrow_schema_util_test.cpp
@@ -0,0 +1,304 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg/arrow_schema_util.h"
+
+#include <arrow/api.h>
+#include <arrow/io/api.h>
+#include <arrow/status.h>
+#include <arrow/type.h>
+#include <arrow/util/key_value_metadata.h>
+#include <gtest/gtest.h>
+#include <parquet/api/reader.h>
+#include <parquet/arrow/writer.h>
+#include <parquet/schema.h>
+
+#include "io/fs/local_file_system.h"
+#include "vec/exec/format/table/iceberg/schema.h"
+#include "vec/exec/format/table/iceberg/schema_parser.h"
+
+namespace doris {
+namespace iceberg {
+
+class ArrowSchemaUtilTest : public testing::Test {
+public:
+    ArrowSchemaUtilTest() = default;
+    virtual ~ArrowSchemaUtilTest() = default;
+};
+
+const std::string_view pfid = "PARQUET:field_id";
+
+TEST(ArrowSchemaUtilTest, test_simple_field) {
+    std::vector<NestedField> nested_fields;
+    nested_fields.reserve(2);
+    NestedField field1(false, 1, "field1", std::make_unique<IntegerType>(), std::nullopt);
+    NestedField field2(false, 2, "field2", std::make_unique<StringType>(), std::nullopt);
+    nested_fields.emplace_back(std::move(field1));
+    nested_fields.emplace_back(std::move(field2));
+
+    Schema schema(1, std::move(nested_fields));
+
+    std::vector<std::shared_ptr<arrow::Field>> fields;
+    Status st;
+    st = ArrowSchemaUtil::convert(&schema, "utc", fields);
+    EXPECT_TRUE(st.ok());
+    EXPECT_EQ(2, fields.size());
+    EXPECT_EQ("field1", fields[0]->name());
+    EXPECT_EQ("field2", fields[1]->name());
+    EXPECT_TRUE(fields[0]->HasMetadata());
+    EXPECT_TRUE(fields[1]->HasMetadata());
+    EXPECT_EQ("1", fields[0]->metadata()->Get(pfid).ValueUnsafe());
+    EXPECT_EQ("2", fields[1]->metadata()->Get(pfid).ValueUnsafe());
+}
+
+TEST(ArrowSchemaUtilTest, test_stuct_field) {
+    // struct_json comes from :
+    //     Schema schema = new Schema(
+    //     Types.NestedField.optional(
+    //         21, "st_col", Types.StructType.of(
+    //             Types.NestedField.optional(32, "st_col_c1", Types.IntegerType.get()),
+    //             Types.NestedField.optional(43, "st_col_c2", Types.StringType.get())
+    //         )
+    //     )
+    // );
+    // StringWriter writer = new StringWriter();
+    // JsonGenerator generator = JsonUtil.factory().createGenerator(writer);
+    // SchemaParser.toJson(schema.asStruct(), generator);
+    // generator.flush();
+    // System.out.println(writer.toString());
+
+    const std::string struct_json = R"({
+        "type": "struct",
+        "fields": [
+            {
+                "id": 21,
+                "name": "st_col",
+                "required": false,
+                "type": {
+                    "type": "struct",
+                    "fields": [
+                        {
+                            "id": 32,
+                            "name": "st_col_c1",
+                            "required": false,
+                            "type": "int"
+                        },
+                        {
+                            "id": 43,
+                            "name": "st_col_c2",
+                            "required": false,
+                            "type": "string"
+                        }
+                    ]
+                }
+            }
+        ]
+    })";
+    std::unique_ptr<Schema> schema = SchemaParser::from_json(struct_json);
+
+    std::vector<std::shared_ptr<arrow::Field>> fields;
+    Status st;
+    st = ArrowSchemaUtil::convert(schema.get(), "utc", fields);
+    EXPECT_TRUE(st.ok());
+    EXPECT_EQ(1, fields.size());
+    EXPECT_EQ("st_col", fields[0]->name());
+    EXPECT_EQ("21", fields[0]->metadata()->Get(pfid).ValueUnsafe());
+
+    arrow::StructType* arrow_struct = dynamic_cast<arrow::StructType*>(fields[0]->type().get());
+    auto map_fields = arrow_struct->fields();
+    EXPECT_EQ(2, arrow_struct->fields().size());
+    EXPECT_EQ("st_col_c1", map_fields.at(0).get()->name());
+    EXPECT_EQ("st_col_c2", map_fields.at(1).get()->name());
+    EXPECT_EQ("32", map_fields.at(0).get()->metadata()->Get(pfid).ValueUnsafe());
+    EXPECT_EQ("43", map_fields.at(1).get()->metadata()->Get(pfid).ValueUnsafe());
+}
+
+TEST(ArrowSchemaUtilTest, test_map_field) {
+    // map_json comes from :
+    // Schema schema = new Schema(
+    //     Types.NestedField.optional(
+    //         21, "map_col", Types.MapType.ofOptional(
+    //             32, 43, Types.IntegerType.get(), Types.StringType.get()
+    //         )
+    //     )
+    // );
+    // StringWriter writer = new StringWriter();
+    // JsonGenerator generator = JsonUtil.factory().createGenerator(writer);
+    // SchemaParser.toJson(schema.asStruct(), generator);
+    // generator.flush();
+    // System.out.println(writer.toString());
+
+    const std::string map_json = R"({
+        "type": "struct",
+        "fields": [
+            {
+                "id": 21,
+                "name": "map_col",
+                "required": false,
+                "type": {
+                    "type": "map",
+                    "key-id": 32,
+                    "key": "int",
+                    "value-id": 43,
+                    "value": "string",
+                    "value-required": false
+                }
+            }
+        ]
+    })";
+    std::unique_ptr<Schema> schema = SchemaParser::from_json(map_json);
+
+    std::vector<std::shared_ptr<arrow::Field>> fields;
+    Status st;
+    st = ArrowSchemaUtil::convert(schema.get(), "utc", fields);
+    EXPECT_TRUE(st.ok());
+    EXPECT_EQ(1, fields.size());
+    EXPECT_EQ("map_col", fields[0]->name());
+    EXPECT_EQ("21", fields[0]->metadata()->Get(pfid).ValueUnsafe());
+
+    arrow::MapType* arrow_map = dynamic_cast<arrow::MapType*>(fields[0]->type().get());
+    auto map_fields = arrow_map->fields();
+    EXPECT_EQ(1, arrow_map->fields().size());
+    EXPECT_EQ("key", arrow_map->key_field()->name());
+    EXPECT_EQ("value", arrow_map->item_field()->name());
+    EXPECT_EQ("32", arrow_map->key_field()->metadata()->Get(pfid).ValueUnsafe());
+    EXPECT_EQ("43", arrow_map->item_field()->metadata()->Get(pfid).ValueUnsafe());
+}
+
+TEST(ArrowSchemaUtilTest, test_list_field) {
+    // list_json comes from :
+    // Schema schema = new Schema(
+    //     Types.NestedField.optional(
+    //         21, "list_col", Types.ListType.ofOptional(
+    //             32, Types.IntegerType.get())));
+    // StringWriter writer = new StringWriter();
+    // JsonGenerator generator = JsonUtil.factory().createGenerator(writer);
+    // SchemaParser.toJson(schema.asStruct(), generator);
+    // generator.flush();
+    // System.out.println(writer.toString());
+
+    const std::string list_json = R"({
+        "type": "struct",
+        "fields": [
+            {
+                "id": 21,
+                "name": "list_col",
+                "required": false,
+                "type": {
+                    "type": "list",
+                    "element-id": 32,
+                    "element": "int",
+                    "element-required": false
+                }
+            }
+        ]
+    })";
+    std::unique_ptr<Schema> schema = SchemaParser::from_json(list_json);
+
+    std::vector<std::shared_ptr<arrow::Field>> fields;
+    Status st;
+    st = ArrowSchemaUtil::convert(schema.get(), "utc", fields);
+    EXPECT_TRUE(st.ok());
+    EXPECT_EQ(1, fields.size());
+    EXPECT_EQ("list_col", fields[0]->name());
+    EXPECT_EQ("21", fields[0]->metadata()->Get(pfid).ValueUnsafe());
+
+    arrow::ListType* arrow_list = dynamic_cast<arrow::ListType*>(fields[0]->type().get());
+    auto map_fields = arrow_list->fields();
+    EXPECT_EQ(1, arrow_list->fields().size());
+    EXPECT_EQ("element", arrow_list->value_field()->name());
+    EXPECT_EQ("32", arrow_list->value_field()->metadata()->Get(pfid).ValueUnsafe());
+}
+
+TEST(ArrowSchemaUtilTest, test_parquet_filed_id) {
+    std::string test_dir = "ut_dir/test_parquet_filed_id";
+    Status st;
+    st = io::global_local_filesystem()->delete_directory(test_dir);
+    ASSERT_TRUE(st.ok()) << st;
+    st = io::global_local_filesystem()->create_directory(test_dir);
+    ASSERT_TRUE(st.ok()) << st;
+
+    std::shared_ptr<arrow::Array> id_array;
+    std::shared_ptr<arrow::Array> name_array;
+
+    arrow::Int32Builder id_builder;
+    ASSERT_TRUE(id_builder.Append(1).ok());
+    ASSERT_TRUE(id_builder.Append(2).ok());
+    ASSERT_TRUE(id_builder.Append(3).ok());
+    auto&& result_id = id_builder.Finish();
+    ASSERT_TRUE(result_id.ok());
+    id_array = std::move(result_id).ValueUnsafe();
+
+    arrow::StringBuilder name_builder;
+    ASSERT_TRUE(name_builder.Append("Alice").ok());
+    ASSERT_TRUE(name_builder.Append("Bob").ok());
+    ASSERT_TRUE(name_builder.Append("Charlie").ok());
+    auto&& result_name = name_builder.Finish();
+    ASSERT_TRUE(result_name.ok());
+    name_array = std::move(result_name).ValueUnsafe();
+
+    // 定义表的 Schema
+    std::vector<NestedField> nested_fields;
+    nested_fields.reserve(2);
+    NestedField field1(false, 17, "field_1", std::make_unique<IntegerType>(), std::nullopt);
+    NestedField field2(false, 36, "field_2", std::make_unique<StringType>(), std::nullopt);
+    nested_fields.emplace_back(std::move(field1));
+    nested_fields.emplace_back(std::move(field2));
+
+    Schema schema(1, std::move(nested_fields));
+
+    std::vector<std::shared_ptr<arrow::Field>> fields;
+    st = ArrowSchemaUtil::convert(&schema, "utc", fields);
+    auto arrow_schema = arrow::schema(fields);
+
+    // create arrow table
+    auto table = arrow::Table::Make(arrow_schema, {id_array, name_array});
+
+    std::string file_path = test_dir + "/f1.parquet";
+    std::shared_ptr<arrow::io::FileOutputStream> outfile;
+    auto&& result_file = arrow::io::FileOutputStream::Open(file_path);
+    ASSERT_TRUE(result_file.ok());
+    outfile = std::move(result_file).ValueUnsafe();
+
+    // arrow table to parquet file
+    PARQUET_THROW_NOT_OK(
+            parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), outfile, 1024));
+
+    // open parquet with parquet's API
+    std::unique_ptr<parquet::ParquetFileReader> parquet_reader =
+            parquet::ParquetFileReader::OpenFile(file_path, false);
+
+    // get MessageType
+    std::shared_ptr<parquet::FileMetaData> file_metadata = parquet_reader->metadata();
+    auto schema_descriptor = file_metadata->schema();
+    const parquet::schema::Node& root = *schema_descriptor->group_node();
+    const auto& group_node = static_cast<const parquet::schema::GroupNode&>(root);
+
+    EXPECT_EQ(2, group_node.field_count());
+    auto filed1 = group_node.field(0);
+    auto filed2 = group_node.field(1);
+    EXPECT_EQ("field_1", filed1->name());
+    EXPECT_EQ(17, filed1->field_id());
+    EXPECT_EQ("field_2", filed2->name());
+    EXPECT_EQ(36, filed2->field_id());
+
+    st = io::global_local_filesystem()->delete_directory(test_dir);
+    EXPECT_TRUE(st.ok()) << st;
+}
+
+} // namespace iceberg
+} // namespace doris
diff --git a/be/test/vec/exec/format/table/iceberg/schema_parser_test.cpp b/be/test/vec/exec/format/table/iceberg/schema_parser_test.cpp
index 4c3f58cdd10491..f464525a7f99bc 100644
--- a/be/test/vec/exec/format/table/iceberg/schema_parser_test.cpp
+++ b/be/test/vec/exec/format/table/iceberg/schema_parser_test.cpp
@@ -78,6 +78,15 @@ const std::string valid_map_json = R"({
     "value-required": true
 })";
 
+const std::string valid_map_json2 = R"({
+    "type": "map",
+    "key-id": 4,
+    "key": "string",
+    "value-id": 5,
+    "value": "int",
+    "value-required": false
+})";
+
 const std::string nested_list_json = R"({
     "type": "list",
     "element-id": 6,
@@ -209,6 +218,21 @@ TEST(SchemaParserTest, parse_valid_map) {
             SchemaParser::_type_from_json(rapidjson::Document().Parse(valid_map_json.c_str()));
     ASSERT_NE(type, nullptr);
     EXPECT_EQ(type->to_string(), "map<string, int>");
+    EXPECT_TRUE(type->is_map_type());
+    MapType* mt = type->as_map_type();
+    EXPECT_TRUE(mt->field(4)->is_required());
+    EXPECT_TRUE(mt->field(5)->is_required());
+}
+
+TEST(SchemaParserTest, parse_valid_map2) {
+    std::unique_ptr<Type> type =
+            SchemaParser::_type_from_json(rapidjson::Document().Parse(valid_map_json2.c_str()));
+    ASSERT_NE(type, nullptr);
+    EXPECT_EQ(type->to_string(), "map<string, int>");
+    EXPECT_TRUE(type->is_map_type());
+    MapType* mt = type->as_map_type();
+    EXPECT_TRUE(mt->field(4)->is_required());
+    EXPECT_TRUE(mt->field(5)->is_optional());
 }
 
 TEST(SchemaParserTest, parse_nested_list) {

From 456a08ba746900ad1167a7c5397775195ade8195 Mon Sep 17 00:00:00 2001
From: starocean999 <lichi@selectdb.com>
Date: Mon, 2 Dec 2024 09:41:10 +0800
Subject: [PATCH 093/399] [refactor](nereids)add NotNullableAggregateFunction
 as base class for all non-nullable agg functions (#44671)

AlwaysNotNullable agg functions must return a non-null value if it's a
scalar agg. So a new base class NotNullableAggregateFunction is
introduced, all sub classes must implement resultForEmptyInput method to
get the non-null value correctly
---
 .../rules/analysis/SubqueryToApply.java       |  8 ++--
 .../functions/AlwaysNotNullable.java          |  9 ----
 .../expressions/functions/agg/ArrayAgg.java   |  5 +-
 .../expressions/functions/agg/BitmapAgg.java  |  5 +-
 .../functions/agg/BitmapIntersect.java        |  5 +-
 .../functions/agg/BitmapUnion.java            |  5 +-
 .../functions/agg/BitmapUnionCount.java       |  5 +-
 .../functions/agg/BitmapUnionInt.java         |  5 +-
 .../functions/agg/CollectList.java            |  5 +-
 .../expressions/functions/agg/CollectSet.java |  5 +-
 .../expressions/functions/agg/Count.java      |  5 +-
 .../functions/agg/CountByEnum.java            |  3 +-
 .../functions/agg/GroupArrayIntersect.java    |  5 +-
 .../expressions/functions/agg/Histogram.java  |  5 +-
 .../expressions/functions/agg/HllUnion.java   |  5 +-
 .../functions/agg/HllUnionAgg.java            |  5 +-
 .../functions/agg/IntersectCount.java         |  5 +-
 .../functions/agg/LinearHistogram.java        |  9 +++-
 .../expressions/functions/agg/MapAgg.java     |  5 +-
 .../functions/agg/MultiDistinctCount.java     |  5 +-
 .../functions/agg/MultiDistinctSum0.java      |  5 +-
 .../trees/expressions/functions/agg/Ndv.java  |  5 +-
 .../agg/NotNullableAggregateFunction.java     | 48 +++++++++++++++++++
 .../agg/OrthogonalBitmapIntersect.java        | 11 +++--
 .../agg/OrthogonalBitmapIntersectCount.java   | 13 +++--
 .../agg/OrthogonalBitmapUnionCount.java       | 11 +++--
 .../functions/agg/PercentileArray.java        |  5 +-
 .../functions/agg/QuantileUnion.java          |  5 +-
 .../functions/agg/SequenceCount.java          |  5 +-
 .../trees/expressions/functions/agg/Sum0.java |  5 +-
 30 files changed, 129 insertions(+), 93 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/NotNullableAggregateFunction.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
index 14700b030d68b7..4bb55eab9d4378 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
@@ -42,9 +42,9 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.SubqueryExpr;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
+import org.apache.doris.nereids.trees.expressions.functions.agg.NotNullableAggregateFunction;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl;
 import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
@@ -424,9 +424,9 @@ private Pair<LogicalPlan, Optional<Expression>> addApply(SubqueryExpr subquery,
                     Map<Expression, Expression> replaceMap = new HashMap<>();
                     NamedExpression agg = ((ScalarSubquery) subquery).getTopLevelScalarAggFunction().get();
                     if (agg instanceof Alias) {
-                        if (((Alias) agg).child() instanceof AlwaysNotNullable) {
-                            AlwaysNotNullable notNullableAggFunc =
-                                    (AlwaysNotNullable) ((Alias) agg).child();
+                        if (((Alias) agg).child() instanceof NotNullableAggregateFunction) {
+                            NotNullableAggregateFunction notNullableAggFunc =
+                                    (NotNullableAggregateFunction) ((Alias) agg).child();
                             if (subquery.getQueryPlan() instanceof LogicalProject) {
                                 LogicalProject logicalProject =
                                         (LogicalProject) subquery.getQueryPlan();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AlwaysNotNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AlwaysNotNullable.java
index 6b12f9cd642992..8fda4d4b020f2b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AlwaysNotNullable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AlwaysNotNullable.java
@@ -17,9 +17,6 @@
 
 package org.apache.doris.nereids.trees.expressions.functions;
 
-import org.apache.doris.nereids.exceptions.AnalysisException;
-import org.apache.doris.nereids.trees.expressions.Expression;
-
 /**
  * nullable is always false.
  *
@@ -30,10 +27,4 @@ public interface AlwaysNotNullable extends ComputeNullable {
     default boolean nullable() {
         return false;
     }
-
-    // return value of this function if the input data is empty.
-    // for example, count(*) of empty table is 0;
-    default Expression resultForEmptyInput() {
-        throw new AnalysisException("should implement resultForEmptyInput() for " + this.getClass());
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/ArrayAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/ArrayAgg.java
index bc91207e31f3a6..6218569261a775 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/ArrayAgg.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/ArrayAgg.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,8 +36,8 @@
 /**
  * AggregateFunction 'array_agg'.
  */
-public class ArrayAgg extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class ArrayAgg extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature {
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(ArrayType.of(new FollowToAnyDataType(0))).args(new AnyDataType(0))
     );
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapAgg.java
index eaf766b908d7af..a0eaf88efaf13b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapAgg.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapAgg.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,8 +36,8 @@
 /**
  * AggregateFunction 'bitmap_agg'.
  */
-public class BitmapAgg extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class BitmapAgg extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature {
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BitmapType.INSTANCE).args(BigIntType.INSTANCE),
             FunctionSignature.ret(BitmapType.INSTANCE).args(IntegerType.INSTANCE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapIntersect.java
index 1b7d2d3c3cddec..80a9887d325634 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapIntersect.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapIntersect.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -35,8 +34,8 @@
 /**
  * AggregateFunction 'bitmap_intersect'. This class is generated by GenerateFunction.
  */
-public class BitmapIntersect extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction {
+public class BitmapIntersect extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature, BitmapFunction {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnion.java
index cd0756a1c9361f..fce2f4fa70c483 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnion.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
@@ -36,8 +35,8 @@
 /**
  * AggregateFunction 'bitmap_union'. This class is generated by GenerateFunction.
  */
-public class BitmapUnion extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction, RollUpTrait {
+public class BitmapUnion extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature, BitmapFunction, RollUpTrait {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java
index 593c814f22d159..d1e2d6010a27a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
@@ -37,8 +36,8 @@
 /**
  * AggregateFunction 'bitmap_union_count'. This class is generated by GenerateFunction.
  */
-public class BitmapUnionCount extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction, RollUpTrait {
+public class BitmapUnionCount extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature, BitmapFunction, RollUpTrait {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionInt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionInt.java
index 2efe1631176c86..af569982c4e62e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionInt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionInt.java
@@ -20,7 +20,6 @@
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -41,8 +40,8 @@
 /**
  * AggregateFunction 'bitmap_union_int'. This class is generated by GenerateFunction.
  */
-public class BitmapUnionInt extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class BitmapUnionInt extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BigIntType.INSTANCE).args(SmallIntType.INSTANCE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectList.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectList.java
index d6cca2d0b90b6d..ed0dd2406211ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectList.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectList.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -38,8 +37,8 @@
 /**
  * AggregateFunction 'collect_list'. This class is generated by GenerateFunction.
  */
-public class CollectList extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class CollectList extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(ArrayType.of(new FollowToAnyDataType(0))).args(new AnyDataType(0)),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectSet.java
index d9e7e7227c6f86..2aba485fc7daaa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectSet.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -38,8 +37,8 @@
 /**
  * AggregateFunction 'collect_set'. This class is generated by GenerateFunction.
  */
-public class CollectSet extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class CollectSet extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(ArrayType.of(new FollowToAnyDataType(0))).args(new AnyDataType(0)),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java
index 10874d47ee3df9..e86e90974da1bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java
@@ -20,7 +20,6 @@
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.functions.window.SupportWindowAnalytic;
@@ -37,8 +36,8 @@
 import java.util.List;
 
 /** count agg function. */
-public class Count extends AggregateFunction
-        implements ExplicitlyCastableSignature, AlwaysNotNullable, SupportWindowAnalytic, RollUpTrait {
+public class Count extends NotNullableAggregateFunction
+        implements ExplicitlyCastableSignature, SupportWindowAnalytic, RollUpTrait {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             // count(*)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CountByEnum.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CountByEnum.java
index 2a4ee7be3f49c2..f956854d6871bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CountByEnum.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CountByEnum.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -31,7 +30,7 @@
 import java.util.List;
 
 /** count_by_enum agg function. */
-public class CountByEnum extends AggregateFunction implements ExplicitlyCastableSignature, AlwaysNotNullable {
+public class CountByEnum extends NotNullableAggregateFunction implements ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(StringType.INSTANCE).varArgs(StringType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupArrayIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupArrayIntersect.java
index 0720d6838bb1b6..1cc0db1bd5ed5a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupArrayIntersect.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupArrayIntersect.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -36,8 +35,8 @@
 /**
  * AggregateFunction 'group_array_intersect'.
  */
-public class GroupArrayIntersect extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class GroupArrayIntersect extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.retArgType(0)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Histogram.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Histogram.java
index 6b0a2759823f3a..827c57facd7498 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Histogram.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Histogram.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
 import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
@@ -37,8 +36,8 @@
 /**
  * AggregateFunction 'histogram'. This class is generated by GenerateFunction.
  */
-public class Histogram extends AggregateFunction
-        implements ExplicitlyCastableSignature, AlwaysNotNullable {
+public class Histogram extends NotNullableAggregateFunction
+        implements ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/HllUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/HllUnion.java
index b81fad270b0cda..8f6224f66b0043 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/HllUnion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/HllUnion.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.HllEmpty;
@@ -36,8 +35,8 @@
 /**
  * AggregateFunction 'hll_union'. This class is generated by GenerateFunction.
  */
-public class HllUnion extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, HllFunction, RollUpTrait {
+public class HllUnion extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature, HllFunction, RollUpTrait {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(HllType.INSTANCE).args(HllType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/HllUnionAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/HllUnionAgg.java
index b14b61b5be07e9..b2c15b402aa2c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/HllUnionAgg.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/HllUnionAgg.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
@@ -37,8 +36,8 @@
 /**
  * AggregateFunction 'hll_union_agg'. This class is generated by GenerateFunction.
  */
-public class HllUnionAgg extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, HllFunction, RollUpTrait {
+public class HllUnionAgg extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature, HllFunction, RollUpTrait {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BigIntType.INSTANCE).args(HllType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/IntersectCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/IntersectCount.java
index c013b2e8b4c31d..3eed9b9a6e4ddb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/IntersectCount.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/IntersectCount.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.BitmapIntersectFunction;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
@@ -37,8 +36,8 @@
 /**
  * AggregateFunction 'intersect_count'. This class is generated by GenerateFunction.
  */
-public class IntersectCount extends AggregateFunction
-        implements ExplicitlyCastableSignature, AlwaysNotNullable, BitmapIntersectFunction {
+public class IntersectCount extends NotNullableAggregateFunction
+        implements ExplicitlyCastableSignature, BitmapIntersectFunction {
 
     public static final List<FunctionSignature> SIGNATURES = DataType.trivialTypes().stream()
             .map(type -> FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE, type, type))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/LinearHistogram.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/LinearHistogram.java
index ac172fc7881c7c..ca8cf3a2f71560 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/LinearHistogram.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/LinearHistogram.java
@@ -20,9 +20,9 @@
 import org.apache.doris.catalog.FunctionSet;
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.DoubleType;
 import org.apache.doris.nereids.types.VarcharType;
@@ -36,7 +36,7 @@
 /**
  * AggregateFunction 'linear_histogram'.
  */
-public class LinearHistogram extends AggregateFunction implements ExplicitlyCastableSignature, AlwaysNotNullable {
+public class LinearHistogram extends NotNullableAggregateFunction implements ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
@@ -86,4 +86,9 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
     public List<FunctionSignature> getSignatures() {
         return SIGNATURES;
     }
+
+    @Override
+    public Expression resultForEmptyInput() {
+        return new VarcharLiteral("{\"num_buckets\":0,\"buckets\":[]}");
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java
index 744d4a23a667e5..798b3677ff127d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.MapLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -36,8 +35,8 @@
 /**
  * AggregateFunction 'map_agg'. This class is generated by GenerateFunction.
  */
-public class MapAgg extends AggregateFunction
-        implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class MapAgg extends NotNullableAggregateFunction
+        implements BinaryExpression, ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(MapType.of(new FollowToAnyDataType(0), new FollowToAnyDataType(1)))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctCount.java
index 68d31e3e7bd19d..81768ef48dd960 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctCount.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctCount.java
@@ -21,7 +21,6 @@
 import org.apache.doris.nereids.analyzer.Unbound;
 import org.apache.doris.nereids.trees.expressions.Cast;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,8 +35,8 @@
 import java.util.List;
 
 /** MultiDistinctCount */
-public class MultiDistinctCount extends AggregateFunction
-        implements AlwaysNotNullable, ExplicitlyCastableSignature, MultiDistinction {
+public class MultiDistinctCount extends NotNullableAggregateFunction
+        implements ExplicitlyCastableSignature, MultiDistinction {
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BigIntType.INSTANCE).varArgs(AnyDataType.INSTANCE_WITHOUT_INDEX)
     );
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctSum0.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctSum0.java
index 2b0eda06b4264a..abd5292e31a233 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctSum0.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctSum0.java
@@ -20,7 +20,6 @@
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ComputePrecisionForSum;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
@@ -39,8 +38,8 @@
 import java.util.List;
 
 /** MultiDistinctSum0 */
-public class MultiDistinctSum0 extends AggregateFunction implements UnaryExpression,
-        ExplicitlyCastableSignature, ComputePrecisionForSum, MultiDistinction, AlwaysNotNullable {
+public class MultiDistinctSum0 extends NotNullableAggregateFunction implements UnaryExpression,
+        ExplicitlyCastableSignature, ComputePrecisionForSum, MultiDistinction {
 
     private final boolean mustUseMultiDistinctAgg;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Ndv.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Ndv.java
index ea90bc587911be..c2a04b4fd9931b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Ndv.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Ndv.java
@@ -21,7 +21,6 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
@@ -38,8 +37,8 @@
 /**
  * AggregateFunction 'ndv'. This class is generated by GenserateFunction.
  */
-public class Ndv extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, RollUpTrait {
+public class Ndv extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature, RollUpTrait {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BigIntType.INSTANCE).args(AnyDataType.INSTANCE_WITHOUT_INDEX)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/NotNullableAggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/NotNullableAggregateFunction.java
new file mode 100644
index 00000000000000..f38decf439c2c6
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/NotNullableAggregateFunction.java
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.agg;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
+
+import java.util.List;
+
+/**
+ * base class for AlwaysNotNullable aggregate function
+ */
+public abstract class NotNullableAggregateFunction extends AggregateFunction implements AlwaysNotNullable {
+    protected NotNullableAggregateFunction(String name, Expression ...expressions) {
+        super(name, false, expressions);
+    }
+
+    protected NotNullableAggregateFunction(String name, List<Expression> expressions) {
+        super(name, false, expressions);
+    }
+
+    protected NotNullableAggregateFunction(String name, boolean distinct, Expression ...expressions) {
+        super(name, distinct, expressions);
+    }
+
+    protected NotNullableAggregateFunction(String name, boolean distinct, List<Expression> expressions) {
+        super(name, distinct, expressions);
+    }
+
+    // return value of this function if the input data is empty.
+    // for example, count(*) of empty table is 0;
+    public abstract Expression resultForEmptyInput();
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java
index 956f585f0a1e4c..7837ea1bec7569 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java
@@ -19,8 +19,8 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.BitmapIntersectFunction;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.BitmapType;
 import org.apache.doris.nereids.util.ExpressionUtils;
@@ -33,8 +33,8 @@
 /**
  * AggregateFunction 'orthogonal_bitmap_intersect'. This class is generated by GenerateFunction.
  */
-public class OrthogonalBitmapIntersect extends AggregateFunction
-        implements AlwaysNotNullable, OrthogonalBitmapFunction, BitmapIntersectFunction {
+public class OrthogonalBitmapIntersect extends NotNullableAggregateFunction
+        implements OrthogonalBitmapFunction, BitmapIntersectFunction {
 
     static final List<FunctionSignature> FUNCTION_SIGNATURES = SUPPORTED_TYPES.stream()
             .map(type -> FunctionSignature.ret(BitmapType.INSTANCE).varArgs(BitmapType.INSTANCE, type, type))
@@ -75,4 +75,9 @@ public OrthogonalBitmapIntersect withDistinctAndChildren(boolean distinct, List<
     public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
         return visitor.visitOrthogonalBitmapIntersect(this, context);
     }
+
+    @Override
+    public Expression resultForEmptyInput() {
+        return new BitmapEmpty();
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersectCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersectCount.java
index 82bfa911c26faf..fd282341249e0a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersectCount.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersectCount.java
@@ -18,8 +18,8 @@
 package org.apache.doris.nereids.trees.expressions.functions.agg;
 
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.BitmapIntersectFunction;
+import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
@@ -30,8 +30,8 @@
 /**
  * AggregateFunction 'orthogonal_bitmap_intersect_count'. This class is generated by GenerateFunction.
  */
-public class OrthogonalBitmapIntersectCount extends AggregateFunction
-        implements AlwaysNotNullable, OrthogonalBitmapFunction, BitmapIntersectFunction {
+public class OrthogonalBitmapIntersectCount extends NotNullableAggregateFunction
+        implements OrthogonalBitmapFunction, BitmapIntersectFunction {
 
     /**
      * constructor with 3 or more arguments.
@@ -44,7 +44,7 @@ public OrthogonalBitmapIntersectCount(Expression arg0, Expression arg1, Expressi
      * constructor with 3 or more arguments.
      */
     public OrthogonalBitmapIntersectCount(boolean distinct, Expression arg0, Expression arg1,
-            Expression arg2, Expression... varArgs) {
+                                          Expression arg2, Expression... varArgs) {
         super("orthogonal_bitmap_intersect_count", distinct,
                 ExpressionUtils.mergeArguments(arg0, arg1, arg2, varArgs));
     }
@@ -63,4 +63,9 @@ public OrthogonalBitmapIntersectCount withDistinctAndChildren(boolean distinct,
     public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
         return visitor.visitOrthogonalBitmapIntersectCount(this, context);
     }
+
+    @Override
+    public Expression resultForEmptyInput() {
+        return new BigIntLiteral(0);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapUnionCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapUnionCount.java
index c08bb08aeecf64..081e5d7f6c4250 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapUnionCount.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapUnionCount.java
@@ -19,8 +19,8 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.BigIntType;
@@ -34,8 +34,8 @@
 /**
  * AggregateFunction 'orthogonal_bitmap_union_count'. This class is generated by GenerateFunction.
  */
-public class OrthogonalBitmapUnionCount extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class OrthogonalBitmapUnionCount extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE)
@@ -73,4 +73,9 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
     public List<FunctionSignature> getSignatures() {
         return SIGNATURES;
     }
+
+    @Override
+    public Expression resultForEmptyInput() {
+        return new BigIntLiteral(0);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java
index bf189d67e89b0b..1abbe4d5450531 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -42,8 +41,8 @@
 /**
  * AggregateFunction 'percentile_array'. This class is generated by GenerateFunction.
  */
-public class PercentileArray extends AggregateFunction
-        implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class PercentileArray extends NotNullableAggregateFunction
+        implements BinaryExpression, ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/QuantileUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/QuantileUnion.java
index 3d0729775a5cec..3d107ec282b606 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/QuantileUnion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/QuantileUnion.java
@@ -20,7 +20,6 @@
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.QuantileStateEmpty;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -36,8 +35,8 @@
 /**
  * AggregateFunction 'quantile_union'. This class is generated by GenerateFunction.
  */
-public class QuantileUnion extends AggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+public class QuantileUnion extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(QuantileStateType.INSTANCE).args(QuantileStateType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java
index 7af112ef8e800f..5ac114f00e83ad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -39,8 +38,8 @@
 /**
  * AggregateFunction 'sequence_count'. This class is generated by GenerateFunction.
  */
-public class SequenceCount extends AggregateFunction
-        implements ExplicitlyCastableSignature, AlwaysNotNullable, SequenceFunction {
+public class SequenceCount extends NotNullableAggregateFunction
+        implements ExplicitlyCastableSignature, SequenceFunction {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(BigIntType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum0.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum0.java
index fd052a69c0e90e..5a1f0f9fb93d34 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum0.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum0.java
@@ -20,7 +20,6 @@
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ComputePrecisionForSum;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
@@ -53,8 +52,8 @@
  * AggregateFunction 'sum0'. sum0 returns the sum of the values which go into it like sum.
  * It differs in that when no non null values are applied zero is returned instead of null.
  */
-public class Sum0 extends AggregateFunction
-        implements UnaryExpression, AlwaysNotNullable, ExplicitlyCastableSignature, ComputePrecisionForSum,
+public class Sum0 extends NotNullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature, ComputePrecisionForSum,
         SupportWindowAnalytic, RollUpTrait {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(

From c0b8478f0a659c0e93dae9b9a86e529085944d99 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Mon, 2 Dec 2024 09:59:52 +0800
Subject: [PATCH 094/399] [Fix](http)Enhanced Security Checks for Audit Log
 File Names (#44612)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Purpose:

To improve the security of audit log files, a new method
checkAuditLogFileName has been added to validate the file name and path
to ensure they meet security requirements. This method is designed to
prevent invalid file names and path traversal attacks, ensuring that
only files within the designated directory can be accessed.↳

### Changes:

#### File Name Validation:

A regular expression check has been added to validate the file name:
^[a-zA-Z0-9._-]+$, restricting the file name to letters, numbers, dots,
underscores, and hyphens.

If the file name contains invalid characters (e.g., spaces, path
traversal characters), a SecurityException is thrown with the message
“Invalid file name.”
Path Validation:

The file name is resolved into a normalized path, and it is checked to
ensure that it is within the allowed directory.

The path is constructed using
Paths.get(Config.audit_log_dir).resolve(logFile).normalize(). If the
path does not start with the specified audit log directory
(Config.audit_log_dir), indicating an attempt to access outside the
permitted directory (e.g., a path traversal attack), a SecurityException
is thrown with the message “Invalid file path: Access outside of
permitted directory.”
---
 .../doris/httpv2/rest/GetLogFileAction.java   | 38 +++++++++++-
 .../doris/httpv2/GetLogFileActionTest.java    | 60 +++++++++++++++++++
 2 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/httpv2/GetLogFileActionTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLogFileAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLogFileAction.java
index 475ee5ace1ea0d..87c4c4cfa90a68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLogFileAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLogFileAction.java
@@ -32,6 +32,8 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Map;
 import java.util.Set;
 import javax.servlet.http.HttpServletRequest;
@@ -51,6 +53,23 @@
  */
 @RestController
 public class GetLogFileAction extends RestBaseController {
+    /**
+     * This method fetches internal logs via HTTP, which is no longer recommended and will
+     * be deprecated in future versions.
+     * <p>
+     * Using HTTP to fetch logs introduces serious security and performance issues:
+     * - **Security Risks**: Log content may expose sensitive information, allowing attackers to exploit the exposed
+     * HTTP endpoints.
+     * - **Performance Problems**: Frequent HTTP requests can cause significant system load, affecting the
+     * responsiveness and stability of the application.
+     * <p>
+     * It is strongly advised not to use this approach for accessing logs. Any new requirements should be
+     * handled using more secure, reliable, and efficient methods such as log aggregation tools (e.g., ELK, Splunk)
+     * or dedicated internal APIs.
+     * <p>
+     * **Note**: No new HTTP endpoints or types for log access will be accepted.
+     * Any further attempts to extend this HTTP-based log retrieval method will not be supported.
+     */
     private final Set<String> logFileTypes = Sets.newHashSet("fe.audit.log");
 
     @RequestMapping(path = "/api/get_log_file", method = {RequestMethod.GET, RequestMethod.HEAD})
@@ -79,7 +98,13 @@ public Object execute(HttpServletRequest request, HttpServletResponse response)
             String fileInfos = getFileInfos(logType);
             response.setHeader("file_infos", fileInfos);
             return ResponseEntityBuilder.ok();
-        } else if (method.equals(RequestMethod.GET.name())) {
+        }
+        if (method.equals(RequestMethod.GET.name())) {
+            try {
+                checkAuditLogFileName(logFile);
+            } catch (SecurityException e) {
+                return ResponseEntityBuilder.internalError(e.getMessage());
+            }
             File log = getLogFile(logType, logFile);
             if (!log.exists() || !log.isFile()) {
                 return ResponseEntityBuilder.okWithCommonError("Log file not exist: " + log.getName());
@@ -97,6 +122,17 @@ public Object execute(HttpServletRequest request, HttpServletResponse response)
         return ResponseEntityBuilder.ok();
     }
 
+    private void checkAuditLogFileName(String logFile) {
+        if (!logFile.matches("^[a-zA-Z0-9._-]+$")) {
+            throw new SecurityException("Invalid file name");
+        }
+        Path normalizedPath = Paths.get(Config.audit_log_dir).resolve(logFile).normalize();
+        // check path is valid or not
+        if (!normalizedPath.startsWith(Config.audit_log_dir)) {
+            throw new SecurityException("Invalid file path: Access outside of permitted directory");
+        }
+    }
+
     private String getFileInfos(String logType) {
         Map<String, Long> fileInfos = Maps.newTreeMap();
         if (logType.equals("fe.audit.log")) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/httpv2/GetLogFileActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/httpv2/GetLogFileActionTest.java
new file mode 100644
index 00000000000000..8d4cac9b6ad9f4
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/httpv2/GetLogFileActionTest.java
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.httpv2;
+
+import org.apache.doris.common.Config;
+import org.apache.doris.httpv2.rest.GetLogFileAction;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+public class GetLogFileActionTest {
+
+    @TempDir
+    public File tempDir;
+
+    @BeforeAll
+    public static void before() {
+        File tempDir = new File("test/audit.log");
+        tempDir.mkdir();
+        Config.audit_log_dir = tempDir.getAbsolutePath();
+    }
+
+    @Test
+    public void testCheckAuditLogFileName() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+        //private method checkAuditLogFileName
+        GetLogFileAction action = new GetLogFileAction();
+        Method method = GetLogFileAction.class.getDeclaredMethod("checkAuditLogFileName", String.class);
+        method.setAccessible(true);
+        method.invoke(action, "audit.log");
+        method.invoke(action, "fe.audit.log.20241104-1");
+        Assertions.assertThrows(InvocationTargetException.class, () -> method.invoke(action, "../etc/passwd"));
+        Assertions.assertThrows(InvocationTargetException.class, () -> method.invoke(action,
+                "fe.audit.log.20241104-1/../../etc/passwd"));
+        Assertions.assertThrows(InvocationTargetException.class,
+                () -> method.invoke(action, "fe.audit.log.20241104-1; rm -rf /"));
+
+
+    }
+}

From 2d9d3f79df01c475a1a0bbc1b68d2084323f04f5 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Mon, 2 Dec 2024 10:26:56 +0800
Subject: [PATCH 095/399] [Bug](runtime-filter) fix publish not inited rf when
 broadcast join meet wake up by downsteam (#44823)

### What problem does this PR solve?
fix publish not inited rf when broadcast join meet wake up by downsteam

related with #44408 #41751
---
 be/src/exprs/runtime_filter.cpp              |  2 +-
 be/src/pipeline/exec/hashjoin_build_sink.cpp | 38 +++++++++++---------
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index e26452c9ef69e5..c983af0fb3ea71 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -1542,7 +1542,7 @@ void IRuntimeFilter::update_runtime_filter_type_to_profile(uint64_t local_merge_
 
 std::string IRuntimeFilter::debug_string() const {
     return fmt::format(
-            "RuntimeFilter: (id = {}, type = {}, is_broadcast: {}, ignored: {}"
+            "RuntimeFilter: (id = {}, type = {}, is_broadcast: {}, ignored: {}, "
             "build_bf_cardinality: {}, dependency: {}, synced_size: {}, has_local_target: {}, "
             "has_remote_target: {},error_msg: [{}]",
             _filter_id, to_string(_runtime_filter_type), _is_broadcast_join,
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index 0a71b86bed0e86..cec0c77da8a61d 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -139,28 +139,32 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
         return Base::close(state, exec_status);
     }
 
-    if (_should_build_hash_table) {
-        if (state->get_task()->wake_up_by_downstream()) {
+    if (state->get_task()->wake_up_by_downstream()) {
+        if (_should_build_hash_table) {
+            // partitial ignore rf to make global rf work
             RETURN_IF_ERROR(_runtime_filter_slots->send_filter_size(state, 0, _finish_dependency));
             RETURN_IF_ERROR(_runtime_filter_slots->ignore_all_filters());
         } else {
-            if (p._shared_hashtable_controller &&
-                !p._shared_hash_table_context->complete_build_stage) {
-                return Status::InternalError("close before sink meet eos");
-            }
-            auto* block = _shared_state->build_block.get();
-            uint64_t hash_table_size = block ? block->rows() : 0;
-            {
-                SCOPED_TIMER(_runtime_filter_init_timer);
-                RETURN_IF_ERROR(_runtime_filter_slots->init_filters(state, hash_table_size));
-                RETURN_IF_ERROR(_runtime_filter_slots->ignore_filters(state));
-            }
-            if (hash_table_size > 1) {
-                SCOPED_TIMER(_runtime_filter_compute_timer);
-                _runtime_filter_slots->insert(block);
-            }
+            // do not publish filter coz local rf not inited and useless
+            return Base::close(state, exec_status);
+        }
+    } else if (_should_build_hash_table) {
+        if (p._shared_hashtable_controller && !p._shared_hash_table_context->complete_build_stage) {
+            return Status::InternalError("close before sink meet eos");
+        }
+        auto* block = _shared_state->build_block.get();
+        uint64_t hash_table_size = block ? block->rows() : 0;
+        {
+            SCOPED_TIMER(_runtime_filter_init_timer);
+            RETURN_IF_ERROR(_runtime_filter_slots->init_filters(state, hash_table_size));
+            RETURN_IF_ERROR(_runtime_filter_slots->ignore_filters(state));
+        }
+        if (hash_table_size > 1) {
+            SCOPED_TIMER(_runtime_filter_compute_timer);
+            _runtime_filter_slots->insert(block);
         }
     }
+
     SCOPED_TIMER(_publish_runtime_filter_timer);
     RETURN_IF_ERROR_OR_CATCH_EXCEPTION(
             _runtime_filter_slots->publish(state, !_should_build_hash_table));

From 3ce581ec0cdad4f2e953b5fd554917e6c3761d7c Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Mon, 2 Dec 2024 10:33:31 +0800
Subject: [PATCH 096/399] [minor](scan) Delete unused function in ScanNode
 (#44733)

---
 .../src/main/java/org/apache/doris/planner/ScanNode.java | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index b4033a0535ef3d..2b10520e67d317 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -727,15 +727,6 @@ public static TScanRangeLocations createSingleScanRangeLocations(FederationBacke
         return scanRangeLocation;
     }
 
-    public boolean ignoreStorageDataDistribution(ConnectContext context, int numBackends) {
-        return context != null
-                && context.getSessionVariable().isIgnoreStorageDataDistribution()
-                && !fragment.hasNullAwareLeftAntiJoin()
-                && getScanRangeNum()
-                < ConnectContext.get().getSessionVariable().getParallelExecInstanceNum()
-                * (numScanBackends() > 0 ? numScanBackends() : numBackends);
-    }
-
     public int numScanBackends() {
         return scanBackendIds.size();
     }

From 59ae8003866a752044b8e450760f2b3077d08e58 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Mon, 2 Dec 2024 10:43:18 +0800
Subject: [PATCH 097/399] [chore](binlog) Save table type in CreateTableRecord
 (#44735)

---
 .../main/java/org/apache/doris/binlog/CreateTableRecord.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/CreateTableRecord.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/CreateTableRecord.java
index 89d11fc014e979..602afe88081d5b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/CreateTableRecord.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/CreateTableRecord.java
@@ -20,6 +20,7 @@
 import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.TableIf.TableType;
 import org.apache.doris.persist.CreateTableInfo;
 import org.apache.doris.persist.gson.GsonUtils;
 
@@ -43,6 +44,8 @@ public class CreateTableRecord {
     private long tableId;
     @SerializedName(value = "tableName")
     private String tableName;
+    @SerializedName(value = "tableType")
+    protected TableType type;
     @SerializedName(value = "sql")
     private String sql;
 
@@ -56,6 +59,8 @@ public CreateTableRecord(long commitSeq, CreateTableInfo info) {
         String dbName = info.getDbName();
         this.dbName = dbName;
 
+        this.type = table.getType();
+
         Database db = Env.getCurrentInternalCatalog().getDbNullable(dbName);
         if (db == null) {
             LOG.warn("db not found. dbId: {}", dbId);

From 4bd1cc1a01b82ea7e74d99b94c5369ae77d08336 Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Mon, 2 Dec 2024 08:26:11 +0530
Subject: [PATCH 098/399] [Enhancement] (nereids)implement
 showCreateDatabaseCommand in nereids (#43034)

Issue Number: close #42739
---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  20 +++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../commands/ShowCreateDatabaseCommand.java   | 122 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../show/test_show_create_db_nereids.out      |   7 +
 .../show/test_show_create_db_nereids.groovy   |  42 ++++++
 7 files changed, 198 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateDatabaseCommand.java
 create mode 100644 regression-test/data/nereids_p0/show/test_show_create_db_nereids.out
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_create_db_nereids.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 1b1e218a958146..f78a1c4b53179a 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -214,6 +214,7 @@ supportedDropStatement
 supportedShowStatement
     : SHOW (GLOBAL | SESSION | LOCAL)? VARIABLES wildWhere?                         #showVariables
     | SHOW AUTHORS                                                                  #showAuthors
+    | SHOW CREATE (DATABASE | SCHEMA) name=multipartIdentifier                      #showCreateDatabase
     | SHOW BROKER                                                                   #showBroker
     | SHOW DYNAMIC PARTITION TABLES ((FROM | IN) database=multipartIdentifier)?     #showDynamicPartition
     | SHOW EVENTS ((FROM | IN) database=multipartIdentifier)? wildWhere?            #showEvents
@@ -295,7 +296,6 @@ unsupportedShowStatement
     | SHOW FULL? PROCESSLIST                                                        #showProcessList
     | SHOW (GLOBAL | SESSION | LOCAL)? STATUS wildWhere?                            #showStatus
     | SHOW CREATE MATERIALIZED VIEW name=multipartIdentifier                        #showMaterializedView
-    | SHOW CREATE (DATABASE | SCHEMA) name=multipartIdentifier                      #showCreateDatabase
     | SHOW CREATE (GLOBAL | SESSION | LOCAL)? FUNCTION functionIdentifier
         LEFT_PAREN functionArguments? RIGHT_PAREN
         ((FROM | IN) database=multipartIdentifier)?                                 #showCreateFunction
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index e5f4322225de53..d52d2696a2951c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -20,6 +20,7 @@
 import org.apache.doris.analysis.ArithmeticExpr.Operator;
 import org.apache.doris.analysis.BrokerDesc;
 import org.apache.doris.analysis.ColumnNullableType;
+import org.apache.doris.analysis.DbName;
 import org.apache.doris.analysis.EncryptKeyName;
 import org.apache.doris.analysis.PassVar;
 import org.apache.doris.analysis.SetType;
@@ -226,6 +227,7 @@
 import org.apache.doris.nereids.DorisParser.ShowConfigContext;
 import org.apache.doris.nereids.DorisParser.ShowConstraintContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateCatalogContext;
+import org.apache.doris.nereids.DorisParser.ShowCreateDatabaseContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateMTMVContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateMaterializedViewContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateProcedureContext;
@@ -512,6 +514,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateCatalogCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCreateDatabaseCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMaterializedViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand;
@@ -4617,6 +4620,23 @@ public LogicalPlan visitShowFrontends(ShowFrontendsContext ctx) {
         return new ShowFrontendsCommand(detail);
     }
 
+    @Override
+    public LogicalPlan visitShowCreateDatabase(ShowCreateDatabaseContext ctx) {
+        List<String> nameParts = visitMultipartIdentifier(ctx.name);
+        String databaseName = "";
+        String catalogName = "";
+        if (nameParts.size() == 2) {
+            // The identifier is in the form "internalcatalog.databasename"
+            catalogName = nameParts.get(0);
+            databaseName = nameParts.get(1);
+        } else if (nameParts.size() == 1) {
+            // The identifier is in the form "databasename"
+            databaseName = nameParts.get(0);
+        }
+
+        return new ShowCreateDatabaseCommand(new DbName(catalogName, databaseName));
+    }
+
     @Override
     public LogicalPlan visitCleanAllProfile(CleanAllProfileContext ctx) {
         return new CleanAllProfileCommand();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 8f474ea395f889..8625d103f3b453 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -198,6 +198,7 @@ public enum PlanType {
     SHOW_BROKER_COMMAND,
     SHOW_CONFIG_COMMAND,
     SHOW_CREATE_CATALOG_COMMAND,
+    SHOW_CREATE_DATABASE_COMMAND,
     SHOW_CREATE_MATERIALIZED_VIEW_COMMAND,
     SHOW_CREATE_TABLE_COMMAND,
     SHOW_CREATE_VIEW_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateDatabaseCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateDatabaseCommand.java
new file mode 100644
index 00000000000000..29186e9e5cddee
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateDatabaseCommand.java
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.DbName;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.cluster.ClusterNamespace;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.PrintableMap;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
+import org.apache.doris.datasource.iceberg.IcebergExternalDatabase;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Represents the command for SHOW CREATE DATABASE.
+ */
+public class ShowCreateDatabaseCommand extends ShowCommand {
+    private static final ShowResultSetMetaData META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("Database", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Create Database", ScalarType.createVarchar(30)))
+                    .build();
+
+    private final String databaseName;
+    private final String catalogName;
+
+    public ShowCreateDatabaseCommand(DbName dbName) {
+        super(PlanType.SHOW_CREATE_DATABASE_COMMAND);
+        this.databaseName = Objects.requireNonNull(dbName.getDb(), "Database name cannot be null");
+        this.catalogName = dbName.getCtl();
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        String ctlgName = catalogName;
+        if (Strings.isNullOrEmpty(catalogName)) {
+            ctlgName = Env.getCurrentEnv().getCurrentCatalog().getName();
+        }
+        if (Strings.isNullOrEmpty(databaseName)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, databaseName);
+        }
+
+        if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), ctlgName, databaseName,
+                PrivPredicate.SHOW)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED_ERROR,
+                    PrivPredicate.SHOW.getPrivs().toString(), databaseName);
+        }
+
+        List<List<String>> rows = Lists.newArrayList();
+
+        StringBuilder sb = new StringBuilder();
+        CatalogIf<?> catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(ctlgName);
+        if (catalog instanceof HMSExternalCatalog) {
+            String simpleDBName = ClusterNamespace.getNameFromFullName(databaseName);
+            org.apache.hadoop.hive.metastore.api.Database db = ((HMSExternalCatalog) catalog).getClient()
+                    .getDatabase(simpleDBName);
+            sb.append("CREATE DATABASE `").append(simpleDBName).append("`")
+                    .append(" LOCATION '")
+                    .append(db.getLocationUri())
+                    .append("'");
+        } else if (catalog instanceof IcebergExternalCatalog) {
+            IcebergExternalDatabase db = (IcebergExternalDatabase) catalog.getDbOrAnalysisException(databaseName);
+            sb.append("CREATE DATABASE `").append(databaseName).append("`")
+                .append(" LOCATION '")
+                .append(db.getLocation())
+                .append("'");
+        } else {
+            DatabaseIf db = catalog.getDbOrAnalysisException(databaseName);
+            sb.append("CREATE DATABASE `").append(ClusterNamespace.getNameFromFullName(databaseName)).append("`");
+            if (db.getDbProperties().getProperties().size() > 0) {
+                sb.append("\nPROPERTIES (\n");
+                sb.append(new PrintableMap<>(db.getDbProperties().getProperties(), "=", true, true, false));
+                sb.append("\n)");
+            }
+        }
+
+        rows.add(Lists.newArrayList(ClusterNamespace.getNameFromFullName(databaseName), sb.toString()));
+        return new ShowResultSet(this.getMetaData(), rows);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowCreateDatabaseCommand(this, context);
+    }
+
+    public ShowResultSetMetaData getMetaData() {
+        return META_DATA;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index c00c0ef90ca018..49485cda51d0c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -75,6 +75,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateCatalogCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCreateDatabaseCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMaterializedViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand;
@@ -398,6 +399,10 @@ default R visitShowCreateMaterializedViewCommand(ShowCreateMaterializedViewComma
         return visitCommand(showCreateMtlzViewCommand, context);
     }
 
+    default R visitShowCreateDatabaseCommand(ShowCreateDatabaseCommand showCreateDatabaseCommand, C context) {
+        return visitCommand(showCreateDatabaseCommand, context);
+    }
+
     default R visitShowCreateViewCommand(ShowCreateViewCommand showCreateViewCommand, C context) {
         return visitCommand(showCreateViewCommand, context);
     }
diff --git a/regression-test/data/nereids_p0/show/test_show_create_db_nereids.out b/regression-test/data/nereids_p0/show/test_show_create_db_nereids.out
new file mode 100644
index 00000000000000..df88b46f8fbe77
--- /dev/null
+++ b/regression-test/data/nereids_p0/show/test_show_create_db_nereids.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !cmd --
+db_test_show_create	CREATE DATABASE `db_test_show_create`
+
+-- !cmd --
+db_test_show_create	CREATE DATABASE `db_test_show_create`\nPROPERTIES (\n"property_key" = "property_value"\n)
+
diff --git a/regression-test/suites/nereids_p0/show/test_show_create_db_nereids.groovy b/regression-test/suites/nereids_p0/show/test_show_create_db_nereids.groovy
new file mode 100644
index 00000000000000..bfc84c9a60f72b
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_create_db_nereids.groovy
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_create_db_nereids", "query,create_database") {
+    String dbName = "db_test_show_create";
+
+    try {
+        // Create a new database to test the SHOW CREATE DATABASE command
+        sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+        
+        // Run the SHOW CREATE DATABASE command and validate the output using checkNereidsExecute and qt_cmd
+        checkNereidsExecute("""SHOW CREATE DATABASE ${dbName}""")
+        qt_cmd("""SHOW CREATE DATABASE ${dbName}""")
+
+        // Drop the database and verify that the command runs successfully
+        sql "DROP DATABASE IF EXISTS ${dbName}"
+        
+        // Re-create the database with additional properties
+        sql "CREATE DATABASE IF NOT EXISTS ${dbName} PROPERTIES ('property_key'='property_value')"
+        
+        // Verify the SHOW CREATE DATABASE command captures the properties using checkNereidsExecute and qt_cmd
+        checkNereidsExecute("""SHOW CREATE DATABASE ${dbName}""")
+        qt_cmd("""SHOW CREATE DATABASE ${dbName}""")
+    } finally {
+        // Clean up by dropping the database if it still exists
+        try_sql("DROP DATABASE IF EXISTS ${dbName}")
+    }
+}

From 70415afa864af41d5028605b897ae2ff67a62c66 Mon Sep 17 00:00:00 2001
From: echo-dundun <50286010+echo-hhj@users.noreply.github.com>
Date: Mon, 2 Dec 2024 10:57:27 +0800
Subject: [PATCH 099/399] [Feat] (Nereids)support showTrash Command (#44684)

Issue Number: close #42763
---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  13 +++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../plans/commands/ShowTrashCommand.java      | 109 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../ddl/show_trash/test_nereids_trash.groovy  |  22 ++++
 6 files changed, 151 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTrashCommand.java
 create mode 100644 regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index f78a1c4b53179a..e5c9a997073df8 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -246,6 +246,7 @@ supportedShowStatement
     | SHOW TABLET DIAGNOSIS tabletId=INTEGER_VALUE                                  #showDiagnoseTablet
     | SHOW FRONTENDS name=identifier?                                               #showFrontends 
     | SHOW TABLE tableId=INTEGER_VALUE                                              #showTableId
+    | SHOW TRASH (ON backend=STRING_LITERAL)?                                       #showTrash
     | SHOW WHITELIST                                                                #showWhitelist
     | SHOW TABLETS BELONG
         tabletIds+=INTEGER_VALUE (COMMA tabletIds+=INTEGER_VALUE)*                  #showTabletsBelong
@@ -332,7 +333,6 @@ unsupportedShowStatement
     | SHOW BRIEF? RESTORE ((FROM | IN) database=multipartIdentifier)? wildWhere?    #showRestore
     | SHOW RESOURCES wildWhere? sortClause? limitClause?                            #showResources
     | SHOW WORKLOAD GROUPS wildWhere?                                               #showWorkloadGroups
-    | SHOW TRASH (ON backend=STRING_LITERAL)?                                       #showTrash
     | SHOW SNAPSHOT ON repo=identifier wildWhere?                                   #showSnapshot
     | SHOW FULL? BUILTIN? FUNCTIONS
         ((FROM | IN) database=multipartIdentifier)? wildWhere?                      #showFunctions
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index d52d2696a2951c..5ab217a1d5eb2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -255,6 +255,7 @@
 import org.apache.doris.nereids.DorisParser.ShowStorageEnginesContext;
 import org.apache.doris.nereids.DorisParser.ShowTableIdContext;
 import org.apache.doris.nereids.DorisParser.ShowTabletsBelongContext;
+import org.apache.doris.nereids.DorisParser.ShowTrashContext;
 import org.apache.doris.nereids.DorisParser.ShowTriggersContext;
 import org.apache.doris.nereids.DorisParser.ShowVariablesContext;
 import org.apache.doris.nereids.DorisParser.ShowViewContext;
@@ -541,6 +542,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTabletsBelongCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTriggersCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowVariablesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowViewCommand;
@@ -4462,6 +4464,17 @@ public LogicalPlan visitShowTriggers(ShowTriggersContext ctx) {
         return new ShowTriggersCommand();
     }
 
+    @Override
+    public LogicalPlan visitShowTrash(ShowTrashContext ctx) {
+        if (ctx.ON() != null) {
+            String backend = stripQuotes(ctx.STRING_LITERAL().getText());
+            new ShowTrashCommand(backend);
+        } else {
+            return new ShowTrashCommand();
+        }
+        return new ShowTrashCommand();
+    }
+
     @Override
     public LogicalPlan visitShowRepositories(ShowRepositoriesContext ctx) {
         return new ShowRepositoriesCommand();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 8625d103f3b453..683af9fe2c4700 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -220,6 +220,7 @@ public enum PlanType {
     SHOW_SMALL_FILES_COMMAND,
     SHOW_STORAGE_ENGINES_COMMAND,
     SHOW_TABLE_ID_COMMAND,
+    SHOW_TRASH_COMMAND,
     SHOW_TRIGGERS_COMMAND,
     SHOW_VARIABLES_COMMAND,
     SHOW_AUTHORS_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTrashCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTrashCommand.java
new file mode 100644
index 00000000000000..bcf30490cc991a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTrashCommand.java
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.proc.TrashProcDir;
+import org.apache.doris.common.util.NetUtils;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.system.Backend;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * show trash command
+ */
+public class ShowTrashCommand extends ShowCommand {
+    private List<Backend> backends = Lists.newArrayList();
+    private String backendQuery;
+
+    public ShowTrashCommand() {
+        super(PlanType.SHOW_TRASH_COMMAND);
+    }
+
+    public ShowTrashCommand(String backendQuery) {
+        super(PlanType.SHOW_TRASH_COMMAND);
+        this.backendQuery = backendQuery;
+    }
+
+    public List<Backend> getBackends() {
+        return backends;
+    }
+
+    public String getBackend() {
+        return backendQuery;
+    }
+
+    public ShowResultSetMetaData getMetaData() {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        for (String title : TrashProcDir.TITLE_NAMES) {
+            builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
+        }
+        return builder.build();
+    }
+
+    private ShowResultSet handleShowTrash(String backendQuery) throws Exception {
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)
+                && !Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(),
+                PrivPredicate.OPERATOR)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR");
+        }
+        ImmutableMap<Long, Backend> backendsInfo = Env.getCurrentSystemInfo().getAllBackendsByAllCluster();
+        if (backendQuery == null || backendQuery.isEmpty()) {
+            for (Backend backend : backendsInfo.values()) {
+                this.backends.add(backend);
+            }
+        } else {
+            for (Backend backend : backendsInfo.values()) {
+                String backendStr = NetUtils.getHostPortInAccessibleFormat(backend.getHost(),
+                        backend.getHeartbeatPort());
+                if (backendQuery.equals(backendStr)) {
+                    this.backends.add(backend);
+                    break;
+                }
+            }
+        }
+        List<List<String>> infos = Lists.newArrayList();
+        TrashProcDir.getTrashInfo(backends, infos);
+        return new ShowResultSet(getMetaData(), infos);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowTrashCommand(this, context);
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        return handleShowTrash(backendQuery);
+    }
+}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 49485cda51d0c2..98050c598e1c55 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -102,6 +102,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTabletsBelongCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTriggersCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowVariablesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowViewCommand;
@@ -370,6 +371,10 @@ default R visitShowPluginsCommand(ShowPluginsCommand showPluginsCommand, C conte
         return visitCommand(showPluginsCommand, context);
     }
 
+    default R visitShowTrashCommand(ShowTrashCommand showTrashCommand, C context) {
+        return visitCommand(showTrashCommand, context);
+    }
+
     default R visitShowTriggersCommand(ShowTriggersCommand showTriggersCommand, C context) {
         return visitCommand(showTriggersCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy b/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
new file mode 100644
index 00000000000000..5d795e6fd119a1
--- /dev/null
+++ b/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("show_trash_nereids") {
+    // can not use qt command since the output change based on cluster and backend ip
+    checkNereidsExecute("""show trash;""")
+    checkNereidsExecute("""show trash on "127.0.0.1:9050";""")
+}

From b8555c4dfa9fbcf122c3df077d919991388dcb3b Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Mon, 2 Dec 2024 08:27:58 +0530
Subject: [PATCH 100/399] [Enhancement] (nereids)implement showCollationCommand
 in nereids (#43157)

Issue Number: close #42748
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    | 15 ++++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/ShowCollationCommand.java  | 89 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../nereids_p0/show/test_show_collation.out   |  5 ++
 .../show/test_show_collation.groovy           | 29 ++++++
 7 files changed, 145 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCollationCommand.java
 create mode 100644 regression-test/data/nereids_p0/show/test_show_collation.out
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_collation.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index e5c9a997073df8..8edcbb40c3c13b 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -236,6 +236,7 @@ supportedShowStatement
     | SHOW FILE ((FROM | IN) database=multipartIdentifier)?                         #showSmallFiles 
     | SHOW STORAGE? ENGINES                                                         #showStorageEngines
     | SHOW CREATE CATALOG name=identifier                                           #showCreateCatalog
+    | SHOW COLLATION wildWhere?                                                     #showCollation
     | SHOW SQL_BLOCK_RULE (FOR ruleName=identifier)?                                #showSqlBlockRule
     | SHOW CREATE VIEW name=multipartIdentifier                                     #showCreateView
     | SHOW CREATE MATERIALIZED VIEW mvName=identifier
@@ -307,7 +308,6 @@ unsupportedShowStatement
     | SHOW CATALOG name=identifier                                                  #showCatalog
     | SHOW FULL? (COLUMNS | FIELDS) (FROM | IN) tableName=multipartIdentifier
         ((FROM | IN) database=multipartIdentifier)? wildWhere?                      #showColumns
-    | SHOW COLLATION wildWhere?                                                     #showCollation
     | SHOW ((CHAR SET) | CHARSET) wildWhere?                                        #showCharset
     | SHOW COUNT LEFT_PAREN ASTERISK RIGHT_PAREN (WARNINGS | ERRORS)                #showWaringErrorCount
     | SHOW (WARNINGS | ERRORS) limitClause?                                         #showWaringErrors
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 5ab217a1d5eb2c..60eda00ce89620 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -224,6 +224,7 @@
 import org.apache.doris.nereids.DorisParser.ShowAuthorsContext;
 import org.apache.doris.nereids.DorisParser.ShowBackendsContext;
 import org.apache.doris.nereids.DorisParser.ShowBrokerContext;
+import org.apache.doris.nereids.DorisParser.ShowCollationContext;
 import org.apache.doris.nereids.DorisParser.ShowConfigContext;
 import org.apache.doris.nereids.DorisParser.ShowConstraintContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateCatalogContext;
@@ -512,6 +513,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowAuthorsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBackendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBrokerCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCollationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateCatalogCommand;
@@ -4782,4 +4784,17 @@ public LogicalPlan visitShowTabletsBelong(ShowTabletsBelongContext ctx) {
         });
         return new ShowTabletsBelongCommand(tabletIdLists);
     }
+
+    @Override
+    public LogicalPlan visitShowCollation(ShowCollationContext ctx) {
+        String wild = null;
+        if (ctx.wildWhere() != null) {
+            if (ctx.wildWhere().LIKE() != null) {
+                wild = stripQuotes(ctx.wildWhere().STRING_LITERAL().getText());
+            } else if (ctx.wildWhere().WHERE() != null) {
+                wild = ctx.wildWhere().expression().getText();
+            }
+        }
+        return new ShowCollationCommand(wild);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 683af9fe2c4700..d07c90b7ccd5e7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -196,6 +196,7 @@ public enum PlanType {
     SHOW_BACKENDS_COMMAND,
     SHOW_BLOCK_RULE_COMMAND,
     SHOW_BROKER_COMMAND,
+    SHOW_COLLATION_COMMAND,
     SHOW_CONFIG_COMMAND,
     SHOW_CREATE_CATALOG_COMMAND,
     SHOW_CREATE_DATABASE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCollationCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCollationCommand.java
new file mode 100644
index 00000000000000..945db68bf5901a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCollationCommand.java
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * Represents the command for SHOW COLLATION
+ */
+public class ShowCollationCommand extends ShowCommand {
+    private static final ShowResultSetMetaData COLLATION_META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("Collation", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Charset", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Id", ScalarType.createVarchar(10)))
+                    .addColumn(new Column("Default", ScalarType.createVarchar(10)))
+                    .addColumn(new Column("Compiled", ScalarType.createVarchar(10)))
+                    .addColumn(new Column("Sortlen", ScalarType.createVarchar(10)))
+                    .build();
+
+    private final String wild;
+
+    public ShowCollationCommand(String wild) {
+        super(PlanType.SHOW_COLLATION_COMMAND);
+        this.wild = wild;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        List<List<String>> rows = Lists.newArrayList();
+        List<String> utf8mb40900Bin = Lists.newArrayList();
+        // | utf8mb4_0900_bin | utf8mb4 | 309 | Yes | Yes | 1 |
+        utf8mb40900Bin.add(ctx.getSessionVariable().getCollationConnection());
+        utf8mb40900Bin.add(ctx.getSessionVariable().getCharsetServer());
+        utf8mb40900Bin.add("309");
+        utf8mb40900Bin.add("Yes");
+        utf8mb40900Bin.add("Yes");
+        utf8mb40900Bin.add("1");
+        rows.add(utf8mb40900Bin);
+        // ATTN: we must have this collation for compatible with some bi tools
+        List<String> utf8mb3GeneralCi = Lists.newArrayList();
+        // | utf8mb3_general_ci | utf8mb3 | 33 | Yes | Yes | 1 |
+        utf8mb3GeneralCi.add("utf8mb3_general_ci");
+        utf8mb3GeneralCi.add("utf8mb3");
+        utf8mb3GeneralCi.add("33");
+        utf8mb3GeneralCi.add("Yes");
+        utf8mb3GeneralCi.add("Yes");
+        utf8mb3GeneralCi.add("1");
+        rows.add(utf8mb3GeneralCi);
+        // Set the result set and send it using the executor
+        return new ShowResultSet(COLLATION_META_DATA, rows);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowCollationCommand(this, context);
+    }
+
+    @Override
+    public String toString() {
+        return "SHOW COLLATION" + (wild != null ? " LIKE '" + wild + "'" : "");
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 98050c598e1c55..f3bf05b2377d1f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -72,6 +72,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowAuthorsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBackendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBrokerCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCollationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateCatalogCommand;
@@ -521,6 +522,10 @@ default R visitShowTabletsBelongCommand(ShowTabletsBelongCommand showTabletBelon
         return visitCommand(showTabletBelongCommand, context);
     }
 
+    default R visitShowCollationCommand(ShowCollationCommand showCollationCommand, C context) {
+        return visitCommand(showCollationCommand, context);
+    }
+
     default R visitCreateRoutineLoadCommand(CreateRoutineLoadCommand createRoutineLoadCommand, C context) {
         return visitCommand(createRoutineLoadCommand, context);
     }
diff --git a/regression-test/data/nereids_p0/show/test_show_collation.out b/regression-test/data/nereids_p0/show/test_show_collation.out
new file mode 100644
index 00000000000000..60bac6866a761e
--- /dev/null
+++ b/regression-test/data/nereids_p0/show/test_show_collation.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !cmd --
+utf8mb4_0900_bin	utf8mb4	309	Yes	Yes	1
+utf8mb3_general_ci	utf8mb3	33	Yes	Yes	1
+
diff --git a/regression-test/suites/nereids_p0/show/test_show_collation.groovy b/regression-test/suites/nereids_p0/show/test_show_collation.groovy
new file mode 100644
index 00000000000000..00837d72161a97
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_collation.groovy
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_collation", "query,collation") {
+    try {
+        // Execute the SHOW COLLATION command and verify the output
+        checkNereidsExecute("SHOW COLLATION")
+        qt_cmd("SHOW COLLATION")
+    } catch (Exception e) {
+        // Log any exceptions that occur during testing
+        log.error("Failed to execute SHOW COLLATION command", e)
+        throw e
+    }
+}
+

From b49374d30f3b515a5e6f86837c7caec7633ebfc5 Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Mon, 2 Dec 2024 08:28:19 +0530
Subject: [PATCH 101/399] [Enhancement] (nereids)implement
 showProcessListCommand in nereids (#43158)

Issue Number: close #42729
---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |   7 +
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../commands/ShowProcessListCommand.java      | 131 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../test_show_process_list_command.groovy     |  30 ++++
 6 files changed, 175 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcessListCommand.java
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_process_list_command.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 8edcbb40c3c13b..d30ef144e52cd0 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -229,6 +229,7 @@ supportedShowStatement
     | SHOW PLUGINS                                                                  #showPlugins    
     | SHOW REPOSITORIES                                                             #showRepositories
     | SHOW BRIEF? CREATE TABLE name=multipartIdentifier                             #showCreateTable
+    | SHOW FULL? PROCESSLIST                                                        #showProcessList
     | SHOW ROLES                                                                    #showRoles        
     | SHOW PARTITION partitionId=INTEGER_VALUE                                      #showPartitionId
     | SHOW PRIVILEGES                                                               #showPrivileges
@@ -295,7 +296,6 @@ unsupportedShowStatement
     | SHOW TABLE STATUS ((FROM | IN) database=multipartIdentifier)? wildWhere?      #showTableStatus
     | SHOW FULL? TABLES ((FROM | IN) database=multipartIdentifier)? wildWhere?      #showTables
     | SHOW FULL? VIEWS ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showViews
-    | SHOW FULL? PROCESSLIST                                                        #showProcessList
     | SHOW (GLOBAL | SESSION | LOCAL)? STATUS wildWhere?                            #showStatus
     | SHOW CREATE MATERIALIZED VIEW name=multipartIdentifier                        #showMaterializedView
     | SHOW CREATE (GLOBAL | SESSION | LOCAL)? FUNCTION functionIdentifier
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 60eda00ce89620..1422d807b71bd9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -248,6 +248,7 @@
 import org.apache.doris.nereids.DorisParser.ShowPrivilegesContext;
 import org.apache.doris.nereids.DorisParser.ShowProcContext;
 import org.apache.doris.nereids.DorisParser.ShowProcedureStatusContext;
+import org.apache.doris.nereids.DorisParser.ShowProcessListContext;
 import org.apache.doris.nereids.DorisParser.ShowReplicaDistributionContext;
 import org.apache.doris.nereids.DorisParser.ShowRepositoriesContext;
 import org.apache.doris.nereids.DorisParser.ShowRolesContext;
@@ -536,6 +537,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowProcessListCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowReplicaDistributionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand;
@@ -4756,6 +4758,11 @@ public LogicalPlan visitShowTableId(ShowTableIdContext ctx) {
         return new ShowTableIdCommand(tableId);
     }
 
+    @Override
+    public LogicalPlan visitShowProcessList(ShowProcessListContext ctx) {
+        return new ShowProcessListCommand(ctx.FULL() != null);
+    }
+
     @Override
     public LogicalPlan visitSync(SyncContext ctx) {
         return new SyncCommand();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index d07c90b7ccd5e7..177e6b4fcff157 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -212,6 +212,7 @@ public enum PlanType {
     SHOW_LAST_INSERT_COMMAND,
     SHOW_LOAD_PROFILE_COMMAND,
     SHOW_PARTITIONID_COMMAND,
+    SHOW_PROCESSLIST_COMMAND,
     SHOW_PROC_COMMAND,
     SHOW_PLUGINS_COMMAND,
     SHOW_PRIVILEGES_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcessListCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcessListCommand.java
new file mode 100644
index 00000000000000..fe04d61eab689e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcessListCommand.java
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.ClientPool;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.proc.FrontendsProcNode;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.thrift.FrontendService;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TShowProcessListRequest;
+import org.apache.doris.thrift.TShowProcessListResult;
+
+import com.google.common.collect.Lists;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+
+/**
+ * Represents the command for SHOW PROCESSLIST
+ */
+public class ShowProcessListCommand extends ShowCommand {
+    private static final Logger LOG = LogManager.getLogger(ShowProcessListCommand.class);
+    private static final ShowResultSetMetaData PROCESSLIST_META_DATA = ShowResultSetMetaData.builder()
+            .addColumn(new Column("CurrentConnected", ScalarType.createVarchar(16)))
+            .addColumn(new Column("Id", ScalarType.createType(PrimitiveType.BIGINT)))
+            .addColumn(new Column("User", ScalarType.createVarchar(16)))
+            .addColumn(new Column("Host", ScalarType.createVarchar(16)))
+            .addColumn(new Column("LoginTime", ScalarType.createVarchar(16)))
+            .addColumn(new Column("Catalog", ScalarType.createVarchar(16)))
+            .addColumn(new Column("Db", ScalarType.createVarchar(16)))
+            .addColumn(new Column("Command", ScalarType.createVarchar(16)))
+            .addColumn(new Column("Time", ScalarType.createType(PrimitiveType.INT)))
+            .addColumn(new Column("State", ScalarType.createVarchar(64)))
+            .addColumn(new Column("QueryId", ScalarType.createVarchar(64)))
+            .addColumn(new Column("Info", ScalarType.STRING))
+            .addColumn(new Column("FE", ScalarType.createVarchar(16)))
+            .addColumn(new Column("CloudCluster", ScalarType.createVarchar(16))).build();
+
+    private final boolean isFull;
+
+    public ShowProcessListCommand(boolean isFull) {
+        super(PlanType.SHOW_PROCESSLIST_COMMAND);
+        this.isFull = isFull;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        boolean isShowFullSql = isFull;
+        boolean isShowAllFe = ConnectContext.get().getSessionVariable().getShowAllFeConnection();
+
+        List<List<String>> rowSet = Lists.newArrayList();
+        List<ConnectContext.ThreadInfo> threadInfos = ctx.getConnectScheduler()
+                .listConnection(ctx.getQualifiedUser(), isShowFullSql);
+        long nowMs = System.currentTimeMillis();
+        for (ConnectContext.ThreadInfo info : threadInfos) {
+            rowSet.add(info.toRow(ctx.getConnectionId(), nowMs));
+        }
+
+        if (isShowAllFe) {
+            try {
+                TShowProcessListRequest request = new TShowProcessListRequest();
+                request.setShowFullSql(isShowFullSql);
+                request.setCurrentUserIdent(ConnectContext.get().getCurrentUserIdentity().toThrift());
+                List<Pair<String, Integer>> frontends = FrontendsProcNode.getFrontendWithRpcPort(Env.getCurrentEnv(),
+                        false);
+                FrontendService.Client client = null;
+                for (Pair<String, Integer> fe : frontends) {
+                    TNetworkAddress thriftAddress = new TNetworkAddress(fe.key(), fe.value());
+                    try {
+                        client = ClientPool.frontendPool.borrowObject(thriftAddress, 3000);
+                    } catch (Exception e) {
+                        LOG.warn("Failed to get frontend {} client. exception: {}", fe.key(), e);
+                        continue;
+                    }
+
+                    boolean isReturnToPool = false;
+                    try {
+                        TShowProcessListResult result = client.showProcessList(request);
+                        if (result.process_list != null && result.process_list.size() > 0) {
+                            rowSet.addAll(result.process_list);
+                        }
+                        isReturnToPool = true;
+                    } catch (Exception e) {
+                        LOG.warn("Failed to request processlist to fe: {} . exception: {}", fe.key(), e);
+                    } finally {
+                        if (isReturnToPool) {
+                            ClientPool.frontendPool.returnObject(thriftAddress, client);
+                        } else {
+                            ClientPool.frontendPool.invalidateObject(thriftAddress, client);
+                        }
+                    }
+                }
+            } catch (Throwable t) {
+                LOG.warn(" fetch process list from other fe failed, ", t);
+            }
+        }
+
+        return new ShowResultSet(PROCESSLIST_META_DATA, rowSet);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowProcessListCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index f3bf05b2377d1f..de5228f498178b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -95,6 +95,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowProcessListCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowReplicaDistributionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand;
@@ -529,4 +530,8 @@ default R visitShowCollationCommand(ShowCollationCommand showCollationCommand, C
     default R visitCreateRoutineLoadCommand(CreateRoutineLoadCommand createRoutineLoadCommand, C context) {
         return visitCommand(createRoutineLoadCommand, context);
     }
+
+    default R visitShowProcessListCommand(ShowProcessListCommand showProcessListCommand, C context) {
+        return visitCommand(showProcessListCommand, context);
+    }
 }
diff --git a/regression-test/suites/nereids_p0/show/test_show_process_list_command.groovy b/regression-test/suites/nereids_p0/show/test_show_process_list_command.groovy
new file mode 100644
index 00000000000000..b67ab1148c0eac
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_process_list_command.groovy
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_process_list_command", "query,process_list") {
+    try {
+        // Execute the SHOW PROCESSLIST command and verify the output
+        checkNereidsExecute("SHOW PROCESSLIST")
+
+        // Execute the SHOW FULL PROCESSLIST command and verify the output
+        checkNereidsExecute("SHOW FULL PROCESSLIST")
+    } catch (Exception e) {
+        // Log any exceptions that occur during testing
+        log.error("Failed to execute SHOW PROCESSLIST command", e)
+        throw e
+    }
+}

From 3e41c91246f0ed615c546e6d7f6dfc8b3e698a87 Mon Sep 17 00:00:00 2001
From: Petrichor <31833513+vinlee19@users.noreply.github.com>
Date: Mon, 2 Dec 2024 10:58:37 +0800
Subject: [PATCH 102/399] [feat](Nereids) support refresh database command
 (#44298)

Issue Number: close #https://github.com/apache/doris/issues/42579
---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  22 ++-
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../refresh/RefreshDatabaseCommand.java       | 149 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../test_nereids_refresh_catalog.out          |  24 ++-
 .../test_nereids_refresh_catalog.groovy       |  28 +++-
 7 files changed, 218 insertions(+), 13 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index d30ef144e52cd0..fd5ae1bd5744f9 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -437,6 +437,7 @@ channelDescription
 
 supportedRefreshStatement
     : REFRESH CATALOG name=identifier propertyClause?                               #refreshCatalog
+    | REFRESH DATABASE name=multipartIdentifier propertyClause?                     #refreshDatabase
     ;
 
 supportedCleanStatement
@@ -445,7 +446,6 @@ supportedCleanStatement
 
 unsupportedRefreshStatement
     : REFRESH TABLE name=multipartIdentifier                                        #refreshTable
-    | REFRESH DATABASE name=multipartIdentifier propertyClause?                     #refreshDatabase
     | REFRESH LDAP (ALL | (FOR user=identifierOrText))                              #refreshLdap
     ;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 1422d807b71bd9..70c4f61e4033c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -188,6 +188,7 @@
 import org.apache.doris.nereids.DorisParser.RecoverPartitionContext;
 import org.apache.doris.nereids.DorisParser.RecoverTableContext;
 import org.apache.doris.nereids.DorisParser.RefreshCatalogContext;
+import org.apache.doris.nereids.DorisParser.RefreshDatabaseContext;
 import org.apache.doris.nereids.DorisParser.RefreshMTMVContext;
 import org.apache.doris.nereids.DorisParser.RefreshMethodContext;
 import org.apache.doris.nereids.DorisParser.RefreshScheduleContext;
@@ -618,6 +619,7 @@
 import org.apache.doris.nereids.trees.plans.commands.load.LoadSequenceClause;
 import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
+import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
 import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
@@ -4351,6 +4353,25 @@ public Object visitRefreshCatalog(RefreshCatalogContext ctx) {
     }
 
     @Override
+    public RefreshDatabaseCommand visitRefreshDatabase(RefreshDatabaseContext ctx) {
+        Map<String, String> properties = visitPropertyClause(ctx.propertyClause()) == null ? Maps.newHashMap()
+                : visitPropertyClause(ctx.propertyClause());
+        List<String> parts = visitMultipartIdentifier(ctx.name);
+        int size = parts.size();
+        if (size == 0) {
+            throw new ParseException("database name can't be empty");
+        }
+        String dbName = parts.get(size - 1);
+
+        // [db].
+        if (size == 1) {
+            return new RefreshDatabaseCommand(dbName, properties);
+        } else if (parts.size() == 2) {  // [ctl,db].
+            return new RefreshDatabaseCommand(parts.get(0), dbName, properties);
+        }
+        throw new ParseException("Only one dot can be in the name: " + String.join(".", parts));
+    }
+
     public LogicalPlan visitShowLastInsert(ShowLastInsertContext ctx) {
         return new ShowLastInsertCommand();
     }
@@ -4379,7 +4400,6 @@ public LogicalPlan visitShowPartitionId(ShowPartitionIdContext ctx) {
             partitionId = Long.parseLong(ctx.partitionId.getText());
         }
         return new ShowPartitionIdCommand(partitionId);
-
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 177e6b4fcff157..6a8fcadaf520c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -187,6 +187,7 @@ public enum PlanType {
     SET_USER_PROPERTIES_COMMAND,
     SET_DEFAULT_STORAGE_VAULT_COMMAND,
     REFRESH_CATALOG_COMMAND,
+    REFRESH_DATABASE_COMMAND,
     PREPARED_COMMAND,
     EXECUTE_COMMAND,
     DROP_SQL_BLOCK_RULE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java
new file mode 100644
index 00000000000000..010adbb05d1cba
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.refresh;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.InfoSchemaDb;
+import org.apache.doris.catalog.MysqlDb;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.ExternalDatabase;
+import org.apache.doris.datasource.ExternalObjectLog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+import java.util.Map;
+
+/**
+ * Refresh database.
+ */
+public class RefreshDatabaseCommand extends Command implements ForwardWithSync {
+    private static final String INVALID_CACHE = "invalid_cache";
+
+    private String catalogName;
+    private String dbName;
+    private Map<String, String> properties;
+    private boolean invalidCache = false;
+
+    public RefreshDatabaseCommand(String dbName, Map<String, String> properties) {
+        super(PlanType.REFRESH_DATABASE_COMMAND);
+        this.dbName = dbName;
+        this.properties = properties;
+    }
+
+    public RefreshDatabaseCommand(String catalogName, String dbName, Map<String, String> properties) {
+        super(PlanType.REFRESH_DATABASE_COMMAND);
+        this.catalogName = catalogName;
+        this.dbName = dbName;
+        this.properties = properties;
+    }
+
+    private void validate(ConnectContext ctx) throws AnalysisException {
+        if (Strings.isNullOrEmpty(catalogName)) {
+            catalogName = ConnectContext.get().getCurrentCatalog().getName();
+        }
+        if (Strings.isNullOrEmpty(dbName)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName);
+        }
+
+        // Don't allow dropping 'information_schema' database
+        if (dbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) {
+
+            ErrorReport.reportAnalysisException(
+                    ErrorCode.ERR_DBACCESS_DENIED_ERROR, ctx.getQualifiedUser(), dbName);
+        }
+        // Don't allow dropping 'mysql' database
+        if (dbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) {
+            ErrorReport.reportAnalysisException(
+                    ErrorCode.ERR_DBACCESS_DENIED_ERROR, ctx.getQualifiedUser(), dbName);
+        }
+        // check access
+        if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), catalogName,
+                dbName, PrivPredicate.SHOW)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED_ERROR,
+                    PrivPredicate.SHOW.getPrivs().toString(), dbName);
+        }
+        String invalidConfig = properties == null ? null : properties.get(INVALID_CACHE);
+        // Default is to invalid cache.
+        invalidCache = invalidConfig == null || invalidConfig.equalsIgnoreCase("true");
+    }
+
+    /**
+     * Refresh database
+     */
+    public void handleRefreshDb() throws DdlException {
+        Env env = Env.getCurrentEnv();
+        CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog();
+        if (catalog == null) {
+            throw new DdlException("Catalog " + catalogName + " doesn't exist.");
+        }
+        if (!(catalog instanceof ExternalCatalog)) {
+            throw new DdlException("Only support refresh database in external catalog");
+        }
+        DatabaseIf db = catalog.getDbOrDdlException(dbName);
+        ((ExternalDatabase<?>) db).setUnInitialized(invalidCache);
+
+        ExternalObjectLog log = new ExternalObjectLog();
+        log.setCatalogId(catalog.getId());
+        log.setDbId(db.getId());
+        log.setInvalidCache(invalidCache);
+        Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log);
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        handleRefreshDb();
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitRefreshDatabaseCommand(this, context);
+    }
+
+    /**
+     * refresh database statement.
+     */
+    public String toSql() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("REFRESH DATABASE ");
+        if (catalogName != null) {
+            sb.append("`").append(catalogName).append("`.");
+        }
+        sb.append("`").append(dbName).append("`");
+        return sb.toString();
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.REFRESH;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index de5228f498178b..fec52fd395f9db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -119,6 +119,7 @@
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.load.CreateRoutineLoadCommand;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
+import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
 
 /** CommandVisitor. */
 public interface CommandVisitor<R, C> {
@@ -353,6 +354,10 @@ default R visitShowViewCommand(ShowViewCommand showViewCommand, C context) {
         return visitCommand(showViewCommand, context);
     }
 
+    default R visitRefreshDatabaseCommand(RefreshDatabaseCommand refreshDatabaseCommand, C context) {
+        return visitCommand(refreshDatabaseCommand, context);
+    }
+
     default R visitShowBackendsCommand(ShowBackendsCommand showBackendsCommand, C context) {
         return visitCommand(showBackendsCommand, context);
     }
diff --git a/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
index 89b47bbe4eb1e7..c3f57d7ba810b8 100644
--- a/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
+++ b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
@@ -1,5 +1,5 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
--- !sql --
+-- !database --
 DORIS
 Doris
 doris
@@ -19,7 +19,7 @@ show_test_do_not_modify
 114	abf
 115	abg
 
--- !sql --
+-- !preceding_create_external_database --
 DORIS
 Doris
 doris
@@ -29,7 +29,7 @@ init_db
 mysql
 show_test_do_not_modify
 
--- !sql --
+-- !subsequent_create_external_database --
 DORIS
 Doris
 doris
@@ -40,7 +40,21 @@ mysql
 new_mysql_db
 show_test_do_not_modify
 
--- !sql --
+-- !sql_show_tables --
+
+-- !preceding_refresh_database --
+
+-- !subsequent_refresh_database --
+new_mysql_table1
+
+-- !preceding_refresh_database --
+new_mysql_table1
+
+-- !subsequent_refresh_database --
+new_mysql_table1
+new_mysql_table2
+
+-- !preceding_drop_external_database --
 DORIS
 Doris
 doris
@@ -51,7 +65,7 @@ mysql
 new_mysql_db
 show_test_do_not_modify
 
--- !sql --
+-- !subsequent_drop_external_database --
 DORIS
 Doris
 doris
diff --git a/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
index f74f78a3709dc2..87dacd4142501e 100644
--- a/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
+++ b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
@@ -28,6 +28,8 @@ suite("test_nereids_refresh_catalog", "p0,external,mysql,external_docker,externa
         String mysql_port = context.config.otherConfigs.get("mysql_57_port");
         String ex_tb0 = "ex_tb0";
         String new_mysql_db = "new_mysql_db";
+        String new_mysql_table1 = "new_mysql_table1";
+        String new_mysql_table2 = "new_mysql_table2";
 
         sql """drop catalog if exists ${catalog_name} """
 
@@ -43,27 +45,41 @@ suite("test_nereids_refresh_catalog", "p0,external,mysql,external_docker,externa
             "driver_class" = "com.mysql.cj.jdbc.Driver"
         );"""
 
-        sql """switch ${catalog_name}"""
         sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists ${new_mysql_db}");"""
+        sql """switch ${catalog_name}"""
 
-        qt_sql """show databases;"""
+        qt_database """show databases;"""
         sql """ use ${ex_db_name}"""
 
         qt_ex_tb0_where """select id from ${ex_tb0} where id = 111;"""
         order_qt_ex_tb0  """ select id, name from ${ex_tb0} order by id; """
         // create database in mysql
         sql """CALL EXECUTE_STMT("${catalog_name}", "create database  ${new_mysql_db} ;");"""
-        qt_sql """show databases;"""
+        qt_preceding_create_external_database """show databases;"""
         checkNereidsExecute("refresh catalog ${catalog_name} ;")
-        qt_sql """show databases;"""
+        qt_subsequent_create_external_database """show databases;"""
 
         checkNereidsExecute("refresh catalog ${catalog_name} properties ('invalid_cache'='true');")
+        sql """use ${new_mysql_db}"""
+        qt_sql_show_tables """show tables;"""
+
+        // create table in mysql external database
+        sql """CALL EXECUTE_STMT("${catalog_name}", "create table ${new_mysql_db}.${new_mysql_table1} (id int, name varchar(20));");"""
+
+        qt_preceding_refresh_database """show tables;"""
+        checkNereidsExecute("refresh database ${new_mysql_db} ;")
+        qt_subsequent_refresh_database """show tables;"""
+
+        sql """CALL EXECUTE_STMT("${catalog_name}", "create table ${new_mysql_db}.${new_mysql_table2} (id int, name varchar(20));");"""
+        qt_preceding_refresh_database """show tables;"""
+        checkNereidsExecute("refresh database ${catalog_name}.${new_mysql_db} ;")
+        qt_subsequent_refresh_database """show tables;"""
 
         sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists ${new_mysql_db} ;");"""
-        qt_sql """show databases;"""
+        qt_preceding_drop_external_database """show databases;"""
 
         checkNereidsExecute("refresh catalog ${catalog_name} properties ('invalid_cache'='true');")
-        qt_sql """show databases;"""
+        qt_subsequent_drop_external_database """show databases;"""
 
         sql """ drop catalog if exists ${catalog_name} ;"""
     }

From 2bd293c560afdb7665f05ae513f1354d6dfb6371 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Mon, 2 Dec 2024 08:30:54 +0530
Subject: [PATCH 103/399] [Enhancement] (nereids)implement
 CreateSqlBlockCommand and AlterSqlBlockCommand in nereids (#44702)

Issue Number: close #42592  #42796
---
 .../org/apache/doris/nereids/DorisParser.g4   |   8 +-
 .../doris/blockrule/SqlBlockRuleMgr.java      |  12 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  18 +++
 .../doris/nereids/trees/plans/PlanType.java   |   2 +
 .../commands/AlterSqlBlockRuleCommand.java    |  87 +++++++++++++
 .../commands/CreateSqlBlockRuleCommand.java   |  99 +++++++++++++++
 .../plans/commands/SqlBlockRuleCommand.java   | 117 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  10 ++
 .../test_sql_block_rule.groovy                |  11 +-
 9 files changed, 349 insertions(+), 15 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterSqlBlockRuleCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateSqlBlockRuleCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SqlBlockRuleCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index fd5ae1bd5744f9..6e2a518ec20982 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -185,7 +185,9 @@ supportedCreateStatement
         ON table=multipartIdentifier
         AS type=(RESTRICTIVE | PERMISSIVE)
         TO (user=userIdentify | ROLE roleName=identifier)
-        USING LEFT_PAREN booleanExpression RIGHT_PAREN                 #createRowPolicy
+        USING LEFT_PAREN booleanExpression RIGHT_PAREN                    #createRowPolicy
+    | CREATE SQL_BLOCK_RULE (IF NOT EXISTS)?
+        name=identifier properties=propertyClause?                        #createSqlBlockRule
     ;
 
 supportedAlterStatement
@@ -197,6 +199,7 @@ supportedAlterStatement
         properties=propertyClause?                                                  #alterWorkloadGroup
     | ALTER WORKLOAD POLICY name=identifierOrText
         properties=propertyClause?                                                  #alterWorkloadPolicy
+    | ALTER SQL_BLOCK_RULE name=identifier properties=propertyClause?               #alterSqlBlockRule
     ;
 
 supportedDropStatement
@@ -573,7 +576,6 @@ unsupportedAlterStatement
         SET LEFT_PAREN propertyItemList RIGHT_PAREN                                 #alterColocateGroup
     | ALTER ROUTINE LOAD FOR name=multipartIdentifier properties=propertyClause?
             (FROM type=identifier LEFT_PAREN propertyItemList RIGHT_PAREN)?         #alterRoutineLoad
-    | ALTER SQL_BLOCK_RULE name=identifier properties=propertyClause?               #alterSqlBlockRule
     | ALTER TABLE name=multipartIdentifier
         SET LEFT_PAREN propertyItemList RIGHT_PAREN                                 #alterTableProperties
     | ALTER STORAGE POLICY name=identifierOrText
@@ -765,8 +767,6 @@ unsupportedCreateStatement
         (ACTIONS LEFT_PAREN workloadPolicyActions RIGHT_PAREN)?
         properties=propertyClause?                                              #createWorkloadPolicy
     | CREATE ENCRYPTKEY (IF NOT EXISTS)? multipartIdentifier AS STRING_LITERAL  #createEncryptkey
-    | CREATE SQL_BLOCK_RULE (IF NOT EXISTS)?
-        name=identifier properties=propertyClause?                              #createSqlBlockRule
     | CREATE STORAGE POLICY (IF NOT EXISTS)?
         name=identifier properties=propertyClause?                              #createStoragePolicy
     | BUILD INDEX name=identifier ON tableName=multipartIdentifier
diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java
index aa3b844f3e38ef..e4e288bc14b2bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java
@@ -116,12 +116,15 @@ private static void verifyLimitations(SqlBlockRule sqlBlockRule) throws DdlExcep
      * Create SqlBlockRule for create stmt.
      **/
     public void createSqlBlockRule(CreateSqlBlockRuleStmt stmt) throws UserException {
+        createSqlBlockRule(SqlBlockRule.fromCreateStmt(stmt), stmt.isIfNotExists());
+    }
+
+    public void createSqlBlockRule(SqlBlockRule sqlBlockRule, boolean isIfNotExists) throws UserException {
         writeLock();
         try {
-            SqlBlockRule sqlBlockRule = SqlBlockRule.fromCreateStmt(stmt);
             String ruleName = sqlBlockRule.getName();
             if (existRule(ruleName)) {
-                if (stmt.isIfNotExists()) {
+                if (isIfNotExists) {
                     return;
                 }
                 throw new DdlException("the sql block rule " + ruleName + " already create");
@@ -146,9 +149,12 @@ public void replayCreate(SqlBlockRule sqlBlockRule) {
      * Alter SqlBlockRule for alter stmt.
      **/
     public void alterSqlBlockRule(AlterSqlBlockRuleStmt stmt) throws AnalysisException, DdlException {
+        alterSqlBlockRule(SqlBlockRule.fromAlterStmt(stmt));
+    }
+
+    public void alterSqlBlockRule(SqlBlockRule sqlBlockRule) throws AnalysisException, DdlException {
         writeLock();
         try {
-            SqlBlockRule sqlBlockRule = SqlBlockRule.fromAlterStmt(stmt);
             String ruleName = sqlBlockRule.getName();
             if (!existRule(ruleName)) {
                 throw new DdlException("the sql block rule " + ruleName + " not exist");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 70c4f61e4033c4..dfe879e7d44dad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -57,6 +57,7 @@
 import org.apache.doris.nereids.DorisParser.AliasedQueryContext;
 import org.apache.doris.nereids.DorisParser.AlterMTMVContext;
 import org.apache.doris.nereids.DorisParser.AlterRoleContext;
+import org.apache.doris.nereids.DorisParser.AlterSqlBlockRuleContext;
 import org.apache.doris.nereids.DorisParser.AlterStorageVaultContext;
 import org.apache.doris.nereids.DorisParser.AlterViewContext;
 import org.apache.doris.nereids.DorisParser.AlterWorkloadGroupContext;
@@ -92,6 +93,7 @@
 import org.apache.doris.nereids.DorisParser.CreateProcedureContext;
 import org.apache.doris.nereids.DorisParser.CreateRoutineLoadContext;
 import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext;
+import org.apache.doris.nereids.DorisParser.CreateSqlBlockRuleContext;
 import org.apache.doris.nereids.DorisParser.CreateTableContext;
 import org.apache.doris.nereids.DorisParser.CreateTableLikeContext;
 import org.apache.doris.nereids.DorisParser.CreateViewContext;
@@ -460,6 +462,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterRoleCommand;
+import org.apache.doris.nereids.trees.plans.commands.AlterSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterStorageVaultCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
@@ -477,6 +480,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableLikeCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateViewCommand;
@@ -4154,6 +4158,20 @@ public LogicalPlan visitShowCreateProcedure(ShowCreateProcedureContext ctx) {
         return ParserUtils.withOrigin(ctx, () -> new ShowCreateProcedureCommand(procedureName));
     }
 
+    @Override
+    public LogicalPlan visitCreateSqlBlockRule(CreateSqlBlockRuleContext ctx) {
+        Map<String, String> properties = ctx.propertyClause() != null
+                        ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+        return new CreateSqlBlockRuleCommand(stripQuotes(ctx.name.getText()), ctx.EXISTS() != null, properties);
+    }
+
+    @Override
+    public LogicalPlan visitAlterSqlBlockRule(AlterSqlBlockRuleContext ctx) {
+        Map<String, String> properties = ctx.propertyClause() != null
+                        ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+        return new AlterSqlBlockRuleCommand(stripQuotes(ctx.name.getText()), properties);
+    }
+
     @Override
     public LogicalPlan visitDropCatalogRecycleBin(DropCatalogRecycleBinContext ctx) {
         String idTypeStr = ctx.idType.getText().substring(1, ctx.idType.getText().length() - 1);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 6a8fcadaf520c5..14a1b13a4667df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -134,6 +134,7 @@ public enum PlanType {
     // commands
     CREATE_POLICY_COMMAND,
     CREATE_TABLE_COMMAND,
+    CREATE_SQL_BLOCK_RULE_COMMAND,
     DELETE_COMMAND,
     EXPLAIN_COMMAND,
     EXPORT_COMMAND,
@@ -194,6 +195,7 @@ public enum PlanType {
     DROP_USER_COMMAND,
     DROP_WORKLOAD_GROUP_NAME,
     DROP_WORKLOAD_POLICY_COMMAND,
+    ALTER_SQL_BLOCK_RULE_COMMAND,
     SHOW_BACKENDS_COMMAND,
     SHOW_BLOCK_RULE_COMMAND,
     SHOW_BROKER_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterSqlBlockRuleCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterSqlBlockRuleCommand.java
new file mode 100644
index 00000000000000..481eb64a58d6d2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterSqlBlockRuleCommand.java
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.blockrule.SqlBlockRule;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.util.SqlBlockUtil;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Map;
+
+/**
+ * alter Sql block rule Commands.
+ */
+public class AlterSqlBlockRuleCommand extends SqlBlockRuleCommand {
+
+    /**
+    * constructor
+    */
+    public AlterSqlBlockRuleCommand(String ruleName, Map<String, String> properties) {
+        super(ruleName, properties, PlanType.ALTER_SQL_BLOCK_RULE_COMMAND);
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        Env.getCurrentEnv().getSqlBlockRuleMgr().alterSqlBlockRule(new SqlBlockRule(ruleName,
+                    sql, sqlHash, partitionNum,
+                    tabletNum, cardinality, global, enable));
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAlterSqlBlockRuleCommand(this, context);
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.ALTER;
+    }
+
+    @Override
+    public void setProperties(Map<String, String> properties) throws AnalysisException {
+        this.sql = properties.getOrDefault(SQL_PROPERTY, SqlBlockRuleCommand.STRING_NOT_SET);
+        this.sqlHash = properties.getOrDefault(SQL_HASH_PROPERTY, SqlBlockRuleCommand.STRING_NOT_SET);
+        String partitionNumString = properties.get(SCANNED_PARTITION_NUM);
+        String tabletNumString = properties.get(SCANNED_TABLET_NUM);
+        String cardinalityString = properties.get(SCANNED_CARDINALITY);
+
+        SqlBlockUtil.checkSqlAndSqlHashSetBoth(sql, sqlHash);
+        SqlBlockUtil.checkSqlAndLimitationsSetBoth(sql, sqlHash,
+                partitionNumString, tabletNumString, cardinalityString);
+        this.partitionNum = Util.getLongPropertyOrDefault(partitionNumString, SqlBlockRuleCommand.LONG_NOT_SET, null,
+                SCANNED_PARTITION_NUM + " should be a long");
+        this.tabletNum = Util.getLongPropertyOrDefault(tabletNumString, SqlBlockRuleCommand.LONG_NOT_SET, null,
+                SCANNED_TABLET_NUM + " should be a long");
+        this.cardinality = Util.getLongPropertyOrDefault(cardinalityString, SqlBlockRuleCommand.LONG_NOT_SET, null,
+                SCANNED_CARDINALITY + " should be a long");
+        // allow null, represents no modification
+        String globalStr = properties.get(GLOBAL_PROPERTY);
+        this.global = StringUtils.isNotEmpty(globalStr) ? Boolean.parseBoolean(globalStr) : null;
+        String enableStr = properties.get(ENABLE_PROPERTY);
+        this.enable = StringUtils.isNotEmpty(enableStr) ? Boolean.parseBoolean(enableStr) : null;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateSqlBlockRuleCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateSqlBlockRuleCommand.java
new file mode 100644
index 00000000000000..ab8e1a1b2689e3
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateSqlBlockRuleCommand.java
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.blockrule.SqlBlockRule;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.SqlBlockUtil;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/**
+ * create Sql block rule Commands.
+ */
+public class CreateSqlBlockRuleCommand extends SqlBlockRuleCommand {
+    private static final String NAME_TYPE = "SQL BLOCK RULE NAME";
+    private final boolean ifNotExists;
+
+    /**
+    * constructor
+    */
+    public CreateSqlBlockRuleCommand(String ruleName, boolean ifNotExists, Map<String, String> properties) {
+        super(ruleName, properties, PlanType.CREATE_SQL_BLOCK_RULE_COMMAND);
+        this.ifNotExists = ifNotExists;
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check name
+        FeNameFormat.checkCommonName(NAME_TYPE, ruleName);
+        // avoid a rule block any ddl for itself
+        if (StringUtils.isNotEmpty(sql) && Pattern.compile(sql).matcher(this.ruleName).find()) {
+            throw new AnalysisException("sql of SQL_BLOCK_RULE should not match its name");
+        }
+        Env.getCurrentEnv().getSqlBlockRuleMgr().createSqlBlockRule(new SqlBlockRule(ruleName, sql,
+                    sqlHash, partitionNum,
+                    tabletNum, cardinality, global, enable), ifNotExists);
+    }
+
+    @Override
+    public void setProperties(Map<String, String> properties) throws UserException {
+        this.sql = properties.getOrDefault(SQL_PROPERTY, SqlBlockRuleCommand.STRING_NOT_SET);
+        this.sqlHash = properties.getOrDefault(SQL_HASH_PROPERTY, SqlBlockRuleCommand.STRING_NOT_SET);
+        String partitionNumString = properties.get(SCANNED_PARTITION_NUM);
+        String tabletNumString = properties.get(SCANNED_TABLET_NUM);
+        String cardinalityString = properties.get(SCANNED_CARDINALITY);
+
+        SqlBlockUtil.checkSqlAndSqlHashSetBoth(sql, sqlHash);
+        SqlBlockUtil.checkPropertiesValidate(sql, sqlHash, partitionNumString, tabletNumString, cardinalityString);
+
+        this.partitionNum = Util.getLongPropertyOrDefault(partitionNumString, 0L, null,
+                SCANNED_PARTITION_NUM + " should be a long");
+        this.tabletNum = Util.getLongPropertyOrDefault(tabletNumString, 0L, null,
+                SCANNED_TABLET_NUM + " should be a long");
+        this.cardinality = Util.getLongPropertyOrDefault(cardinalityString, 0L, null,
+                SCANNED_CARDINALITY + " should be a long");
+
+        this.global = Util.getBooleanPropertyOrDefault(properties.get(GLOBAL_PROPERTY), false,
+                GLOBAL_PROPERTY + " should be a boolean");
+        this.enable = Util.getBooleanPropertyOrDefault(properties.get(ENABLE_PROPERTY), true,
+                ENABLE_PROPERTY + " should be a boolean");
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCreateSqlBlockRuleCommand(this, context);
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.CREATE;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SqlBlockRuleCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SqlBlockRuleCommand.java
new file mode 100644
index 00000000000000..785da1aeb5861e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SqlBlockRuleCommand.java
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.SqlBlockUtil;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Common class for SqlBlockRule Commands.
+ */
+public abstract class SqlBlockRuleCommand extends Command {
+    public static final String SQL_PROPERTY = "sql";
+
+    public static final String SQL_HASH_PROPERTY = "sqlHash";
+
+    public static final String SCANNED_PARTITION_NUM = "partition_num";
+
+    public static final String SCANNED_TABLET_NUM = "tablet_num";
+
+    public static final String SCANNED_CARDINALITY = "cardinality";
+
+    public static final String GLOBAL_PROPERTY = "global";
+
+    public static final String ENABLE_PROPERTY = "enable";
+
+    public static final Long LONG_NOT_SET = SqlBlockUtil.LONG_MINUS_ONE;
+
+    public static final String STRING_NOT_SET = SqlBlockUtil.STRING_DEFAULT;
+
+    private static final ImmutableSet<String> PROPERTIES_SET = new ImmutableSet.Builder<String>().add(SQL_PROPERTY)
+                        .add(SQL_HASH_PROPERTY).add(GLOBAL_PROPERTY).add(ENABLE_PROPERTY).add(SCANNED_PARTITION_NUM)
+                        .add(SCANNED_TABLET_NUM).add(SCANNED_CARDINALITY).build();
+
+    protected final String ruleName;
+
+    protected String sql;
+
+    protected String sqlHash;
+
+    protected Long partitionNum;
+
+    protected Long tabletNum;
+
+    protected Long cardinality;
+
+    // whether effective global, default is false
+    protected Boolean global;
+
+    // whether to use the rule, default is true
+    protected Boolean enable;
+
+    protected final Map<String, String> properties;
+
+    /**
+    * constructor
+    */
+    public SqlBlockRuleCommand(String ruleName, Map<String, String> properties, PlanType planType) {
+        super(planType);
+        this.ruleName = ruleName;
+        this.properties = properties;
+    }
+
+    private static void checkCommonProperties(Map<String, String> properties) throws UserException {
+        if (properties == null || properties.isEmpty()) {
+            throw new AnalysisException("Not set properties");
+        }
+        Optional<String> optional = properties.keySet().stream().filter(entity -> !PROPERTIES_SET.contains(entity))
+                .findFirst();
+        if (optional.isPresent()) {
+            throw new AnalysisException(optional.get() + " is invalid property");
+        }
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+        // check properties
+        checkCommonProperties(properties);
+        setProperties(properties);
+        doRun(ctx, executor);
+    }
+
+    public abstract void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception;
+
+    public abstract void setProperties(Map<String, String> properties) throws UserException;
+}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index fec52fd395f9db..d386d097a93b49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -21,6 +21,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AlterJobStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterRoleCommand;
+import org.apache.doris.nereids.trees.plans.commands.AlterSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadPolicyCommand;
@@ -36,6 +37,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableLikeCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateViewCommand;
@@ -471,6 +473,14 @@ default R visitShowLoadProfileCommand(ShowLoadProfileCommand showLoadProfileComm
         return visitCommand(showLoadProfileCommand, context);
     }
 
+    default R visitAlterSqlBlockRuleCommand(AlterSqlBlockRuleCommand dropRoleCommand, C context) {
+        return visitCommand(dropRoleCommand, context);
+    }
+
+    default R visitCreateSqlBlockRuleCommand(CreateSqlBlockRuleCommand dropRoleCommand, C context) {
+        return visitCommand(dropRoleCommand, context);
+    }
+
     default R visitDropRoleCommand(DropRoleCommand dropRoleCommand, C context) {
         return visitCommand(dropRoleCommand, context);
     }
diff --git a/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy b/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
index 001c64fbc0d0ad..f90b89dfb6e5a7 100644
--- a/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
+++ b/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
@@ -47,10 +47,7 @@ suite("test_sql_block_rule", "nonConcurrent") {
     """
     sql """ INSERT INTO table_2 VALUES ('H220427011909850160918','2022-04-27 16:00:33'),('T220427400109910160949','2022-04-27 16:00:54'),('T220427400123770120058','2022-04-27 16:00:56'),('T220427400126530112854','2022-04-27 16:00:34'),('T220427400127160144672','2022-04-27 16:00:10'),('T220427400127900184511','2022-04-27 16:00:34'),('T220427400129940120380','2022-04-27 16:00:23'),('T220427400139720192986','2022-04-27 16:00:34'),('T220427400140260152375','2022-04-27 16:00:02'),('T220427400153170104281','2022-04-27 16:00:31'),('H220427011909800104411','2022-04-27 16:00:14'),('H220427011909870184823','2022-04-27 16:00:36'),('T220427400115770144416','2022-04-27 16:00:12'),('T220427400126390112736','2022-04-27 16:00:19'),('T220427400128350120717','2022-04-27 16:00:56'),('T220427400129680120838','2022-04-27 16:00:39'),('T220427400136970192083','2022-04-27 16:00:51'),('H220427011909770192580','2022-04-27 16:00:04'),('H220427011909820192943','2022-04-27 16:00:23'),('T220427400109110184990','2022-04-27 16:00:29'),('T220427400109930192249','2022-04-27 16:00:56'),('T220427400123050168464','2022-04-27 16:00:37'),('T220427400124330112931','2022-04-27 16:00:56'),('T220427400124430144718','2022-04-27 16:00:07'),('T220427400130570160488','2022-04-27 16:00:34'),('T220427400130610112671','2022-04-27 16:00:30'),('T220427400137600160704','2022-04-27 16:00:35'),('T220427400144590176969','2022-04-27 16:00:49'),('T220427400146320176530','2022-04-27 16:00:34'),('T220427601780480120027','2022-04-27 16:00:58');"""
 
-    sql """
-                CREATE SQL_BLOCK_RULE if not exists test_rule_sql
-                PROPERTIES("sql"="SELECT \\\\* FROM table_2", "global"= "true", "enable"= "true")
-              """
+    checkNereidsExecute("CREATE SQL_BLOCK_RULE if not exists test_rule_sql PROPERTIES(\"sql\"=\"SELECT \\\\* FROM table_2\", \"global\"= \"true\", \"enable\"= \"true\")")
 
     test {
         sql("SELECT * FROM table_2", false)
@@ -66,10 +63,8 @@ suite("test_sql_block_rule", "nonConcurrent") {
         exception "sql match regex sql block rule: test_rule_sql"
     }
 
-    sql """
-        ALTER SQL_BLOCK_RULE test_rule_sql PROPERTIES("enable"="false")
-        """
-
+    checkNereidsExecute("ALTER SQL_BLOCK_RULE test_rule_sql PROPERTIES(\"enable\"=\"false\")")
+      
     sql "SELECT * FROM table_2"
 
     sql """

From 7d7f7fec5b3ca0ae9d30e8fd798e11da96c077ee Mon Sep 17 00:00:00 2001
From: lihangyu <lihangyu@selectdb.com>
Date: Mon, 2 Dec 2024 11:08:46 +0800
Subject: [PATCH 104/399] [Opt](SegmentIterator) clear and release iterators
 memory footprint in advance when EOF (#44768)

Otherwise the memory will hold until BetaRowsetReader released
---
 be/src/olap/rowset/segment_v2/segment_iterator.cpp | 8 ++++++++
 be/src/olap/rowset/segment_v2/segment_iterator.h   | 2 ++
 2 files changed, 10 insertions(+)

diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 96b0bea2ae8212..4ee73547c117e9 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1998,6 +1998,12 @@ Status SegmentIterator::copy_column_data_by_selector(vectorized::IColumn* input_
     return input_col_ptr->filter_by_selector(sel_rowid_idx, select_size, output_col);
 }
 
+void SegmentIterator::_clear_iterators() {
+    _column_iterators.clear();
+    _bitmap_index_iterators.clear();
+    _inverted_index_iterators.clear();
+}
+
 Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
     bool is_mem_reuse = block->mem_reuse();
     DCHECK(is_mem_reuse);
@@ -2104,6 +2110,8 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
             }
         }
         block->clear_column_data();
+        // clear and release iterators memory footprint in advance
+        _clear_iterators();
         return Status::EndOfFile("no more data in segment");
     }
 
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index 5588661302dd06..5b4c8f6d73d0cd 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -383,6 +383,8 @@ class SegmentIterator : public RowwiseIterator {
 
     void _calculate_expr_in_remaining_conjunct_root();
 
+    void _clear_iterators();
+
     class BitmapRangeIterator;
     class BackwardBitmapRangeIterator;
 

From 1abfd10c34934bd24b3b208975c3441751f0608d Mon Sep 17 00:00:00 2001
From: lihangyu <lihangyu@selectdb.com>
Date: Mon, 2 Dec 2024 11:09:02 +0800
Subject: [PATCH 105/399] [Fix](Variant) fix some nested explode_variant_array
 bug and add more test (#44533)

---
 be/src/vec/exprs/table_function/vexplode.cpp  | 40 +++++++++++---
 be/src/vec/exprs/table_function/vexplode.h    |  1 +
 .../array/function_array_contains_all.cpp     |  5 +-
 .../functions/array/function_array_distance.h |  4 +-
 .../functions/array/function_array_utils.cpp  | 13 ++++-
 .../functions/array/function_array_utils.h    |  6 ++-
 regression-test/data/variant_p0/nested.out    | 20 +++++++
 .../suites/variant_p0/nested.groovy           | 53 ++++++++++++++++++-
 8 files changed, 128 insertions(+), 14 deletions(-)

diff --git a/be/src/vec/exprs/table_function/vexplode.cpp b/be/src/vec/exprs/table_function/vexplode.cpp
index feef58cd27726c..5fa378f63517ec 100644
--- a/be/src/vec/exprs/table_function/vexplode.cpp
+++ b/be/src/vec/exprs/table_function/vexplode.cpp
@@ -23,12 +23,17 @@
 
 #include "common/status.h"
 #include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nothing.h"
 #include "vec/columns/column_object.h"
 #include "vec/core/block.h"
 #include "vec/core/column_with_type_and_name.h"
 #include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_nothing.h"
 #include "vec/exprs/vexpr.h"
 #include "vec/exprs/vexpr_context.h"
+#include "vec/functions/function_helpers.h"
 
 namespace doris::vectorized {
 #include "common/compile_check_begin.h"
@@ -37,6 +42,34 @@ VExplodeTableFunction::VExplodeTableFunction() {
     _fn_name = "vexplode";
 }
 
+Status VExplodeTableFunction::_process_init_variant(Block* block, int value_column_idx) {
+    // explode variant array
+    const auto& variant_column = check_and_get_column<ColumnObject>(
+            remove_nullable(block->get_by_position(value_column_idx)
+                                    .column->convert_to_full_column_if_const())
+                    .get());
+    _detail.output_as_variant = true;
+    if (!variant_column->is_null_root()) {
+        _array_column = variant_column->get_root();
+        // We need to wrap the output nested column within a variant column.
+        // Otherwise the type is missmatched
+        const auto* array_type = check_and_get_data_type<DataTypeArray>(
+                remove_nullable(variant_column->get_root_type()).get());
+        if (array_type == nullptr) {
+            return Status::NotSupported("explode not support none array type {}",
+                                        variant_column->get_root_type()->get_name());
+        }
+        _detail.nested_type = array_type->get_nested_type();
+    } else {
+        // null root, use nothing type
+        _array_column = ColumnNullable::create(ColumnArray::create(ColumnNothing::create(0)),
+                                               ColumnUInt8::create(0));
+        _array_column->assume_mutable()->insert_many_defaults(variant_column->size());
+        _detail.nested_type = std::make_shared<DataTypeNothing>();
+    }
+    return Status::OK();
+}
+
 Status VExplodeTableFunction::process_init(Block* block, RuntimeState* state) {
     CHECK(_expr_context->root()->children().size() == 1)
             << "VExplodeTableFunction only support 1 child but has "
@@ -47,12 +80,7 @@ Status VExplodeTableFunction::process_init(Block* block, RuntimeState* state) {
                                                                   &value_column_idx));
     if (WhichDataType(remove_nullable(block->get_by_position(value_column_idx).type))
                 .is_variant_type()) {
-        // explode variant array
-        const auto& variant_column = check_and_get_column<ColumnObject>(
-                remove_nullable(block->get_by_position(value_column_idx)
-                                        .column->convert_to_full_column_if_const())
-                        .get());
-        _array_column = variant_column->get_root();
+        RETURN_IF_ERROR(_process_init_variant(block, value_column_idx));
     } else {
         _array_column =
                 block->get_by_position(value_column_idx).column->convert_to_full_column_if_const();
diff --git a/be/src/vec/exprs/table_function/vexplode.h b/be/src/vec/exprs/table_function/vexplode.h
index 17b67d07824fa0..7b53926ae2c31e 100644
--- a/be/src/vec/exprs/table_function/vexplode.h
+++ b/be/src/vec/exprs/table_function/vexplode.h
@@ -47,6 +47,7 @@ class VExplodeTableFunction : public TableFunction {
     int get_value(MutableColumnPtr& column, int max_step) override;
 
 private:
+    Status _process_init_variant(Block* block, int value_column_idx);
     ColumnPtr _array_column;
     ColumnArrayExecutionData _detail;
     size_t _array_offset; // start offset of array[row_idx]
diff --git a/be/src/vec/functions/array/function_array_contains_all.cpp b/be/src/vec/functions/array/function_array_contains_all.cpp
index 67279babd5f49b..c65ec57e3d6572 100644
--- a/be/src/vec/functions/array/function_array_contains_all.cpp
+++ b/be/src/vec/functions/array/function_array_contains_all.cpp
@@ -231,8 +231,9 @@ class FunctionArrayContainsAll : public IFunction {
                         is_equal_value = false;
                     } else {
                         // all is not null, check the data is equal
-                        const auto* left_column = assert_cast<const T*>(left_data.nested_col);
-                        const auto* right_column = assert_cast<const T*>(right_data.nested_col);
+                        const auto* left_column = assert_cast<const T*>(left_data.nested_col.get());
+                        const auto* right_column =
+                                assert_cast<const T*>(right_data.nested_col.get());
                         auto res = left_column->compare_at(left_nested_loop_pos, right_pos,
                                                            *right_column, -1);
                         is_equal_value = (res == 0);
diff --git a/be/src/vec/functions/array/function_array_distance.h b/be/src/vec/functions/array/function_array_distance.h
index e03e52a0ce191f..0984479ce3404b 100644
--- a/be/src/vec/functions/array/function_array_distance.h
+++ b/be/src/vec/functions/array/function_array_distance.h
@@ -128,8 +128,8 @@ class FunctionArrayDistance : public IFunction {
 
         const auto& offsets1 = *arr1.offsets_ptr;
         const auto& offsets2 = *arr2.offsets_ptr;
-        const auto& nested_col1 = assert_cast<const ColumnFloat64*>(arr1.nested_col);
-        const auto& nested_col2 = assert_cast<const ColumnFloat64*>(arr2.nested_col);
+        const auto& nested_col1 = assert_cast<const ColumnFloat64*>(arr1.nested_col.get());
+        const auto& nested_col2 = assert_cast<const ColumnFloat64*>(arr2.nested_col.get());
         for (ssize_t row = 0; row < offsets1.size(); ++row) {
             if (arr1.array_nullmap_data && arr1.array_nullmap_data[row]) {
                 dst_null_data[row] = true;
diff --git a/be/src/vec/functions/array/function_array_utils.cpp b/be/src/vec/functions/array/function_array_utils.cpp
index ab999aa21ccdd1..d25904baf931f4 100644
--- a/be/src/vec/functions/array/function_array_utils.cpp
+++ b/be/src/vec/functions/array/function_array_utils.cpp
@@ -24,7 +24,9 @@
 
 #include "vec/columns/column.h"
 #include "vec/columns/column_nullable.h"
+#include "vec/columns/column_object.h"
 #include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
 
 namespace doris::vectorized {
 
@@ -45,12 +47,19 @@ bool extract_column_array_info(const IColumn& src, ColumnArrayExecutionData& dat
 
     // extract array offsets and nested column
     data.offsets_ptr = &data.array_col->get_offsets();
-    data.nested_col = &data.array_col->get_data();
+    data.nested_col = data.array_col->get_data_ptr();
     // extract nested column is nullable
     if (data.nested_col->is_nullable()) {
         const auto& nested_null_col = reinterpret_cast<const ColumnNullable&>(*data.nested_col);
         data.nested_nullmap_data = nested_null_col.get_null_map_data().data();
-        data.nested_col = nested_null_col.get_nested_column_ptr().get();
+        data.nested_col = nested_null_col.get_nested_column_ptr();
+    }
+    if (data.output_as_variant &&
+        !WhichDataType(remove_nullable(data.nested_type)).is_variant_type()) {
+        // set variant root column/type to from column/type
+        auto variant = ColumnObject::create(true /*always nullable*/);
+        variant->create_root(data.nested_type, make_nullable(data.nested_col)->assume_mutable());
+        data.nested_col = variant->get_ptr();
     }
     return true;
 }
diff --git a/be/src/vec/functions/array/function_array_utils.h b/be/src/vec/functions/array/function_array_utils.h
index dc522ed813760a..36217c02452214 100644
--- a/be/src/vec/functions/array/function_array_utils.h
+++ b/be/src/vec/functions/array/function_array_utils.h
@@ -16,6 +16,7 @@
 // under the License.
 #pragma once
 
+#include "vec/columns/column.h"
 #include "vec/columns/column_array.h"
 #include "vec/columns/column_nullable.h"
 #include "vec/columns/columns_number.h"
@@ -55,7 +56,10 @@ struct ColumnArrayExecutionData {
     const ColumnArray* array_col = nullptr;
     const ColumnArray::Offsets64* offsets_ptr = nullptr;
     const UInt8* nested_nullmap_data = nullptr;
-    const IColumn* nested_col = nullptr;
+    ColumnPtr nested_col = nullptr;
+    DataTypePtr nested_type = nullptr;
+    // wrap the nested column as variant column
+    bool output_as_variant = false;
 
     ColumnArrayMutableData to_mutable_data() const {
         ColumnArrayMutableData dst;
diff --git a/regression-test/data/variant_p0/nested.out b/regression-test/data/variant_p0/nested.out
index 2c105a68778a0d..d0e0e9c822ddc2 100644
--- a/regression-test/data/variant_p0/nested.out
+++ b/regression-test/data/variant_p0/nested.out
@@ -174,3 +174,23 @@ v.xx	tinyint	Yes	false	\N	NONE
 1	{"callLimit":3,"number":"02124713252","type":"HOME"}
 1	{"callLimit":5,"number":"5550219210","type":"GSM"}
 
+-- !sql --
+2	{"nested":[{"ba":"11111"},{"a":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+4	{"nested":[{"baaa":"11111"},{"ax1111":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+5	{"nested":[{"ba":"11111"},{"a":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+6	{"nested":[{"mmm":"11111"},{"ax1111":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+7	{"nested":[{"ba":"11111"},{"a":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+8	{"nested":[{"yyy":"11111"},{"ax1111":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+9	{"nested":[{"yyy":"11111"},{"ax1111":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+11	{"nested":[{"yyy":"11111"},{"ax1111":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+12	{"nested":[{"yyy":"11111"},{"ax1111":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+13	{"nested":[{"yyy":"11111"},{"ax1111":"1111"},{"axxxb":100,"xxxy111":111},{"aaa":"11","ddsss":1024},{"xx":10}]}
+
+-- !explode_sql --
+
+-- !explode_sql --
+19	10
+
+-- !explode_sql --
+2	10
+
diff --git a/regression-test/suites/variant_p0/nested.groovy b/regression-test/suites/variant_p0/nested.groovy
index 90728df2532668..25bd682d43aca4 100644
--- a/regression-test/suites/variant_p0/nested.groovy
+++ b/regression-test/suites/variant_p0/nested.groovy
@@ -164,13 +164,64 @@ parallel_pipeline_task_num=7,parallel_fragment_exec_instance_num=4,profile_level
         sql """insert into var_nested2 select * from var_nested order by k limit 1024"""
         qt_sql """select  /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=5,parallel_pipeline_task_num=1,profile_level=1,enable_pipeline_engine=false,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=false,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */  * from var_nested2 order by k limit 10;"""
         qt_sql """select v['nested'] from var_nested2 where k < 10 order by k limit 10;"""
-        // explode variant array
+        // 0. nomal explode variant array
         order_qt_explode_sql """select count(),cast(vv['xx'] as int) from var_nested lateral view explode_variant_array(v['nested']) tmp as vv where vv['xx'] = 10 group by cast(vv['xx'] as int)"""
         sql """truncate table var_nested2"""
         sql """insert into var_nested2 values(1119111, '{"eventId":1,"firstName":"Name1","lastName":"Surname1","body":{"phoneNumbers":[{"number":"5550219210","type":"GSM","callLimit":5},{"number":"02124713252","type":"HOME","callLimit":3},{"number":"05550219211","callLimit":2,"type":"WORK"}]}}
 ')"""
         order_qt_explode_sql """select v['eventId'], phone_numbers from var_nested2 lateral view explode_variant_array(v['body']['phoneNumbers']) tmp1 as phone_numbers
 where phone_numbers['type'] = 'GSM' OR phone_numbers['type'] = 'HOME' and phone_numbers['callLimit'] > 2;"""
+
+        // test array_function
+        sql "DROP TABLE IF EXISTS var_nested_array_agg"
+        sql """
+                CREATE TABLE IF NOT EXISTS var_nested_array_agg(
+                    k bigint,
+                    v variant
+                )
+                UNIQUE KEY(`k`)
+                DISTRIBUTED BY HASH(k) BUCKETS 1
+                properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "variant_enable_flatten_nested" = "true");
+            """
+        sql "insert into var_nested_array_agg select * from var_nested"
+        // 1. array_contains
+        qt_sql "select * from var_nested_array_agg where array_contains(cast(v['nested']['xx'] as array<int>), 10) order by k limit 10"
+        // 2. array_agg scalar
+        sql "select k, array_agg(cast(v['nested'] as text))  from var_nested_array_agg group by k limit 10"
+
+        // test explode_variant_array with abonomal case
+        sql "DROP TABLE IF EXISTS var_nested_explode_variant_with_abnomal"
+        sql """
+                CREATE TABLE IF NOT EXISTS var_nested_explode_variant_with_abnomal(
+                    k bigint,
+                    v variant
+                )
+                UNIQUE KEY(`k`)
+                DISTRIBUTED BY HASH(k) BUCKETS 1
+                properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "variant_enable_flatten_nested" = "true");
+            """
+        sql "insert into var_nested_explode_variant_with_abnomal select * from var_nested"
+        // 1. v['nested']['x'] is null root
+        order_qt_explode_sql """select count(),cast(vv as int) from var_nested_explode_variant_with_abnomal lateral view explode_variant_array(v['nested']['x']) tmp as vv where vv = 10 group by cast(vv as int)"""
+        // 2. v['nested']['xx'] is normal array
+        order_qt_explode_sql """select count(),cast(vv as int) from var_nested_explode_variant_with_abnomal lateral view explode_variant_array(v['nested']['xx']) tmp as vv where vv = 10 group by cast(vv as int)"""
+        // 3. v['xx'] is none array scalar type 
+        test {
+            sql """select count(),cast(vv as int) from var_nested_explode_variant_with_abnomal lateral view explode_variant_array(v['xx']) tmp as vv where vv = 10 group by cast(vv as int)"""
+            exception("explode not support none array type")
+        }
+        // 4. v['k1'] is json scalar type 
+        test {
+            sql """select count(),cast(vv as int) from var_nested_explode_variant_with_abnomal lateral view explode_variant_array(v['k1']) tmp as vv where vv = 10 group by cast(vv as int)"""
+            exception("explode not support none array type")
+        }
+        // 5. toplevel nested array
+        sql "truncate table var_nested_explode_variant_with_abnomal"
+        sql """insert into var_nested_explode_variant_with_abnomal values(1, '[{"a" : 10}, {"b" : "20", "c" :1024, "a" : 11}]')"""
+        sql """insert into var_nested_explode_variant_with_abnomal values(2, '[{"a" : 10}, {"b" : "20", "a" : 150}]')"""
+        order_qt_explode_sql """select count(),cast(vv as int) from var_nested_explode_variant_with_abnomal lateral view explode_variant_array(v['a']) tmp as vv where vv = 10 group by cast(vv as int)"""
+        // FIXME after refator
+        // order_qt_explode_sql """select count(),cast(vv as int) from var_nested_explode_variant_with_abnomal lateral view explode_variant_array(v) tmp as vv where vv['a'] = 10 group by cast(vv as int)"""
     } finally {
         // reset flags
     }

From 7526b9c3a3e5f4e477cda5615d7943ed0d34f11a Mon Sep 17 00:00:00 2001
From: Sun Chenyang <sunchenyang@selectdb.com>
Date: Mon, 2 Dec 2024 11:27:17 +0800
Subject: [PATCH 106/399] [fix](inverted index) add a sequence number to an
 enum variable (#44605)

Related PR: #36757

Problem Summary:
add a sequence number to an enum variable
---
 gensrc/thrift/Types.thrift | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index 235c1cb28378e2..1912f950587dbf 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -122,9 +122,9 @@ enum TStorageBackendType {
 // This enum is used to distinguish between different organizational methods
 // of inverted index data, affecting how the index is stored and accessed.
 enum TInvertedIndexFileStorageFormat {
-    DEFAULT, // Default format, unspecified storage method.
-    V1,      // Index per idx: Each index is stored separately based on its identifier.
-    V2       // Segment id per idx: Indexes are organized based on segment identifiers, grouping indexes by their associated segment.
+    DEFAULT = 0, // Default format, unspecified storage method.
+    V1 = 1,      // Index per idx: Each index is stored separately based on its identifier.
+    V2 = 2      // Segment id per idx: Indexes are organized based on segment identifiers, grouping indexes by their associated segment.
 }
 
 struct TScalarType {

From ed4f7fb5995b9726bcf05430ecdabd77ff6cbd21 Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Mon, 2 Dec 2024 12:00:06 +0800
Subject: [PATCH 107/399] [feat](nereids) add rewrite rule
 :EliminateGroupByKeyByUniform (#43391)

This PR introduces two main changes:
1. Adds an optional constant value to the uniform attribute in
DataTrait. A slot with a constant value that is not null will be
considered uniform and not null.
2. Introduces a new transform rule: EliminateGroupByKeyByUniform, which
utilizes the newly added part of the uniform attribute. Following is
example transformation:

from

 +--aggregate(group by a,b output a,b,max(c))
(a is uniform and not null: e.g. a is projection 2 as a in
logicalProject)

to

+--aggregate(group by b output b,any_value(a) as a,max(c))
---
 .../doris/nereids/jobs/executor/Rewriter.java |   2 +
 .../doris/nereids/properties/DataTrait.java   | 211 +++++++++++--
 .../apache/doris/nereids/rules/RuleType.java  |   7 +
 .../rules/expression/ExpressionRewrite.java   |   2 +-
 .../rewrite/EliminateGroupByKeyByUniform.java | 148 +++++++++
 .../nereids/rules/rewrite/ExprIdRewriter.java | 284 ++++++++++++++++++
 .../plans/commands/info/CreateMTMVInfo.java   |   3 +-
 .../trees/plans/logical/LogicalFilter.java    |   7 +-
 .../trees/plans/logical/LogicalHaving.java    |   7 +-
 .../trees/plans/logical/LogicalJoin.java      |  34 ++-
 .../trees/plans/logical/LogicalProject.java   |  14 +-
 .../trees/plans/logical/LogicalQualify.java   |   7 +-
 .../doris/nereids/util/ExpressionUtils.java   |  11 +-
 .../mv/MaterializedViewUtilsTest.java         |   2 +-
 .../EliminateGroupByKeyByUniformTest.java     | 250 +++++++++++++++
 .../rewrite/EliminateGroupByKeyTest.java      |   4 +-
 .../eliminate_group_by_key_by_uniform.out     | 269 +++++++++++++++++
 .../eliminate_group_by_key_by_uniform.groovy  | 221 ++++++++++++++
 .../aggregate_without_roll_up.groovy          |   6 +-
 19 files changed, 1428 insertions(+), 61 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniform.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExprIdRewriter.java
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java
 create mode 100644 regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out
 create mode 100644 regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index f4ca9a972a6814..fa00a4e9343325 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -64,6 +64,7 @@
 import org.apache.doris.nereids.rules.rewrite.EliminateFilter;
 import org.apache.doris.nereids.rules.rewrite.EliminateGroupBy;
 import org.apache.doris.nereids.rules.rewrite.EliminateGroupByKey;
+import org.apache.doris.nereids.rules.rewrite.EliminateGroupByKeyByUniform;
 import org.apache.doris.nereids.rules.rewrite.EliminateJoinByFK;
 import org.apache.doris.nereids.rules.rewrite.EliminateJoinByUnique;
 import org.apache.doris.nereids.rules.rewrite.EliminateJoinCondition;
@@ -356,6 +357,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
                     topDown(new EliminateJoinByUnique())
                 ),
                 topic("eliminate Aggregate according to fd items",
+                        custom(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM, EliminateGroupByKeyByUniform::new),
                         topDown(new EliminateGroupByKey()),
                         topDown(new PushDownAggThroughJoinOnPkFk()),
                         topDown(new PullUpJoinFromUnionAll())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java
index e97fad6f479047..ff4756979e450e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java
@@ -17,18 +17,23 @@
 
 package org.apache.doris.nereids.properties;
 
+import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.util.ImmutableEqualSet;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -46,16 +51,16 @@
 public class DataTrait {
 
     public static final DataTrait EMPTY_TRAIT
-            = new DataTrait(new NestedSet().toImmutable(),
-                    new NestedSet().toImmutable(), new ImmutableSet.Builder<FdItem>().build(),
+            = new DataTrait(new UniqueDescription().toImmutable(),
+                    new UniformDescription().toImmutable(), new ImmutableSet.Builder<FdItem>().build(),
                     ImmutableEqualSet.empty(), new FuncDepsDG.Builder().build());
-    private final NestedSet uniqueSet;
-    private final NestedSet uniformSet;
+    private final UniqueDescription uniqueSet;
+    private final UniformDescription uniformSet;
     private final ImmutableSet<FdItem> fdItems;
     private final ImmutableEqualSet<Slot> equalSet;
     private final FuncDepsDG fdDg;
 
-    private DataTrait(NestedSet uniqueSet, NestedSet uniformSet, ImmutableSet<FdItem> fdItems,
+    private DataTrait(UniqueDescription uniqueSet, UniformDescription uniformSet, ImmutableSet<FdItem> fdItems,
             ImmutableEqualSet<Slot> equalSet, FuncDepsDG fdDg) {
         this.uniqueSet = uniqueSet;
         this.uniformSet = uniformSet;
@@ -86,8 +91,7 @@ public boolean isUniform(Slot slot) {
     }
 
     public boolean isUniform(Set<Slot> slotSet) {
-        return !slotSet.isEmpty()
-                && uniformSet.slots.containsAll(slotSet);
+        return uniformSet.contains(slotSet);
     }
 
     public boolean isUniqueAndNotNull(Slot slot) {
@@ -102,11 +106,25 @@ public boolean isUniqueAndNotNull(Set<Slot> slotSet) {
     }
 
     public boolean isUniformAndNotNull(Slot slot) {
-        return !slot.nullable() && isUniform(slot);
+        return uniformSet.isUniformAndNotNull(slot);
     }
 
+    /** isUniformAndNotNull for slot set */
     public boolean isUniformAndNotNull(ImmutableSet<Slot> slotSet) {
-        return slotSet.stream().noneMatch(Slot::nullable) && isUniform(slotSet);
+        for (Slot slot : slotSet) {
+            if (!uniformSet.isUniformAndNotNull(slot)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public boolean isUniformAndHasConstValue(Slot slot) {
+        return uniformSet.isUniformAndHasConstValue(slot);
+    }
+
+    public Optional<Expression> getUniformValue(Slot slot) {
+        return uniformSet.slotUniformValue.get(slot);
     }
 
     public boolean isNullSafeEqual(Slot l, Slot r) {
@@ -143,23 +161,23 @@ public String toString() {
      * Builder of trait
      */
     public static class Builder {
-        private final NestedSet uniqueSet;
-        private final NestedSet uniformSet;
+        private final UniqueDescription uniqueSet;
+        private final UniformDescription uniformSet;
         private ImmutableSet<FdItem> fdItems;
         private final ImmutableEqualSet.Builder<Slot> equalSetBuilder;
         private final FuncDepsDG.Builder fdDgBuilder;
 
         public Builder() {
-            uniqueSet = new NestedSet();
-            uniformSet = new NestedSet();
+            uniqueSet = new UniqueDescription();
+            uniformSet = new UniformDescription();
             fdItems = new ImmutableSet.Builder<FdItem>().build();
             equalSetBuilder = new ImmutableEqualSet.Builder<>();
             fdDgBuilder = new FuncDepsDG.Builder();
         }
 
         public Builder(DataTrait other) {
-            this.uniformSet = new NestedSet(other.uniformSet);
-            this.uniqueSet = new NestedSet(other.uniqueSet);
+            this.uniformSet = new UniformDescription(other.uniformSet);
+            this.uniqueSet = new UniqueDescription(other.uniqueSet);
             this.fdItems = ImmutableSet.copyOf(other.fdItems);
             equalSetBuilder = new ImmutableEqualSet.Builder<>(other.equalSet);
             fdDgBuilder = new FuncDepsDG.Builder(other.fdDg);
@@ -173,6 +191,14 @@ public void addUniformSlot(DataTrait dataTrait) {
             uniformSet.add(dataTrait.uniformSet);
         }
 
+        public void addUniformSlotForOuterJoinNullableSide(DataTrait dataTrait) {
+            uniformSet.addUniformSlotForOuterJoinNullableSide(dataTrait.uniformSet);
+        }
+
+        public void addUniformSlotAndLiteral(Slot slot, Expression literal) {
+            uniformSet.add(slot, literal);
+        }
+
         public void addUniqueSlot(Slot slot) {
             uniqueSet.add(slot);
         }
@@ -261,8 +287,21 @@ public void addUniqueByEqualSet(Set<Slot> equalSet) {
          * if there is a uniform slot in the equivalence set, then all slots of an equivalence set are uniform
          */
         public void addUniformByEqualSet(Set<Slot> equalSet) {
-            if (uniformSet.isIntersect(uniformSet.slots, equalSet)) {
-                uniformSet.slots.addAll(equalSet);
+            List<Slot> intersectionList = uniformSet.slotUniformValue.keySet().stream()
+                    .filter(equalSet::contains)
+                    .collect(Collectors.toList());
+            if (intersectionList.isEmpty()) {
+                return;
+            }
+            Expression expr = null;
+            for (Slot slot : intersectionList) {
+                if (uniformSet.slotUniformValue.get(slot).isPresent()) {
+                    expr = uniformSet.slotUniformValue.get(slot).get();
+                    break;
+                }
+            }
+            for (Slot equal : equalSet) {
+                uniformSet.add(equal, expr);
             }
         }
 
@@ -293,9 +332,11 @@ public List<Set<Slot>> getAllUniqueAndNotNull() {
          */
         public List<Set<Slot>> getAllUniformAndNotNull() {
             List<Set<Slot>> res = new ArrayList<>();
-            for (Slot s : uniformSet.slots) {
-                if (!s.nullable()) {
-                    res.add(ImmutableSet.of(s));
+            for (Map.Entry<Slot, Optional<Expression>> entry : uniformSet.slotUniformValue.entrySet()) {
+                if (!entry.getKey().nullable()) {
+                    res.add(ImmutableSet.of(entry.getKey()));
+                } else if (entry.getValue().isPresent() && !entry.getValue().get().nullable()) {
+                    res.add(ImmutableSet.of(entry.getKey()));
                 }
             }
             return res;
@@ -338,21 +379,21 @@ public void replaceFuncDepsBy(Map<Slot, Slot> replaceMap) {
         }
     }
 
-    static class NestedSet {
+    static class UniqueDescription {
         Set<Slot> slots;
         Set<ImmutableSet<Slot>> slotSets;
 
-        NestedSet() {
+        UniqueDescription() {
             slots = new HashSet<>();
             slotSets = new HashSet<>();
         }
 
-        NestedSet(NestedSet o) {
+        UniqueDescription(UniqueDescription o) {
             this.slots = new HashSet<>(o.slots);
             this.slotSets = new HashSet<>(o.slotSets);
         }
 
-        NestedSet(Set<Slot> slots, Set<ImmutableSet<Slot>> slotSets) {
+        UniqueDescription(Set<Slot> slots, Set<ImmutableSet<Slot>> slotSets) {
             this.slots = slots;
             this.slotSets = slotSets;
         }
@@ -408,9 +449,9 @@ public void add(ImmutableSet<Slot> slotSet) {
             slotSets.add(slotSet);
         }
 
-        public void add(NestedSet nestedSet) {
-            slots.addAll(nestedSet.slots);
-            slotSets.addAll(nestedSet.slotSets);
+        public void add(UniqueDescription uniqueDescription) {
+            slots.addAll(uniqueDescription.slots);
+            slotSets.addAll(uniqueDescription.slotSets);
         }
 
         public boolean isIntersect(Set<Slot> set1, Set<Slot> set2) {
@@ -446,8 +487,120 @@ public void replace(Map<Slot, Slot> replaceMap) {
                     .collect(Collectors.toSet());
         }
 
-        public NestedSet toImmutable() {
-            return new NestedSet(ImmutableSet.copyOf(slots), ImmutableSet.copyOf(slotSets));
+        public UniqueDescription toImmutable() {
+            return new UniqueDescription(ImmutableSet.copyOf(slots), ImmutableSet.copyOf(slotSets));
+        }
+    }
+
+    static class UniformDescription {
+        // slot and its uniform expression(literal or const expression)
+        // some slot can get uniform values, others can not.
+        // e.g.select a from t where a=10 group by a, b;
+        // in LogicalAggregate, a UniformDescription with map {a : 10} can be obtained.
+        // which means a is uniform and the uniform value is 10.
+        Map<Slot, Optional<Expression>> slotUniformValue;
+
+        public UniformDescription() {
+            slotUniformValue = new LinkedHashMap<>();
+        }
+
+        public UniformDescription(UniformDescription ud) {
+            slotUniformValue = new LinkedHashMap<>(ud.slotUniformValue);
+        }
+
+        public UniformDescription(Map<Slot, Optional<Expression>> slotUniformValue) {
+            this.slotUniformValue = slotUniformValue;
+        }
+
+        public UniformDescription toImmutable() {
+            return new UniformDescription(ImmutableMap.copyOf(slotUniformValue));
+        }
+
+        public boolean isEmpty() {
+            return slotUniformValue.isEmpty();
+        }
+
+        public boolean contains(Slot slot) {
+            return slotUniformValue.containsKey(slot);
+        }
+
+        public boolean contains(Set<Slot> slots) {
+            return !slots.isEmpty() && slotUniformValue.keySet().containsAll(slots);
+        }
+
+        public void add(Slot slot) {
+            slotUniformValue.putIfAbsent(slot, Optional.empty());
+        }
+
+        public void add(Set<Slot> slots) {
+            for (Slot s : slots) {
+                slotUniformValue.putIfAbsent(s, Optional.empty());
+            }
+        }
+
+        public void add(UniformDescription ud) {
+            slotUniformValue.putAll(ud.slotUniformValue);
+            for (Map.Entry<Slot, Optional<Expression>> entry : ud.slotUniformValue.entrySet()) {
+                add(entry.getKey(), entry.getValue().orElse(null));
+            }
+        }
+
+        public void add(Slot slot, Expression literal) {
+            if (null == literal) {
+                slotUniformValue.putIfAbsent(slot, Optional.empty());
+            } else {
+                slotUniformValue.put(slot, Optional.of(literal));
+            }
+        }
+
+        public void addUniformSlotForOuterJoinNullableSide(UniformDescription ud) {
+            for (Map.Entry<Slot, Optional<Expression>> entry : ud.slotUniformValue.entrySet()) {
+                if ((!entry.getValue().isPresent() && entry.getKey().nullable())
+                        || (entry.getValue().isPresent() && entry.getValue().get() instanceof NullLiteral)) {
+                    add(entry.getKey(), entry.getValue().orElse(null));
+                }
+            }
+        }
+
+        public void removeNotContain(Set<Slot> slotSet) {
+            if (slotSet.isEmpty()) {
+                return;
+            }
+            Map<Slot, Optional<Expression>> newSlotUniformValue = new LinkedHashMap<>();
+            for (Map.Entry<Slot, Optional<Expression>> entry : slotUniformValue.entrySet()) {
+                if (slotSet.contains(entry.getKey())) {
+                    newSlotUniformValue.put(entry.getKey(), entry.getValue());
+                }
+            }
+            this.slotUniformValue = newSlotUniformValue;
+        }
+
+        public void replace(Map<Slot, Slot> replaceMap) {
+            Map<Slot, Optional<Expression>> newSlotUniformValue = new LinkedHashMap<>();
+            for (Map.Entry<Slot, Optional<Expression>> entry : slotUniformValue.entrySet()) {
+                Slot newKey = replaceMap.getOrDefault(entry.getKey(), entry.getKey());
+                newSlotUniformValue.put(newKey, entry.getValue());
+            }
+            slotUniformValue = newSlotUniformValue;
+        }
+
+        // The current implementation logic is: if a slot key exists in map slotUniformValue,
+        // its value is present and is not nullable,
+        // or if a slot key exists in map slotUniformValue and the slot is not nullable
+        // it indicates that this slot is uniform and not null.
+        public boolean isUniformAndNotNull(Slot slot) {
+            return slotUniformValue.containsKey(slot)
+                    && (!slot.nullable() || slotUniformValue.get(slot).isPresent()
+                    && !slotUniformValue.get(slot).get().nullable());
+        }
+
+        public boolean isUniformAndHasConstValue(Slot slot) {
+            return slotUniformValue.containsKey(slot) && slotUniformValue.get(slot).isPresent();
+        }
+
+        @Override
+        public String toString() {
+            return "{" + slotUniformValue + "}";
         }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index beb8bd43655743..c81f56f85b6add 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -220,6 +220,12 @@ public enum RuleType {
     REWRITE_HAVING_EXPRESSION(RuleTypeClass.REWRITE),
     REWRITE_REPEAT_EXPRESSION(RuleTypeClass.REWRITE),
     REWRITE_OLAP_TABLE_SINK_EXPRESSION(RuleTypeClass.REWRITE),
+    REWRITE_SINK_EXPRESSION(RuleTypeClass.REWRITE),
+    REWRITE_WINDOW_EXPRESSION(RuleTypeClass.REWRITE),
+    REWRITE_SET_OPERATION_EXPRESSION(RuleTypeClass.REWRITE),
+    REWRITE_PARTITION_TOPN_EXPRESSION(RuleTypeClass.REWRITE),
+    REWRITE_QUALIFY_EXPRESSION(RuleTypeClass.REWRITE),
+    REWRITE_TOPN_EXPRESSION(RuleTypeClass.REWRITE),
     EXTRACT_FILTER_FROM_JOIN(RuleTypeClass.REWRITE),
     REORDER_JOIN(RuleTypeClass.REWRITE),
     MERGE_PERCENTILE_TO_ARRAY(RuleTypeClass.REWRITE),
@@ -249,6 +255,7 @@ public enum RuleType {
     ELIMINATE_JOIN_BY_UK(RuleTypeClass.REWRITE),
     ELIMINATE_JOIN_BY_FK(RuleTypeClass.REWRITE),
     ELIMINATE_GROUP_BY_KEY(RuleTypeClass.REWRITE),
+    ELIMINATE_GROUP_BY_KEY_BY_UNIFORM(RuleTypeClass.REWRITE),
     ELIMINATE_FILTER_GROUP_BY_KEY(RuleTypeClass.REWRITE),
     ELIMINATE_DEDUP_JOIN_CONDITION(RuleTypeClass.REWRITE),
     ELIMINATE_NULL_AWARE_LEFT_ANTI_JOIN(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
index e79dd6705c0513..e5b74ee26bcb02 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
@@ -56,7 +56,7 @@
  * expression of plan rewrite rule.
  */
 public class ExpressionRewrite implements RewriteRuleFactory {
-    private final ExpressionRuleExecutor rewriter;
+    protected final ExpressionRuleExecutor rewriter;
 
     public ExpressionRewrite(ExpressionRewriteRule... rules) {
         this.rewriter = new ExpressionRuleExecutor(ImmutableList.copyOf(rules));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniform.java
new file mode 100644
index 00000000000000..4cb39c2a9341ae
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniform.java
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.jobs.JobContext;
+import org.apache.doris.nereids.properties.DataTrait;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.CTEId;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * +--aggregate(group by a, b output a#0 ,b#1, max(c) as max(c)#2)
+ * (a is uniform and not null: e.g. a is projection 2 as a in logicalProject)
+ * ->
+ * +--aggregate(group by b output b#1, any_value(a#0) as a#3, max(c)#2)
+ * if output any_value(a#0) as a#0, the uniqueness of ExprId #0 is violated, because #0 is both any_value(a#0) and a#0
+ * error will occurs in other module(e.g. mv rewrite).
+ * As a result, new aggregate outputs #3 instead of #0, but upper plan refer slot #0,
+ * therefore, all references to #0 in the upper plan need to be changed to #3.
+ * use ExprIdRewriter to do this ExprId rewrite, and use CustomRewriter to rewrite upward。
+ * */
+public class EliminateGroupByKeyByUniform extends DefaultPlanRewriter<Map<ExprId, ExprId>> implements CustomRewriter {
+    private ExprIdRewriter exprIdReplacer;
+
+    @Override
+    public Plan rewriteRoot(Plan plan, JobContext jobContext) {
+        Optional<CTEId> cteId = jobContext.getCascadesContext().getCurrentTree();
+        if (cteId.isPresent()) {
+            return plan;
+        }
+        Map<ExprId, ExprId> replaceMap = new HashMap<>();
+        ExprIdRewriter.ReplaceRule replaceRule = new ExprIdRewriter.ReplaceRule(replaceMap);
+        exprIdReplacer = new ExprIdRewriter(replaceRule, jobContext);
+        return plan.accept(this, replaceMap);
+    }
+
+    @Override
+    public Plan visit(Plan plan, Map<ExprId, ExprId> replaceMap) {
+        plan = visitChildren(this, plan, replaceMap);
+        plan = exprIdReplacer.rewriteExpr(plan, replaceMap);
+        return plan;
+    }
+
+    @Override
+    public Plan visitLogicalAggregate(LogicalAggregate<? extends Plan> aggregate, Map<ExprId, ExprId> replaceMap) {
+        aggregate = visitChildren(this, aggregate, replaceMap);
+        aggregate = (LogicalAggregate<? extends Plan>) exprIdReplacer.rewriteExpr(aggregate, replaceMap);
+
+        if (aggregate.getGroupByExpressions().isEmpty() || aggregate.getSourceRepeat().isPresent()) {
+            return aggregate;
+        }
+        DataTrait aggChildTrait = aggregate.child().getLogicalProperties().getTrait();
+        // Get the Group by column of agg. If there is a uniform one, delete the group by key.
+        Set<Expression> removedExpression = new LinkedHashSet<>();
+        List<Expression> newGroupBy = new ArrayList<>();
+        for (Expression groupBy : aggregate.getGroupByExpressions()) {
+            if (!(groupBy instanceof Slot)) {
+                newGroupBy.add(groupBy);
+                continue;
+            }
+            if (aggChildTrait.isUniformAndNotNull((Slot) groupBy)) {
+                removedExpression.add(groupBy);
+            } else {
+                newGroupBy.add(groupBy);
+            }
+        }
+        if (removedExpression.isEmpty()) {
+            return aggregate;
+        }
+        // when newGroupBy is empty, need retain one expr in group by, otherwise the result may be wrong in empty table
+        if (newGroupBy.isEmpty()) {
+            Expression expr = removedExpression.iterator().next();
+            newGroupBy.add(expr);
+            removedExpression.remove(expr);
+        }
+        if (removedExpression.isEmpty()) {
+            return aggregate;
+        }
+        List<NamedExpression> newOutputs = new ArrayList<>();
+        // If this output appears in the removedExpression column, replace it with any_value
+        for (NamedExpression output : aggregate.getOutputExpressions()) {
+            if (output instanceof Slot) {
+                if (removedExpression.contains(output)) {
+                    Alias alias = new Alias(new AnyValue(false, output), output.getName());
+                    newOutputs.add(alias);
+                    replaceMap.put(output.getExprId(), alias.getExprId());
+                } else {
+                    newOutputs.add(output);
+                }
+            } else if (output instanceof Alias) {
+                if (removedExpression.contains(output.child(0))) {
+                    newOutputs.add(new Alias(
+                            new AnyValue(false, output.child(0)), output.getName()));
+                } else {
+                    newOutputs.add(output);
+                }
+            } else {
+                newOutputs.add(output);
+            }
+        }
+
+        // Adjust the order of this new output so that aggregate functions are placed at the back
+        // and non-aggregated functions are placed at the front.
+        List<NamedExpression> aggFuncs = new ArrayList<>();
+        List<NamedExpression> orderOutput = new ArrayList<>();
+        for (NamedExpression output : newOutputs) {
+            if (output.anyMatch(e -> e instanceof AggregateFunction)) {
+                aggFuncs.add(output);
+            } else {
+                orderOutput.add(output);
+            }
+        }
+        orderOutput.addAll(aggFuncs);
+        return aggregate.withGroupByAndOutput(newGroupBy, orderOutput);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExprIdRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExprIdRewriter.java
new file mode 100644
index 00000000000000..60c9da4bc6eec5
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExprIdRewriter.java
@@ -0,0 +1,284 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.jobs.JobContext;
+import org.apache.doris.nereids.pattern.MatchingContext;
+import org.apache.doris.nereids.pattern.Pattern;
+import org.apache.doris.nereids.properties.OrderKey;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.rules.expression.ExpressionRewrite;
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.OrderExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/** replace SlotReference ExprId in logical plans */
+public class ExprIdRewriter extends ExpressionRewrite {
+    private final List<Rule> rules;
+    private final JobContext jobContext;
+
+    public ExprIdRewriter(ReplaceRule replaceRule, JobContext jobContext) {
+        super(new ExpressionRuleExecutor(ImmutableList.of(bottomUp(replaceRule))));
+        rules = buildRules();
+        this.jobContext = jobContext;
+    }
+
+    @Override
+    public List<Rule> buildRules() {
+        ImmutableList.Builder<Rule> builder = ImmutableList.builder();
+        builder.addAll(super.buildRules());
+        builder.addAll(ImmutableList.of(
+                new LogicalPartitionTopNExpressionRewrite().build(),
+                new LogicalTopNExpressionRewrite().build(),
+                new LogicalSetOperationRewrite().build(),
+                new LogicalWindowRewrite().build(),
+                new LogicalResultSinkRewrite().build(),
+                new LogicalFileSinkRewrite().build(),
+                new LogicalHiveTableSinkRewrite().build(),
+                new LogicalIcebergTableSinkRewrite().build(),
+                new LogicalJdbcTableSinkRewrite().build(),
+                new LogicalOlapTableSinkRewrite().build(),
+                new LogicalDeferMaterializeResultSinkRewrite().build()
+                ));
+        return builder.build();
+    }
+
+    /**rewriteExpr*/
+    public Plan rewriteExpr(Plan plan, Map<ExprId, ExprId> replaceMap) {
+        if (replaceMap.isEmpty()) {
+            return plan;
+        }
+        for (Rule rule : rules) {
+            Pattern<Plan> pattern = (Pattern<Plan>) rule.getPattern();
+            if (pattern.matchPlanTree(plan)) {
+                List<Plan> newPlans = rule.transform(plan, jobContext.getCascadesContext());
+                Plan newPlan = newPlans.get(0);
+                if (!newPlan.deepEquals(plan)) {
+                    return newPlan;
+                }
+            }
+        }
+        return plan;
+    }
+
+    /**
+     * Iteratively rewrites IDs using the replaceMap:
+     * 1. For a given SlotReference with initial ID, retrieve the corresponding value ID from the replaceMap.
+     * 2. If the value ID exists within the replaceMap, continue the lookup process using the value ID
+     * until it no longer appears in the replaceMap.
+     * 3. return SlotReference final value ID as the result of the rewrite.
+     * e.g. replaceMap:{0:3, 1:6, 6:7}
+     * SlotReference:a#0 -> a#3, a#1 -> a#7
+     * */
+    public static class ReplaceRule implements ExpressionPatternRuleFactory {
+        private final Map<ExprId, ExprId> replaceMap;
+
+        public ReplaceRule(Map<ExprId, ExprId> replaceMap) {
+            this.replaceMap = replaceMap;
+        }
+
+        @Override
+        public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
+            return ImmutableList.of(
+                    matchesType(SlotReference.class).thenApply(ctx -> {
+                        Slot slot = ctx.expr;
+                        if (replaceMap.containsKey(slot.getExprId())) {
+                            ExprId newId = replaceMap.get(slot.getExprId());
+                            while (replaceMap.containsKey(newId)) {
+                                newId = replaceMap.get(newId);
+                            }
+                            return slot.withExprId(newId);
+                        }
+                        return slot;
+                    })
+            );
+        }
+    }
+
+    private class LogicalResultSinkRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalResultSink().thenApply(ExprIdRewriter.this::applyRewrite)
+                    .toRule(RuleType.REWRITE_SINK_EXPRESSION);
+        }
+    }
+
+    private class LogicalFileSinkRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalFileSink().thenApply(ExprIdRewriter.this::applyRewrite)
+                    .toRule(RuleType.REWRITE_SINK_EXPRESSION);
+        }
+    }
+
+    private class LogicalHiveTableSinkRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalHiveTableSink().thenApply(ExprIdRewriter.this::applyRewrite)
+                    .toRule(RuleType.REWRITE_SINK_EXPRESSION);
+        }
+    }
+
+    private class LogicalIcebergTableSinkRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalIcebergTableSink().thenApply(ExprIdRewriter.this::applyRewrite)
+                    .toRule(RuleType.REWRITE_SINK_EXPRESSION);
+        }
+    }
+
+    private class LogicalJdbcTableSinkRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalJdbcTableSink().thenApply(ExprIdRewriter.this::applyRewrite)
+                    .toRule(RuleType.REWRITE_SINK_EXPRESSION);
+        }
+    }
+
+    private class LogicalOlapTableSinkRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalOlapTableSink().thenApply(ExprIdRewriter.this::applyRewrite)
+                    .toRule(RuleType.REWRITE_SINK_EXPRESSION);
+        }
+    }
+
+    private class LogicalDeferMaterializeResultSinkRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalDeferMaterializeResultSink().thenApply(ExprIdRewriter.this::applyRewrite)
+                    .toRule(RuleType.REWRITE_SINK_EXPRESSION);
+        }
+    }
+
+    private class LogicalSetOperationRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalSetOperation().thenApply(ctx -> {
+                LogicalSetOperation setOperation = ctx.root;
+                List<List<SlotReference>> slotsList = setOperation.getRegularChildrenOutputs();
+                List<List<SlotReference>> newSlotsList = new ArrayList<>();
+                ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
+                for (List<SlotReference> slots : slotsList) {
+                    List<SlotReference> newSlots = rewriteAll(slots, rewriter, context);
+                    newSlotsList.add(newSlots);
+                }
+                if (newSlotsList.equals(slotsList)) {
+                    return setOperation;
+                }
+                return setOperation.withChildrenAndTheirOutputs(setOperation.children(), newSlotsList);
+            })
+            .toRule(RuleType.REWRITE_SET_OPERATION_EXPRESSION);
+        }
+    }
+
+    private class LogicalWindowRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalWindow().thenApply(ctx -> {
+                LogicalWindow<Plan> window = ctx.root;
+                List<NamedExpression> windowExpressions = window.getWindowExpressions();
+                ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
+                List<NamedExpression> newWindowExpressions = rewriteAll(windowExpressions, rewriter, context);
+                if (newWindowExpressions.equals(windowExpressions)) {
+                    return window;
+                }
+                return window.withExpressionsAndChild(newWindowExpressions, window.child());
+            })
+            .toRule(RuleType.REWRITE_WINDOW_EXPRESSION);
+        }
+    }
+
+    private class LogicalTopNExpressionRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalTopN().thenApply(ctx -> {
+                LogicalTopN<Plan> topN = ctx.root;
+                List<OrderKey> orderKeys = topN.getOrderKeys();
+                ImmutableList.Builder<OrderKey> rewrittenOrderKeys
+                        = ImmutableList.builderWithExpectedSize(orderKeys.size());
+                ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
+                boolean changed = false;
+                for (OrderKey k : orderKeys) {
+                    Expression expression = rewriter.rewrite(k.getExpr(), context);
+                    changed |= expression != k.getExpr();
+                    rewrittenOrderKeys.add(new OrderKey(expression, k.isAsc(), k.isNullFirst()));
+                }
+                return changed ? topN.withOrderKeys(rewrittenOrderKeys.build()) : topN;
+            }).toRule(RuleType.REWRITE_TOPN_EXPRESSION);
+        }
+    }
+
+    private class LogicalPartitionTopNExpressionRewrite extends OneRewriteRuleFactory {
+        @Override
+        public Rule build() {
+            return logicalPartitionTopN().thenApply(ctx -> {
+                LogicalPartitionTopN<Plan> partitionTopN = ctx.root;
+                ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
+                List<OrderExpression> newOrderExpressions = new ArrayList<>();
+                boolean changed = false;
+                for (OrderExpression orderExpression : partitionTopN.getOrderKeys()) {
+                    OrderKey orderKey = orderExpression.getOrderKey();
+                    Expression expr = rewriter.rewrite(orderKey.getExpr(), context);
+                    changed |= expr != orderKey.getExpr();
+                    OrderKey newOrderKey = new OrderKey(expr, orderKey.isAsc(), orderKey.isNullFirst());
+                    newOrderExpressions.add(new OrderExpression(newOrderKey));
+                }
+                List<Expression> newPartitionKeys = rewriteAll(partitionTopN.getPartitionKeys(), rewriter, context);
+                if (!newPartitionKeys.equals(partitionTopN.getPartitionKeys())) {
+                    changed = true;
+                }
+                if (!changed) {
+                    return partitionTopN;
+                }
+                return partitionTopN.withPartitionKeysAndOrderKeys(newPartitionKeys, newOrderExpressions);
+            }).toRule(RuleType.REWRITE_PARTITION_TOPN_EXPRESSION);
+        }
+    }
+
+    private LogicalSink<Plan> applyRewrite(MatchingContext<? extends LogicalSink<Plan>> ctx) {
+        LogicalSink<Plan> sink = ctx.root;
+        ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
+        List<NamedExpression> outputExprs = sink.getOutputExprs();
+        List<NamedExpression> newOutputExprs = rewriteAll(outputExprs, rewriter, context);
+        if (outputExprs.equals(newOutputExprs)) {
+            return sink;
+        }
+        return sink.withOutputExprs(newOutputExprs);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index 8c44b42a5ccfbd..5c6c71c92e8111 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -103,7 +103,8 @@
  */
 public class CreateMTMVInfo {
     public static final Logger LOG = LogManager.getLogger(CreateMTMVInfo.class);
-    public static final String MTMV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION";
+    public static final String MTMV_PLANER_DISABLE_RULES = "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION,"
+            + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM";
     private final boolean ifNotExists;
     private final TableNameInfo mvName;
     private List<String> keys;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
index d23ea3d2395f05..efd7e90c13615e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
@@ -37,6 +37,7 @@
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -154,9 +155,9 @@ public void computeUnique(Builder builder) {
     @Override
     public void computeUniform(Builder builder) {
         for (Expression e : getConjuncts()) {
-            Set<Slot> uniformSlots = ExpressionUtils.extractUniformSlot(e);
-            for (Slot slot : uniformSlots) {
-                builder.addUniformSlot(slot);
+            Map<Slot, Expression> uniformSlots = ExpressionUtils.extractUniformSlot(e);
+            for (Map.Entry<Slot, Expression> entry : uniformSlots.entrySet()) {
+                builder.addUniformSlotAndLiteral(entry.getKey(), entry.getValue());
             }
         }
         builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
index f4f2178840b6ab..680988b39f6bb1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
@@ -35,6 +35,7 @@
 import com.google.common.collect.ImmutableSet;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -125,9 +126,9 @@ public void computeUnique(Builder builder) {
     @Override
     public void computeUniform(Builder builder) {
         for (Expression e : getConjuncts()) {
-            Set<Slot> uniformSlots = ExpressionUtils.extractUniformSlot(e);
-            for (Slot slot : uniformSlots) {
-                builder.addUniformSlot(slot);
+            Map<Slot, Expression> uniformSlots = ExpressionUtils.extractUniformSlot(e);
+            for (Map.Entry<Slot, Expression> entry : uniformSlots.entrySet()) {
+                builder.addUniformSlotAndLiteral(entry.getKey(), entry.getValue());
             }
         }
         builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
index f557b07d3b646e..c583360c3d8a76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
@@ -535,11 +535,35 @@ public void computeUniform(Builder builder) {
             // TODO disable function dependence calculation for mark join, but need re-think this in future.
             return;
         }
-        if (!joinType.isLeftSemiOrAntiJoin()) {
-            builder.addUniformSlot(right().getLogicalProperties().getTrait());
-        }
-        if (!joinType.isRightSemiOrAntiJoin()) {
-            builder.addUniformSlot(left().getLogicalProperties().getTrait());
+        switch (joinType) {
+            case INNER_JOIN:
+            case CROSS_JOIN:
+                builder.addUniformSlot(left().getLogicalProperties().getTrait());
+                builder.addUniformSlot(right().getLogicalProperties().getTrait());
+                break;
+            case LEFT_SEMI_JOIN:
+            case LEFT_ANTI_JOIN:
+            case NULL_AWARE_LEFT_ANTI_JOIN:
+                builder.addUniformSlot(left().getLogicalProperties().getTrait());
+                break;
+            case RIGHT_SEMI_JOIN:
+            case RIGHT_ANTI_JOIN:
+                builder.addUniformSlot(right().getLogicalProperties().getTrait());
+                break;
+            case LEFT_OUTER_JOIN:
+                builder.addUniformSlot(left().getLogicalProperties().getTrait());
+                builder.addUniformSlotForOuterJoinNullableSide(right().getLogicalProperties().getTrait());
+                break;
+            case RIGHT_OUTER_JOIN:
+                builder.addUniformSlot(right().getLogicalProperties().getTrait());
+                builder.addUniformSlotForOuterJoinNullableSide(left().getLogicalProperties().getTrait());
+                break;
+            case FULL_OUTER_JOIN:
+                builder.addUniformSlotForOuterJoinNullableSide(left().getLogicalProperties().getTrait());
+                builder.addUniformSlotForOuterJoinNullableSide(right().getLogicalProperties().getTrait());
+                break;
+            default:
+                break;
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
index 171a355d8452d8..612231340f16ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
@@ -238,14 +238,18 @@ public void computeUnique(DataTrait.Builder builder) {
     public void computeUniform(DataTrait.Builder builder) {
         builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
         for (NamedExpression proj : getProjects()) {
-            if (proj.children().isEmpty()) {
+            if (!(proj instanceof Alias)) {
                 continue;
             }
             if (proj.child(0).isConstant()) {
-                builder.addUniformSlot(proj.toSlot());
-            } else if (ExpressionUtils.isInjective(proj.child(0))) {
-                ImmutableSet<Slot> inputs = ImmutableSet.copyOf(proj.getInputSlots());
-                if (child(0).getLogicalProperties().getTrait().isUniform(inputs)) {
+                builder.addUniformSlotAndLiteral(proj.toSlot(), proj.child(0));
+            } else if (proj.child(0) instanceof Slot) {
+                Slot slot = (Slot) proj.child(0);
+                DataTrait childTrait = child(0).getLogicalProperties().getTrait();
+                if (childTrait.isUniformAndHasConstValue(slot)) {
+                    builder.addUniformSlotAndLiteral(proj.toSlot(),
+                            child(0).getLogicalProperties().getTrait().getUniformValue(slot).get());
+                } else if (childTrait.isUniform(slot)) {
                     builder.addUniformSlot(proj.toSlot());
                 }
             }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalQualify.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalQualify.java
index ced6730dfb54a2..904c66f6482ee5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalQualify.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalQualify.java
@@ -35,6 +35,7 @@
 import com.google.common.collect.ImmutableSet;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -130,9 +131,9 @@ public void computeUnique(Builder builder) {
     @Override
     public void computeUniform(Builder builder) {
         for (Expression e : getConjuncts()) {
-            Set<Slot> uniformSlots = ExpressionUtils.extractUniformSlot(e);
-            for (Slot slot : uniformSlots) {
-                builder.addUniformSlot(slot);
+            Map<Slot, Expression> uniformSlots = ExpressionUtils.extractUniformSlot(e);
+            for (Map.Entry<Slot, Expression> entry : uniformSlots.entrySet()) {
+                builder.addUniformSlotAndLiteral(entry.getKey(), entry.getValue());
             }
         }
         builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index e3909b87bd3726..53ab8a50683ecb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -73,6 +73,7 @@
 import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -744,15 +745,15 @@ public static <E> List<E> collectToList(Collection<? extends Expression> express
     /**
      * extract uniform slot for the given predicate, such as a = 1 and b = 2
      */
-    public static ImmutableSet<Slot> extractUniformSlot(Expression expression) {
-        ImmutableSet.Builder<Slot> builder = new ImmutableSet.Builder<>();
+    public static ImmutableMap<Slot, Expression> extractUniformSlot(Expression expression) {
+        ImmutableMap.Builder<Slot, Expression> builder = new ImmutableMap.Builder<>();
         if (expression instanceof And) {
-            builder.addAll(extractUniformSlot(expression.child(0)));
-            builder.addAll(extractUniformSlot(expression.child(1)));
+            builder.putAll(extractUniformSlot(expression.child(0)));
+            builder.putAll(extractUniformSlot(expression.child(1)));
         }
         if (expression instanceof EqualTo) {
             if (isInjective(expression.child(0)) && expression.child(1).isConstant()) {
-                builder.add((Slot) expression.child(0));
+                builder.put((Slot) expression.child(0), expression.child(1));
             }
         }
         return builder.build();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
index f824a40eda6474..45e1190412d0a4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
@@ -248,7 +248,7 @@ protected void runBeforeAll() throws Exception {
                 + "\"replication_allocation\" = \"tag.location.default: 1\"\n"
                 + ");\n");
         // Should not make scan to empty relation when the table used by materialized view has no data
-        connectContext.getSessionVariable().setDisableNereidsRules("OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION");
+        connectContext.getSessionVariable().setDisableNereidsRules("OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION,ELIMINATE_GROUP_BY_KEY_BY_UNIFORM");
     }
 
     // Test when join both side are all partition table and partition column name is same
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java
new file mode 100644
index 00000000000000..78d8034e3fdfed
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyByUniformTest.java
@@ -0,0 +1,250 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Test;
+
+public class EliminateGroupByKeyByUniformTest extends TestWithFeService implements MemoPatternMatchSupported {
+    @Override
+    protected void runBeforeAll() throws Exception {
+        createDatabase("test");
+        createTable("create table test.eli_gbk_by_uniform_t(a int null, b int not null,"
+                + "c varchar(10) null, d date, dt datetime)\n"
+                + "distributed by hash(a) properties('replication_num' = '1');");
+        connectContext.setDatabase("test");
+        connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
+    }
+
+    @Test
+    void testEliminateByFilter() {
+        PlanChecker.from(connectContext)
+                .analyze("select a, min(a), sum(a),b from eli_gbk_by_uniform_t where a = 1 group by a,b")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg ->
+                        agg.getGroupByExpressions().size() == 1
+                                && agg.getGroupByExpressions().get(0).toSql().equals("b")));
+
+    }
+
+    @Test
+    void testNotEliminateWhenOnlyOneGbyKey() {
+        PlanChecker.from(connectContext)
+                .analyze("select a, min(a), sum(a) from eli_gbk_by_uniform_t where a = 1 group by a")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg ->
+                        agg.getGroupByExpressions().size() == 1
+                                && agg.getGroupByExpressions().get(0).toSql().equals("a")));
+
+    }
+
+    @Test
+    void testEliminateByProjectConst() {
+        PlanChecker.from(connectContext)
+                .analyze("select  sum(c1), c2 from (select a c1,1 c2, d c3 from eli_gbk_by_uniform_t) t group by c2,c3 ")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg ->
+                        agg.getGroupByExpressions().size() == 1
+                                && agg.getGroupByExpressions().get(0).toSql().equals("c3")));
+    }
+
+    @Test
+    void testEliminateByProjectUniformSlot() {
+        PlanChecker.from(connectContext)
+                .analyze("select  max(c3), c1,c2,c3 from (select a c1,1 c2, d c3 from eli_gbk_by_uniform_t where a=1) t group by c1,c2,c3")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg ->
+                        agg.getGroupByExpressions().size() == 1
+                                && agg.getGroupByExpressions().get(0).toSql().equals("c3")));
+    }
+
+    @Test
+    void testEliminateDate() {
+        PlanChecker.from(connectContext)
+                .analyze("select  d, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where d = '2023-01-06' group by d,a")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg ->
+                        agg.getGroupByExpressions().size() == 1
+                                && agg.getGroupByExpressions().get(0).toSql().equals("a")));
+    }
+
+    @Test
+    void testSaveOneExpr() {
+        PlanChecker.from(connectContext)
+                .analyze("select  a, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where a = 1 and b=100 group by a, b,'abc'")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg ->
+                        agg.getGroupByExpressions().size() == 1
+                                && agg.getGroupByExpressions().get(0).toSql().equals("a")));
+    }
+
+    @Test
+    void testSaveOneExprProjectConst() {
+        PlanChecker.from(connectContext)
+                .analyze("select  c2 from (select a c1,1 c2, 3 c3 from eli_gbk_by_uniform_t) t group by c2,c3 order by 1;")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg ->
+                        agg.getGroupByExpressions().size() == 1
+                                && agg.getGroupByExpressions().get(0).toSql().equals("c2")));
+    }
+
+    @Test
+    void testNotRewriteWhenHasRepeat() {
+        PlanChecker.from(connectContext)
+                .analyze("select  c2 from (select a c1,1 c2, 3 c3 from eli_gbk_by_uniform_t) t group by grouping sets((c2),(c3)) order by 1;")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 3));
+    }
+
+    @Test
+    void testInnerJoin() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b,t2.b from eli_gbk_by_uniform_t t1 inner join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t2.b,t2.c;")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1));
+    }
+
+    @Test
+    void testLeftJoinOnConditionNotRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t2.b,t2.c;")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 3));
+    }
+
+    @Test
+    void testLeftJoinWhereConditionRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b where t1.b=100 group by t1.b,t2.b,t2.c;")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 2));
+    }
+
+    @Test
+    void testRightJoinOnConditionNullableSideFilterNotRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b,t2.b from eli_gbk_by_uniform_t t1 right join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t2.b,t2.c;")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 3));
+    }
+
+    @Test
+    void testRightJoinOnConditionNonNullableSideFilterNotRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b,t2.b from eli_gbk_by_uniform_t t1 right join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t1.b,t2.b,t2.c;")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 3));
+    }
+
+    @Test
+    void testRightJoinWhereConditionToInnerRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b,t2.b from eli_gbk_by_uniform_t t1 right join eli_gbk_by_uniform_t t2 on t1.b=t2.b where t1.b=100 group by t1.b,t2.b,t2.c;")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1));
+    }
+
+    @Test
+    void testLeftSemiJoinWhereConditionRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b from eli_gbk_by_uniform_t t1 left semi join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t1.b,t1.a")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1));
+    }
+
+    @Test
+    void testLeftSemiJoinRetainOneSlotInGroupBy() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b from eli_gbk_by_uniform_t t1 left semi join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t1.b")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1));
+    }
+
+    @Test
+    void testRightSemiJoinWhereConditionRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t2.b from eli_gbk_by_uniform_t t1 right semi join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t2.b,t2.a")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1));
+    }
+
+    @Test
+    void testRightSemiJoinRetainOneSlotInGroupBy() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t2.b from eli_gbk_by_uniform_t t1 right semi join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t2.b")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1));
+    }
+
+    @Test
+    void testLeftAntiJoinOnConditionNotRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b from eli_gbk_by_uniform_t t1 left anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t1.a")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 2));
+    }
+
+    @Test
+    void testLeftAntiJoinWhereConditionRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t1.b from eli_gbk_by_uniform_t t1 left anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b where t1.b=100 group by t1.b,t1.c")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1));
+    }
+
+    @Test
+    void testRightAntiJoinOnConditionNotRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t2.b from eli_gbk_by_uniform_t t1 right anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t2.b,t2.a")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 2));
+    }
+
+    @Test
+    void testRightAntiJoinWhereConditionRewrite() {
+        PlanChecker.from(connectContext)
+                .analyze("select  t2.b from eli_gbk_by_uniform_t t1 right anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b where t2.b=100 group by t2.b,t2.c")
+                .rewrite()
+                .printlnTree()
+                .matches(logicalAggregate().when(agg -> agg.getGroupByExpressions().size() == 1));
+    }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java
index 5a9e15cf4774d1..103e074c73bfd5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java
@@ -111,7 +111,7 @@ void testProjectAlias() {
                 .rewrite()
                 .printlnTree()
                 .matches(logicalAggregate().when(agg ->
-                        agg.getGroupByExpressions().size() == 2));
+                        agg.getGroupByExpressions().size() == 1));
         PlanChecker.from(connectContext)
                 .analyze("select id as c, name as n from t1 group by name, id")
                 .rewrite()
@@ -123,7 +123,7 @@ void testProjectAlias() {
                 .rewrite()
                 .printlnTree()
                 .matches(logicalAggregate().when(agg ->
-                        agg.getGroupByExpressions().size() == 2));
+                        agg.getGroupByExpressions().size() == 1));
     }
 
     @Test
diff --git a/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out b/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out
new file mode 100644
index 00000000000000..94cebafa4e9aad
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out
@@ -0,0 +1,269 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !empty_tranform_not_to_scalar_agg --
+
+-- !empty_tranform_multi_column --
+
+-- !empty_tranform_multi_column --
+1	1	2	2
+
+-- !tranform_to_scalar_agg_not_null_column --
+
+-- !project_const --
+\N	1
+\N	1
+2	1
+2	1
+4	1
+6	1
+6	1
+10	1
+
+-- !project_slot_uniform --
+2023-01-01	1	1	2023-01-01
+
+-- !upper_refer --
+
+-- !upper_refer_varchar_alias --
+cherry	3
+
+-- !upper_refer_date --
+2023-01-06
+
+-- !upper_refer_datetime_not_to_scalar_agg --
+2023-01-06T15:00
+
+-- !upper_refer_datetime --
+2023-01-06T15:00
+
+-- !project_no_other_agg_func --
+1
+1
+1
+1
+1
+1
+1
+1
+
+-- !project_const_not_to_scalar_agg_multi --
+1
+
+-- !not_to_scalar_agg_multi --
+1	1	2	2
+
+-- !conflict_equal_value --
+
+-- !project_slot_uniform_confict_value --
+
+-- !inner_join_left_has_filter --
+100	100
+
+-- !inner_join_right_has_filter --
+100	100
+
+-- !left_join_right_has_filter --
+100	100
+101	\N
+102	\N
+103	\N
+104	\N
+105	\N
+106	\N
+107	\N
+
+-- !left_join_left_has_filter --
+100	100
+101	\N
+102	\N
+103	\N
+104	\N
+105	\N
+106	\N
+107	\N
+
+-- !right_join_right_has_filter --
+\N	101
+\N	102
+\N	103
+\N	104
+\N	105
+\N	106
+\N	107
+100	100
+
+-- !right_join_left_has_filter --
+\N	101
+\N	102
+\N	103
+\N	104
+\N	105
+\N	106
+\N	107
+100	100
+
+-- !left_semi_join_right_has_filter --
+100
+
+-- !left_semi_join_left_has_filter --
+100
+
+-- !left_anti_join_right_has_on_filter --
+101
+102
+103
+104
+105
+106
+107
+
+-- !left_anti_join_left_has_on_filter --
+101
+102
+103
+104
+105
+106
+107
+
+-- !left_anti_join_left_has_where_filter --
+
+-- !right_semi_join_right_has_filter --
+100
+
+-- !right_semi_join_left_has_filter --
+100
+
+-- !right_anti_join_right_has_on_filter --
+101
+102
+103
+104
+105
+106
+107
+
+-- !right_anti_join_left_has_on_filter --
+101
+102
+103
+104
+105
+106
+107
+
+-- !right_anti_join_right_has_where_filter --
+
+-- !cross_join_left_has_filter --
+100	100
+100	101
+100	102
+100	103
+100	104
+100	105
+100	106
+100	107
+
+-- !cross_join_right_has_filter --
+100	100
+101	100
+102	100
+103	100
+104	100
+105	100
+106	100
+107	100
+
+-- !union --
+1	100
+5	105
+
+-- !union_all --
+1	100
+1	100
+5	105
+
+-- !intersect --
+
+-- !except --
+
+-- !set_op_mixed --
+1	100
+
+-- !window --
+
+-- !partition_topn --
+
+-- !partition_topn_qualifiy --
+
+-- !cte_producer --
+1	1	100
+
+-- !cte_multi_producer --
+
+-- !cte_consumer --
+
+-- !filter --
+1	100
+
+-- !topn --
+1	100
+
+-- !sink --
+\N	103	date	2023-01-04	2023-01-04T13:00
+\N	107	grape	2023-01-08	2023-01-08T17:00
+1	100	apple	2023-01-01	2023-01-01T10:00
+1	100	apple	2023-01-01	2023-01-01T10:00
+1	100	apple	2023-01-01	2023-01-01T10:00
+2	101	banana	2023-01-02	2023-01-02T11:00
+3	102	cherry	2023-01-03	2023-01-03T12:00
+3	102	cherry	2023-01-03	2023-01-03T12:00
+4	104	elderberry	2023-01-05	2023-01-05T14:00
+5	105	\N	2023-01-06	2023-01-06T15:00
+5	105	\N	2023-01-06	2023-01-06T15:00
+6	106	fig	2023-01-07	2023-01-07T16:00
+
+-- !nest_exprid_replace --
+2023-10-17	2	2023-10-17	2	6
+2023-10-17	2	2023-10-18	2	6
+2023-10-17	2	2023-10-21	2	6
+2023-10-18	2	2023-10-17	2	6
+2023-10-18	2	2023-10-18	2	6
+2023-10-18	2	2023-10-21	2	6
+
+-- !full_join_uniform_should_not_eliminate_group_by_key --
+\N	1
+105	1
+
+-- !full2 --
+1	\N
+1	105
+
+-- !left_join_right_side_should_not_eliminate_group_by_key --
+\N	1
+105	1
+
+-- !left_join_left_side_should_eliminate_group_by_key --
+\N	1
+105	1
+
+-- !right_join_left_side_should_not_eliminate_group_by_key --
+1	\N
+1	105
+
+-- !right_join_right_side_should_eliminate_group_by_key --
+1	\N
+1	105
+
+-- !left_semi_left_side --
+1
+1
+
+-- !left_anti_left_side --
+1
+
+-- !right_semi_right_side --
+105
+105
+
+-- !right_anti_right_side --
+
diff --git a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy
new file mode 100644
index 00000000000000..4375aac316d5d1
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy
@@ -0,0 +1,221 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("eliminate_group_by_key_by_uniform") {
+    sql "set enable_nereids_rules = 'ELIMINATE_GROUP_BY_KEY_BY_UNIFORM'"
+    sql "drop table if exists eli_gbk_by_uniform_t"
+    sql """create table eli_gbk_by_uniform_t(a int null, b int not null, c varchar(10) null, d date, dt datetime)
+    distributed by hash(a) properties("replication_num"="1");
+    """
+    qt_empty_tranform_not_to_scalar_agg "select a, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where a = 1 group by a"
+    qt_empty_tranform_multi_column "select a,b, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where a = 1 and b=2 group by a,b"
+
+    sql """
+    INSERT INTO eli_gbk_by_uniform_t (a, b, c, d, dt) VALUES
+    (1, 100, 'apple', '2023-01-01', '2023-01-01 10:00:00'),
+    (1, 100, 'apple', '2023-01-01', '2023-01-01 10:00:00'),
+    (2, 101, 'banana', '2023-01-02', '2023-01-02 11:00:00'),
+    (3, 102, 'cherry', '2023-01-03', '2023-01-03 12:00:00'),
+    (3, 102, 'cherry', '2023-01-03', '2023-01-03 12:00:00'), 
+    (NULL, 103, 'date', '2023-01-04', '2023-01-04 13:00:00'),
+    (4, 104, 'elderberry', '2023-01-05', '2023-01-05 14:00:00'),
+    (5, 105, NULL, '2023-01-06', '2023-01-06 15:00:00'),
+    (5, 105, NULL, '2023-01-06', '2023-01-06 15:00:00'),
+    (6, 106, 'fig', '2023-01-07', '2023-01-07 16:00:00'),
+    (NULL, 107, 'grape', '2023-01-08', '2023-01-08 17:00:00');
+    """
+    qt_empty_tranform_multi_column "select a, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where a = 1 group by a, b,'abc' order by 1,2,3,4"
+    qt_tranform_to_scalar_agg_not_null_column "select b, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where b = 1 group by a, b order by 1,2,3,4"
+
+    qt_project_const "select sum(c1), c2 from (select a c1,1 c2, d c3 from eli_gbk_by_uniform_t) t group by c2,c3 order by 1,2;"
+    qt_project_slot_uniform "select max(c3), c1,c2,c3 from (select a c1,1 c2, d c3 from eli_gbk_by_uniform_t where a=1) t group by c1,c2,c3 order by 1,2,3,4;"
+
+    qt_upper_refer "select b from (select b, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where b = 1 group by a, b) t order by b"
+    qt_upper_refer_varchar_alias "select c1,c2 from (select c as c1, min(a) c2, sum(a), count(a) from eli_gbk_by_uniform_t where c = 'cherry' group by a, b,c) t order  by c1,c2"
+    qt_upper_refer_date "select d from (select d, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where d = '2023-01-06' group by d,a) t order by  1"
+    qt_upper_refer_datetime_not_to_scalar_agg "select dt from (select dt, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where dt = '2023-01-06 15:00:00' group by dt) t order by 1"
+    qt_upper_refer_datetime "select dt from (select dt, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where dt = '2023-01-06 15:00:00' group by dt, a) t order by 1"
+
+    qt_project_no_other_agg_func "select c2 from (select a c1,1 c2, d c3 from eli_gbk_by_uniform_t) t group by c2,c3 order by 1;"
+    qt_project_const_not_to_scalar_agg_multi "select c2 from (select a c1,1 c2, 3 c3 from eli_gbk_by_uniform_t) t group by c2,c3 order by 1;"
+    qt_not_to_scalar_agg_multi "select a, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where a = 1 and b=100 group by a, b,'abc' order by 1,2,3,4"
+    qt_conflict_equal_value "select a, min(a), sum(a), count(a) from eli_gbk_by_uniform_t where a = 1 and a=2 group by a, b,'abc' order by 1,2,3,4"
+    qt_project_slot_uniform_confict_value "select max(c3), c1,c2,c3 from (select a c1,1 c2, d c3 from eli_gbk_by_uniform_t where a=1) t where c2=2 group by c1,c2,c3 order by 1,2,3,4;"
+
+    // test join
+    qt_inner_join_left_has_filter "select t1.b,t2.b from eli_gbk_by_uniform_t t1 inner join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t2.b,t2.c order by 1,2"
+    qt_inner_join_right_has_filter "select t1.b,t2.b from eli_gbk_by_uniform_t t1 inner join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t1.b,t2.b,t2.c order by 1,2"
+    qt_left_join_right_has_filter "select t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t1.b,t2.b,t2.c order by 1,2"
+    qt_left_join_left_has_filter "select t1.b,t2.b from eli_gbk_by_uniform_t t1 left join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t2.b,t2.c order by 1,2"
+    qt_right_join_right_has_filter "select t1.b,t2.b from eli_gbk_by_uniform_t t1 right join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t1.b,t2.b,t2.c order by 1,2"
+    qt_right_join_left_has_filter "select t1.b,t2.b from eli_gbk_by_uniform_t t1 right join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t2.b,t2.c order by 1,2"
+    qt_left_semi_join_right_has_filter "select t1.b from eli_gbk_by_uniform_t t1 left semi join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t1.b,t1.a order by 1"
+    qt_left_semi_join_left_has_filter "select t1.b from eli_gbk_by_uniform_t t1 left semi join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by  t1.b,t1.a order by 1"
+    qt_left_anti_join_right_has_on_filter "select t1.b from eli_gbk_by_uniform_t t1 left anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t1.b,t1.a order by 1"
+    qt_left_anti_join_left_has_on_filter "select t1.b from eli_gbk_by_uniform_t t1 left anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t1.b,t1.a order by 1"
+    qt_left_anti_join_left_has_where_filter "select t1.b from eli_gbk_by_uniform_t t1 left anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b where t1.b=100 group by t1.b,t1.a order by 1"
+    qt_right_semi_join_right_has_filter "select t2.b from eli_gbk_by_uniform_t t1 right semi join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t2.b,t2.c order by 1"
+    qt_right_semi_join_left_has_filter "select t2.b from eli_gbk_by_uniform_t t1 right semi join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t2.b,t2.c order by 1"
+    qt_right_anti_join_right_has_on_filter "select t2.b from eli_gbk_by_uniform_t t1 right anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t2.b=100 group by t2.b,t2.c order by 1"
+    qt_right_anti_join_left_has_on_filter "select t2.b from eli_gbk_by_uniform_t t1 right anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b and t1.b=100 group by t2.b,t2.c order by 1"
+    qt_right_anti_join_right_has_where_filter "select t2.b from eli_gbk_by_uniform_t t1 right anti join eli_gbk_by_uniform_t t2 on t1.b=t2.b where t2.b=100 group by t2.b,t2.c order by 1"
+    qt_cross_join_left_has_filter "select t1.b,t2.b from eli_gbk_by_uniform_t t1 cross join eli_gbk_by_uniform_t t2 where t1.b=100 group by t1.b,t2.b,t2.c order by 1,2"
+    qt_cross_join_right_has_filter "select t1.b,t2.b from eli_gbk_by_uniform_t t1 cross join eli_gbk_by_uniform_t t2 where t2.b=100 group by t1.b,t2.b,t2.c order by 1,2"
+
+    //test union
+    qt_union "select * from (select a,b from eli_gbk_by_uniform_t where a=1 group by a,b union select a,b from eli_gbk_by_uniform_t where b=100 group by a,b union select a,b from eli_gbk_by_uniform_t where a=5 group by a,b) t order by 1,2,3,4,5"
+    qt_union_all "select * from (select a,b from eli_gbk_by_uniform_t where a=1 group by a,b union all select a,b from eli_gbk_by_uniform_t where b=100 group by a,b union all select a,b from eli_gbk_by_uniform_t where a=5 group by a,b) t order by 1,2,3,4,5"
+    qt_intersect "select * from (select a,b from eli_gbk_by_uniform_t where a=1 group by a,b intersect select a,b from eli_gbk_by_uniform_t where b=100 group by a,b intersect select a,b from eli_gbk_by_uniform_t where a=5 group by a,b) t order by 1,2,3,4,5"
+    qt_except "select * from (select a,b from eli_gbk_by_uniform_t where a=1 group by a,b except select a,b from eli_gbk_by_uniform_t where b=100 group by a,b except select a,b from eli_gbk_by_uniform_t where a=5 group by a,b) t order by 1,2,3,4,5"
+    qt_set_op_mixed "select * from (select a,b from eli_gbk_by_uniform_t where a=1 group by a,b union select a,b from eli_gbk_by_uniform_t where b=100 group by a,b except select a,b from eli_gbk_by_uniform_t where a=5 group by a,b) t order by 1,2,3,4,5"
+
+    //test window
+    qt_window "select max(a) over(partition by a order by a) from eli_gbk_by_uniform_t where a=10 group by a,b order by 1"
+    //test partition topn
+    qt_partition_topn "select r from (select rank() over(partition by a order by a) r from eli_gbk_by_uniform_t where a=10 group by a,b) t where r<2 order by 1"
+    qt_partition_topn_qualifiy "select rank() over(partition by a order by a) r from eli_gbk_by_uniform_t where a=10 group by a,b qualify r<2 order by 1"
+    //test cte
+    qt_cte_producer "with t as (select a,b,count(*) from eli_gbk_by_uniform_t where a=1 group by a,b) select t1.a,t2.a,t2.b from t t1 inner join t t2 on t1.a=t2.a order by 1,2,3"
+    qt_cte_multi_producer "with t as (select a,b,count(*) from eli_gbk_by_uniform_t where a=1 group by a,b), tt as (select a,b,count(*) from eli_gbk_by_uniform_t where b=10 group by a,b) select t1.a,t2.a,t2.b from t t1 inner join tt t2 on t1.a=t2.a order by 1,2,3"
+    qt_cte_consumer "with t as (select * from eli_gbk_by_uniform_t) select t1.a,t2.b from t t1 inner join t t2 on t1.a=t2.a where t1.a=10 group by t1.a,t2.b order by 1,2 "
+
+    //test filter
+    qt_filter "select * from (select a,b from eli_gbk_by_uniform_t where a=1 group by a,b) t where a>0 order by 1,2"
+
+    //test topn
+    qt_topn "select a,b from eli_gbk_by_uniform_t where a=1 group by a,b order by a limit 10 offset 0"
+
+    //olap table sink
+    sql "insert into eli_gbk_by_uniform_t select a,b,c,d,dt from eli_gbk_by_uniform_t where a = 1 group by a,b,c,d,dt"
+    qt_sink "select * from eli_gbk_by_uniform_t order by 1,2,3,4,5"
+
+    sql """
+    drop table if exists orders_inner_1
+    """
+
+    sql """CREATE TABLE `orders_inner_1` (
+      `o_orderkey` BIGINT not NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    PARTITION BY list(o_orderkey) (
+    PARTITION p1 VALUES in ('1'),
+    PARTITION p2 VALUES in ('2'),
+    PARTITION p3 VALUES in ('3'),
+    PARTITION p4 VALUES in ('4')
+    )
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists lineitem_inner_1
+    """
+
+    sql """CREATE TABLE `lineitem_inner_1` (
+      `l_orderkey` BIGINT not NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    PARTITION BY list(l_orderkey) (
+    PARTITION p1 VALUES in ('1'),
+    PARTITION p2 VALUES in ('2'),
+    PARTITION p3 VALUES in ('3')
+    )
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into orders_inner_1 values 
+    (2, 1, 'o', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (1, null, 'k', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'o', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'k', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'o', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into lineitem_inner_1 values 
+    (2, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    qt_nest_exprid_replace """
+    select l_shipdate, l_orderkey, t.O_ORDERDATE, t.o_orderkey,
+            count(t.O_ORDERDATE) over (partition by lineitem_inner_1.l_orderkey order by lineitem_inner_1.l_orderkey) as window_count
+    from lineitem_inner_1
+    inner join (select O_ORDERDATE, o_orderkey, count(O_ORDERDATE) over (partition by O_ORDERDATE order by o_orderkey ) from orders_inner_1 where o_orderkey=2 group by O_ORDERDATE, o_orderkey) as t
+    on lineitem_inner_1.l_orderkey = t.o_orderkey
+    where t.o_orderkey=2
+    group by l_shipdate, l_orderkey, t.O_ORDERDATE, t.o_orderkey
+    order by 1,2,3,4,5
+    """
+    sql "drop table if exists test1"
+    sql "drop table if exists test2"
+    sql "create table test1(a int, b int) distributed by hash(a) properties('replication_num'='1');"
+    sql "insert into test1 values(1,1),(2,1),(3,1);"
+    sql "create table test2(a int, b int) distributed by hash(a) properties('replication_num'='1');"
+    sql "insert into test2 values(1,105),(2,105);"
+    qt_full_join_uniform_should_not_eliminate_group_by_key "select t2.b,t1.b from test1 t1 full join (select * from test2 where b=105)  t2 on t1.a=t2.a group by t2.b,t1.b order by 1,2;"
+    qt_full2 "select t2.b,t1.b from (select * from test2 where b=105)  t1  full join test1 t2 on t1.a=t2.a group by t2.b,t1.b order by 1,2;"
+
+    qt_left_join_right_side_should_not_eliminate_group_by_key "select t2.b,t1.b from test1 t1 left join (select * from test2 where b=105)  t2 on t1.a=t2.a group by t2.b,t1.b order by 1,2;"
+    qt_left_join_left_side_should_eliminate_group_by_key "select t2.b,t1.b from test1 t1 left join (select * from test2 where b=105)  t2 on t1.a=t2.a where t1.b=1 group by t2.b,t1.b order by 1,2;"
+
+    qt_right_join_left_side_should_not_eliminate_group_by_key "select t2.b,t1.b from (select * from test2 where b=105)  t1  right join test1 t2 on t1.a=t2.a group by t2.b,t1.b order by 1,2;"
+    qt_right_join_right_side_should_eliminate_group_by_key "select t2.b,t1.b from (select * from test2 where b=105)  t1  right join test1 t2 on t1.a=t2.a where t2.b=1 group by t2.b,t1.b order by 1,2;"
+
+    qt_left_semi_left_side "select t1.b from test1 t1 left semi join (select * from test2 where b=105)  t2 on t1.a=t2.a where t1.b=1 group by t1.b,t1.a order by 1;"
+    qt_left_anti_left_side "select t1.b from test1 t1 left anti join (select * from test2 where b=105)  t2 on t1.a=t2.a where t1.b=1 group by t1.b,t1.a order by 1;"
+    qt_right_semi_right_side "select t2.b from test1 t1 right semi join (select * from test2 where b=105)  t2 on t1.a=t2.a  group by t2.b,t2.a order by 1;"
+    qt_right_anti_right_side "select t2.b from test1 t1 right anti join (select * from test2 where b=105)  t2 on t1.a=t2.a  group by t2.b,t2.a order by 1;"
+}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
index c43bde59acaa3d..92f54f3dd456e8 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
@@ -373,7 +373,7 @@ suite("aggregate_without_roll_up") {
             "max(o_totalprice) as max_total, " +
             "min(o_totalprice) as min_total, " +
             "count(*) as count_all, " +
-            "count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) as distinct_count " +
+            "bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end))  as distinct_count " +
             "from lineitem " +
             "left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate " +
             "group by " +
@@ -571,7 +571,7 @@ suite("aggregate_without_roll_up") {
             "max(o_totalprice) as max_total, " +
             "min(o_totalprice) as min_total, " +
             "count(*) as count_all, " +
-            "count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) as distinct_count " +
+            "bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as distinct_count " +
             "from lineitem " +
             "left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate " +
             "group by " +
@@ -661,7 +661,7 @@ suite("aggregate_without_roll_up") {
             "max(o_totalprice) as max_total, " +
             "min(o_totalprice) as min_total, " +
             "count(*) as count_all, " +
-            "count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) as distinct_count " +
+            "bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as distinct_count " +
             "from lineitem " +
             "left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate " +
             "group by " +

From 8514df82ca87716dd1e5b175817b6beac411fb60 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Mon, 2 Dec 2024 14:29:04 +0800
Subject: [PATCH 108/399] [Bug](runtime-filter) check rf dependency is set and
 fix wrong throw status (#44838)

### What problem does this PR solve?
1. check rf dependency is set
2. fix wrong throw status, introduced by
https://github.com/apache/doris/issues/44697
3. catch throw exception on buildsink::close
---
 be/src/exprs/runtime_filter.h                | 5 +++--
 be/src/pipeline/exec/hashjoin_build_sink.cpp | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index a1fdfbf07d239d..63ef3e2dbd4653 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -355,8 +355,9 @@ class IRuntimeFilter {
             const std::shared_ptr<pipeline::CountedFinishDependency>& dependency);
 
     int64_t get_synced_size() const {
-        if (_synced_size == -1) {
-            throw Status::InternalError("sync filter size meet error, filter: {}", debug_string());
+        if (_synced_size == -1 || !_dependency) {
+            throw Exception(doris::ErrorCode::INTERNAL_ERROR,
+                            "sync filter size meet error, filter: {}", debug_string());
         }
         return _synced_size;
     }
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index cec0c77da8a61d..864927e1721923 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -156,7 +156,8 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
         uint64_t hash_table_size = block ? block->rows() : 0;
         {
             SCOPED_TIMER(_runtime_filter_init_timer);
-            RETURN_IF_ERROR(_runtime_filter_slots->init_filters(state, hash_table_size));
+            RETURN_IF_ERROR_OR_CATCH_EXCEPTION(
+                    _runtime_filter_slots->init_filters(state, hash_table_size));
             RETURN_IF_ERROR(_runtime_filter_slots->ignore_filters(state));
         }
         if (hash_table_size > 1) {

From fa13a700770349521c39ac88e75b5da7f313c4a9 Mon Sep 17 00:00:00 2001
From: Sun Chenyang <sunchenyang@selectdb.com>
Date: Mon, 2 Dec 2024 14:49:30 +0800
Subject: [PATCH 109/399] [feat] (inverted index) show index file size (#44120)

### What problem does this PR solve?

Problem Summary:

1. Show the data in detail, including the size of the inverted index
file and the data file.

```
mysql > show data all;

+------------------------+-----------+--------------+-------------+-----------------+-----------------+-----------------+-----------------+
| TableName              | ReplicaCount      | LocalTotalSize| LocalDataSize  | LocalIndexSize | RemoteTotalSize| RemoteDataSize | RemoteIndexSize |
+------------------------+-----------+--------------+-------------+-----------------+-----------------+-----------------+-----------------+
| test_show_index_data_p2 |  1 | 291.534 MB          | 133.697 MB      | 157.837 MB       | 0.000      | 0.000           | 0.000           |
| Total                  | 1 | 291.534 MB          | 133.697 MB       | 157.837 MB       |   0.000      | 0.000           | 0.000           |
| Quota                  | 1024.000 TB | 1073741824  |             |                  |                 |                 |                 |
| Left                   | 1024.000 TB | 1073741823  |             |                  |                 |                 |                 |
+------------------------+-----------+--------------+-------------+-----------------+-----------------+-----------------+-----------------+
4 rows in set (0.00 sec)
```

```
msql> show data all from test_show_index_data_p2;
+------------------------+------------------------+-----------+--------------+----------+-------------+-----------------+-----------------+-----------------+-----------------+
| TableName              | IndexName              | ReplicaCount      |  RowCount| LocalTotalSize | LocalDataSize  | LocalIndexSize | RemoteTotalSize | RemoteDataSize | RemoteIndexSize |
+------------------------+------------------------+-----------+--------------+----------+-------------+-----------------+-----------------+-----------------+-----------------+
| test_show_index_data_p2 | test_show_index_data_p2 | 1 | 19697882          | 291.534 MB | 157.837 MB       |     133.697 MN    | 0.000       | 0.000           | 0.000           |
|                        | Total                  | 1 |            |    291.534 MB      | 133.697 MB       | 157.837 MB       |   0.000    | 0.000           | 0.000           |
+------------------------+------------------------+-----------+--------------+----------+-------------+-----------------+-----------------+-----------------+-----------------+
2 rows in set (0.00 sec)
```

2. It is possible to obtain the sizes of data and index files by
querying the system tables

```
msyql > select * from information_schema.tables where TABLE_NAME = "test_show_index_data_p2";
+-------------------------+-------------------------------+---------------------+----------------+-------------+-------------+-------------+------------+------------+---------------------+---------------------+------------+---------+------------+---------------+----------------+
| TABLE_CATALOG           | TABLE_SCHEMA                                    | TABLE_NAME          | TABLE_TYPE     | ENGINE      | VERSION     | ROW_FORMAT  | TABLE_ROWS | AVG_ROW_LENGTH | DATA_LENGTH | MAX_DATA_LENGTH | INDEX_LENGTH | DATA_FREE | AUTO_INCREMENT | CREATE_TIME          | UPDATE_TIME          | CHECK_TIME | TABLE_COLLATION | CHECKSUM | CREATE_OPTIONS | TABLE_COMMENT |
+-------------------------+-------------------------------+---------------------+----------------+-------------+-------------+-------------+------------+--------------+-------------+----------------+--------------+-----------+---------------+---------------------+---------------------+------------+----------------+---------+---------------+---------------+
| internal                | regression_test_inverted_index_p2_show_data | test_show_index_data_p2 | BASE TABLE  | Doris       | NULL        | NULL        | 19697882    | 15            | 140191631   | NULL          | 165504277   | NULL      | NULL          | 2024-11-18 15:22:32 | 2024-11-18 15:24:52 | NULL       | utf-8           | NULL     | NULL          |               |
+-------------------------+-------------------------------+---------------------+----------------+-------------+-------------+-------------+------------+--------------+-------------+----------------+--------------+-----------+---------------+---------------------+---------------------+------------+----------------+---------+---------------+---------------+
1 row in set (0.02 sec)

```

### Release note

1. Added `show data all;` to retrieve the detailed file size.
2. Fixed the semantics of `DATA_LENGTH` and `INDEX_LENGTH` in the system
`table information_schema.tables`.
---
 be/src/cloud/cloud_base_compaction.cpp        |   5 +
 be/src/cloud/cloud_cumulative_compaction.cpp  |   4 +
 be/src/cloud/cloud_full_compaction.cpp        |   4 +
 be/src/cloud/cloud_schema_change_job.cpp      |   6 +
 .../schema_scanner/schema_tables_scanner.cpp  |  16 +-
 be/src/olap/tablet.cpp                        |   4 +
 be/src/olap/tablet_manager.cpp                |   4 +
 be/src/olap/tablet_meta.h                     |  46 ++
 cloud/src/meta-service/keys.cpp               |   8 +
 cloud/src/meta-service/keys.h                 |   6 +
 cloud/src/meta-service/meta_service.cpp       |   2 +
 cloud/src/meta-service/meta_service_job.cpp   |  22 +-
 .../meta_service_tablet_stats.cpp             | 111 +++-
 .../meta-service/meta_service_tablet_stats.h  |   2 +
 cloud/src/meta-service/meta_service_txn.cpp   |  26 +-
 cloud/src/meta-service/txn_lazy_committer.cpp |   4 +-
 cloud/test/http_encode_key_test.cpp           |  32 +-
 cloud/test/keys_test.cpp                      |  34 ++
 cloud/test/meta_service_http_test.cpp         |  14 +
 cloud/test/meta_service_job_test.cpp          |  60 ++-
 cloud/test/meta_service_test.cpp              |  24 +
 cloud/test/schema_kv_test.cpp                 |   6 +
 .../org/apache/doris/nereids/DorisParser.g4   |   3 +-
 fe/fe-core/src/main/cup/sql_parser.cup        |  21 +-
 .../apache/doris/analysis/ShowDataStmt.java   | 484 +++++++++++-------
 .../doris/catalog/CloudTabletStatMgr.java     |  20 +-
 .../doris/catalog/MaterializedIndex.java      |  40 ++
 .../org/apache/doris/catalog/OlapTable.java   |  55 +-
 .../org/apache/doris/catalog/Replica.java     |  19 +
 .../java/org/apache/doris/catalog/Table.java  |   3 +
 .../org/apache/doris/catalog/TableIf.java     |   2 +
 .../apache/doris/catalog/TabletStatMgr.java   |  17 +-
 .../doris/datasource/ExternalTable.java       |   5 +
 .../doris/service/FrontendServiceImpl.java    |   1 +
 .../doris/analysis/ShowDataStmtTest.java      |   6 +-
 gensrc/proto/cloud.proto                      |  10 +-
 gensrc/thrift/BackendService.thrift           |   7 +-
 gensrc/thrift/FrontendService.thrift          |   1 +
 gensrc/thrift/MasterService.thrift            |   4 +
 .../system/test_query_sys_tables.out          |   6 +-
 .../query_p0/system/test_query_sys_tables.out |   6 +-
 .../show_data/test_show_index_data.groovy     | 399 +++++++++++++++
 .../show_data/test_show_index_data_p2.groovy  | 397 ++++++++++++++
 43 files changed, 1702 insertions(+), 244 deletions(-)
 create mode 100644 regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy
 create mode 100644 regression-test/suites/inverted_index_p2/show_data/test_show_index_data_p2.groovy

diff --git a/be/src/cloud/cloud_base_compaction.cpp b/be/src/cloud/cloud_base_compaction.cpp
index 88d83000e95dfa..9742e57dcf9d34 100644
--- a/be/src/cloud/cloud_base_compaction.cpp
+++ b/be/src/cloud/cloud_base_compaction.cpp
@@ -125,6 +125,7 @@ Status CloudBaseCompaction::prepare_compact() {
         _input_row_num += rs->num_rows();
         _input_segments += rs->num_segments();
         _input_rowsets_data_size += rs->data_disk_size();
+        _input_rowsets_index_size += rs->index_disk_size();
         _input_rowsets_total_size += rs->total_disk_size();
     }
     LOG_INFO("start CloudBaseCompaction, tablet_id={}, range=[{}-{}]", _tablet->tablet_id(),
@@ -320,6 +321,10 @@ Status CloudBaseCompaction::modify_rowsets() {
     compaction_job->add_output_versions(_output_rowset->end_version());
     compaction_job->add_txn_id(_output_rowset->txn_id());
     compaction_job->add_output_rowset_ids(_output_rowset->rowset_id().to_string());
+    compaction_job->set_index_size_input_rowsets(_input_rowsets_index_size);
+    compaction_job->set_segment_size_input_rowsets(_input_rowsets_data_size);
+    compaction_job->set_index_size_output_rowsets(_output_rowset->index_disk_size());
+    compaction_job->set_segment_size_output_rowsets(_output_rowset->data_disk_size());
 
     DeleteBitmapPtr output_rowset_delete_bitmap = nullptr;
     if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp
index 2f08082f51b5f3..c466c35e2a2ab6 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -263,6 +263,10 @@ Status CloudCumulativeCompaction::modify_rowsets() {
     compaction_job->add_output_versions(_output_rowset->end_version());
     compaction_job->add_txn_id(_output_rowset->txn_id());
     compaction_job->add_output_rowset_ids(_output_rowset->rowset_id().to_string());
+    compaction_job->set_index_size_input_rowsets(_input_rowsets_index_size);
+    compaction_job->set_segment_size_input_rowsets(_input_rowsets_data_size);
+    compaction_job->set_index_size_output_rowsets(_output_rowset->index_disk_size());
+    compaction_job->set_segment_size_output_rowsets(_output_rowset->data_disk_size());
 
     DBUG_EXECUTE_IF("CloudCumulativeCompaction::modify_rowsets.enable_spin_wait", {
         LOG(INFO) << "CloudCumulativeCompaction::modify_rowsets.enable_spin_wait, start";
diff --git a/be/src/cloud/cloud_full_compaction.cpp b/be/src/cloud/cloud_full_compaction.cpp
index c27b728c93d29b..bce00c9a2e74f6 100644
--- a/be/src/cloud/cloud_full_compaction.cpp
+++ b/be/src/cloud/cloud_full_compaction.cpp
@@ -216,6 +216,10 @@ Status CloudFullCompaction::modify_rowsets() {
     compaction_job->add_output_versions(_output_rowset->end_version());
     compaction_job->add_txn_id(_output_rowset->txn_id());
     compaction_job->add_output_rowset_ids(_output_rowset->rowset_id().to_string());
+    compaction_job->set_index_size_input_rowsets(_input_rowsets_index_size);
+    compaction_job->set_segment_size_input_rowsets(_input_rowsets_data_size);
+    compaction_job->set_index_size_output_rowsets(_output_rowset->index_disk_size());
+    compaction_job->set_segment_size_output_rowsets(_output_rowset->data_disk_size());
 
     DeleteBitmapPtr output_rowset_delete_bitmap = nullptr;
     if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp
index 0bab742c3ad6e5..1cc4d052a81d69 100644
--- a/be/src/cloud/cloud_schema_change_job.cpp
+++ b/be/src/cloud/cloud_schema_change_job.cpp
@@ -340,17 +340,23 @@ Status CloudSchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParam
         int64_t num_output_rows = 0;
         int64_t size_output_rowsets = 0;
         int64_t num_output_segments = 0;
+        int64_t index_size_output_rowsets = 0;
+        int64_t segment_size_output_rowsets = 0;
         for (auto& rs : _output_rowsets) {
             sc_job->add_txn_ids(rs->txn_id());
             sc_job->add_output_versions(rs->end_version());
             num_output_rows += rs->num_rows();
             size_output_rowsets += rs->total_disk_size();
             num_output_segments += rs->num_segments();
+            index_size_output_rowsets += rs->index_disk_size();
+            segment_size_output_rowsets += rs->data_disk_size();
         }
         sc_job->set_num_output_rows(num_output_rows);
         sc_job->set_size_output_rowsets(size_output_rowsets);
         sc_job->set_num_output_segments(num_output_segments);
         sc_job->set_num_output_rowsets(_output_rowsets.size());
+        sc_job->set_index_size_output_rowsets(index_size_output_rowsets);
+        sc_job->set_segment_size_output_rowsets(segment_size_output_rowsets);
     }
     _output_cumulative_point = std::min(_output_cumulative_point, sc_job->alter_version() + 1);
     sc_job->set_output_cumulative_point(_output_cumulative_point);
diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp
index 23710b81971c15..3aba0dfcc4f83c 100644
--- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp
@@ -236,7 +236,7 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) {
         std::vector<int64_t> srcs(table_num);
         for (int i = 0; i < table_num; ++i) {
             const TTableStatus& tbl_status = _table_result.tables[i];
-            if (tbl_status.__isset.avg_row_length) {
+            if (tbl_status.__isset.data_length) {
                 srcs[i] = tbl_status.data_length;
                 datas[i] = srcs.data() + i;
             } else {
@@ -248,7 +248,19 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) {
     // max_data_length
     { RETURN_IF_ERROR(fill_dest_column_for_range(block, 10, null_datas)); }
     // index_length
-    { RETURN_IF_ERROR(fill_dest_column_for_range(block, 11, null_datas)); }
+    {
+        std::vector<int64_t> srcs(table_num);
+        for (int i = 0; i < table_num; ++i) {
+            const TTableStatus& tbl_status = _table_result.tables[i];
+            if (tbl_status.__isset.index_length) {
+                srcs[i] = tbl_status.index_length;
+                datas[i] = srcs.data() + i;
+            } else {
+                datas[i] = nullptr;
+            }
+        }
+        RETURN_IF_ERROR(fill_dest_column_for_range(block, 11, datas));
+    }
     // data_free
     { RETURN_IF_ERROR(fill_dest_column_for_range(block, 12, null_datas)); }
     // auto_increment
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 0d04984d0e06ba..379fb6eec3cb43 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1692,6 +1692,10 @@ void Tablet::build_tablet_report_info(TTabletInfo* tablet_info,
         // tablet may not have cooldowned data, but the storage policy is set
         tablet_info->__set_cooldown_term(_cooldown_conf.term);
     }
+    tablet_info->__set_local_index_size(_tablet_meta->tablet_local_index_size());
+    tablet_info->__set_local_segment_size(_tablet_meta->tablet_local_segment_size());
+    tablet_info->__set_remote_index_size(_tablet_meta->tablet_remote_index_size());
+    tablet_info->__set_remote_segment_size(_tablet_meta->tablet_remote_segment_size());
 }
 
 void Tablet::report_error(const Status& st) {
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index d6a944dbc39853..a18dc0b4a161b6 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1083,6 +1083,10 @@ void TabletManager::build_all_report_tablets_info(std::map<TTabletId, TTablet>*
         t_tablet_stat.__set_total_version_count(tablet_info.total_version_count);
         t_tablet_stat.__set_visible_version_count(tablet_info.visible_version_count);
         t_tablet_stat.__set_visible_version(tablet_info.version);
+        t_tablet_stat.__set_local_index_size(tablet_info.local_index_size);
+        t_tablet_stat.__set_local_segment_size(tablet_info.local_segment_size);
+        t_tablet_stat.__set_remote_index_size(tablet_info.remote_index_size);
+        t_tablet_stat.__set_remote_segment_size(tablet_info.remote_segment_size);
     };
     for_each_tablet(handler, filter_all_tablets);
 
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index fb0895604a19fe..34794ef6d0a760 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -170,6 +170,12 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
     size_t tablet_local_size() const;
     // Remote disk space occupied by tablet.
     size_t tablet_remote_size() const;
+
+    size_t tablet_local_index_size() const;
+    size_t tablet_local_segment_size() const;
+    size_t tablet_remote_index_size() const;
+    size_t tablet_remote_segment_size() const;
+
     size_t version_count() const;
     size_t stale_version_count() const;
     size_t version_count_cross_with_range(const Version& range) const;
@@ -668,6 +674,46 @@ inline size_t TabletMeta::tablet_remote_size() const {
     return total_size;
 }
 
+inline size_t TabletMeta::tablet_local_index_size() const {
+    size_t total_size = 0;
+    for (auto& rs : _rs_metas) {
+        if (rs->is_local()) {
+            total_size += rs->index_disk_size();
+        }
+    }
+    return total_size;
+}
+
+inline size_t TabletMeta::tablet_local_segment_size() const {
+    size_t total_size = 0;
+    for (auto& rs : _rs_metas) {
+        if (rs->is_local()) {
+            total_size += rs->data_disk_size();
+        }
+    }
+    return total_size;
+}
+
+inline size_t TabletMeta::tablet_remote_index_size() const {
+    size_t total_size = 0;
+    for (auto& rs : _rs_metas) {
+        if (!rs->is_local()) {
+            total_size += rs->index_disk_size();
+        }
+    }
+    return total_size;
+}
+
+inline size_t TabletMeta::tablet_remote_segment_size() const {
+    size_t total_size = 0;
+    for (auto& rs : _rs_metas) {
+        if (!rs->is_local()) {
+            total_size += rs->data_disk_size();
+        }
+    }
+    return total_size;
+}
+
 inline size_t TabletMeta::version_count() const {
     return _rs_metas.size();
 }
diff --git a/cloud/src/meta-service/keys.cpp b/cloud/src/meta-service/keys.cpp
index 820d349084e35f..a518b6e264d20d 100644
--- a/cloud/src/meta-service/keys.cpp
+++ b/cloud/src/meta-service/keys.cpp
@@ -382,6 +382,14 @@ void stats_tablet_num_segs_key(const StatsTabletKeyInfo& in, std::string* out) {
     stats_tablet_key(in, out);
     encode_bytes(STATS_KEY_SUFFIX_NUM_SEGS, out);
 }
+void stats_tablet_index_size_key(const StatsTabletKeyInfo& in, std::string* out) {
+    stats_tablet_key(in, out);
+    encode_bytes(STATS_KEY_SUFFIX_INDEX_SIZE, out);
+}
+void stats_tablet_segment_size_key(const StatsTabletKeyInfo& in, std::string* out) {
+    stats_tablet_key(in, out);
+    encode_bytes(STATS_KEY_SUFFIX_SEGMENT_SIZE, out);
+}
 
 //==============================================================================
 // Job keys
diff --git a/cloud/src/meta-service/keys.h b/cloud/src/meta-service/keys.h
index c63af925b8f4de..855171c1dc648a 100644
--- a/cloud/src/meta-service/keys.h
+++ b/cloud/src/meta-service/keys.h
@@ -51,6 +51,8 @@
 // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "num_rows"    -> int64
 // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "num_rowsets" -> int64
 // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "num_segs"    -> int64
+// 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "index_size"  -> int64
+// 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "segment_size"-> int64
 //
 // 0x01 "recycle" ${instance_id} "index" ${index_id}                                       -> RecycleIndexPB
 // 0x01 "recycle" ${instance_id} "partition" ${partition_id}                               -> RecyclePartitionPB
@@ -83,6 +85,8 @@ static constexpr std::string_view STATS_KEY_SUFFIX_DATA_SIZE = "data_size";
 static constexpr std::string_view STATS_KEY_SUFFIX_NUM_ROWS = "num_rows";
 static constexpr std::string_view STATS_KEY_SUFFIX_NUM_ROWSETS = "num_rowsets";
 static constexpr std::string_view STATS_KEY_SUFFIX_NUM_SEGS = "num_segs";
+static constexpr std::string_view STATS_KEY_SUFFIX_INDEX_SIZE = "index_size";
+static constexpr std::string_view STATS_KEY_SUFFIX_SEGMENT_SIZE = "segment_size";
 
 // clang-format off
 /**
@@ -247,6 +251,8 @@ void stats_tablet_data_size_key(const StatsTabletKeyInfo& in, std::string* out);
 void stats_tablet_num_rows_key(const StatsTabletKeyInfo& in, std::string* out);
 void stats_tablet_num_rowsets_key(const StatsTabletKeyInfo& in, std::string* out);
 void stats_tablet_num_segs_key(const StatsTabletKeyInfo& in, std::string* out);
+void stats_tablet_index_size_key(const StatsTabletKeyInfo& in, std::string* out);
+void stats_tablet_segment_size_key(const StatsTabletKeyInfo& in, std::string* out);
 static inline std::string stats_tablet_key(const StatsTabletKeyInfo& in) { std::string s; stats_tablet_key(in, &s); return s; }
 
 void job_recycle_key(const JobRecycleKeyInfo& in, std::string* out);
diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index 974b3091a73e95..ddf4ee3889c00e 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -1642,6 +1642,8 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro
 #ifdef NDEBUG
         // Force data size >= 0 to reduce the losses caused by bugs
         if (tablet_stats->data_size() < 0) tablet_stats->set_data_size(0);
+        if (tablet_stats->index_size() < 0) tablet_stats->set_index_size(0);
+        if (tablet_stats->segment_size() < 0) tablet_stats->set_segment_size(0);
 #endif
     }
 }
diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp
index d1c8df15870de7..cc80d5bd138980 100644
--- a/cloud/src/meta-service/meta_service_job.cpp
+++ b/cloud/src/meta-service/meta_service_job.cpp
@@ -701,6 +701,8 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string
         stats->set_data_size(stats->data_size() + (compaction.size_output_rowsets() - compaction.size_input_rowsets()));
         stats->set_num_rowsets(stats->num_rowsets() + (compaction.num_output_rowsets() - compaction.num_input_rowsets()));
         stats->set_num_segments(stats->num_segments() + (compaction.num_output_segments() - compaction.num_input_segments()));
+        stats->set_index_size(stats->index_size() + (compaction.index_size_output_rowsets() - compaction.index_size_input_rowsets()));
+        stats->set_segment_size(stats->segment_size() + (compaction.segment_size_output_rowsets() - compaction.segment_size_input_rowsets()));
         stats->set_last_cumu_compaction_time_ms(now * 1000);
         // clang-format on
     } else if (compaction.type() == TabletCompactionJobPB::BASE) {
@@ -710,6 +712,8 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string
         stats->set_data_size(stats->data_size() + (compaction.size_output_rowsets() - compaction.size_input_rowsets()));
         stats->set_num_rowsets(stats->num_rowsets() + (compaction.num_output_rowsets() - compaction.num_input_rowsets()));
         stats->set_num_segments(stats->num_segments() + (compaction.num_output_segments() - compaction.num_input_segments()));
+        stats->set_index_size(stats->index_size() + (compaction.index_size_output_rowsets() - compaction.index_size_input_rowsets()));
+        stats->set_segment_size(stats->segment_size() + (compaction.segment_size_output_rowsets() - compaction.segment_size_input_rowsets()));
         stats->set_last_base_compaction_time_ms(now * 1000);
         // clang-format on
     } else if (compaction.type() == TabletCompactionJobPB::FULL) {
@@ -724,6 +728,8 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string
         stats->set_data_size(stats->data_size() + (compaction.size_output_rowsets() - compaction.size_input_rowsets()));
         stats->set_num_rowsets(stats->num_rowsets() + (compaction.num_output_rowsets() - compaction.num_input_rowsets()));
         stats->set_num_segments(stats->num_segments() + (compaction.num_output_segments() - compaction.num_input_segments()));
+        stats->set_index_size(stats->index_size() + (compaction.index_size_output_rowsets() - compaction.index_size_input_rowsets()));
+        stats->set_segment_size(stats->segment_size() + (compaction.segment_size_output_rowsets() - compaction.segment_size_input_rowsets()));
         stats->set_last_full_compaction_time_ms(now * 1000);
         // clang-format on
     } else {
@@ -738,10 +744,14 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string
                << " stats.data_size=" << stats->data_size()
                << " stats.num_rowsets=" << stats->num_rowsets()
                << " stats.num_segments=" << stats->num_segments()
+               << " stats.index_size=" << stats->index_size()
+               << " stats.segment_size=" << stats->segment_size()
                << " detached_stats.num_rows=" << detached_stats.num_rows
                << " detached_stats.data_size=" << detached_stats.data_size
                << " detached_stats.num_rowset=" << detached_stats.num_rowsets
                << " detached_stats.num_segments=" << detached_stats.num_segs
+               << " detached_stats.index_size=" << detached_stats.index_size
+               << " detached_stats.segment_size=" << detached_stats.segment_size
                << " compaction.size_output_rowsets=" << compaction.size_output_rowsets()
                << " compaction.size_input_rowsets=" << compaction.size_input_rowsets();
     txn->put(stats_key, stats_val);
@@ -752,10 +762,14 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string
                             << " stats.data_size=" << stats->data_size()
                             << " stats.num_rowsets=" << stats->num_rowsets()
                             << " stats.num_segments=" << stats->num_segments()
+                            << " stats.index_size=" << stats->index_size()
+                            << " stats.segment_size=" << stats->segment_size()
                             << " detached_stats.num_rows=" << detached_stats.num_rows
                             << " detached_stats.data_size=" << detached_stats.data_size
                             << " detached_stats.num_rowset=" << detached_stats.num_rowsets
                             << " detached_stats.num_segments=" << detached_stats.num_segs
+                            << " detached_stats.index_size=" << detached_stats.index_size
+                            << " detached_stats.segment_size=" << detached_stats.segment_size
                             << " compaction.size_output_rowsets="
                             << compaction.size_output_rowsets()
                             << " compaction.size_input_rowsets=" << compaction.size_input_rowsets();
@@ -1133,6 +1147,8 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str
     int64_t size_remove_rowsets = 0;
     int64_t num_remove_rowsets = 0;
     int64_t num_remove_segments = 0;
+    int64_t index_size_remove_rowsets = 0;
+    int64_t segment_size_remove_rowsets = 0;
 
     auto rs_start = meta_rowset_key({instance_id, new_tablet_id, 2});
     auto rs_end = meta_rowset_key({instance_id, new_tablet_id, schema_change.alter_version() + 1});
@@ -1162,9 +1178,11 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str
             }
 
             num_remove_rows += rs.num_rows();
-            size_remove_rowsets += rs.data_disk_size();
+            size_remove_rowsets += rs.total_disk_size();
             ++num_remove_rowsets;
             num_remove_segments += rs.num_segments();
+            index_size_remove_rowsets += rs.index_disk_size();
+            segment_size_remove_rowsets += rs.data_disk_size();
 
             auto recycle_key = recycle_rowset_key({instance_id, new_tablet_id, rs.rowset_id_v2()});
             RecycleRowsetPB recycle_rowset;
@@ -1199,6 +1217,8 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str
     stats->set_data_size(stats->data_size() + (schema_change.size_output_rowsets() - size_remove_rowsets));
     stats->set_num_rowsets(stats->num_rowsets() + (schema_change.num_output_rowsets() - num_remove_rowsets));
     stats->set_num_segments(stats->num_segments() + (schema_change.num_output_segments() - num_remove_segments));
+    stats->set_index_size(stats->index_size() + (schema_change.index_size_output_rowsets() - index_size_remove_rowsets));
+    stats->set_segment_size(stats->segment_size() + (schema_change.segment_size_output_rowsets() - segment_size_remove_rowsets));
     // clang-format on
     auto stats_key = stats_tablet_key(
             {instance_id, new_table_id, new_index_id, new_partition_id, new_tablet_id});
diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp b/cloud/src/meta-service/meta_service_tablet_stats.cpp
index cecccbd67673ad..4cbf629c9a3185 100644
--- a/cloud/src/meta-service/meta_service_tablet_stats.cpp
+++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp
@@ -45,7 +45,8 @@ void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg, Transact
     auto end_key = stats_tablet_key({instance_id, idx.table_id(), idx.index_id(), idx.partition_id(), idx.tablet_id() + 1});
     // clang-format on
     std::vector<std::pair<std::string, std::string>> stats_kvs;
-    stats_kvs.reserve(5); // aggregate + data_size + num_rows + num_rowsets + num_segments
+    stats_kvs.reserve(
+            7); // aggregate + data_size + num_rows + num_rowsets + num_segments + index_size + segment_size
 
     std::unique_ptr<RangeGetIterator> it;
     do {
@@ -93,7 +94,7 @@ int get_detached_tablet_stats(const std::vector<std::pair<std::string, std::stri
                               TabletStats& detached_stats) {
     bool unexpected_size = false;
     // clang-format off
-    if (stats_kvs.size() != 5    // aggregated stats and 4 splitted stats: num_rowsets num_segs data_size num_rows
+    if (stats_kvs.size() != 7    // aggregated stats and 4 splitted stats: num_rowsets num_segs data_size num_rows index_size segment_size
         && stats_kvs.size() != 2 // aggregated stats and 1 splitted stats: num_rowsets
         && stats_kvs.size() != 1 // aggregated stats only (nothing has been imported since created)
         ) {
@@ -137,6 +138,10 @@ int get_detached_tablet_stats(const std::vector<std::pair<std::string, std::stri
             detached_stats.num_rowsets = val;
         } else if (*suffix == STATS_KEY_SUFFIX_NUM_SEGS) {
             detached_stats.num_segs = val;
+        } else if (*suffix == STATS_KEY_SUFFIX_INDEX_SIZE) {
+            detached_stats.index_size = val;
+        } else if (*suffix == STATS_KEY_SUFFIX_SEGMENT_SIZE) {
+            detached_stats.segment_size = val;
         } else {
             LOG(WARNING) << "unknown tablet stats suffix=" << *suffix << " key=" << hex(k);
         }
@@ -155,6 +160,8 @@ void merge_tablet_stats(TabletStatsPB& stats, const TabletStats& detached_stats)
     stats.set_num_rows(stats.num_rows() + detached_stats.num_rows);
     stats.set_num_rowsets(stats.num_rowsets() + detached_stats.num_rowsets);
     stats.set_num_segments(stats.num_segments() + detached_stats.num_segs);
+    stats.set_index_size(stats.index_size() + detached_stats.index_size);
+    stats.set_segment_size(stats.segment_size() + detached_stats.segment_size);
 }
 
 void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg, Transaction* txn,
@@ -253,14 +260,20 @@ MetaServiceResponseStatus fix_tablet_stats_internal(
             return st;
         }
         int64_t total_disk_size = 0;
+        int64_t index_disk_size = 0;
+        int64_t data_disk_size = 0;
         for (const auto& rs_meta : resp.rowset_meta()) {
             total_disk_size += rs_meta.total_disk_size();
+            index_disk_size += rs_meta.index_disk_size();
+            data_disk_size += rs_meta.data_disk_size();
         }
 
         // set new disk size to tabletPB and write it back
         TabletStatsPB tablet_stat;
         tablet_stat.CopyFrom(*tablet_stat_ptr);
         tablet_stat.set_data_size(total_disk_size);
+        tablet_stat.set_index_size(index_disk_size);
+        tablet_stat.set_segment_size(data_disk_size);
         // record tablet stats batch
         tablet_stat_shared_ptr_vec_batch.emplace_back(std::make_shared<TabletStatsPB>(tablet_stat));
         std::string tablet_stat_key;
@@ -312,6 +325,30 @@ MetaServiceResponseStatus fix_tablet_stats_internal(
             memcpy(tablet_stat_data_size_value.data(), &tablet_stat_data_size,
                    sizeof(tablet_stat_data_size));
             txn->put(tablet_stat_data_size_key, tablet_stat_data_size_value);
+            // set tablet stats index size = 0
+            // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "index_size" -> int64
+            std::string tablet_stat_index_size_key;
+            stats_tablet_index_size_key(
+                    {instance_id, tablet_stat.idx().table_id(), tablet_stat.idx().index_id(),
+                     tablet_stat.idx().partition_id(), tablet_stat.idx().tablet_id()},
+                    &tablet_stat_index_size_key);
+            int64_t tablet_stat_index_size = 0;
+            std::string tablet_stat_index_size_value(sizeof(tablet_stat_index_size), '\0');
+            memcpy(tablet_stat_index_size_value.data(), &tablet_stat_index_size,
+                   sizeof(tablet_stat_index_size));
+            txn->put(tablet_stat_index_size_key, tablet_stat_index_size_value);
+            // set tablet stats segment size = 0
+            // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "segment_size" -> int64
+            std::string tablet_stat_segment_size_key;
+            stats_tablet_segment_size_key(
+                    {instance_id, tablet_stat.idx().table_id(), tablet_stat.idx().index_id(),
+                     tablet_stat.idx().partition_id(), tablet_stat.idx().tablet_id()},
+                    &tablet_stat_segment_size_key);
+            int64_t tablet_stat_segment_size = 0;
+            std::string tablet_stat_segment_size_value(sizeof(tablet_stat_segment_size), '\0');
+            memcpy(tablet_stat_segment_size_value.data(), &tablet_stat_segment_size,
+                   sizeof(tablet_stat_segment_size));
+            txn->put(tablet_stat_segment_size_key, tablet_stat_segment_size_value);
         }
     }
 
@@ -396,6 +433,76 @@ MetaServiceResponseStatus check_new_tablet_stats(
                     .tag("data size", tablet_stat_data_size)
                     .tag("check data size", tablet_stat_data_size_check);
         }
+
+        // check index size
+        std::string tablet_stat_index_size_key;
+        stats_tablet_index_size_key(
+                {instance_id, tablet_stat_ptr->idx().table_id(), tablet_stat_ptr->idx().index_id(),
+                 tablet_stat_ptr->idx().partition_id(), tablet_stat_ptr->idx().tablet_id()},
+                &tablet_stat_index_size_key);
+        int64_t tablet_stat_index_size = 0;
+        std::string tablet_stat_index_size_value(sizeof(tablet_stat_index_size), '\0');
+        err = txn->get(tablet_stat_index_size_key, &tablet_stat_index_size_value);
+        if (err != TxnErrorCode::TXN_OK && err != TxnErrorCode::TXN_KEY_NOT_FOUND) {
+            st.set_code(cast_as<ErrCategory::READ>(err));
+            return st;
+        }
+        int64_t tablet_stat_index_size_check;
+
+        if (tablet_stat_index_size_value.size() != sizeof(tablet_stat_index_size_check))
+                [[unlikely]] {
+            LOG(WARNING) << " malformed tablet stats value v.index_size="
+                         << tablet_stat_index_size_value.size()
+                         << " value=" << hex(tablet_stat_index_size_value);
+        }
+        std::memcpy(&tablet_stat_index_size_check, tablet_stat_index_size_value.data(),
+                    sizeof(tablet_stat_index_size_check));
+        if constexpr (std::endian::native == std::endian::big) {
+            tablet_stat_index_size_check = bswap_64(tablet_stat_index_size_check);
+        }
+        if (tablet_stat_index_size_check != tablet_stat_index_size &&
+            // ditto
+            (tablet_stat_index_size_check > 2 * tablet_stat_index_size ||
+             tablet_stat_index_size > 2 * tablet_stat_index_size_check)) {
+            LOG_WARNING("[fix tablet stats]:index size check failed")
+                    .tag("index size", tablet_stat_index_size)
+                    .tag("check index size", tablet_stat_index_size_check);
+        }
+
+        // check data size
+        std::string tablet_stat_segment_size_key;
+        stats_tablet_segment_size_key(
+                {instance_id, tablet_stat_ptr->idx().table_id(), tablet_stat_ptr->idx().index_id(),
+                 tablet_stat_ptr->idx().partition_id(), tablet_stat_ptr->idx().tablet_id()},
+                &tablet_stat_segment_size_key);
+        int64_t tablet_stat_segment_size = 0;
+        std::string tablet_stat_segment_size_value(sizeof(tablet_stat_segment_size), '\0');
+        err = txn->get(tablet_stat_segment_size_key, &tablet_stat_segment_size_value);
+        if (err != TxnErrorCode::TXN_OK && err != TxnErrorCode::TXN_KEY_NOT_FOUND) {
+            st.set_code(cast_as<ErrCategory::READ>(err));
+            return st;
+        }
+        int64_t tablet_stat_segment_size_check;
+
+        if (tablet_stat_segment_size_value.size() != sizeof(tablet_stat_segment_size_check))
+                [[unlikely]] {
+            LOG(WARNING) << " malformed tablet stats value v.segment_size="
+                         << tablet_stat_segment_size_value.size()
+                         << " value=" << hex(tablet_stat_segment_size_value);
+        }
+        std::memcpy(&tablet_stat_segment_size_check, tablet_stat_segment_size_value.data(),
+                    sizeof(tablet_stat_segment_size_check));
+        if constexpr (std::endian::native == std::endian::big) {
+            tablet_stat_segment_size_check = bswap_64(tablet_stat_segment_size_check);
+        }
+        if (tablet_stat_segment_size_check != tablet_stat_segment_size &&
+            // ditto
+            (tablet_stat_segment_size_check > 2 * tablet_stat_segment_size ||
+             tablet_stat_segment_size > 2 * tablet_stat_segment_size_check)) {
+            LOG_WARNING("[fix tablet stats]:segment size check failed")
+                    .tag("segment size", tablet_stat_segment_size)
+                    .tag("check segment size", tablet_stat_segment_size_check);
+        }
     }
 
     return st;
diff --git a/cloud/src/meta-service/meta_service_tablet_stats.h b/cloud/src/meta-service/meta_service_tablet_stats.h
index a7aea5885a8e1a..30cbaf56f6dc80 100644
--- a/cloud/src/meta-service/meta_service_tablet_stats.h
+++ b/cloud/src/meta-service/meta_service_tablet_stats.h
@@ -31,6 +31,8 @@ struct TabletStats {
     int64_t num_rows = 0;
     int64_t num_rowsets = 0;
     int64_t num_segs = 0;
+    int64_t index_size = 0;
+    int64_t segment_size = 0;
 };
 
 // Get tablet stats and detached tablet stats via `txn`. If an error occurs, `code` will be set to non OK.
diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp
index 58930f6edfc63b..5d696220b72dab 100644
--- a/cloud/src/meta-service/meta_service_txn.cpp
+++ b/cloud/src/meta-service/meta_service_txn.cpp
@@ -880,6 +880,12 @@ void update_tablet_stats(const StatsTabletKeyInfo& info, const TabletStats& stat
             std::string num_segs_key;
             stats_tablet_num_segs_key(info, &num_segs_key);
             txn->atomic_add(num_segs_key, stats.num_segs);
+            std::string index_size_key;
+            stats_tablet_index_size_key(info, &index_size_key);
+            txn->atomic_add(index_size_key, stats.index_size);
+            std::string segment_size_key;
+            stats_tablet_segment_size_key(info, &segment_size_key);
+            txn->atomic_add(segment_size_key, stats.segment_size);
         }
         std::string num_rowsets_key;
         stats_tablet_num_rowsets_key(info, &num_rowsets_key);
@@ -906,6 +912,8 @@ void update_tablet_stats(const StatsTabletKeyInfo& info, const TabletStats& stat
         stats_pb.set_num_rows(stats_pb.num_rows() + stats.num_rows);
         stats_pb.set_num_rowsets(stats_pb.num_rowsets() + stats.num_rowsets);
         stats_pb.set_num_segments(stats_pb.num_segments() + stats.num_segs);
+        stats_pb.set_index_size(stats_pb.index_size() + stats.index_size);
+        stats_pb.set_segment_size(stats_pb.segment_size() + stats.segment_size);
         stats_pb.SerializeToString(&val);
         txn->put(key, val);
         LOG(INFO) << "put stats_tablet_key key=" << hex(key);
@@ -1234,6 +1242,8 @@ void commit_txn_immediately(
             stats.num_rows += i.num_rows();
             ++stats.num_rowsets;
             stats.num_segs += i.num_segments();
+            stats.index_size += i.index_disk_size();
+            stats.segment_size += i.data_disk_size();
         } // for tmp_rowsets_meta
 
         process_mow_when_commit_txn(request, instance_id, code, msg, txn, table_id_tablet_ids);
@@ -1864,10 +1874,12 @@ void commit_txn_eventually(
         for (auto& [_, i] : tmp_rowsets_meta) {
             // Accumulate affected rows
             auto& stats = tablet_stats[i.tablet_id()];
-            stats.data_size += i.data_disk_size();
+            stats.data_size += i.total_disk_size();
             stats.num_rows += i.num_rows();
             ++stats.num_rowsets;
             stats.num_segs += i.num_segments();
+            stats.index_size += i.index_disk_size();
+            stats.segment_size += i.data_disk_size();
         }
 
         // calculate table stats from tablets stats
@@ -2238,10 +2250,12 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse*
 
             // Accumulate affected rows
             auto& stats = tablet_stats[tablet_id];
-            stats.data_size += i.data_disk_size();
+            stats.data_size += i.total_disk_size();
             stats.num_rows += i.num_rows();
             ++stats.num_rowsets;
             stats.num_segs += i.num_segments();
+            stats.index_size += i.index_disk_size();
+            stats.segment_size += i.data_disk_size();
         } // for tmp_rowsets_meta
     }
 
@@ -2351,6 +2365,12 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse*
                 auto& num_segs_key = kv_pool.emplace_back();
                 stats_tablet_num_segs_key(info, &num_segs_key);
                 txn->atomic_add(num_segs_key, stats.num_segs);
+                auto& index_size_key = kv_pool.emplace_back();
+                stats_tablet_index_size_key(info, &index_size_key);
+                txn->atomic_add(index_size_key, stats.index_size);
+                auto& segment_size_key = kv_pool.emplace_back();
+                stats_tablet_segment_size_key(info, &segment_size_key);
+                txn->atomic_add(segment_size_key, stats.segment_size);
             }
             auto& num_rowsets_key = kv_pool.emplace_back();
             stats_tablet_num_rowsets_key(info, &num_rowsets_key);
@@ -2379,6 +2399,8 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse*
             stats_pb.set_num_rows(stats_pb.num_rows() + stats.num_rows);
             stats_pb.set_num_rowsets(stats_pb.num_rowsets() + stats.num_rowsets);
             stats_pb.set_num_segments(stats_pb.num_segments() + stats.num_segs);
+            stats_pb.set_index_size(stats_pb.index_size() + stats.index_size);
+            stats_pb.set_segment_size(stats_pb.segment_size() + stats.segment_size);
             stats_pb.SerializeToString(&val);
             txn->put(key, val);
             LOG(INFO) << "put stats_tablet_key, key=" << hex(key);
diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp
index 25d36faab06796..9859c2b0ed1ae3 100644
--- a/cloud/src/meta-service/txn_lazy_committer.cpp
+++ b/cloud/src/meta-service/txn_lazy_committer.cpp
@@ -174,10 +174,12 @@ void convert_tmp_rowsets(
 
         // Accumulate affected rows
         auto& stats = tablet_stats[tmp_rowset_pb.tablet_id()];
-        stats.data_size += tmp_rowset_pb.data_disk_size();
+        stats.data_size += tmp_rowset_pb.total_disk_size();
         stats.num_rows += tmp_rowset_pb.num_rows();
         ++stats.num_rowsets;
         stats.num_segs += tmp_rowset_pb.num_segments();
+        stats.index_size += tmp_rowset_pb.index_disk_size();
+        stats.segment_size += tmp_rowset_pb.data_disk_size();
     }
 
     for (auto& [tablet_id, stats] : tablet_stats) {
diff --git a/cloud/test/http_encode_key_test.cpp b/cloud/test/http_encode_key_test.cpp
index 77f800846076a9..d245682474349d 100644
--- a/cloud/test/http_encode_key_test.cpp
+++ b/cloud/test/http_encode_key_test.cpp
@@ -310,7 +310,7 @@ txn_id=126419752960)",
         },
         R"({"creation_time":"12345","label":"label_1"})",
     },
-    Input { // aggregated_stats + full detached_stats, there are 5 KVs in total
+    Input { // aggregated_stats + full detached_stats, there are 7 KVs in total
         "StatsTabletKey",
         "instance_id=gavin-instance&table_id=10086&index_id=10010&part_id=10000&tablet_id=1008601",
         {
@@ -327,15 +327,17 @@ txn_id=126419752960)",
             idx->set_index_id(100010);
             idx->set_partition_id(10000);
             idx->set_tablet_id(1008601);
-            pb.set_data_size(1);
+            pb.set_data_size(2);
             pb.set_num_rows(10);
             pb.set_num_rowsets(11);
             pb.set_num_segments(12);
+            pb.set_index_size(1);
+            pb.set_segment_size(1);
             return {pb.SerializeAsString(), {"\x01\x00\x00\x00\x00\x00\x00\x00",8}, {"\x02\x00\x00\x00\x00\x00\x00\x00",8}, {"\x03\x00\x00\x00\x00\x00\x00\x00",8}, {"\x04\x00\x00\x00\x00\x00\x00\x00",8}};
         },
-        R"(aggregated_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"1","num_rows":"10","num_rowsets":"11","num_segments":"12"}
-detached_stats: {"data_size":"1","num_rows":"2","num_rowsets":"3","num_segments":"4"}
-merged_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"2","num_rows":"12","num_rowsets":"14","num_segments":"16"}
+        R"(aggregated_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"2","num_rows":"10","num_rowsets":"11","num_segments":"12","index_size":"1","segment_size":"1"}
+detached_stats: {"data_size":"1","num_rows":"2","num_rowsets":"3","num_segments":"4","index_size":"0","segment_size":"0"}
+merged_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"3","num_rows":"12","num_rowsets":"14","num_segments":"16","index_size":"1","segment_size":"1"}
 )",
     },
     Input { // aggregated_stats + half detached_stats (num_segs == 0, there is num_rowsets detached stats)
@@ -352,15 +354,17 @@ merged_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"100
             idx->set_index_id(100010);
             idx->set_partition_id(10000);
             idx->set_tablet_id(1008602);
-            pb.set_data_size(1);
+            pb.set_data_size(2);
             pb.set_num_rows(10);
             pb.set_num_rowsets(11);
             pb.set_num_segments(12);
+            pb.set_index_size(1);
+            pb.set_segment_size(1);
             return {pb.SerializeAsString(), {"\x03\x00\x00\x00\x00\x00\x00\x00",8}};
         },
-        R"(aggregated_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"1","num_rows":"10","num_rowsets":"11","num_segments":"12"}
-detached_stats: {"data_size":"0","num_rows":"0","num_rowsets":"3","num_segments":"0"}
-merged_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"1","num_rows":"10","num_rowsets":"14","num_segments":"12"}
+        R"(aggregated_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"2","num_rows":"10","num_rowsets":"11","num_segments":"12","index_size":"1","segment_size":"1"}
+detached_stats: {"data_size":"0","num_rows":"0","num_rowsets":"3","num_segments":"0","index_size":"0","segment_size":"0"}
+merged_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"2","num_rows":"10","num_rowsets":"14","num_segments":"12","index_size":"1","segment_size":"1"}
 )",
     },
     Input { // aggregated_stats only, the legacy
@@ -376,15 +380,17 @@ merged_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"100
             idx->set_index_id(100010);
             idx->set_partition_id(10000);
             idx->set_tablet_id(1008602);
-            pb.set_data_size(1);
+            pb.set_data_size(2);
             pb.set_num_rows(10);
             pb.set_num_rowsets(11);
             pb.set_num_segments(12);
+            pb.set_index_size(1);
+            pb.set_segment_size(1);
             return {pb.SerializeAsString()};
         },
-        R"(aggregated_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"1","num_rows":"10","num_rowsets":"11","num_segments":"12"}
-detached_stats: {"data_size":"0","num_rows":"0","num_rowsets":"0","num_segments":"0"}
-merged_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"1","num_rows":"10","num_rowsets":"11","num_segments":"12"}
+        R"(aggregated_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"2","num_rows":"10","num_rowsets":"11","num_segments":"12","index_size":"1","segment_size":"1"}
+detached_stats: {"data_size":"0","num_rows":"0","num_rowsets":"0","num_segments":"0","index_size":"0","segment_size":"0"}
+merged_stats: {"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"2","num_rows":"10","num_rowsets":"11","num_segments":"12","index_size":"1","segment_size":"1"}
 )",
     },
     Input {
diff --git a/cloud/test/keys_test.cpp b/cloud/test/keys_test.cpp
index a92a685cda45f8..52d840e1f5ce6b 100644
--- a/cloud/test/keys_test.cpp
+++ b/cloud/test/keys_test.cpp
@@ -803,6 +803,40 @@ TEST(KeysTest, StatsKeysTest) {
 
         EXPECT_EQ("num_segs", dec_stats_suffix);
     }
+
+    // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "index_size"   -> int64
+    {
+        StatsTabletKeyInfo stats_key {instance_id, table_id, index_id, partition_id, tablet_id};
+        std::string encoded_stats_key0;
+        stats_tablet_index_size_key(stats_key, &encoded_stats_key0);
+        std::cout << hex(encoded_stats_key0) << std::endl;
+
+        std::string dec_stats_suffix;
+
+        std::string_view key_sv(encoded_stats_key0);
+        expect_stats_prefix(key_sv);
+        ASSERT_EQ(decode_bytes(&key_sv, &dec_stats_suffix), 0);
+        ASSERT_TRUE(key_sv.empty());
+
+        EXPECT_EQ("index_size", dec_stats_suffix);
+    }
+
+    // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "segment_size"   -> int64
+    {
+        StatsTabletKeyInfo stats_key {instance_id, table_id, index_id, partition_id, tablet_id};
+        std::string encoded_stats_key0;
+        stats_tablet_segment_size_key(stats_key, &encoded_stats_key0);
+        std::cout << hex(encoded_stats_key0) << std::endl;
+
+        std::string dec_stats_suffix;
+
+        std::string_view key_sv(encoded_stats_key0);
+        expect_stats_prefix(key_sv);
+        ASSERT_EQ(decode_bytes(&key_sv, &dec_stats_suffix), 0);
+        ASSERT_TRUE(key_sv.empty());
+
+        EXPECT_EQ("segment_size", dec_stats_suffix);
+    }
 }
 
 TEST(KeysTest, JobKeysTest) {
diff --git a/cloud/test/meta_service_http_test.cpp b/cloud/test/meta_service_http_test.cpp
index 4360efeb4422a9..81c322303a5a8c 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -1257,6 +1257,8 @@ TEST(MetaServiceHttpTest, GetTabletStatsTest) {
     EXPECT_EQ(res.tablet_stats(0).num_rows(), 0);
     EXPECT_EQ(res.tablet_stats(0).num_rowsets(), 1);
     EXPECT_EQ(res.tablet_stats(0).num_segments(), 0);
+    EXPECT_EQ(res.tablet_stats(0).index_size(), 0);
+    EXPECT_EQ(res.tablet_stats(0).segment_size(), 0);
     {
         GetTabletStatsRequest req;
         auto idx = req.add_tablet_idx();
@@ -1288,6 +1290,16 @@ TEST(MetaServiceHttpTest, GetTabletStatsTest) {
                                &data_size_key);
     ASSERT_EQ(txn->get(data_size_key, &data_size_val), TxnErrorCode::TXN_OK);
     EXPECT_EQ(*(int64_t*)data_size_val.data(), 22000);
+    std::string index_size_key, index_size_val;
+    stats_tablet_index_size_key({mock_instance, table_id, index_id, partition_id, tablet_id},
+                                &index_size_key);
+    ASSERT_EQ(txn->get(index_size_key, &index_size_val), TxnErrorCode::TXN_OK);
+    EXPECT_EQ(*(int64_t*)index_size_val.data(), 2000);
+    std::string segment_size_key, segment_size_val;
+    stats_tablet_segment_size_key({mock_instance, table_id, index_id, partition_id, tablet_id},
+                                  &segment_size_key);
+    ASSERT_EQ(txn->get(segment_size_key, &segment_size_val), TxnErrorCode::TXN_OK);
+    EXPECT_EQ(*(int64_t*)segment_size_val.data(), 20000);
     std::string num_rows_key, num_rows_val;
     stats_tablet_num_rows_key({mock_instance, table_id, index_id, partition_id, tablet_id},
                               &num_rows_key);
@@ -1312,6 +1324,8 @@ TEST(MetaServiceHttpTest, GetTabletStatsTest) {
     EXPECT_EQ(res.tablet_stats(0).num_rows(), 400);
     EXPECT_EQ(res.tablet_stats(0).num_rowsets(), 5);
     EXPECT_EQ(res.tablet_stats(0).num_segments(), 4);
+    EXPECT_EQ(res.tablet_stats(0).index_size(), 4000);
+    EXPECT_EQ(res.tablet_stats(0).segment_size(), 40000);
     {
         GetTabletStatsRequest req;
         auto idx = req.add_tablet_idx();
diff --git a/cloud/test/meta_service_job_test.cpp b/cloud/test/meta_service_job_test.cpp
index f0323eebb790be..e1dbb0089e2e73 100644
--- a/cloud/test/meta_service_job_test.cpp
+++ b/cloud/test/meta_service_job_test.cpp
@@ -95,7 +95,9 @@ doris::RowsetMetaCloudPB create_rowset(int64_t tablet_id, int64_t start_version,
     rowset.set_end_version(end_version);
     rowset.set_num_segments(1);
     rowset.set_num_rows(num_rows);
-    rowset.set_data_disk_size(num_rows * 100);
+    rowset.set_total_disk_size(num_rows * 100);
+    rowset.set_data_disk_size(num_rows * 50);
+    rowset.set_index_disk_size(num_rows * 50);
     rowset.mutable_tablet_schema()->set_schema_version(0);
     rowset.set_txn_expiration(::time(nullptr)); // Required by DCHECK
     return rowset;
@@ -114,11 +116,13 @@ void insert_rowsets(TxnKv* txn_kv, int64_t table_id, int64_t index_id, int64_t p
     std::unique_ptr<Transaction> txn;
     ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK) << tablet_id;
     std::deque<std::string> buffer;
-    int data_size = 0, num_rows = 0, num_seg = 0;
+    int data_size = 0, num_rows = 0, num_seg = 0, index_size = 0, segment_size = 0;
     for (auto& rowset : rowsets) {
-        data_size += rowset.data_disk_size();
+        data_size += rowset.total_disk_size();
         num_rows += rowset.num_rows();
         num_seg += rowset.num_segments();
+        index_size += rowset.index_disk_size();
+        segment_size += rowset.data_disk_size();
         auto& key = buffer.emplace_back();
         auto& val = buffer.emplace_back();
         meta_rowset_key({instance_id, tablet_id, rowset.end_version()}, &key);
@@ -138,6 +142,12 @@ void insert_rowsets(TxnKv* txn_kv, int64_t table_id, int64_t index_id, int64_t p
     std::string num_segs_key;
     stats_tablet_num_segs_key(info, &num_segs_key);
     txn->atomic_add(num_segs_key, num_seg);
+    std::string index_size_key;
+    stats_tablet_index_size_key(info, &index_size_key);
+    txn->atomic_add(index_size_key, index_size);
+    std::string segment_size_key;
+    stats_tablet_segment_size_key(info, &segment_size_key);
+    txn->atomic_add(segment_size_key, segment_size);
     ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK) << tablet_id;
 }
 
@@ -241,7 +251,11 @@ void finish_schema_change_job(
             sc->add_output_versions(rowset.end_version());
             sc->set_num_output_rows(sc->num_output_rows() + rowset.num_rows());
             sc->set_num_output_segments(sc->num_output_segments() + rowset.num_segments());
-            sc->set_size_output_rowsets(sc->size_output_rowsets() + rowset.data_disk_size());
+            sc->set_size_output_rowsets(sc->size_output_rowsets() + rowset.total_disk_size());
+            sc->set_index_size_output_rowsets(sc->index_size_output_rowsets() +
+                                              rowset.index_disk_size());
+            sc->set_segment_size_output_rowsets(sc->segment_size_output_rowsets() +
+                                                rowset.data_disk_size());
         }
         sc->set_num_output_rowsets(output_rowsets.size());
     }
@@ -853,6 +867,10 @@ TEST(MetaServiceJobTest, CompactionJobTest) {
         compaction->set_num_input_segments(dist(rng));
         compaction->set_size_input_rowsets(dist(rng));
         compaction->set_size_output_rowsets(dist(rng));
+        compaction->set_index_size_input_rowsets(dist(rng));
+        compaction->set_segment_size_output_rowsets(dist(rng));
+        compaction->set_index_size_input_rowsets(dist(rng));
+        compaction->set_segment_size_output_rowsets(dist(rng));
         compaction->set_type(type);
 
         tablet_stats_pb.set_cumulative_compaction_cnt(dist(rng));
@@ -863,6 +881,8 @@ TEST(MetaServiceJobTest, CompactionJobTest) {
         tablet_stats_pb.set_data_size(dist(rng) + compaction->size_input_rowsets());
         tablet_stats_pb.set_num_rowsets(dist(rng) + compaction->num_input_rowsets());
         tablet_stats_pb.set_num_segments(dist(rng) + compaction->num_input_segments());
+        tablet_stats_pb.set_index_size(dist(rng) + compaction->index_size_input_rowsets());
+        tablet_stats_pb.set_segment_size(dist(rng) + compaction->segment_size_input_rowsets());
 
         std::string tablet_stats_val = tablet_stats_pb.SerializeAsString();
         ASSERT_FALSE(tablet_stats_val.empty());
@@ -981,6 +1001,8 @@ TEST(MetaServiceJobTest, CompactionJobTest) {
         EXPECT_EQ(stats.data_size()                , tablet_stats_pb.data_size() + (req.job().compaction(0).size_output_rowsets() - req.job().compaction(0).size_input_rowsets()));
         EXPECT_EQ(stats.num_rowsets()              , tablet_stats_pb.num_rowsets() + (req.job().compaction(0).num_output_rowsets() - req.job().compaction(0).num_input_rowsets()));
         EXPECT_EQ(stats.num_segments()             , tablet_stats_pb.num_segments() + (req.job().compaction(0).num_output_segments() - req.job().compaction(0).num_input_segments()));
+        EXPECT_EQ(stats.index_size()               , tablet_stats_pb.index_size() + (req.job().compaction(0).index_size_output_rowsets() - req.job().compaction(0).index_size_input_rowsets()));
+        EXPECT_EQ(stats.segment_size()             , tablet_stats_pb.segment_size() + (req.job().compaction(0).segment_size_output_rowsets() - req.job().compaction(0).segment_size_input_rowsets()));
         // clang-format on
 
         // Check job removed, tablet meta updated
@@ -1157,6 +1179,8 @@ TEST(MetaServiceJobTest, CompactionJobWithMoWTest) {
         tablet_stats_pb.set_data_size(dist(rng) + compaction->size_input_rowsets());
         tablet_stats_pb.set_num_rowsets(dist(rng) + compaction->num_input_rowsets());
         tablet_stats_pb.set_num_segments(dist(rng) + compaction->num_input_segments());
+        tablet_stats_pb.set_index_size(dist(rng) + compaction->index_size_input_rowsets());
+        tablet_stats_pb.set_segment_size(dist(rng) + compaction->segment_size_input_rowsets());
 
         std::string tablet_stats_val = tablet_stats_pb.SerializeAsString();
         ASSERT_FALSE(tablet_stats_val.empty());
@@ -1391,12 +1415,16 @@ TEST(MetaServiceJobTest, SchemaChangeJobTest) {
         EXPECT_EQ(res.stats().num_rowsets(), 6);
         EXPECT_EQ(res.stats().num_segments(), 5);
         EXPECT_EQ(res.stats().data_size(), 50000);
+        EXPECT_EQ(res.stats().index_size(), 25000);
+        EXPECT_EQ(res.stats().segment_size(), 25000);
         TabletStatsPB tablet_stats;
         get_tablet_stats(meta_service.get(), new_tablet_id, tablet_stats);
         EXPECT_EQ(tablet_stats.num_rows(), 500);
         EXPECT_EQ(tablet_stats.num_rowsets(), 6);
         EXPECT_EQ(tablet_stats.num_segments(), 5);
         EXPECT_EQ(tablet_stats.data_size(), 50000);
+        EXPECT_EQ(tablet_stats.index_size(), 25000);
+        EXPECT_EQ(tablet_stats.segment_size(), 25000);
 
         std::unique_ptr<Transaction> txn;
         ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
@@ -1465,12 +1493,16 @@ TEST(MetaServiceJobTest, SchemaChangeJobTest) {
         EXPECT_EQ(res.stats().num_rowsets(), 6);
         EXPECT_EQ(res.stats().num_segments(), 5);
         EXPECT_EQ(res.stats().data_size(), 50000);
+        EXPECT_EQ(res.stats().index_size(), 25000);
+        EXPECT_EQ(res.stats().segment_size(), 25000);
         TabletStatsPB tablet_stats;
         get_tablet_stats(meta_service.get(), new_tablet_id, tablet_stats);
         EXPECT_EQ(tablet_stats.num_rows(), 500);
         EXPECT_EQ(tablet_stats.num_rowsets(), 6);
         EXPECT_EQ(tablet_stats.num_segments(), 5);
         EXPECT_EQ(tablet_stats.data_size(), 50000);
+        EXPECT_EQ(tablet_stats.index_size(), 25000);
+        EXPECT_EQ(tablet_stats.segment_size(), 25000);
 
         std::unique_ptr<Transaction> txn;
         ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
@@ -1628,12 +1660,16 @@ TEST(MetaServiceJobTest, RetrySchemaChangeJobTest) {
     EXPECT_EQ(res.stats().num_rowsets(), 6);
     EXPECT_EQ(res.stats().num_segments(), 5);
     EXPECT_EQ(res.stats().data_size(), 50000);
+    EXPECT_EQ(res.stats().index_size(), 25000);
+    EXPECT_EQ(res.stats().segment_size(), 25000);
     TabletStatsPB tablet_stats;
     get_tablet_stats(meta_service.get(), new_tablet_id, tablet_stats);
     EXPECT_EQ(tablet_stats.num_rows(), 500);
     EXPECT_EQ(tablet_stats.num_rowsets(), 6);
     EXPECT_EQ(tablet_stats.num_segments(), 5);
     EXPECT_EQ(tablet_stats.data_size(), 50000);
+    EXPECT_EQ(tablet_stats.index_size(), 25000);
+    EXPECT_EQ(tablet_stats.segment_size(), 25000);
 
     std::unique_ptr<Transaction> txn;
     ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
@@ -1942,10 +1978,14 @@ TEST(MetaServiceJobTest, ConcurrentCompactionTest) {
         compaction->add_output_rowset_ids(output_rowset.rowset_id_v2());
         compaction->set_output_cumulative_point(11);
         compaction->set_size_input_rowsets(60000);
+        compaction->set_index_size_input_rowsets(30000);
+        compaction->set_segment_size_input_rowsets(30000);
         compaction->set_num_input_rows(600);
         compaction->set_num_input_rowsets(6);
         compaction->set_num_input_segments(6);
         compaction->set_size_output_rowsets(10000);
+        compaction->set_index_size_output_rowsets(5000);
+        compaction->set_segment_size_output_rowsets(5000);
         compaction->set_num_output_rows(100);
         compaction->set_num_output_rowsets(1);
         compaction->set_num_output_segments(1);
@@ -1958,12 +1998,16 @@ TEST(MetaServiceJobTest, ConcurrentCompactionTest) {
         EXPECT_EQ(res.stats().num_rowsets(), 6);
         EXPECT_EQ(res.stats().num_segments(), 5);
         EXPECT_EQ(res.stats().data_size(), 50000);
+        EXPECT_EQ(res.stats().index_size(), 25000);
+        EXPECT_EQ(res.stats().segment_size(), 25000);
         TabletStatsPB tablet_stats;
         get_tablet_stats(meta_service.get(), tablet_id, tablet_stats);
         EXPECT_EQ(tablet_stats.num_rows(), 500);
         EXPECT_EQ(tablet_stats.num_rowsets(), 6);
         EXPECT_EQ(tablet_stats.num_segments(), 5);
         EXPECT_EQ(tablet_stats.data_size(), 50000);
+        EXPECT_EQ(tablet_stats.index_size(), 25000);
+        EXPECT_EQ(tablet_stats.segment_size(), 25000);
 
         ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
         // Check tmp rowsets
@@ -2046,10 +2090,14 @@ TEST(MetaServiceJobTest, ConcurrentCompactionTest) {
         compaction->add_output_rowset_ids(output_rowset.rowset_id_v2());
         compaction->set_output_cumulative_point(5);
         compaction->set_size_input_rowsets(30000);
+        compaction->set_index_size_input_rowsets(15000);
+        compaction->set_segment_size_input_rowsets(15000);
         compaction->set_num_input_rows(300);
         compaction->set_num_input_rowsets(3);
         compaction->set_num_input_segments(3);
         compaction->set_size_output_rowsets(10000);
+        compaction->set_index_size_output_rowsets(5000);
+        compaction->set_segment_size_output_rowsets(5000);
         compaction->set_num_output_rows(100);
         compaction->set_num_output_rowsets(1);
         compaction->set_num_output_segments(1);
@@ -2062,12 +2110,16 @@ TEST(MetaServiceJobTest, ConcurrentCompactionTest) {
         EXPECT_EQ(res.stats().num_rowsets(), 4);
         EXPECT_EQ(res.stats().num_segments(), 3);
         EXPECT_EQ(res.stats().data_size(), 30000);
+        EXPECT_EQ(res.stats().index_size(), 15000);
+        EXPECT_EQ(res.stats().segment_size(), 15000);
         TabletStatsPB tablet_stats;
         get_tablet_stats(meta_service.get(), tablet_id, tablet_stats);
         EXPECT_EQ(tablet_stats.num_rows(), 300);
         EXPECT_EQ(tablet_stats.num_rowsets(), 4);
         EXPECT_EQ(tablet_stats.num_segments(), 3);
         EXPECT_EQ(tablet_stats.data_size(), 30000);
+        EXPECT_EQ(tablet_stats.index_size(), 15000);
+        EXPECT_EQ(tablet_stats.segment_size(), 15000);
 
         ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
         // Check tmp rowsets
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 1f93b228c6de08..d2dd80f6871e3e 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -4429,6 +4429,8 @@ TEST(MetaServiceTest, GetTabletStatsTest) {
     EXPECT_EQ(res.tablet_stats(0).num_rows(), 0);
     EXPECT_EQ(res.tablet_stats(0).num_rowsets(), 1);
     EXPECT_EQ(res.tablet_stats(0).num_segments(), 0);
+    EXPECT_EQ(res.tablet_stats(0).index_size(), 0);
+    EXPECT_EQ(res.tablet_stats(0).segment_size(), 0);
     // Insert rowset
     config::split_tablet_stats = false;
     ASSERT_NO_FATAL_FAILURE(
@@ -4448,6 +4450,16 @@ TEST(MetaServiceTest, GetTabletStatsTest) {
                                &data_size_key);
     ASSERT_EQ(txn->get(data_size_key, &data_size_val), TxnErrorCode::TXN_OK);
     EXPECT_EQ(*(int64_t*)data_size_val.data(), 22000);
+    std::string index_size_key, index_size_val;
+    stats_tablet_index_size_key({mock_instance, table_id, index_id, partition_id, tablet_id},
+                                &index_size_key);
+    ASSERT_EQ(txn->get(index_size_key, &index_size_val), TxnErrorCode::TXN_OK);
+    EXPECT_EQ(*(int64_t*)index_size_val.data(), 2000);
+    std::string segment_size_key, segment_size_val;
+    stats_tablet_segment_size_key({mock_instance, table_id, index_id, partition_id, tablet_id},
+                                  &segment_size_key);
+    ASSERT_EQ(txn->get(segment_size_key, &segment_size_val), TxnErrorCode::TXN_OK);
+    EXPECT_EQ(*(int64_t*)segment_size_val.data(), 20000);
     std::string num_rows_key, num_rows_val;
     stats_tablet_num_rows_key({mock_instance, table_id, index_id, partition_id, tablet_id},
                               &num_rows_key);
@@ -4472,6 +4484,8 @@ TEST(MetaServiceTest, GetTabletStatsTest) {
     EXPECT_EQ(res.tablet_stats(0).num_rows(), 400);
     EXPECT_EQ(res.tablet_stats(0).num_rowsets(), 5);
     EXPECT_EQ(res.tablet_stats(0).num_segments(), 4);
+    EXPECT_EQ(res.tablet_stats(0).index_size(), 4000);
+    EXPECT_EQ(res.tablet_stats(0).segment_size(), 40000);
 }
 
 TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
@@ -7771,6 +7785,8 @@ TEST(MetaServiceTest, UpdateTmpRowsetTest) {
         // simulate that there are new segments added to this rowset
         rowset.set_num_segments(rowset.num_segments() + 3);
         rowset.set_num_rows(rowset.num_rows() + 1000);
+        rowset.set_total_disk_size(rowset.total_disk_size() + 11000);
+        rowset.set_index_disk_size(rowset.index_disk_size() + 1000);
         rowset.set_data_disk_size(rowset.data_disk_size() + 10000);
 
         ASSERT_NO_FATAL_FAILURE(update_tmp_rowset(meta_service.get(), rowset, res));
@@ -7789,6 +7805,8 @@ TEST(MetaServiceTest, UpdateTmpRowsetTest) {
         ASSERT_EQ(doris::BEGIN_PARTIAL_UPDATE, fetchedRowsetMeta.rowset_state());
         ASSERT_EQ(rowset.num_segments(), fetchedRowsetMeta.num_segments());
         ASSERT_EQ(rowset.num_rows(), fetchedRowsetMeta.num_rows());
+        ASSERT_EQ(rowset.total_disk_size(), fetchedRowsetMeta.total_disk_size());
+        ASSERT_EQ(rowset.index_disk_size(), fetchedRowsetMeta.index_disk_size());
         ASSERT_EQ(rowset.data_disk_size(), fetchedRowsetMeta.data_disk_size());
 
         ASSERT_NO_FATAL_FAILURE(commit_txn(meta_service.get(), db_id, txn_id, label));
@@ -7819,6 +7837,8 @@ TEST(MetaServiceTest, UpdateTmpRowsetTest) {
         // simulate that there are new segments added to this rowset
         rowset.set_num_segments(rowset.num_segments() + 3);
         rowset.set_num_rows(rowset.num_rows() + 1000);
+        rowset.set_total_disk_size(rowset.total_disk_size() + 11000);
+        rowset.set_index_disk_size(rowset.index_disk_size() + 1000);
         rowset.set_data_disk_size(rowset.data_disk_size() + 10000);
 
         // repeated calls to update_tmp_rowset will all success
@@ -7844,6 +7864,8 @@ TEST(MetaServiceTest, UpdateTmpRowsetTest) {
         ASSERT_EQ(doris::BEGIN_PARTIAL_UPDATE, fetchedRowsetMeta.rowset_state());
         ASSERT_EQ(rowset.num_segments(), fetchedRowsetMeta.num_segments());
         ASSERT_EQ(rowset.num_rows(), fetchedRowsetMeta.num_rows());
+        ASSERT_EQ(rowset.total_disk_size(), fetchedRowsetMeta.total_disk_size());
+        ASSERT_EQ(rowset.index_disk_size(), fetchedRowsetMeta.index_disk_size());
         ASSERT_EQ(rowset.data_disk_size(), fetchedRowsetMeta.data_disk_size());
 
         ASSERT_NO_FATAL_FAILURE(commit_txn(meta_service.get(), db_id, txn_id, label));
@@ -7869,6 +7891,8 @@ TEST(MetaServiceTest, UpdateTmpRowsetTest) {
         // simulate that there are new segments added to this rowset
         rowset.set_num_segments(rowset.num_segments() + 3);
         rowset.set_num_rows(rowset.num_rows() + 1000);
+        rowset.set_total_disk_size(rowset.total_disk_size() + 11000);
+        rowset.set_index_disk_size(rowset.index_disk_size() + 1000);
         rowset.set_data_disk_size(rowset.data_disk_size() + 10000);
 
         ASSERT_NO_FATAL_FAILURE(update_tmp_rowset(meta_service.get(), rowset, res));
diff --git a/cloud/test/schema_kv_test.cpp b/cloud/test/schema_kv_test.cpp
index 07f658175c806f..52e54f5e494b7f 100644
--- a/cloud/test/schema_kv_test.cpp
+++ b/cloud/test/schema_kv_test.cpp
@@ -210,6 +210,8 @@ TEST(DetachSchemaKVTest, TabletTest) {
         EXPECT_EQ(get_rowset_res.stats().num_rowsets(), 1);
         EXPECT_EQ(get_rowset_res.stats().num_segments(), 0);
         EXPECT_EQ(get_rowset_res.stats().data_size(), 0);
+        EXPECT_EQ(get_rowset_res.stats().index_size(), 0);
+        EXPECT_EQ(get_rowset_res.stats().segment_size(), 0);
     }
 
     // new MS batch create tablets with write_schema_kv=true
@@ -481,6 +483,8 @@ TEST(DetachSchemaKVTest, RowsetTest) {
         EXPECT_EQ(get_rowset_res.stats().num_rowsets(), 2);
         EXPECT_EQ(get_rowset_res.stats().num_segments(), 1);
         EXPECT_EQ(get_rowset_res.stats().data_size(), 11000);
+        EXPECT_EQ(get_rowset_res.stats().index_size(), 1000);
+        EXPECT_EQ(get_rowset_res.stats().segment_size(), 10000);
     }
 
     // new MS read rowsets committed by both old and new MS
@@ -530,6 +534,8 @@ TEST(DetachSchemaKVTest, RowsetTest) {
         EXPECT_EQ(get_rowset_res->stats().num_rowsets(), 26);
         EXPECT_EQ(get_rowset_res->stats().num_segments(), 25);
         EXPECT_EQ(get_rowset_res->stats().data_size(), 275000);
+        EXPECT_EQ(get_rowset_res->stats().index_size(), 25000);
+        EXPECT_EQ(get_rowset_res->stats().segment_size(), 250000);
         if (schema != nullptr) {
             auto schema_version = get_rowset_res->rowset_meta(10).schema_version();
             get_rowset_res->mutable_rowset_meta(10)->mutable_tablet_schema()->set_schema_version(3);
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 6e2a518ec20982..a372a993ac6f22 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -324,7 +324,8 @@ unsupportedShowStatement
         ((FROM | IN) database=multipartIdentifier)? wildWhere?
         sortClause? limitClause?                                                    #showAlterTable
     | SHOW DATA SKEW FROM baseTableRef                                              #showDataSkew
-    | SHOW DATA (FROM tableName=multipartIdentifier)? sortClause? propertyClause?   #showData
+    | SHOW DATA (ALL)? (FROM tableName=multipartIdentifier)?
+        sortClause? propertyClause?                                                 #showData
     | SHOW TEMPORARY? PARTITIONS FROM tableName=multipartIdentifier
         wildWhere? sortClause? limitClause?                                         #showPartitions
     | SHOW TABLET tabletId=INTEGER_VALUE                                            #showTabletId
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 03f1f02f158f84..05804343bc3417 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -1014,6 +1014,8 @@ nonterminal List<Expr> copy_select_expr_list;
 //genearted column
 nonterminal Boolean opt_generated_always;
 
+nonterminal Boolean opt_detailed;
+
 precedence nonassoc COMMA;
 precedence nonassoc STRING_LITERAL;
 precedence nonassoc KW_COLUMNS;
@@ -4493,13 +4495,13 @@ show_param ::=
         RESULT = new ShowDataSkewStmt(table_ref);
     :}
     /* Show data statement: used to show data size of specified range */
-     | KW_DATA order_by_clause:orderByClause opt_properties:prop
+     | KW_DATA opt_detailed:detailed order_by_clause:orderByClause opt_properties:prop
     {:
-        RESULT = new ShowDataStmt(null, orderByClause, prop);
+        RESULT = new ShowDataStmt(null, orderByClause, prop, detailed);
     :}
-    | KW_DATA KW_FROM table_name:dbTblName order_by_clause:orderByClause
+    | KW_DATA opt_detailed:detailed KW_FROM table_name:dbTblName order_by_clause:orderByClause
     {:
-        RESULT = new ShowDataStmt(dbTblName, orderByClause, null);
+        RESULT = new ShowDataStmt(dbTblName, orderByClause, null, detailed);
     :}
     | opt_tmp:tmp KW_PARTITIONS KW_FROM table_name:tblName opt_wild_where order_by_clause:orderByClause limit_clause: limitClause
     {:
@@ -5063,6 +5065,17 @@ cancel_param ::=
     :}
     ;
 
+opt_detailed ::=
+     /* empty */
+    {:
+        RESULT = false;
+    :}
+    | KW_ALL
+    {:
+        RESULT = true;
+    :}
+    ;
+
 // Delete stmt
 delete_stmt ::=
     KW_DELETE KW_FROM table_name:table opt_partition_names:partitionNames opt_table_alias:alias opt_using_clause:fromClause where_clause:wherePredicate
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
index 26b77e0b3d5d26..398b0d7ec05d42 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
@@ -100,26 +100,64 @@ public class ShowDataStmt extends ShowStmt implements NotFallbackInParser {
             new ImmutableList.Builder<String>().add("TableName").add("IndexName").add("Size").add("ReplicaCount")
                     .add("RowCount").add("RemoteSize").build();
 
+    private static final ShowResultSetMetaData SHOW_DETAILED_TABLE_DATA_META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("TableName", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("ReplicaCount", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("LocalTotalSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("LocalDataSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("LocalIndexSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("RemoteTotalSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("RemoteDataSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("RemoteIndexSize", ScalarType.createVarchar(30)))
+                    .build();
+
+    private static final ShowResultSetMetaData SHOW_DETAILED_INDEX_DATA_META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("TableName", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("IndexName", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("ReplicaCount", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("RowCount", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("LocalTotalSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("LocalDataSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("LocalIndexSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("RemoteTotalSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("RemoteDataSize", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("RemoteIndexSize", ScalarType.createVarchar(30)))
+                    .build();
+
     TableName tableName;
     String dbName;
-    List<List<String>> totalRows;
+    List<List<String>> totalRows = Lists.newArrayList();
     List<List<Object>> totalRowsObject = Lists.newArrayList();
 
-    private List<OrderByElement> orderByElements;
-    private List<OrderByPair> orderByPairs;
+    private List<OrderByElement> orderByElements = null;
+    private List<OrderByPair> orderByPairs = null;
 
-    private final Map<String, String> properties;
+    private Map<String, String> properties = null;
+    private boolean detailed = true;
 
     private static final String WAREHOUSE = "entire_warehouse";
     private static final String DB_LIST = "db_names";
 
-    public ShowDataStmt(TableName tableName, List<OrderByElement> orderByElements, Map<String, String> properties) {
+    private long totalSize = 0;
+    private long totalReplicaCount = 0;
+    private long totalRemoteSize = 0;
+    private long totalLocalInvertedSize = 0;
+    private long totalLocalSegmentSize = 0;
+    private long totalRemoteInvertedSize = 0;
+    private long totalRemoteSegmentSize = 0;
+
+    public ShowDataStmt(TableName tableName, List<OrderByElement> orderByElements,
+                                    Map<String, String> properties, boolean detailed) {
         this.tableName = tableName;
-        this.totalRows = Lists.newArrayList();
         this.orderByElements = orderByElements;
         this.properties = properties;
+        this.detailed = detailed;
     }
 
+    public ShowDataStmt() {}
+
     @Override
     public void analyze(Analyzer analyzer) throws UserException {
         super.analyze(analyzer);
@@ -156,208 +194,245 @@ public void analyze(Analyzer analyzer) throws UserException {
             }
         }
 
-        if (tableName == null) {
-            db.readLock();
-            try {
-                long totalSize = 0;
-                long totalReplicaCount = 0;
-                long totalRemoteSize = 0;
-                // sort by table name
-                List<Table> tables = db.getTables();
-                SortedSet<Table> sortedTables = new TreeSet<>(new Comparator<Table>() {
-                    @Override
-                    public int compare(Table t1, Table t2) {
-                        return t1.getName().compareTo(t2.getName());
-                    }
-                });
-
-                for (Table table : tables) {
-                    if (!Env.getCurrentEnv().getAccessManager()
-                            .checkTblPriv(ConnectContext.get(), InternalCatalog.INTERNAL_CATALOG_NAME, dbName,
-                                    table.getName(),
-                                    PrivPredicate.SHOW)) {
-                        continue;
-                    }
-                    sortedTables.add(table);
-                }
+        if (hasTable()) {
+            if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), tableName,
+                    PrivPredicate.SHOW)) {
+                ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW DATA",
+                        ConnectContext.get().getQualifiedUser(),
+                        ConnectContext.get().getRemoteIP(),
+                        dbName + ": " + tableName);
+            }
+            OlapTable table = (OlapTable) db.getTableOrMetaException(tableName.getTbl(), TableType.OLAP);
+            getSingleTableStats(table);
+        } else {
+            getSingleDbStats(db);
+        }
+        return;
+    }
 
-                for (Table table : sortedTables) {
-                    if (!table.isManagedTable()) {
-                        continue;
-                    }
+    private void collectDbStats(Database db) {
+        // sort by table name
+        List<Table> tables = db.getTables();
+        SortedSet<Table> sortedTables = new TreeSet<>(new Comparator<Table>() {
+            @Override
+            public int compare(Table t1, Table t2) {
+                return t1.getName().compareTo(t2.getName());
+            }
+        });
 
-                    OlapTable olapTable = (OlapTable) table;
-                    long tableSize = 0;
-                    long replicaCount = 0;
-                    long remoteSize = 0;
+        for (Table table : tables) {
+            if (!Env.getCurrentEnv().getAccessManager()
+                    .checkTblPriv(ConnectContext.get(), InternalCatalog.INTERNAL_CATALOG_NAME, dbName,
+                            table.getName(),
+                            PrivPredicate.SHOW)) {
+                continue;
+            }
+            sortedTables.add(table);
+        }
 
-                    tableSize = olapTable.getDataSize();
-                    replicaCount = olapTable.getReplicaCount();
-                    remoteSize = olapTable.getRemoteDataSize();
+        for (Table table : sortedTables) {
+            if (!table.isManagedTable()) {
+                continue;
+            }
 
-                    //|TableName|Size|ReplicaCount|RemoteSize
-                    List<Object> row = Arrays.asList(table.getName(), tableSize, replicaCount, remoteSize);
-                    totalRowsObject.add(row);
+            OlapTable olapTable = (OlapTable) table;
+            long tableSize = 0;
+            long replicaCount = 0;
+            long remoteSize = 0;
 
-                    totalSize += tableSize;
-                    totalReplicaCount += replicaCount;
-                    totalRemoteSize += remoteSize;
-                } // end for tables
+            tableSize = olapTable.getDataSize();
+            replicaCount = olapTable.getReplicaCount();
+            remoteSize = olapTable.getRemoteDataSize();
 
-                // sort by
-                if (orderByPairs != null && !orderByPairs.isEmpty()) {
-                    // k-> index, v-> isDesc
-                    Map<Integer, Boolean> sortMap = Maps.newLinkedHashMap();
-                    for (OrderByPair orderByPair : orderByPairs) {
-                        sortMap.put(orderByPair.getIndex(), orderByPair.isDesc());
+            if (!detailed) {
+                totalRowsObject.add(Arrays.asList(table.getName(), tableSize, replicaCount, remoteSize));
+            } else {
+                long localIndexSize = olapTable.getLocalIndexFileSize();
+                long localSegmentSize = olapTable.getLocalSegmentSize();
+                long remoteIndexSize = olapTable.getRemoteIndexFileSize();
+                long remoteSegmentSize = olapTable.getRemoteSegmentSize();
+                totalRowsObject.add(Arrays.asList(table.getName(), tableSize, replicaCount, remoteSize,
+                        localIndexSize, localSegmentSize, remoteIndexSize, remoteSegmentSize));
+                totalLocalInvertedSize += localIndexSize;
+                totalLocalSegmentSize += localSegmentSize;
+                totalRemoteInvertedSize += remoteIndexSize;
+                totalRemoteSegmentSize += remoteSegmentSize;
+            }
 
-                    }
-                    totalRowsObject.sort(sortRows(sortMap));
-                }
+            totalSize += tableSize;
+            totalReplicaCount += replicaCount;
+            totalRemoteSize += remoteSize;
+        } // end for tables
+    }
 
-                // for output
-                for (List<Object> row : totalRowsObject) {
-                    //|TableName|Size|ReplicaCount|RemoteSize
-                    Pair<Double, String> tableSizePair = DebugUtil.getByteUint((long) row.get(1));
-                    String readableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(tableSizePair.first) + " "
-                            + tableSizePair.second;
-                    Pair<Double, String> remoteSizePair = DebugUtil.getByteUint((long) row.get(3));
-                    String remoteReadableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(remoteSizePair.first) + " "
-                            + remoteSizePair.second;
-                    List<String> result = Arrays.asList(String.valueOf(row.get(0)),
-                            readableSize, String.valueOf(row.get(2)), remoteReadableSize);
-                    totalRows.add(result);
-                }
+    private void collectTableStats(OlapTable table) {
+        // sort by index name
+        Map<String, Long> indexNames = table.getIndexNameToId();
+        Map<String, Long> sortedIndexNames = new TreeMap<String, Long>();
+        for (Map.Entry<String, Long> entry : indexNames.entrySet()) {
+            sortedIndexNames.put(entry.getKey(), entry.getValue());
+        }
 
-                Pair<Double, String> totalSizePair = DebugUtil.getByteUint(totalSize);
-                String readableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalSizePair.first) + " "
-                        + totalSizePair.second;
-                Pair<Double, String> totalRemoteSizePair = DebugUtil.getByteUint(totalRemoteSize);
-                String remoteReadableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalRemoteSizePair.first) + " "
-                        + totalRemoteSizePair.second;
-                List<String> total = Arrays.asList("Total", readableSize, String.valueOf(totalReplicaCount),
-                         remoteReadableSize);
-                totalRows.add(total);
-
-                // quota
-                long quota = db.getDataQuota();
-                long replicaQuota = db.getReplicaQuota();
-                Pair<Double, String> quotaPair = DebugUtil.getByteUint(quota);
-                String readableQuota = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(quotaPair.first) + " "
-                        + quotaPair.second;
-
-                List<String> quotaRow = Arrays.asList("Quota", readableQuota, String.valueOf(replicaQuota), "");
-                totalRows.add(quotaRow);
-
-                // left
-                long left = Math.max(0, quota - totalSize);
-                long replicaCountLeft = Math.max(0, replicaQuota - totalReplicaCount);
-                Pair<Double, String> leftPair = DebugUtil.getByteUint(left);
-                String readableLeft = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(leftPair.first) + " "
-                        + leftPair.second;
-                List<String> leftRow = Arrays.asList("Left", readableLeft, String.valueOf(replicaCountLeft), "");
-                totalRows.add(leftRow);
-            } finally {
-                db.readUnlock();
+        for (Long indexId : sortedIndexNames.values()) {
+            long indexSize = 0;
+            long indexReplicaCount = 0;
+            long indexRowCount = 0;
+            long indexRemoteSize = 0;
+            long localIndexSize = 0;
+            long localSegmentSize = 0;
+            long remoteIndexSize = 0;
+            long remoteSegmentSize = 0;
+            for (Partition partition : table.getAllPartitions()) {
+                MaterializedIndex mIndex = partition.getIndex(indexId);
+                indexSize += mIndex.getDataSize(false);
+                indexReplicaCount += mIndex.getReplicaCount();
+                indexRowCount += mIndex.getRowCount() == -1 ? 0 : mIndex.getRowCount();
+                indexRemoteSize += mIndex.getRemoteDataSize();
+                localIndexSize += mIndex.getLocalIndexSize();
+                localSegmentSize += mIndex.getLocalSegmentSize();
+                remoteIndexSize += mIndex.getRemoteIndexSize();
+                remoteSegmentSize += mIndex.getRemoteSegmentSize();
             }
-        } else {
-            if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), tableName,
-                    PrivPredicate.SHOW)) {
-                ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW DATA",
-                        ConnectContext.get().getQualifiedUser(),
-                        ConnectContext.get().getRemoteIP(),
-                        dbName + ": " + tableName);
+
+            String indexName = table.getIndexNameById(indexId);
+            if (!detailed) {
+                totalRowsObject.add(Arrays.asList(tableName, indexName, indexSize, indexReplicaCount,
+                        indexRowCount, indexRemoteSize));
+            } else {
+                totalRowsObject.add(Arrays.asList(tableName, indexName, indexSize, indexReplicaCount, indexRowCount,
+                            indexRemoteSize, localIndexSize, localSegmentSize, remoteIndexSize, remoteSegmentSize));
             }
 
-            OlapTable olapTable = (OlapTable) db
-                    .getTableOrMetaException(tableName.getTbl(), TableType.OLAP);
-            long totalSize = 0;
-            long totalReplicaCount = 0;
-            long totalRemoteSize = 0;
-            olapTable.readLock();
-            try {
-                // sort by index name
-                Map<String, Long> indexNames = olapTable.getIndexNameToId();
-                Map<String, Long> sortedIndexNames = new TreeMap<String, Long>();
-                for (Map.Entry<String, Long> entry : indexNames.entrySet()) {
-                    sortedIndexNames.put(entry.getKey(), entry.getValue());
-                }
+            totalSize += indexSize;
+            totalReplicaCount += indexReplicaCount;
+            totalRemoteSize += indexRemoteSize;
+            totalLocalInvertedSize += localIndexSize;
+            totalLocalSegmentSize += localSegmentSize;
+            totalRemoteInvertedSize += remoteIndexSize;
+            totalRemoteSegmentSize += remoteSegmentSize;
+        } // end for indices
+    }
 
-                for (Long indexId : sortedIndexNames.values()) {
-                    long indexSize = 0;
-                    long indexReplicaCount = 0;
-                    long indexRowCount = 0;
-                    long indexRemoteSize = 0;
-                    for (Partition partition : olapTable.getAllPartitions()) {
-                        MaterializedIndex mIndex = partition.getIndex(indexId);
-                        indexSize += mIndex.getDataSize(false);
-                        indexReplicaCount += mIndex.getReplicaCount();
-                        indexRowCount += mIndex.getRowCount() == -1 ? 0 : mIndex.getRowCount();
-                        indexRemoteSize += mIndex.getRemoteDataSize();
-                    }
+    private void sortResult() {
+        if (orderByPairs != null && !orderByPairs.isEmpty()) {
+            // k-> index, v-> isDesc
+            Map<Integer, Boolean> sortMap = Maps.newLinkedHashMap();
+            for (OrderByPair orderByPair : orderByPairs) {
+                sortMap.put(orderByPair.getIndex(), orderByPair.isDesc());
 
-                    String indexName = olapTable.getIndexNameById(indexId);
-                    // .add("TableName").add("IndexName").add("Size").add("ReplicaCount").add("RowCount")
-                    //      .add("RemoteSize")
-                    List<Object> row = Arrays.asList(tableName, indexName, indexSize, indexReplicaCount,
-                             indexRowCount, indexRemoteSize);
-                    totalRowsObject.add(row);
-
-                    totalSize += indexSize;
-                    totalReplicaCount += indexReplicaCount;
-                    totalRemoteSize += indexRemoteSize;
-                } // end for indices
-
-                // sort by
-                if (orderByPairs != null && !orderByPairs.isEmpty()) {
-                    // k-> index, v-> isDesc
-                    Map<Integer, Boolean> sortMap = Maps.newLinkedHashMap();
-                    for (OrderByPair orderByPair : orderByPairs) {
-                        sortMap.put(orderByPair.getIndex(), orderByPair.isDesc());
+            }
+            totalRowsObject.sort(sortRows(sortMap));
+        }
+    }
 
-                    }
-                    totalRowsObject.sort(sortRows(sortMap));
-                }
+    private void buildDbStatsOutput(long quota, long replicaQuota) {
+        // for output
+        for (List<Object> row : totalRowsObject) {
+            if (!detailed) {
+                totalRows.add(Arrays.asList(String.valueOf(row.get(0)),
+                        DebugUtil.printByteWithUnit((long) row.get(1)), String.valueOf(row.get(2)),
+                        DebugUtil.printByteWithUnit((long) row.get(3))));
+            } else {
+                totalRows.add(Arrays.asList(String.valueOf(row.get(0)), String.valueOf(row.get(2)),
+                        DebugUtil.printByteWithUnit((long) row.get(1)), DebugUtil.printByteWithUnit((long) row.get(5)),
+                        DebugUtil.printByteWithUnit((long) row.get(4)), DebugUtil.printByteWithUnit((long) row.get(3)),
+                        DebugUtil.printByteWithUnit((long) row.get(7)),
+                        DebugUtil.printByteWithUnit((long) row.get(6))));
+            }
+        }
 
-                // for output
-                for (int index = 0; index <= totalRowsObject.size() - 1; index++) {
-                    //| TableName| IndexName | Size | ReplicaCount | RowCount | RemoteSize
-                    List<Object> row = totalRowsObject.get(index);
-                    List<String> result;
-                    Pair<Double, String> tableSizePair = DebugUtil.getByteUint((long) row.get(2));
-                    String readableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(tableSizePair.first)
-                            + " " + tableSizePair.second;
-                    Pair<Double, String> remoteSizePair = DebugUtil.getByteUint((long) row.get(5));
-                    String remoteReadableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(remoteSizePair.first) + " "
-                            + remoteSizePair.second;
-                    if (index == 0) {
-                        result = Arrays.asList(tableName.getTbl(), String.valueOf(row.get(1)),
-                                readableSize, String.valueOf(row.get(3)),
-                                String.valueOf(row.get(4)), remoteReadableSize);
-                    } else {
-                        result = Arrays.asList("", String.valueOf(row.get(1)),
-                                readableSize, String.valueOf(row.get(3)),
-                                String.valueOf(row.get(4)), remoteReadableSize);
-                    }
-                    totalRows.add(result);
-                }
+        long left = Math.max(0, quota - totalSize);
+        long replicaCountLeft = Math.max(0, replicaQuota - totalReplicaCount);
 
-                Pair<Double, String> totalSizePair = DebugUtil.getByteUint(totalSize);
-                String readableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalSizePair.first) + " "
-                        + totalSizePair.second;
-                Pair<Double, String> totalRemoteSizePair = DebugUtil.getByteUint(totalRemoteSize);
-                String remoteReadableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalRemoteSizePair.first) + " "
-                        + totalRemoteSizePair.second;
-                List<String> row = Arrays.asList("", "Total", readableSize, String.valueOf(totalReplicaCount), "",
-                         remoteReadableSize);
-                totalRows.add(row);
-            } finally {
-                olapTable.readUnlock();
+        if (!detailed) {
+            totalRows.add(Arrays.asList("Total", DebugUtil.printByteWithUnit(totalSize),
+                    String.valueOf(totalReplicaCount), DebugUtil.printByteWithUnit(totalRemoteSize)));
+            totalRows.add(Arrays.asList("Quota", DebugUtil.printByteWithUnit(quota),
+                                                                    String.valueOf(replicaQuota), ""));
+            totalRows.add(Arrays.asList("Left", DebugUtil.printByteWithUnit(left),
+                                                                String.valueOf(replicaCountLeft), ""));
+        } else {
+            totalRows.add(Arrays.asList("Total", String.valueOf(totalReplicaCount),
+                    DebugUtil.printByteWithUnit(totalSize),
+                    DebugUtil.printByteWithUnit(totalLocalSegmentSize),
+                    DebugUtil.printByteWithUnit(totalLocalInvertedSize),
+                    DebugUtil.printByteWithUnit(totalRemoteSize),
+                    DebugUtil.printByteWithUnit(totalRemoteSegmentSize),
+                    DebugUtil.printByteWithUnit(totalRemoteInvertedSize)));
+            totalRows.add(Arrays.asList("Quota", String.valueOf(replicaQuota),
+                                                DebugUtil.printByteWithUnit(quota), "", "", "", "", ""));
+            totalRows.add(Arrays.asList("Left", String.valueOf(replicaCountLeft),
+                                                DebugUtil.printByteWithUnit(left), "", "", "", "", ""));
+        }
+    }
+
+    private void buildTableStatsOutput() {
+        for (int index = 0; index < totalRowsObject.size(); index++) {
+            List<Object> row = totalRowsObject.get(index);
+            String indexName = index == 0 ? tableName.getTbl() : "";
+            if (!detailed) {
+                totalRows.add(Arrays.asList(indexName, String.valueOf(row.get(1)),
+                        DebugUtil.printByteWithUnit((long) row.get(2)), String.valueOf(row.get(3)),
+                        String.valueOf(row.get(4)), DebugUtil.printByteWithUnit((long) row.get(5))));
+            } else {
+                totalRows.add(Arrays.asList(indexName, String.valueOf(row.get(1)),
+                        String.valueOf(row.get(3)), String.valueOf(row.get(4)),
+                        DebugUtil.printByteWithUnit((long) row.get(2)), DebugUtil.printByteWithUnit((long) row.get(7)),
+                        DebugUtil.printByteWithUnit((long) row.get(6)), DebugUtil.printByteWithUnit((long) row.get(5)),
+                        DebugUtil.printByteWithUnit((long) row.get(9)),
+                        DebugUtil.printByteWithUnit((long) row.get(8))));
             }
         }
+
+        // Total
+        if (!detailed) {
+            totalRows.add(Arrays.asList("", "Total", DebugUtil.printByteWithUnit(totalSize),
+                    String.valueOf(totalReplicaCount), "", DebugUtil.printByteWithUnit(totalRemoteSize)));
+        } else {
+            totalRows.add(Arrays.asList("", "Total", String.valueOf(totalReplicaCount), "",
+                    DebugUtil.printByteWithUnit(totalSize), DebugUtil.printByteWithUnit(totalLocalSegmentSize),
+                    DebugUtil.printByteWithUnit(totalLocalInvertedSize),
+                    DebugUtil.printByteWithUnit(totalRemoteSize),
+                    DebugUtil.printByteWithUnit(totalRemoteSegmentSize),
+                    DebugUtil.printByteWithUnit(totalRemoteInvertedSize)));
+        }
+    }
+
+    // |TableName|Size|ReplicaCount|RemoteSize|
+    // |TableName|ReplicaCount|LocalTotalSize|LocalDataSize|LocalIndexSize|
+    //                        |RemoteTotalSize|RemoteDataSize|RemoteIndexSize|
+    private void getSingleDbStats(Database db) {
+        db.readLock();
+        long quota = 0;
+        long replicaQuota = 0;
+        try {
+            collectDbStats(db);
+            quota = db.getDataQuota();
+            replicaQuota = db.getReplicaQuota();
+        } finally {
+            db.readUnlock();
+        }
+        // sort by
+        sortResult();
+        buildDbStatsOutput(quota, replicaQuota);
     }
 
-    public static int analyzeColumn(String columnName, String tableName) throws AnalysisException {
+    // |TableName|IndexName|Size|ReplicaCount|RowCount|RemoteSize|
+    // |TableName|IndexName|ReplicaCount||RowCount|LocalTotalSize |LocalDataSize |LocalIndexSize|
+    //                                            |RemoteTotalSize|RemoteDataSize|RemoteIndexSize|
+    private void getSingleTableStats(OlapTable table) {
+        table.readLock();
+        try {
+            collectTableStats(table);
+        } finally {
+            table.readUnlock();
+        }
+        // sort by
+        sortResult();
+        buildTableStatsOutput();
+    }
+
+    private int analyzeColumn(String columnName, String tableName) throws AnalysisException {
         ImmutableList<String> titles = SHOW_TABLE_DATA_META_DATA_ORIGIN;
         if (tableName != null) {
             titles = SHOW_INDEX_DATA_META_DATA_ORIGIN;
@@ -371,7 +446,7 @@ public static int analyzeColumn(String columnName, String tableName) throws Anal
         throw new AnalysisException("Title name[" + columnName + "] does not exist");
     }
 
-    private static Comparator<List<Object>> sortRows(Map<Integer, Boolean> sortMap) {
+    private Comparator<List<Object>> sortRows(Map<Integer, Boolean> sortMap) {
         Ordering ordering = Ordering.natural();
 
         return new Comparator<List<Object>>() {
@@ -400,28 +475,41 @@ public List<List<String>> getResultRows() throws AnalysisException {
 
     @Override
     public ShowResultSetMetaData getMetaData() {
-        String value = null;
         if (properties != null) {
-            value = properties.get(WAREHOUSE);
-        }
-        if (value != null && value.equals("true")) {
-            return SHOW_WAREHOUSE_DATA_META_DATA;
+            String value = properties.get(WAREHOUSE);
+            if (value != null && value.equals("true")) {
+                return SHOW_WAREHOUSE_DATA_META_DATA;
+            }
         }
 
         if (Strings.isNullOrEmpty(dbName)) {
             return SHOW_DATABASE_DATA_META_DATA;
         }
-        if (tableName != null) {
-            return SHOW_INDEX_DATA_META_DATA;
+
+        if (hasTable()) {
+            if (!detailed) {
+                return SHOW_INDEX_DATA_META_DATA;
+            } else {
+                return SHOW_DETAILED_INDEX_DATA_META_DATA;
+            }
         } else {
-            return SHOW_TABLE_DATA_META_DATA;
+            if (!detailed) {
+                return SHOW_TABLE_DATA_META_DATA;
+            } else {
+                return SHOW_DETAILED_TABLE_DATA_META_DATA;
+            }
         }
     }
 
     @Override
     public String toSql() {
         StringBuilder builder = new StringBuilder();
-        builder.append("SHOW DATA");
+        builder.append("SHOW ");
+        if (detailed) {
+            builder.append("DETAILED DATA");
+        } else {
+            builder.append("DATA");
+        }
 
         if (tableName != null) {
             builder.append(" FROM ");
@@ -445,6 +533,7 @@ public String toString() {
         return toSql();
     }
 
+    // |DBName|DataSize|RecycleSize|
     private boolean getDbStatsByProperties() {
         if (properties == null) {
             return false;
@@ -507,6 +596,7 @@ private boolean getDbStatsByProperties() {
         return false;
     }
 
+    // |DbId|DbName|Size|RemoteSize|RecycleSize|RecycleRemoteSize|
     private void getAllDbStats() throws AnalysisException {
         // check auth
         if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
index 309b433572c942..3babb0e001a204 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
@@ -165,6 +165,8 @@ protected void runAfterCatalogReady() {
 
                 Long tableDataSize = 0L;
                 Long tableTotalReplicaDataSize = 0L;
+                Long tableTotalLocalIndexSize = 0L;
+                Long tableTotalLocalSegmentSize = 0L;
 
                 Long tableReplicaCount = 0L;
 
@@ -185,6 +187,8 @@ protected void runAfterCatalogReady() {
                                 long tabletRowsetCount = 0L;
                                 long tabletSegmentCount = 0L;
                                 long tabletRowCount = 0L;
+                                long tabletIndexSize = 0L;
+                                long tabletSegmentSize = 0L;
 
                                 for (Replica replica : tablet.getReplicas()) {
                                     if (replica.getDataSize() > tabletDataSize) {
@@ -204,6 +208,13 @@ protected void runAfterCatalogReady() {
                                         tabletSegmentCount = replica.getSegmentCount();
                                     }
 
+                                    if (replica.getLocalInvertedIndexSize() > tabletIndexSize) {
+                                        tabletIndexSize = replica.getLocalInvertedIndexSize();
+                                    }
+                                    if (replica.getLocalSegmentSize() > tabletSegmentSize) {
+                                        tabletSegmentSize = replica.getLocalSegmentSize();
+                                    }
+
                                     tableReplicaCount++;
                                 }
 
@@ -214,6 +225,8 @@ protected void runAfterCatalogReady() {
 
                                 tableRowsetCount += tabletRowsetCount;
                                 tableSegmentCount += tabletSegmentCount;
+                                tableTotalLocalIndexSize += tabletIndexSize;
+                                tableTotalLocalSegmentSize += tabletSegmentSize;
                             } // end for tablets
                             index.setRowCountReported(true);
                             index.setRowCount(indexRowCount);
@@ -223,7 +236,8 @@ protected void runAfterCatalogReady() {
                     //  this is only one thread to update table statistics, readLock is enough
                     olapTable.setStatistics(new OlapTable.Statistics(db.getName(),
                             table.getName(), tableDataSize, tableTotalReplicaDataSize, 0L,
-                            tableReplicaCount, tableRowCount, tableRowsetCount, tableSegmentCount));
+                            tableReplicaCount, tableRowCount, tableRowsetCount, tableSegmentCount,
+                            tableTotalLocalIndexSize, tableTotalLocalSegmentSize, 0L, 0L));
                     LOG.debug("finished to set row num for table: {} in database: {}",
                              table.getName(), db.getFullName());
                 } finally {
@@ -232,7 +246,7 @@ protected void runAfterCatalogReady() {
 
                 newCloudTableStatsMap.put(Pair.of(dbId, table.getId()), new OlapTable.Statistics(db.getName(),
                         table.getName(), tableDataSize, tableTotalReplicaDataSize, 0L,
-                        tableReplicaCount, tableRowCount, tableRowsetCount, tableSegmentCount));
+                        tableReplicaCount, tableRowCount, tableRowsetCount, tableSegmentCount, 0L, 0L, 0L, 0L));
             }
         }
         this.cloudTableStatsMap = newCloudTableStatsMap;
@@ -253,6 +267,8 @@ private void updateTabletStat(GetTabletStatsResponse response) {
                 replica.setRowsetCount(stat.getNumRowsets());
                 replica.setSegmentCount(stat.getNumSegments());
                 replica.setRowCount(stat.getNumRows());
+                replica.setLocalInvertedIndexSize(stat.getIndexSize());
+                replica.setLocalSegmentSize(stat.getSegmentSize());
             }
         }
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
index 6e4f6a2248e719..5e367b538cd74c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
@@ -197,6 +197,46 @@ public long getReplicaCount() {
         return replicaCount;
     }
 
+    public long getLocalIndexSize() {
+        long localIndexSize = 0;
+        for (Tablet tablet : getTablets()) {
+            for (Replica replica : tablet.getReplicas()) {
+                localIndexSize += replica.getLocalInvertedIndexSize();
+            }
+        }
+        return localIndexSize;
+    }
+
+    public long getLocalSegmentSize() {
+        long localSegmentSize = 0;
+        for (Tablet tablet : getTablets()) {
+            for (Replica replica : tablet.getReplicas()) {
+                localSegmentSize += replica.getLocalSegmentSize();
+            }
+        }
+        return localSegmentSize;
+    }
+
+    public long getRemoteIndexSize() {
+        long remoteIndexSize = 0;
+        for (Tablet tablet : getTablets()) {
+            for (Replica replica : tablet.getReplicas()) {
+                remoteIndexSize += replica.getRemoteInvertedIndexSize();
+            }
+        }
+        return remoteIndexSize;
+    }
+
+    public long getRemoteSegmentSize() {
+        long remoteSegmentSize = 0;
+        for (Tablet tablet : getTablets()) {
+            for (Replica replica : tablet.getReplicas()) {
+                remoteSegmentSize += replica.getRemoteSegmentSize();
+            }
+        }
+        return remoteSegmentSize;
+    }
+
     public int getTabletOrderIdx(long tabletId) {
         int idx = 0;
         for (Tablet tablet : tablets) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 8925e483c29df7..50b1dc6ab3954f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1688,11 +1688,22 @@ public long getAvgRowLength() {
     public long getDataLength() {
         long dataSize = 0;
         for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
-            dataSize += entry.getValue().getBaseIndex().getDataSize(false);
+            dataSize += entry.getValue().getBaseIndex().getLocalSegmentSize();
+            dataSize += entry.getValue().getBaseIndex().getRemoteSegmentSize();
         }
         return dataSize;
     }
 
+    @Override
+    public long getIndexLength() {
+        long indexSize = 0;
+        for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
+            indexSize += entry.getValue().getBaseIndex().getLocalIndexSize();
+            indexSize += entry.getValue().getBaseIndex().getRemoteIndexSize();
+        }
+        return indexSize;
+    }
+
     // Get the signature string of this table with specified partitions.
     // This method is used to determine whether the tables have the same schema.
     // Contains:
@@ -3387,6 +3398,18 @@ public static class Statistics {
         @Getter
         private Long segmentCount;
 
+        @Getter
+        private Long localInvertedIndexSize;   // multi replicas
+
+        @Getter
+        private Long localSegmentSize;         // multi replicas
+
+        @Getter
+        private Long remoteInvertedIndexSize;  // single replica
+
+        @Getter
+        private Long remoteSegmentSize;        // single replica
+
         public Statistics() {
             this.dbName = null;
             this.tableName = null;
@@ -3401,13 +3424,18 @@ public Statistics() {
             this.rowCount = 0L;
             this.rowsetCount = 0L;
             this.segmentCount = 0L;
-
+            this.localInvertedIndexSize = 0L;
+            this.localSegmentSize = 0L;
+            this.remoteInvertedIndexSize = 0L;
+            this.remoteSegmentSize = 0L;
         }
 
         public Statistics(String dbName, String tableName,
                 Long dataSize, Long totalReplicaDataSize,
                 Long remoteDataSize, Long replicaCount, Long rowCount,
-                Long rowsetCount, Long segmentCount) {
+                Long rowsetCount, Long segmentCount,
+                Long localInvertedIndexSize, Long localSegmentSize,
+                Long remoteInvertedIndexSize, Long remoteSegmentSize) {
 
             this.dbName = dbName;
             this.tableName = tableName;
@@ -3422,6 +3450,11 @@ public Statistics(String dbName, String tableName,
             this.rowCount = rowCount;
             this.rowsetCount = rowsetCount;
             this.segmentCount = segmentCount;
+
+            this.localInvertedIndexSize = localInvertedIndexSize;
+            this.localSegmentSize = localSegmentSize;
+            this.remoteInvertedIndexSize = remoteInvertedIndexSize;
+            this.remoteSegmentSize = remoteSegmentSize;
         }
     }
 
@@ -3445,6 +3478,22 @@ public long getReplicaCount() {
         return statistics.getReplicaCount();
     }
 
+    public long getLocalIndexFileSize() {
+        return statistics.getLocalInvertedIndexSize();
+    }
+
+    public long getLocalSegmentSize() {
+        return statistics.getLocalSegmentSize();
+    }
+
+    public long getRemoteIndexFileSize() {
+        return statistics.getRemoteInvertedIndexSize();
+    }
+
+    public long getRemoteSegmentSize() {
+        return statistics.getRemoteSegmentSize();
+    }
+
     public boolean isShadowIndex(long indexId) {
         String indexName = getIndexNameById(indexId);
         if (indexName != null && indexName.startsWith(org.apache.doris.alter.SchemaChangeHandler.SHADOW_NAME_PREFIX)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
index e86bd25cb5e926..d248cc40839988 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
@@ -26,6 +26,8 @@
 import org.apache.doris.thrift.TUniqueId;
 
 import com.google.gson.annotations.SerializedName;
+import lombok.Getter;
+import lombok.Setter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -122,6 +124,23 @@ public static class ReplicaContext {
     @SerializedName(value = "lsvh", alternate = {"lastSuccessVersionHash"})
     private long lastSuccessVersionHash = 0L;
 
+    @Setter
+    @Getter
+    @SerializedName(value = "lis", alternate = {"localInvertedIndexSize"})
+    private Long localInvertedIndexSize = 0L;
+    @Setter
+    @Getter
+    @SerializedName(value = "lss", alternate = {"localSegmentSize"})
+    private Long localSegmentSize = 0L;
+    @Setter
+    @Getter
+    @SerializedName(value = "ris", alternate = {"remoteInvertedIndexSize"})
+    private Long remoteInvertedIndexSize = 0L;
+    @Setter
+    @Getter
+    @SerializedName(value = "rss", alternate = {"remoteSegmentSize"})
+    private Long remoteSegmentSize = 0L;
+
     private volatile long totalVersionCount = -1;
     private volatile long visibleVersionCount = -1;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index ed37a936e10819..0f1d91087d2988 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -446,6 +446,9 @@ public long getDataLength() {
         return 0;
     }
 
+    public long getIndexLength() {
+        return 0;
+    }
 
     public TTableDescriptor toThrift() {
         return null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index 8f6e924f44a54d..4761ac9d86db2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -174,6 +174,8 @@ default int getBaseColumnIdxByName(String colName) {
 
     long getAvgRowLength();
 
+    long getIndexLength();
+
     long getLastCheckTime();
 
     String getComment(boolean escapeQuota);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index aa46c362e38ba9..f79ed89215b4d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -114,6 +114,11 @@ protected void runAfterCatalogReady() {
                 Long tableDataSize = 0L;
                 Long tableTotalReplicaDataSize = 0L;
 
+                Long tableTotalLocalIndexSize = 0L;
+                Long tableTotalLocalSegmentSize = 0L;
+                Long tableTotalRemoteIndexSize = 0L;
+                Long tableTotalRemoteSegmentSize = 0L;
+
                 Long tableRemoteDataSize = 0L;
 
                 Long tableReplicaCount = 0L;
@@ -171,6 +176,10 @@ protected void runAfterCatalogReady() {
                                         tabletRemoteDataSize = replica.getRemoteDataSize();
                                     }
                                     tableReplicaCount++;
+                                    tableTotalLocalIndexSize += replica.getLocalInvertedIndexSize();
+                                    tableTotalLocalSegmentSize += replica.getLocalSegmentSize();
+                                    tableTotalRemoteIndexSize += replica.getRemoteInvertedIndexSize();
+                                    tableTotalRemoteSegmentSize += replica.getRemoteSegmentSize();
                                 }
 
                                 tableDataSize += tabletDataSize;
@@ -196,7 +205,9 @@ protected void runAfterCatalogReady() {
                     // this is only one thread to update table statistics, readLock is enough
                     olapTable.setStatistics(new OlapTable.Statistics(db.getName(), table.getName(),
                             tableDataSize, tableTotalReplicaDataSize,
-                            tableRemoteDataSize, tableReplicaCount, tableRowCount, 0L, 0L));
+                            tableRemoteDataSize, tableReplicaCount, tableRowCount, 0L, 0L,
+                            tableTotalLocalIndexSize, tableTotalLocalSegmentSize,
+                            tableTotalRemoteIndexSize, tableTotalRemoteSegmentSize));
 
                     if (LOG.isDebugEnabled()) {
                         LOG.debug("finished to set row num for table: {} in database: {}",
@@ -220,6 +231,10 @@ private void updateTabletStat(Long beId, TTabletStatResult result) {
                     if (replica != null) {
                         replica.setDataSize(stat.getDataSize());
                         replica.setRemoteDataSize(stat.getRemoteDataSize());
+                        replica.setLocalInvertedIndexSize(stat.getLocalIndexSize());
+                        replica.setLocalSegmentSize(stat.getLocalSegmentSize());
+                        replica.setRemoteInvertedIndexSize(stat.getRemoteIndexSize());
+                        replica.setRemoteSegmentSize(stat.getRemoteSegmentSize());
                         replica.setRowCount(stat.getRowCount());
                         replica.setTotalVersionCount(stat.getTotalVersionCount());
                         replica.setVisibleVersionCount(stat.isSetVisibleVersionCount() ? stat.getVisibleVersionCount()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 6c72d0a67d4732..bd1e36e7bc968b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -239,6 +239,11 @@ public long getDataLength() {
         return 0;
     }
 
+    @Override
+    public long getIndexLength() {
+        return 0;
+    }
+
     @Override
     public long getCreateTime() {
         return 0;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index e35fd2dc852322..cf9e8e82ce112f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -659,6 +659,7 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE
                             status.setRows(table.getCachedRowCount());
                             status.setDataLength(table.getDataLength());
                             status.setAvgRowLength(table.getAvgRowLength());
+                            status.setIndexLength(table.getIndexLength());
                             tablesResult.add(status);
                         } finally {
                             table.readUnlock();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowDataStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowDataStmtTest.java
index 9b421de0c397bd..83d230274f6f29 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowDataStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowDataStmtTest.java
@@ -139,7 +139,7 @@ public void setUp() throws UserException {
 
     @Test
     public void testNormal() throws AnalysisException, UserException {
-        ShowDataStmt stmt = new ShowDataStmt(null, null, null);
+        ShowDataStmt stmt = new ShowDataStmt(null, null, null, false);
         stmt.analyze(analyzer);
         Assert.assertEquals("SHOW DATA", stmt.toString());
         Assert.assertEquals(4, stmt.getMetaData().getColumnCount());
@@ -151,7 +151,7 @@ public void testNormal() throws AnalysisException, UserException {
         OrderByElement orderByElementTwo = new OrderByElement(slotRefTwo, false, false);
 
         stmt = new ShowDataStmt(new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, "testDb", "test_tbl"),
-                Arrays.asList(orderByElementOne, orderByElementTwo), null);
+                Arrays.asList(orderByElementOne, orderByElementTwo), null, false);
         stmt.analyze(analyzer);
         Assert.assertEquals(
                 "SHOW DATA FROM `testDb`.`test_tbl` ORDER BY `ReplicaCount` DESC, `Size` DESC",
@@ -159,7 +159,7 @@ public void testNormal() throws AnalysisException, UserException {
         Assert.assertEquals(6, stmt.getMetaData().getColumnCount());
         Assert.assertEquals(true, stmt.hasTable());
 
-        stmt = new ShowDataStmt(null, Arrays.asList(orderByElementOne, orderByElementTwo), null);
+        stmt = new ShowDataStmt(null, Arrays.asList(orderByElementOne, orderByElementTwo), null, false);
         stmt.analyze(analyzer);
         Assert.assertEquals("SHOW DATA ORDER BY `ReplicaCount` DESC, `Size` DESC",
                 stmt.toString());
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index 381075074335c5..c113868a2c3286 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -544,6 +544,10 @@ message TabletCompactionJobPB {
     optional int64 delete_bitmap_lock_initiator = 24;
     optional int64 full_compaction_cnt = 25;  // prepare
     optional bool check_input_versions_range = 26;
+    optional int64 index_size_input_rowsets = 27;
+    optional int64 segment_size_input_rowsets = 28;
+    optional int64 index_size_output_rowsets = 29;
+    optional int64 segment_size_output_rowsets = 30;
 }
 
 message TabletSchemaChangeJobPB {
@@ -563,6 +567,8 @@ message TabletSchemaChangeJobPB {
     optional int64 output_cumulative_point = 12;
     optional bool is_inverted_index_change = 13 [default = false];
     optional int64 delete_bitmap_lock_initiator = 14;
+    optional int64 index_size_output_rowsets = 15;
+    optional int64 segment_size_output_rowsets = 16;
 }
 
 message TabletJobInfoPB {
@@ -576,7 +582,7 @@ message TabletJobInfoPB {
 //==============================================================================
 message TabletStatsPB {
     optional TabletIndexPB idx = 1;
-    optional int64 data_size = 2;
+    optional int64 data_size = 2; // data_size = index_size + segment_size
     optional int64 num_rows = 3;
     optional int64 num_rowsets = 4;
     optional int64 num_segments = 5;
@@ -588,6 +594,8 @@ message TabletStatsPB {
     optional int64 last_cumu_compaction_time_ms = 11;
     optional int64 full_compaction_cnt = 12;
     optional int64 last_full_compaction_time_ms = 13;
+    optional int64 index_size = 14;
+    optional int64 segment_size = 15; 
 }
 
 message ObjectFilePB {
diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift
index 7f073b2b71095a..aed248adfe891a 100644
--- a/gensrc/thrift/BackendService.thrift
+++ b/gensrc/thrift/BackendService.thrift
@@ -32,13 +32,18 @@ struct TExportTaskRequest {
 
 struct TTabletStat {
     1: required i64 tablet_id
-    // local data size
+    // local data size = local inverted index file size + local segment file size
     2: optional i64 data_size
     3: optional i64 row_count
     4: optional i64 total_version_count
+    // remote data size = remote inverted index file size + remote segment file size
     5: optional i64 remote_data_size
     6: optional i64 visible_version_count
     7: optional i64 visible_version
+    8: optional i64 local_index_size = 0      // .idx
+    9: optional i64 local_segment_size = 0    // .dat
+    10: optional i64 remote_index_size = 0    // .idx
+    11: optional i64 remote_segment_size = 0  // .dat
 }
 
 struct TTabletStatResult {
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index e2af8937425d0c..246cc999562934 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -350,6 +350,7 @@ struct TTableStatus {
     11: optional i64 rows;
     12: optional i64 avg_row_length
     13: optional i64 data_length;
+    14: optional i64 index_length;
 }
 
 struct TListTableStatusResult {
diff --git a/gensrc/thrift/MasterService.thrift b/gensrc/thrift/MasterService.thrift
index 9d8cd9111ba5c1..a045fa3adcb5bd 100644
--- a/gensrc/thrift/MasterService.thrift
+++ b/gensrc/thrift/MasterService.thrift
@@ -47,6 +47,10 @@ struct TTabletInfo {
     19: optional i64 cooldown_term
     20: optional Types.TUniqueId cooldown_meta_id
     21: optional i64 visible_version_count
+    22: optional i64 local_index_size = 0      // .idx
+    23: optional i64 local_segment_size = 0    // .dat
+    24: optional i64 remote_index_size = 0     // .idx
+    25: optional i64 remote_segment_size = 0   // .dat
 
     // For cloud
     1000: optional bool is_persistent
diff --git a/regression-test/data/nereids_p0/system/test_query_sys_tables.out b/regression-test/data/nereids_p0/system/test_query_sys_tables.out
index a43229a87adead..16b71543a11302 100644
--- a/regression-test/data/nereids_p0/system/test_query_sys_tables.out
+++ b/regression-test/data/nereids_p0/system/test_query_sys_tables.out
@@ -25,9 +25,9 @@ internal	test_query_sys_db_5	\N
 internal	test_query_sys_db_6	\N
 
 -- !tables --
-internal	test_query_sys_tb_4	BASE TABLE	0	\N	\N
-internal	test_query_sys_tb_5	BASE TABLE	0	\N	\N
-internal	test_query_sys_tb_6	BASE TABLE	0	\N	\N
+internal	test_query_sys_tb_4	BASE TABLE	0	\N	0
+internal	test_query_sys_tb_5	BASE TABLE	0	\N	0
+internal	test_query_sys_tb_6	BASE TABLE	0	\N	0
 
 -- !session_variables --
 wait_timeout	30000
diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out
index cdd89914994c63..0100314ef799a1 100644
--- a/regression-test/data/query_p0/system/test_query_sys_tables.out
+++ b/regression-test/data/query_p0/system/test_query_sys_tables.out
@@ -158,9 +158,9 @@ internal	test_query_sys_db_2	\N
 internal	test_query_sys_db_3	\N
 
 -- !tables --
-internal	test_query_sys_tb_1	BASE TABLE	0	\N	\N
-internal	test_query_sys_tb_2	BASE TABLE	0	\N	\N
-internal	test_query_sys_tb_3	BASE TABLE	0	\N	\N
+internal	test_query_sys_tb_1	BASE TABLE	0	\N	0
+internal	test_query_sys_tb_2	BASE TABLE	0	\N	0
+internal	test_query_sys_tb_3	BASE TABLE	0	\N	0
 
 -- !session_variables --
 wait_timeout	30000
diff --git a/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy b/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy
new file mode 100644
index 00000000000000..671fe907782f2c
--- /dev/null
+++ b/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy
@@ -0,0 +1,399 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
+enum FileSizeChange {
+    LARGER,
+    SMALLER,
+    UNCHANGED
+}
+
+suite("test_show_index_data", "p1") {
+
+    def show_table_name = "test_show_index_data"
+
+    def load_json_data = { table_name, file_name ->
+        // load the json data
+        streamLoad {
+            table "${table_name}"
+
+            // set http request header params
+            set 'read_json_by_line', 'true' 
+            set 'format', 'json' 
+            set 'max_filter_ratio', '0.1'
+            file file_name // import json file
+            time 10000 // limit inflight 10s
+
+            // if declared a check callback, the default check condition will ignore.
+            // So you must check all condition
+
+            check { result, exception, startTime, endTime ->
+                if (exception != null) {
+                        throw exception
+                }
+                logger.info("Stream load ${file_name} result: ${result}".toString())
+                def json = parseJson(result)
+                assertEquals("success", json.Status.toLowerCase())
+                // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows)
+                assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+            }
+        }
+    }
+
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+    
+
+    def convert_size = { str ->
+        def matcher = str =~ /(\d+(\.\d+)?)\s*(KB|MB|GB|TB|B)/
+        if (matcher) {
+            def value = matcher[0][1] as double
+            def unit = matcher[0][3]
+            logger.info("value is: " + value + " unit is: " + unit)
+            def result = 0.0
+            switch (unit) {
+                case 'KB':
+                    result = value * 1024
+                    break
+                case 'MB':
+                    result = value * 1024 * 1024 
+                    break
+                case 'GB':
+                    result = value * 1024 * 1024 * 1024
+                    break
+                case 'B':
+                    result = value
+                    break
+                default:
+                    throw new IllegalArgumentException("Unknown unit: $unit")
+            }
+            
+            return result
+        } else {
+           return 0
+        }
+    }
+
+    sql "DROP TABLE IF EXISTS ${show_table_name}"
+    sql """ 
+        CREATE TABLE ${show_table_name} (
+            `@timestamp` int(11) NULL,
+            `clientip` varchar(20) NULL,
+            `request` varchar(500) NULL,
+            `status` int NULL,
+            `size` int NULL,
+            INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '',
+            INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "unicode") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`@timestamp`, `clientip`)
+        DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1
+        PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "compaction_policy" = "time_series",
+            "time_series_compaction_file_count_threshold" = "20",
+            "disable_auto_compaction" = "true"
+        );
+    """
+
+    def compaction = {
+
+        def tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+
+        for (def tablet in tablets) {
+            int beforeSegmentCount = 0
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                beforeSegmentCount += Integer.parseInt(rowset.split(" ")[1])
+            }
+            assertEquals(beforeSegmentCount, 10)
+        }
+
+        // trigger compactions for all tablets in ${tableName}
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            backend_id = tablet.BackendId
+            (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+            logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def compactJson = parseJson(out.trim())
+            assertEquals("success", compactJson.status.toLowerCase())
+        }
+
+        // wait for all compactions done
+        for (def tablet in tablets) {
+            Awaitility.await().atMost(30, TimeUnit.MINUTES).untilAsserted(() -> {
+                Thread.sleep(30000)
+                String tablet_id = tablet.TabletId
+                backend_id = tablet.BackendId
+                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def compactionStatus = parseJson(out.trim())
+                assertEquals("compaction task for this tablet is not running", compactionStatus.msg.toLowerCase())
+            });
+        }
+
+        
+        for (def tablet in tablets) {
+            int afterSegmentCount = 0
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                logger.info("rowset is: " + rowset)
+                afterSegmentCount += Integer.parseInt(rowset.split(" ")[1])
+            }
+            assertEquals(afterSegmentCount, 1)
+        }
+    }
+
+    double localIndexSize = 0
+    double localSegmentSize = 0
+
+    def check_size_equal = { double result1, double result2 ->
+        double tolerance = 0.1 * Math.max(result1, result2);
+        return Math.abs(result1 - result2) <= tolerance;
+    }
+
+    def check_show_data = { FileSizeChange expect_idx, FileSizeChange expect_data ->
+        Thread.sleep(90000)
+        Awaitility.await().atMost(5, TimeUnit.MINUTES).untilAsserted(() -> {
+            Thread.sleep(10000)
+            def result = sql """ show data all;"""
+            logger.info("show data all; result is: ${result}")
+            def currentLocalIndexSize = convert_size.call(result[0][4])
+            def currentSegmentIndexSize = convert_size.call(result[0][3])
+
+            if (expect_idx == FileSizeChange.LARGER) {
+                assertTrue(currentLocalIndexSize > localIndexSize)
+            } else if (expect_idx == FileSizeChange.SMALLER) {
+                assertTrue(currentLocalIndexSize < localIndexSize)
+            } else {
+                assertTrue(check_size_equal(currentLocalIndexSize, localIndexSize))
+            }
+
+            if (expect_data == FileSizeChange.LARGER) {
+                assertTrue(currentSegmentIndexSize > localSegmentSize)
+            } else if (expect_data == FileSizeChange.SMALLER) {
+                assertTrue(currentSegmentIndexSize < localSegmentSize)
+            } else {
+                assertTrue(check_size_equal(currentSegmentIndexSize, localSegmentSize))
+            }
+            assertTrue(currentLocalIndexSize != 0)
+            assertTrue(currentSegmentIndexSize != 0)
+            localIndexSize = currentLocalIndexSize
+            localSegmentSize = currentSegmentIndexSize
+
+            def result2 = sql """ select * from information_schema.tables where TABLE_NAME = '${show_table_name}' """
+            logger.info("result 2 is: ${result2}")
+            def currentLocalIndexSize2 = result2[0][11] as double
+            def currentSegmentIndexSize2 = result2[0][9] as double
+            logger.info("currentLocalIndexSize2 is: ${currentLocalIndexSize2}, currentSegmentIndexSize2 is: ${currentSegmentIndexSize2}")
+            assertTrue(check_size_equal(currentLocalIndexSize, currentLocalIndexSize2))
+            assertTrue(check_size_equal(currentSegmentIndexSize, currentSegmentIndexSize2))
+            logger.info("show data all localIndexSize is: " + localIndexSize)
+            logger.info("show data all localSegmentSize is: " + localSegmentSize)
+        });
+    }
+
+    def schema_change = {
+        def tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+        Set<String> rowsetids = new HashSet<>();
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                if (segmentCount == 0) {
+                    continue;
+                }
+                String rowsetid = rowset.split(" ")[4];
+                rowsetids.add(rowsetid)
+                logger.info("rowsetid: " + rowsetid)
+            }
+        }
+        sql """ alter table ${show_table_name} drop column clientip"""
+        Awaitility.await().atMost(30, TimeUnit.MINUTES).untilAsserted(() -> {
+            Thread.sleep(30000)
+            tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+            for (def tablet in tablets) {
+                String tablet_id = tablet.TabletId
+                (code, out, err) = curl("GET", tablet.CompactionStatus)
+                logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def tabletJson = parseJson(out.trim())
+                assert tabletJson.rowsets instanceof List
+                for (String rowset in (List<String>) tabletJson.rowsets) {
+                    int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                    if (segmentCount == 0) {
+                        continue;
+                    }
+                    String rowsetid = rowset.split(" ")[4];
+                    logger.info("rowsetid: " + rowsetid)
+                    assertTrue(!rowsetids.contains(rowsetid))
+                }
+            }
+        });
+    }
+
+    def build_index = {
+
+        def tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+        Set<String> rowsetids = new HashSet<>();
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                if (segmentCount == 0) {
+                    continue;
+                }
+                String rowsetid = rowset.split(" ")[4];
+                rowsetids.add(rowsetid)
+                logger.info("rowsetid: " + rowsetid)
+            }
+        }
+        sql """ ALTER TABLE ${show_table_name} ADD INDEX status_idx (status) using inverted; """
+        if (!isCloudMode()) {
+            sql """ build index status_idx on ${show_table_name}"""
+        }
+        Awaitility.await().atMost(30, TimeUnit.MINUTES).untilAsserted(() -> {
+            Thread.sleep(30000)
+            tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+            for (def tablet in tablets) {
+                String tablet_id = tablet.TabletId
+                (code, out, err) = curl("GET", tablet.CompactionStatus)
+                logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def tabletJson = parseJson(out.trim())
+                assert tabletJson.rowsets instanceof List
+                for (String rowset in (List<String>) tabletJson.rowsets) {
+                    int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                    if (segmentCount == 0) {
+                        continue;
+                    }
+                    String rowsetid = rowset.split(" ")[4];
+                    logger.info("rowsetid: " + rowsetid)
+                    assertTrue(!rowsetids.contains(rowsetid))
+                }
+            }
+        });
+    }
+
+    def drop_index = {
+        def tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+        Set<String> rowsetids = new HashSet<>();
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                if (segmentCount == 0) {
+                    continue;
+                }
+                String rowsetid = rowset.split(" ")[4];
+                rowsetids.add(rowsetid)
+                logger.info("rowsetid: " + rowsetid)
+            }
+        }
+        sql """ DROP INDEX status_idx on ${show_table_name}"""
+        Awaitility.await().atMost(30, TimeUnit.MINUTES).untilAsserted(() -> {
+            Thread.sleep(30000)
+            tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+            for (def tablet in tablets) {
+                String tablet_id = tablet.TabletId
+                (code, out, err) = curl("GET", tablet.CompactionStatus)
+                logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def tabletJson = parseJson(out.trim())
+                assert tabletJson.rowsets instanceof List
+                for (String rowset in (List<String>) tabletJson.rowsets) {
+                    int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                    if (segmentCount == 0) {
+                        continue;
+                    }
+                    String rowsetid = rowset.split(" ")[4];
+                    logger.info("rowsetid: " + rowsetid)
+                    assertTrue(!rowsetids.contains(rowsetid))
+                }
+            }
+        });
+    }
+
+    // 1. load data
+    def executor = Executors.newFixedThreadPool(5)
+    (1..10).each { i ->
+        executor.submit {
+            def fileName = "documents-" + i + ".json"
+            load_json_data.call(show_table_name, """${getS3Url()}/regression/inverted_index_cases/httplogs/${fileName}""")
+        }
+    }
+    executor.shutdown()
+    executor.awaitTermination(30, TimeUnit.MINUTES)
+
+    // 2. check show data
+    check_show_data.call(FileSizeChange.LARGER, FileSizeChange.LARGER)
+
+    // 3. compaction
+    compaction.call()
+
+    // 4. check show data
+    check_show_data.call(FileSizeChange.SMALLER, FileSizeChange.SMALLER)
+
+    // 5. schema change
+    schema_change.call()
+
+    // 6.check show data
+    check_show_data.call(FileSizeChange.SMALLER, FileSizeChange.SMALLER)
+
+    // 7. build index
+    build_index.call()
+
+    // 8.check show data
+    check_show_data.call(FileSizeChange.LARGER, FileSizeChange.UNCHANGED)
+
+    // 9. drop index
+    drop_index.call()
+
+    // 10.check show data
+    check_show_data.call(FileSizeChange.SMALLER, FileSizeChange.UNCHANGED)
+}
diff --git a/regression-test/suites/inverted_index_p2/show_data/test_show_index_data_p2.groovy b/regression-test/suites/inverted_index_p2/show_data/test_show_index_data_p2.groovy
new file mode 100644
index 00000000000000..2839a8a47b04aa
--- /dev/null
+++ b/regression-test/suites/inverted_index_p2/show_data/test_show_index_data_p2.groovy
@@ -0,0 +1,397 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
+enum FileSizeChange {
+    LARGER,
+    SMALLER,
+    UNCHANGED
+}
+
+suite("test_show_index_data_p2", "p2") {
+
+    def show_table_name = "test_show_index_data_p2"
+
+    def load_json_data = { table_name, file_name ->
+        // load the json data
+        streamLoad {
+            table "${table_name}"
+
+            // set http request header params
+            set 'read_json_by_line', 'true' 
+            set 'format', 'json' 
+            set 'max_filter_ratio', '0.1'
+            file file_name // import json file
+            time 10000 // limit inflight 10s
+
+            // if declared a check callback, the default check condition will ignore.
+            // So you must check all condition
+
+            check { result, exception, startTime, endTime ->
+                if (exception != null) {
+                        throw exception
+                }
+                logger.info("Stream load ${file_name} result: ${result}".toString())
+                def json = parseJson(result)
+                assertEquals("success", json.Status.toLowerCase())
+                // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows)
+                assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+            }
+        }
+    }
+
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+    
+
+    def convert_size = { str ->
+        def matcher = str =~ /(\d+(\.\d+)?)\s*(KB|MB|GB|TB|B)/
+        if (matcher) {
+            def value = matcher[0][1] as double
+            def unit = matcher[0][3]
+            logger.info("value is: " + value + " unit is: " + unit)
+            def result = 0.0
+            switch (unit) {
+                case 'KB':
+                    result = value * 1024
+                    break
+                case 'MB':
+                    result = value * 1024 * 1024 
+                    break
+                case 'GB':
+                    result = value * 1024 * 1024 * 1024
+                    break
+                case 'B':
+                    result = value
+                    break
+                default:
+                    throw new IllegalArgumentException("Unknown unit: $unit")
+            }
+            
+            return result
+        } else {
+           return 0
+        }
+    }
+
+    sql "DROP TABLE IF EXISTS ${show_table_name}"
+    sql """ 
+        CREATE TABLE ${show_table_name} (
+            `@timestamp` int(11) NULL,
+            `clientip` varchar(20) NULL,
+            `request` varchar(500) NULL,
+            `status` int NULL,
+            `size` int NULL,
+            INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '',
+            INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "unicode") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`@timestamp`, `clientip`)
+        DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1
+        PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "compaction_policy" = "time_series",
+            "time_series_compaction_file_count_threshold" = "20",
+            "disable_auto_compaction" = "true"
+        );
+    """
+
+    def compaction = {
+        def tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+        for (def tablet in tablets) {
+            int beforeSegmentCount = 0
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                beforeSegmentCount += Integer.parseInt(rowset.split(" ")[1])
+            }
+            assertEquals(beforeSegmentCount, 110)
+        }
+
+        // trigger compactions for all tablets in ${tableName}
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            backend_id = tablet.BackendId
+            (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+            logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def compactJson = parseJson(out.trim())
+            assertEquals("success", compactJson.status.toLowerCase())
+        }
+
+        // wait for all compactions done
+        for (def tablet in tablets) {
+            Awaitility.await().atMost(60, TimeUnit.MINUTES).untilAsserted(() -> {
+                Thread.sleep(30000)
+                String tablet_id = tablet.TabletId
+                backend_id = tablet.BackendId
+                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def compactionStatus = parseJson(out.trim())
+                assertEquals("compaction task for this tablet is not running", compactionStatus.msg.toLowerCase())
+            });
+        }
+
+        for (def tablet in tablets) {
+            int afterSegmentCount = 0
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                logger.info("rowset is: " + rowset)
+                afterSegmentCount += Integer.parseInt(rowset.split(" ")[1])
+            }
+            assertEquals(afterSegmentCount, 1)
+        }
+        
+    }
+
+    double localIndexSize = 0
+    double localSegmentSize = 0
+
+    def check_size_equal = { double result1, double result2 ->
+        double tolerance = 0.1 * Math.max(result1, result2);
+        return Math.abs(result1 - result2) <= tolerance;
+    }
+
+    def check_show_data = { FileSizeChange expect_idx, FileSizeChange expect_data ->
+        Thread.sleep(90000)
+        Awaitility.await().atMost(10, TimeUnit.MINUTES).untilAsserted(() -> {
+            Thread.sleep(10000)
+            def result = sql """ show data all;"""
+            logger.info("show data all; result is: ${result}")
+            def currentLocalIndexSize = convert_size.call(result[0][4])
+            def currentSegmentIndexSize = convert_size.call(result[0][3])
+
+            if (expect_idx == FileSizeChange.LARGER) {
+                assertTrue(currentLocalIndexSize > localIndexSize)
+            } else if (expect_idx == FileSizeChange.SMALLER) {
+                assertTrue(currentLocalIndexSize < localIndexSize)
+            } else {
+                assertTrue(check_size_equal(currentLocalIndexSize, localIndexSize))
+            }
+
+            if (expect_data == FileSizeChange.LARGER) {
+                assertTrue(currentSegmentIndexSize > localSegmentSize)
+            } else if (expect_data == FileSizeChange.SMALLER) {
+                assertTrue(currentSegmentIndexSize < localSegmentSize)
+            } else {
+                assertTrue(check_size_equal(currentSegmentIndexSize, localSegmentSize))
+            }
+
+            assertTrue(currentLocalIndexSize != 0)
+            assertTrue(currentSegmentIndexSize != 0)
+            localIndexSize = currentLocalIndexSize
+            localSegmentSize = currentSegmentIndexSize
+
+            def result2 = sql """ select * from information_schema.tables where TABLE_NAME = '${show_table_name}' """
+            logger.info("result 2 is: ${result2}")
+            def currentLocalIndexSize2 = result2[0][11] as double
+            def currentSegmentIndexSize2 = result2[0][9] as double
+            logger.info("currentLocalIndexSize2 is: ${currentLocalIndexSize2}, currentSegmentIndexSize2 is: ${currentSegmentIndexSize2}")
+            assertTrue(check_size_equal(currentLocalIndexSize, currentLocalIndexSize2))
+            assertTrue(check_size_equal(currentSegmentIndexSize, currentSegmentIndexSize2))
+            logger.info("show data all localIndexSize is: " + localIndexSize)
+            logger.info("show data all localSegmentSize is: " + localSegmentSize)
+        });
+    }
+
+    def schema_change = {
+        def tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+        Set<String> rowsetids = new HashSet<>();
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                if (segmentCount == 0) {
+                    continue;
+                }
+                String rowsetid = rowset.split(" ")[4];
+                rowsetids.add(rowsetid)
+                logger.info("rowsetid: " + rowsetid)
+            }
+        }
+        sql """ alter table ${show_table_name} drop column clientip"""
+        Awaitility.await().atMost(60, TimeUnit.MINUTES).untilAsserted(() -> {
+            Thread.sleep(30000)
+            tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+            for (def tablet in tablets) {
+                String tablet_id = tablet.TabletId
+                (code, out, err) = curl("GET", tablet.CompactionStatus)
+                logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def tabletJson = parseJson(out.trim())
+                assert tabletJson.rowsets instanceof List
+                for (String rowset in (List<String>) tabletJson.rowsets) {
+                    int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                    if (segmentCount == 0) {
+                        continue;
+                    }
+                    String rowsetid = rowset.split(" ")[4];
+                    logger.info("rowsetid: " + rowsetid)
+                    assertTrue(!rowsetids.contains(rowsetid))
+                }
+            }
+        });
+    }
+
+    def build_index = {
+        def tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+        Set<String> rowsetids = new HashSet<>();
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                if (segmentCount == 0) {
+                    continue;
+                }
+                String rowsetid = rowset.split(" ")[4];
+                rowsetids.add(rowsetid)
+                logger.info("rowsetid: " + rowsetid)
+            }
+        }
+        sql """ ALTER TABLE ${show_table_name} ADD INDEX status_idx (status) using inverted; """
+        if (!isCloudMode()) {
+            sql """ build index status_idx on ${show_table_name}"""
+        }
+        Awaitility.await().atMost(60, TimeUnit.MINUTES).untilAsserted(() -> {
+            Thread.sleep(30000)
+            tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+            for (def tablet in tablets) {
+                String tablet_id = tablet.TabletId
+                (code, out, err) = curl("GET", tablet.CompactionStatus)
+                logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def tabletJson = parseJson(out.trim())
+                assert tabletJson.rowsets instanceof List
+                for (String rowset in (List<String>) tabletJson.rowsets) {
+                    int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                    if (segmentCount == 0) {
+                        continue;
+                    }
+                    String rowsetid = rowset.split(" ")[4];
+                    logger.info("rowsetid: " + rowsetid)
+                    assertTrue(!rowsetids.contains(rowsetid))
+                }
+            }
+        });
+    }
+
+    def drop_index = {
+        def tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+        Set<String> rowsetids = new HashSet<>();
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            for (String rowset in (List<String>) tabletJson.rowsets) {
+                int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                if (segmentCount == 0) {
+                    continue;
+                }
+                String rowsetid = rowset.split(" ")[4];
+                rowsetids.add(rowsetid)
+                logger.info("rowsetid: " + rowsetid)
+            }
+        }
+        sql """ DROP INDEX status_idx on ${show_table_name}"""
+        Awaitility.await().atMost(60, TimeUnit.MINUTES).untilAsserted(() -> {
+            Thread.sleep(30000)
+            tablets = sql_return_maparray """ show tablets from ${show_table_name}; """
+            for (def tablet in tablets) {
+                String tablet_id = tablet.TabletId
+                (code, out, err) = curl("GET", tablet.CompactionStatus)
+                logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def tabletJson = parseJson(out.trim())
+                assert tabletJson.rowsets instanceof List
+                for (String rowset in (List<String>) tabletJson.rowsets) {
+                    int segmentCount = Integer.parseInt(rowset.split(" ")[1])
+                    if (segmentCount == 0) {
+                        continue;
+                    }
+                    String rowsetid = rowset.split(" ")[4];
+                    logger.info("rowsetid: " + rowsetid)
+                    assertTrue(!rowsetids.contains(rowsetid))
+                }
+            }
+        });
+    }
+
+    // 1. load data
+    def executor = Executors.newFixedThreadPool(5)
+    (1..110).each { i ->
+        executor.submit {
+            def fileName = "documents-" + i + ".json"
+            load_json_data.call(show_table_name, """${getS3Url()}/regression/inverted_index_cases/httplogs/${fileName}""")
+        }
+    }
+    executor.shutdown()
+    executor.awaitTermination(60, TimeUnit.MINUTES)
+
+    // 2. check show data
+    check_show_data.call(FileSizeChange.LARGER, FileSizeChange.LARGER)
+
+    // 3. compaction
+    compaction.call()
+
+    // 4. check show data
+    check_show_data.call(FileSizeChange.SMALLER, FileSizeChange.LARGER)
+
+    // 5. schema change
+    schema_change.call()
+
+    // 6.check show data
+    check_show_data.call(FileSizeChange.SMALLER, FileSizeChange.SMALLER)
+
+    // 7. build index
+    build_index.call()
+
+    // 8.check show data
+    check_show_data.call(FileSizeChange.LARGER, FileSizeChange.UNCHANGED)
+
+    // 9. drop index
+    drop_index.call()
+
+    // 10.check show data
+    check_show_data.call(FileSizeChange.SMALLER, FileSizeChange.UNCHANGED)
+}

From b2b0ac76c6c0e8533673486ba6db7b04b9ec4627 Mon Sep 17 00:00:00 2001
From: airborne12 <jiangkai@selectdb.com>
Date: Mon, 2 Dec 2024 14:54:22 +0800
Subject: [PATCH 110/399] [feat](test) add bloomfilter ut case (#44351)

add bloomfilter index UT test case for BE
---
 be/src/olap/bloom_filter.hpp                  | 272 ----------
 .../segment_v2/block_split_bloom_filter.h     |   1 -
 be/src/olap/rowset/segment_v2/bloom_filter.h  |   2 +-
 .../segment_v2/bloom_filter_index_reader.cpp  |   1 +
 .../segment_v2/bloom_filter_index_writer.cpp  |  29 +-
 be/test/olap/bloom_filter_test.cpp            | 160 ------
 .../bloom_filter_index_reader_writer_test.cpp | 485 +++++++++++++++++-
 7 files changed, 488 insertions(+), 462 deletions(-)
 delete mode 100644 be/src/olap/bloom_filter.hpp
 delete mode 100644 be/test/olap/bloom_filter_test.cpp

diff --git a/be/src/olap/bloom_filter.hpp b/be/src/olap/bloom_filter.hpp
deleted file mode 100644
index 5c7cb5f9e6419f..00000000000000
--- a/be/src/olap/bloom_filter.hpp
+++ /dev/null
@@ -1,272 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef DORIS_BE_SRC_OLAP_COLUMN_FILE_BLOOM_FILTER_HPP
-#define DORIS_BE_SRC_OLAP_COLUMN_FILE_BLOOM_FILTER_HPP
-
-#include <math.h>
-
-#include <sstream>
-#include <string>
-
-#include "olap/olap_define.h"
-#include "olap/utils.h"
-#include "util/hash_util.hpp"
-
-namespace doris {
-
-static const uint64_t DEFAULT_SEED = 104729;
-static const uint64_t BLOOM_FILTER_NULL_HASHCODE = 2862933555777941757ULL;
-
-struct BloomFilterIndexHeader {
-    uint64_t block_count;
-    BloomFilterIndexHeader() : block_count(0) {}
-} __attribute__((packed));
-
-// Bare metal bit set implementation. For performance reasons, this implementation does not
-// check for index bounds nor expand the bit set if the specified index is greater than the size.
-class BitSet {
-public:
-    BitSet() : _data(nullptr), _data_len(0) {}
-
-    ~BitSet() { SAFE_DELETE_ARRAY(_data); }
-
-    // Init BitSet with given bit_num, which will align up to uint64_t
-    bool init(uint32_t bit_num) {
-        if (bit_num <= 0) {
-            return false;
-        }
-
-        _data_len = (bit_num + sizeof(uint64_t) * 8 - 1) / (sizeof(uint64_t) * 8);
-        _data = new (std::nothrow) uint64_t[_data_len];
-        if (_data == nullptr) {
-            return false;
-        }
-
-        memset(_data, 0, _data_len * sizeof(uint64_t));
-        return true;
-    }
-
-    // Init BitSet with given buffer
-    bool init(uint64_t* data, uint32_t data_len) {
-        _data = data;
-        _data_len = data_len;
-        return true;
-    }
-
-    // Set the bit specified by param, note that uint64_t type contains 2^6 bits
-    void set(uint32_t index) { _data[index >> 6] |= 1L << (index % 64); }
-
-    // Return true if the bit specified by param is set
-    bool get(uint32_t index) const { return (_data[index >> 6] & (1L << (index % 64))) != 0; }
-
-    // Merge with another BitSet by byte, return false when the length is not equal
-    bool merge(const BitSet& set) {
-        if (_data_len != set.data_len()) {
-            return false;
-        }
-
-        for (uint32_t i = 0; i < _data_len; ++i) {
-            _data[i] |= set.data()[i];
-        }
-
-        return true;
-    }
-
-    // Convert BitSet to string to convenient debug and test
-    std::string to_string() const {
-        uint32_t bit_num = _data_len * sizeof(uint64_t) * 8;
-        std::string str(bit_num, '0');
-        for (uint32_t i = 0; i < bit_num; ++i) {
-            if ((_data[i >> 6] & (1L << i)) != 0) {
-                str[i] = '1';
-            }
-        }
-
-        return str;
-    }
-
-    uint64_t* data() const { return _data; }
-
-    uint32_t data_len() const { return _data_len; }
-
-    uint32_t bit_num() const { return _data_len * sizeof(uint64_t) * 8; }
-
-    void clear() { memset(_data, 0, _data_len * sizeof(uint64_t)); }
-
-    void reset() {
-        _data = NULL;
-        _data_len = 0;
-    }
-
-private:
-    uint64_t* _data;
-    uint32_t _data_len;
-};
-
-class BloomFilter {
-public:
-    BloomFilter() : _bit_num(0), _hash_function_num(0) {}
-    ~BloomFilter() {}
-
-    // Create BloomFilter with given entry num and fpp, which is used for loading data
-    bool init(int64_t expected_entries, double fpp) {
-        uint32_t bit_num = _optimal_bit_num(expected_entries, fpp);
-        if (!_bit_set.init(bit_num)) {
-            return false;
-        }
-
-        _bit_num = _bit_set.bit_num();
-        _hash_function_num = _optimal_hash_function_num(expected_entries, _bit_num);
-        return true;
-    }
-
-    // Create BloomFilter with given entry num and default fpp
-    bool init(int64_t expected_entries) {
-        return this->init(expected_entries, BLOOM_FILTER_DEFAULT_FPP);
-    }
-
-    // Init BloomFilter with given buffer, which is used for query
-    bool init(uint64_t* data, uint32_t len, uint32_t hash_function_num) {
-        _bit_num = sizeof(uint64_t) * 8 * len;
-        _hash_function_num = hash_function_num;
-        return _bit_set.init(data, len);
-    }
-
-    // Compute hash value of given buffer and add to BloomFilter
-    void add_bytes(const char* buf, uint32_t len) {
-        uint64_t hash = buf == nullptr ? BLOOM_FILTER_NULL_HASHCODE
-                                       : HashUtil::hash64(buf, len, DEFAULT_SEED);
-        add_hash(hash);
-    }
-
-    // Generate multiple hash value according to following rule:
-    //     new_hash_value = hash_high_part + (i * hash_low_part)
-    void add_hash(uint64_t hash) {
-        uint32_t hash1 = (uint32_t)hash;
-        uint32_t hash2 = (uint32_t)(hash >> 32);
-
-        for (uint32_t i = 0; i < _hash_function_num; ++i) {
-            uint64_t combine_hash = hash1 + hash2 * i;
-            uint32_t index = combine_hash % _bit_num;
-            _bit_set.set(index);
-        }
-    }
-
-    // Compute hash value of given buffer and verify whether exist in BloomFilter
-    bool test_bytes(const char* buf, uint32_t len) const {
-        uint64_t hash = buf == nullptr ? BLOOM_FILTER_NULL_HASHCODE
-                                       : HashUtil::hash64(buf, len, DEFAULT_SEED);
-        return test_hash(hash);
-    }
-
-    // Verify whether hash value in BloomFilter
-    bool test_hash(uint64_t hash) const {
-        uint32_t hash1 = (uint32_t)hash;
-        uint32_t hash2 = (uint32_t)(hash >> 32);
-
-        for (uint32_t i = 0; i < _hash_function_num; ++i) {
-            uint64_t combine_hash = hash1 + hash2 * i;
-            uint32_t index = combine_hash % _bit_num;
-            if (!_bit_set.get(index)) {
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    // Merge with another BloomFilter, return false when the length
-    //     and hash function number is not equal
-    bool merge(const BloomFilter& that) {
-        if (_bit_num == that.bit_num() && _hash_function_num == that.hash_function_num()) {
-            _bit_set.merge(that.bit_set());
-            return true;
-        }
-
-        return false;
-    }
-
-    void clear() { _bit_set.clear(); }
-
-    void reset() {
-        _bit_num = 0;
-        _hash_function_num = 0;
-        _bit_set.reset();
-    }
-
-    uint32_t bit_num() const { return _bit_num; }
-
-    uint32_t hash_function_num() const { return _hash_function_num; }
-
-    const BitSet& bit_set() const { return _bit_set; }
-
-    uint64_t* bit_set_data() const { return _bit_set.data(); }
-
-    uint32_t bit_set_data_len() const { return _bit_set.data_len(); }
-
-    // Convert BloomFilter to string to convenient debug and test
-    std::string to_string() const {
-        std::stringstream bf_stream;
-        bf_stream << "bit_num:" << _bit_num << " hash_function_num:" << _hash_function_num
-                  << " bit_set:" << _bit_set.to_string();
-        return bf_stream.str();
-    }
-
-    // Get points which set by given buffer in the BitSet
-    std::string get_bytes_points_string(const char* buf, uint32_t len) const {
-        uint64_t hash = buf == nullptr ? BLOOM_FILTER_NULL_HASHCODE
-                                       : HashUtil::hash64(buf, len, DEFAULT_SEED);
-        uint32_t hash1 = (uint32_t)hash;
-        uint32_t hash2 = (uint32_t)(hash >> 32);
-
-        std::stringstream stream;
-        for (uint32_t i = 0; i < _hash_function_num; ++i) {
-            if (i != 0) {
-                stream << "-";
-            }
-
-            uint32_t combine_hash = hash1 + hash2 * i;
-            uint32_t index = combine_hash % _bit_num;
-            stream << index;
-        }
-
-        return stream.str();
-    }
-
-private:
-    // Compute the optimal bit number according to the following rule:
-    //     m = -n * ln(fpp) / (ln(2) ^ 2)
-    uint32_t _optimal_bit_num(int64_t n, double fpp) {
-        return (uint32_t)(-n * log(fpp) / (log(2) * log(2)));
-    }
-
-    // Compute the optimal hash function number according to the following rule:
-    //     k = round(m * ln(2) / n)
-    uint32_t _optimal_hash_function_num(int64_t n, uint32_t m) {
-        uint32_t k = (uint32_t)round(m * log(2) / n);
-        return k > 1 ? k : 1;
-    }
-
-    BitSet _bit_set;
-    uint32_t _bit_num;
-    uint32_t _hash_function_num;
-};
-
-} // namespace doris
-
-#endif // DORIS_BE_SRC_OLAP_COLUMN_FILE_BLOOM_FILTER_HPP
diff --git a/be/src/olap/rowset/segment_v2/block_split_bloom_filter.h b/be/src/olap/rowset/segment_v2/block_split_bloom_filter.h
index f68ddd7e74bfc5..8dc470d9da4f88 100644
--- a/be/src/olap/rowset/segment_v2/block_split_bloom_filter.h
+++ b/be/src/olap/rowset/segment_v2/block_split_bloom_filter.h
@@ -34,7 +34,6 @@ class BlockSplitBloomFilter : public BloomFilter {
     void add_hash(uint64_t hash) override;
 
     bool test_hash(uint64_t hash) const override;
-    bool contains(const BloomFilter&) const override { return true; }
 
 private:
     // Bytes in a tiny Bloom filter block.
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter.h b/be/src/olap/rowset/segment_v2/bloom_filter.h
index a7845d1ca36704..4f4adf0fd12283 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter.h
+++ b/be/src/olap/rowset/segment_v2/bloom_filter.h
@@ -186,7 +186,7 @@ class BloomFilter {
 
     /// Checks if this contains everything from another bloom filter.
     /// Bloom filters must have equal size and seed.
-    virtual bool contains(const BloomFilter& bf_) const = 0;
+    virtual bool contains(const BloomFilter& bf_) const { return true; };
 
     virtual char* data() const { return _data; }
 
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
index 609d21ce4f5c22..8c63c25d20acee 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
@@ -70,6 +70,7 @@ Status BloomFilterIndexIterator::read_bloom_filter(rowid_t ordinal,
     auto column = data_type->create_column();
 
     RETURN_IF_ERROR(_bloom_filter_iter.seek_to_ordinal(ordinal));
+    DCHECK(current_bloom_filter_index() == ordinal);
     size_t num_read = num_to_read;
     RETURN_IF_ERROR(_bloom_filter_iter.next_batch(&num_read, column));
     DCHECK(num_to_read == num_read);
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
index edc6102703f492..3f9fb94df0a844 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
@@ -21,6 +21,7 @@
 #include <string.h>
 
 #include <algorithm>
+#include <memory>
 #include <set>
 #include <string>
 #include <utility>
@@ -68,15 +69,12 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter {
 
     explicit BloomFilterIndexWriterImpl(const BloomFilterOptions& bf_options,
                                         const TypeInfo* type_info)
-            : _bf_options(bf_options),
-              _type_info(type_info),
-              _has_null(false),
-              _bf_buffer_size(0) {}
+            : _bf_options(bf_options), _type_info(type_info) {}
 
     ~BloomFilterIndexWriterImpl() override = default;
 
     Status add_values(const void* values, size_t count) override {
-        const CppType* v = (const CppType*)values;
+        const auto* v = (const CppType*)values;
         for (int i = 0; i < count; ++i) {
             if (_values.find(*v) == _values.end()) {
                 if constexpr (_is_slice_type()) {
@@ -105,7 +103,7 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter {
         bf->set_has_null(_has_null);
         for (auto& v : _values) {
             if constexpr (_is_slice_type()) {
-                Slice* s = (Slice*)&v;
+                auto* s = (Slice*)&v;
                 bf->add_bytes(s->data, s->size);
             } else {
                 bf->add_bytes((char*)&v, sizeof(CppType));
@@ -160,11 +158,11 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter {
     static constexpr bool _is_int128() { return field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT; }
 
 private:
-    BloomFilterOptions _bf_options;
-    const TypeInfo* _type_info;
+    BloomFilterOptions _bf_options {};
+    const TypeInfo* _type_info = nullptr;
     vectorized::Arena _arena;
-    bool _has_null;
-    uint64_t _bf_buffer_size;
+    bool _has_null = false;
+    uint64_t _bf_buffer_size = 0;
     // distinct values
     ValueDict _values;
     std::vector<std::unique_ptr<BloomFilter>> _bfs;
@@ -173,7 +171,7 @@ class BloomFilterIndexWriterImpl : public BloomFilterIndexWriter {
 } // namespace
 
 Status PrimaryKeyBloomFilterIndexWriterImpl::add_values(const void* values, size_t count) {
-    const Slice* v = (const Slice*)values;
+    const auto* v = (const Slice*)values;
     for (int i = 0; i < count; ++i) {
         Slice new_value;
         RETURN_IF_CATCH_EXCEPTION(_type_info->deep_copy(&new_value, v, &_arena));
@@ -189,7 +187,7 @@ Status PrimaryKeyBloomFilterIndexWriterImpl::flush() {
     RETURN_IF_ERROR(bf->init(_values.size(), _bf_options.fpp, _bf_options.strategy));
     bf->set_has_null(_has_null);
     for (auto& v : _values) {
-        Slice* s = (Slice*)&v;
+        auto* s = (Slice*)&v;
         bf->add_bytes(s->data, s->size);
     }
     _bf_buffer_size += bf->size();
@@ -205,7 +203,7 @@ Status PrimaryKeyBloomFilterIndexWriterImpl::flush() {
 
 Status PrimaryKeyBloomFilterIndexWriterImpl::finish(io::FileWriter* file_writer,
                                                     ColumnIndexMetaPB* index_meta) {
-    if (_values.size() > 0) {
+    if (!_values.empty()) {
         RETURN_IF_ERROR(flush());
     }
     index_meta->set_type(BLOOM_FILTER_INDEX);
@@ -246,7 +244,7 @@ NGramBloomFilterIndexWriterImpl::NGramBloomFilterIndexWriterImpl(
 }
 
 Status NGramBloomFilterIndexWriterImpl::add_values(const void* values, size_t count) {
-    const Slice* src = reinterpret_cast<const Slice*>(values);
+    const auto* src = reinterpret_cast<const Slice*>(values);
     for (int i = 0; i < count; ++i, ++src) {
         if (src->size < _gram_size) {
             continue;
@@ -339,7 +337,8 @@ Status NGramBloomFilterIndexWriterImpl::create(const BloomFilterOptions& bf_opti
     case FieldType::OLAP_FIELD_TYPE_CHAR:
     case FieldType::OLAP_FIELD_TYPE_VARCHAR:
     case FieldType::OLAP_FIELD_TYPE_STRING:
-        res->reset(new NGramBloomFilterIndexWriterImpl(bf_options, gram_size, gram_bf_size));
+        *res = std::make_unique<NGramBloomFilterIndexWriterImpl>(bf_options, gram_size,
+                                                                 gram_bf_size);
         break;
     default:
         return Status::NotSupported("unsupported type for ngram bloom filter index:{}",
diff --git a/be/test/olap/bloom_filter_test.cpp b/be/test/olap/bloom_filter_test.cpp
deleted file mode 100644
index 32bf88e2c4d684..00000000000000
--- a/be/test/olap/bloom_filter_test.cpp
+++ /dev/null
@@ -1,160 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/bloom_filter.hpp"
-
-#include <glog/logging.h>
-#include <gtest/gtest-message.h>
-#include <gtest/gtest-test-part.h>
-
-#include <string>
-
-#include "gtest/gtest_pred_impl.h"
-
-using std::string;
-
-namespace doris {
-
-class TestBloomFilter : public testing::Test {
-public:
-    virtual ~TestBloomFilter() {}
-
-    virtual void SetUp() {}
-    virtual void TearDown() {}
-};
-
-// Init BloomFilter with different item number and fpp,
-//     and verify bit_num and hash_function_num calculated by BloomFilter
-TEST_F(TestBloomFilter, init_bloom_filter) {
-    {
-        BloomFilter bf;
-        bf.init(1024);
-        EXPECT_EQ(6400, bf.bit_num());
-        EXPECT_EQ(4, bf.hash_function_num());
-    }
-
-    {
-        BloomFilter bf;
-        bf.init(1024, 0.01);
-        EXPECT_EQ(9856, bf.bit_num());
-        EXPECT_EQ(7, bf.hash_function_num());
-    }
-
-    {
-        BloomFilter bf;
-        bf.init(10240, 0.1);
-        EXPECT_EQ(49088, bf.bit_num());
-        EXPECT_EQ(3, bf.hash_function_num());
-    }
-
-    {
-        BloomFilter bf;
-        uint32_t data_len = 100;
-        uint32_t hash_function_num = 4;
-        uint64_t* data = new uint64_t[data_len];
-
-        bf.init(data, data_len, hash_function_num);
-        EXPECT_EQ(6400, bf.bit_num());
-        EXPECT_EQ(4, bf.hash_function_num());
-        EXPECT_EQ(data, bf.bit_set_data());
-
-        bf.reset();
-        EXPECT_EQ(0, bf.bit_num());
-        EXPECT_EQ(0, bf.hash_function_num());
-        EXPECT_EQ(nullptr, bf.bit_set_data());
-        delete[] data;
-    }
-}
-
-// Add different buffer to BloomFilter and verify existence
-TEST_F(TestBloomFilter, add_and_test_bytes) {
-    string bytes;
-    BloomFilter bf;
-    bf.init(1024);
-
-    bf.add_bytes(nullptr, 0);
-    EXPECT_TRUE(bf.test_bytes(nullptr, 0));
-
-    bytes = "hello";
-    bf.add_bytes(bytes.c_str(), bytes.size());
-    EXPECT_TRUE(bf.test_bytes(bytes.c_str(), bytes.size()));
-
-    bytes = "doris";
-    bf.add_bytes(bytes.c_str(), bytes.size());
-    EXPECT_TRUE(bf.test_bytes(bytes.c_str(), bytes.size()));
-
-    BloomFilter new_bf;
-    new_bf.init(1024);
-
-    bytes = "world";
-    new_bf.add_bytes(bytes.c_str(), bytes.size());
-    EXPECT_TRUE(bf.merge(new_bf));
-    EXPECT_TRUE(bf.test_bytes(bytes.c_str(), bytes.size()));
-}
-
-// Print bloom filter buffer and points of specified string
-TEST_F(TestBloomFilter, bloom_filter_info) {
-    string bytes;
-    BloomFilter bf;
-    bf.init(8, 0.1);
-
-    bytes = "doris";
-    bf.add_bytes(bytes.c_str(), bytes.size());
-    string buffer_expect =
-            "bit_num:64 hash_function_num:6 "
-            "bit_set:0000100000000000100000010000000000010000001000000000000000000100";
-    string buffer = bf.to_string();
-    EXPECT_TRUE(buffer_expect == buffer);
-
-    string points_expect = "4-23-42-61-16-35";
-    string points = bf.get_bytes_points_string(bytes.c_str(), bytes.size());
-    EXPECT_TRUE(points_expect == points);
-
-    bytes = "a";
-    points = bf.get_bytes_points_string(bytes.c_str(), bytes.size());
-    LOG(WARNING) << "bytes=" << bytes << " points=" << points;
-
-    bytes = "ab";
-    points = bf.get_bytes_points_string(bytes.c_str(), bytes.size());
-    LOG(WARNING) << "bytes=" << bytes << " points=" << points;
-
-    bytes = "b";
-    points = bf.get_bytes_points_string(bytes.c_str(), bytes.size());
-    LOG(WARNING) << "bytes=" << bytes << " points=" << points;
-
-    bytes = "ba";
-    points = bf.get_bytes_points_string(bytes.c_str(), bytes.size());
-    LOG(WARNING) << "bytes=" << bytes << " points=" << points;
-
-    bytes = "c";
-    points = bf.get_bytes_points_string(bytes.c_str(), bytes.size());
-    LOG(WARNING) << "bytes=" << bytes << " points=" << points;
-
-    bytes = "bc";
-    points = bf.get_bytes_points_string(bytes.c_str(), bytes.size());
-    LOG(WARNING) << "bytes=" << bytes << " points=" << points;
-
-    bytes = "ac";
-    points = bf.get_bytes_points_string(bytes.c_str(), bytes.size());
-    LOG(WARNING) << "bytes=" << bytes << " points=" << points;
-
-    bytes = "abc";
-    points = bf.get_bytes_points_string(bytes.c_str(), bytes.size());
-    LOG(WARNING) << "bytes=" << bytes << " points=" << points;
-}
-
-} // namespace doris
diff --git a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
index 69cb343f04bf91..813952595efcfd 100644
--- a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
@@ -32,6 +32,7 @@
 #include "io/fs/file_writer.h"
 #include "io/fs/local_file_system.h"
 #include "olap/decimal12.h"
+#include "olap/itoken_extractor.h"
 #include "olap/olap_common.h"
 #include "olap/rowset/segment_v2/bloom_filter.h"
 #include "olap/rowset/segment_v2/bloom_filter_index_reader.h"
@@ -62,18 +63,19 @@ template <FieldType type>
 Status write_bloom_filter_index_file(const std::string& file_name, const void* values,
                                      size_t value_count, size_t null_count,
                                      ColumnIndexMetaPB* index_meta,
-                                     bool use_primary_key_bloom_filter = false) {
+                                     bool use_primary_key_bloom_filter = false, double fpp = 0.05) {
     const auto* type_info = get_scalar_type_info<type>();
     using CppType = typename CppTypeTraits<type>::CppType;
     std::string fname = dname + "/" + file_name;
     auto fs = io::global_local_filesystem();
     {
+        size_t expect_size = 0;
         io::FileWriterPtr file_writer;
         RETURN_IF_ERROR(fs->create_file(fname, &file_writer));
 
         std::unique_ptr<BloomFilterIndexWriter> bloom_filter_index_writer;
         BloomFilterOptions bf_options;
-
+        bf_options.fpp = fpp; // Set the expected FPP
         if (use_primary_key_bloom_filter) {
             RETURN_IF_ERROR(PrimaryKeyBloomFilterIndexWriterImpl::create(
                     bf_options, type_info, &bloom_filter_index_writer));
@@ -91,12 +93,26 @@ Status write_bloom_filter_index_file(const std::string& file_name, const void* v
                 bloom_filter_index_writer->add_nulls(null_count);
             }
             RETURN_IF_ERROR(bloom_filter_index_writer->flush());
+            auto bf_size = BloomFilter::optimal_bit_num(num, fpp) / 8;
+            expect_size += bf_size + 1;
             i += 1024;
         }
+        if (value_count == 3072) {
+            RETURN_IF_ERROR(bloom_filter_index_writer->add_values(vals + 3071, 1));
+            auto bf_size = BloomFilter::optimal_bit_num(1, fpp) / 8;
+            expect_size += bf_size + 1;
+        }
         RETURN_IF_ERROR(bloom_filter_index_writer->finish(file_writer.get(), index_meta));
         EXPECT_TRUE(file_writer->close().ok());
         EXPECT_EQ(BLOOM_FILTER_INDEX, index_meta->type());
         EXPECT_EQ(bf_options.strategy, index_meta->bloom_filter_index().hash_strategy());
+        if constexpr (!field_is_slice_type(type)) {
+            EXPECT_EQ(expect_size, bloom_filter_index_writer->size());
+        }
+        if (use_primary_key_bloom_filter) {
+            std::cout << "primary key bf size is " << bloom_filter_index_writer->size()
+                      << std::endl;
+        }
     }
     return Status::OK();
 }
@@ -128,7 +144,7 @@ Status test_bloom_filter_index_reader_writer_template(
         BloomFilterIndexReader* reader = nullptr;
         std::unique_ptr<BloomFilterIndexIterator> iter;
         get_bloom_filter_reader_iter(file_name, meta, &reader, &iter);
-
+        EXPECT_EQ(reader->algorithm(), BloomFilterAlgorithmPB::BLOCK_BLOOM_FILTER);
         // page 0
         std::unique_ptr<BloomFilter> bf;
         RETURN_IF_ERROR(iter->read_bloom_filter(0, &bf));
@@ -171,7 +187,7 @@ Status test_bloom_filter_index_reader_writer_template(
 }
 
 TEST_F(BloomFilterIndexReaderWriterTest, test_int) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     int* val = new int[num];
     for (int i = 0; i < num; ++i) {
         // there will be 3 bloom filter pages
@@ -186,8 +202,58 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_int) {
     delete[] val;
 }
 
+TEST_F(BloomFilterIndexReaderWriterTest, test_string) {
+    size_t num = 1024 * 3;
+    std::vector<std::string> val_strings(num);
+    for (size_t i = 0; i < num; ++i) {
+        val_strings[i] = "string_test_" + std::to_string(i + 1);
+    }
+    Slice* val = new Slice[num];
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = Slice(val_strings[i]);
+    }
+
+    std::string file_name = "bloom_filter_string";
+    Slice not_exist_value("string_test_not_exist");
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_STRING>(
+            file_name, val, num, 1, &not_exist_value, true);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_unsigned_int) {
+    size_t num = 1024 * 3;
+    uint32_t* val = new uint32_t[num];
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = static_cast<uint32_t>(i + 1);
+    }
+
+    std::string file_name = "bloom_filter_unsigned_int";
+    uint32_t not_exist_value = 0xFFFFFFFF;
+    auto st =
+            test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT>(
+                    file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_smallint) {
+    size_t num = 1024 * 3;
+    int16_t* val = new int16_t[num];
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = static_cast<int16_t>(i + 1);
+    }
+
+    std::string file_name = "bloom_filter_smallint";
+    int16_t not_exist_value = -1;
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_SMALLINT>(
+            file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
 TEST_F(BloomFilterIndexReaderWriterTest, test_bigint) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     int64_t* val = new int64_t[num];
     for (int i = 0; i < num; ++i) {
         // there will be 3 bloom filter pages
@@ -203,7 +269,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_bigint) {
 }
 
 TEST_F(BloomFilterIndexReaderWriterTest, test_largeint) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     int128_t* val = new int128_t[num];
     for (int i = 0; i < num; ++i) {
         // there will be 3 bloom filter pages
@@ -219,7 +285,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_largeint) {
 }
 
 TEST_F(BloomFilterIndexReaderWriterTest, test_varchar_type) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     std::string* val = new std::string[num];
     for (int i = 0; i < num; ++i) {
         // there will be 3 bloom filter pages
@@ -240,7 +306,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_varchar_type) {
 }
 
 TEST_F(BloomFilterIndexReaderWriterTest, test_char) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     std::string* val = new std::string[num];
     for (int i = 0; i < num; ++i) {
         // there will be 3 bloom filter pages
@@ -261,7 +327,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_char) {
 }
 
 TEST_F(BloomFilterIndexReaderWriterTest, test_date) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     uint24_t* val = new uint24_t[num];
     for (int i = 0; i < num; ++i) {
         // there will be 3 bloom filter pages
@@ -277,7 +343,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_date) {
 }
 
 TEST_F(BloomFilterIndexReaderWriterTest, test_datetime) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     int64_t* val = new int64_t[num];
     for (int i = 0; i < num; ++i) {
         // there will be 3 bloom filter pages
@@ -293,7 +359,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_datetime) {
 }
 
 TEST_F(BloomFilterIndexReaderWriterTest, test_decimal) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     decimal12_t* val = new decimal12_t[num];
     for (int i = 0; i < num; ++i) {
         // there will be 3 bloom filter pages
@@ -308,8 +374,29 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_decimal) {
     delete[] val;
 }
 
+TEST_F(BloomFilterIndexReaderWriterTest, test_primary_key_bloom_filter_index_char) {
+    size_t num = 1024 * 3;
+    std::string* val = new std::string[num];
+    for (int i = 0; i < num; ++i) {
+        // there will be 3 bloom filter pages
+        val[i] = "primary_key_" + std::to_string(10000 + i);
+    }
+    Slice* slices = new Slice[num];
+    for (int i = 0; i < num; ++i) {
+        // there will be 3 bloom filter pages
+        slices[i] = Slice(val[i].c_str(), val[i].size());
+    }
+    std::string file_name = "primary_key_bloom_filter_index_char";
+    Slice not_exist_value("primary_key_not_exist_char");
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_CHAR>(
+            file_name, slices, num, 1, &not_exist_value, true, true);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+    delete[] slices;
+}
+
 TEST_F(BloomFilterIndexReaderWriterTest, test_primary_key_bloom_filter_index) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     std::vector<std::string> val_strings(num);
     for (size_t i = 0; i < num; ++i) {
         val_strings[i] = "primary_key_" + std::to_string(i);
@@ -328,7 +415,7 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_primary_key_bloom_filter_index) {
 }
 
 TEST_F(BloomFilterIndexReaderWriterTest, test_primary_key_bloom_filter_index_int) {
-    size_t num = 1024 * 3 - 1;
+    size_t num = 1024 * 3;
     int* val = new int[num];
     for (int i = 0; i < num; ++i) {
         // there will be 3 bloom filter pages
@@ -344,5 +431,377 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_primary_key_bloom_filter_index_int
     delete[] val;
 }
 
+TEST_F(BloomFilterIndexReaderWriterTest, test_datev2) {
+    size_t num = 1024 * 3;
+    uint32_t* val = new uint32_t[num];
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = 20210101 + i; // YYYYMMDD
+    }
+
+    std::string file_name = "bloom_filter_datev2";
+    uint32_t not_exist_value = 20211231;
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_DATEV2>(
+            file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_datetimev2) {
+    size_t num = 1024 * 3;
+    uint64_t* val = new uint64_t[num];
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = 20210101000000 + i; // YYYYMMDDHHMMSS
+    }
+
+    std::string file_name = "bloom_filter_datetimev2";
+    uint64_t not_exist_value = 20211231235959;
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>(
+            file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_decimal32) {
+    size_t num = 1024 * 3;
+    int32_t* val = new int32_t[num];
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = static_cast<int32_t>(i * 100 + 1);
+    }
+
+    std::string file_name = "bloom_filter_decimal32";
+    int32_t not_exist_value = 99999;
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_DECIMAL32>(
+            file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_decimal64) {
+    size_t num = 1024 * 3;
+    ;
+    int64_t* val = new int64_t[num];
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = static_cast<int64_t>(i * 1000 + 123);
+    }
+
+    std::string file_name = "bloom_filter_decimal64";
+    int64_t not_exist_value = 9999999;
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_DECIMAL64>(
+            file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_ipv4) {
+    size_t num = 1024 * 3; // 3072
+    uint32_t* val = new uint32_t[num];
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = (192 << 24) | (168 << 16) | (i & 0xFFFF);
+    }
+
+    std::string file_name = "bloom_filter_ipv4";
+    uint32_t not_exist_value = (10 << 24) | (0 << 16) | (0 << 8) | 1; // 10.0.0.1
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_IPV4>(
+            file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_decimal128i) {
+    size_t num = 1024 * 3;
+    int128_t* val = new int128_t[num];
+
+    int128_t base_value = int128_t(1000000000ULL) * int128_t(1000000000ULL);
+
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = base_value + int128_t(i);
+    }
+
+    std::string file_name = "bloom_filter_decimal128i";
+    int128_t not_exist_value = int128_t(9999999999999999999ULL);
+
+    auto st =
+            test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_DECIMAL128I>(
+                    file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_decimal256) {
+    size_t num = 1024 * 3;
+    using Decimal256Type = wide::Int256;
+
+    Decimal256Type* val = new Decimal256Type[num];
+
+    Decimal256Type base_value = Decimal256Type(1000000000ULL); // 1e9
+    base_value *= Decimal256Type(1000000000ULL);               // base_value = 1e18
+    base_value *= Decimal256Type(100000000ULL);                // base_value = 1e26
+    base_value *= Decimal256Type(100000000ULL);                // base_value = 1e34
+    base_value *= Decimal256Type(10000ULL);                    // base_value = 1e38
+
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = base_value + Decimal256Type(i);
+    }
+
+    std::string file_name = "bloom_filter_decimal256";
+
+    Decimal256Type not_exist_value = base_value + Decimal256Type(9999999ULL);
+
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_DECIMAL256>(
+            file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_ipv6) {
+    size_t num = 1024 * 3;
+    uint128_t* val = new uint128_t[num];
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = (uint128_t(0x20010DB800000000) << 64) | uint128_t(i);
+    }
+
+    std::string file_name = "bloom_filter_ipv6";
+    uint128_t not_exist_value = (uint128_t(0x20010DB800000000) << 64) | uint128_t(999999);
+
+    auto st = test_bloom_filter_index_reader_writer_template<FieldType::OLAP_FIELD_TYPE_IPV6>(
+            file_name, val, num, 1, &not_exist_value);
+    EXPECT_TRUE(st.ok());
+    delete[] val;
+}
+
+template <FieldType type>
+Status write_ngram_bloom_filter_index_file(const std::string& file_name, Slice* values,
+                                           size_t num_values, const TypeInfo* type_info,
+                                           BloomFilterIndexWriter* bf_index_writer,
+                                           ColumnIndexMetaPB* meta) {
+    auto fs = io::global_local_filesystem();
+    std::string fname = dname + "/" + file_name;
+    io::FileWriterPtr file_writer;
+    Status st = fs->create_file(fname, &file_writer);
+    EXPECT_TRUE(st.ok()) << st.to_string();
+
+    size_t i = 0;
+    while (i < num_values) {
+        size_t num = std::min(static_cast<size_t>(1024), num_values - i);
+        st = bf_index_writer->add_values(values + i, num);
+        EXPECT_TRUE(st.ok());
+        st = bf_index_writer->flush();
+        EXPECT_TRUE(st.ok());
+        i += num;
+    }
+    bf_index_writer->add_nulls(1);
+    st = bf_index_writer->finish(file_writer.get(), meta);
+    EXPECT_TRUE(st.ok()) << "Writer finish status: " << st.to_string();
+    EXPECT_TRUE(file_writer->close().ok());
+
+    return Status::OK();
+}
+
+Status read_and_test_ngram_bloom_filter_index_file(const std::string& file_name, size_t num_values,
+                                                   uint8_t gram_size, uint16_t bf_size,
+                                                   const ColumnIndexMetaPB& meta,
+                                                   const std::vector<std::string>& test_patterns) {
+    BloomFilterIndexReader* reader = nullptr;
+    std::unique_ptr<BloomFilterIndexIterator> iter;
+    get_bloom_filter_reader_iter(file_name, meta, &reader, &iter);
+    EXPECT_EQ(reader->algorithm(), BloomFilterAlgorithmPB::NGRAM_BLOOM_FILTER);
+
+    NgramTokenExtractor extractor(gram_size);
+    uint16_t gram_bf_size = bf_size;
+
+    size_t total_pages = (num_values + 1023) / 1024;
+    for (size_t page = 0; page < total_pages; ++page) {
+        std::unique_ptr<BloomFilter> bf;
+        auto st = iter->read_bloom_filter(page, &bf);
+        EXPECT_TRUE(st.ok());
+
+        for (const auto& pattern : test_patterns) {
+            std::unique_ptr<BloomFilter> query_bf;
+            st = BloomFilter::create(NGRAM_BLOOM_FILTER, &query_bf, gram_bf_size);
+            EXPECT_TRUE(st.ok());
+
+            if (extractor.string_like_to_bloom_filter(pattern.data(), pattern.size(), *query_bf)) {
+                bool contains = bf->contains(*query_bf);
+                bool expected = false;
+                if ((page == 0 && (pattern == "ngram15" || pattern == "ngram1000")) ||
+                    (page == 1 && pattern == "ngram1499")) {
+                    expected = true;
+                }
+                EXPECT_EQ(contains, expected) << "Pattern: " << pattern << ", Page: " << page;
+            }
+        }
+    }
+
+    delete reader;
+    return Status::OK();
+}
+
+template <FieldType type>
+Status test_ngram_bloom_filter_index_reader_writer(const std::string& file_name, Slice* values,
+                                                   size_t num_values, uint8_t gram_size,
+                                                   uint16_t bf_size) {
+    const auto* type_info = get_scalar_type_info<type>();
+    ColumnIndexMetaPB meta;
+
+    BloomFilterOptions bf_options;
+    std::unique_ptr<BloomFilterIndexWriter> bf_index_writer;
+    RETURN_IF_ERROR(NGramBloomFilterIndexWriterImpl::create(bf_options, type_info, gram_size,
+                                                            bf_size, &bf_index_writer));
+
+    RETURN_IF_ERROR(write_ngram_bloom_filter_index_file<type>(
+            file_name, values, num_values, type_info, bf_index_writer.get(), &meta));
+
+    std::vector<std::string> test_patterns = {"ngram15", "ngram1000", "ngram1499",
+                                              "non-existent-string"};
+
+    RETURN_IF_ERROR(read_and_test_ngram_bloom_filter_index_file(file_name, num_values, gram_size,
+                                                                bf_size, meta, test_patterns));
+
+    return Status::OK();
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_ngram_bloom_filter) {
+    size_t num = 1500;
+    std::vector<std::string> val(num);
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = "ngram" + std::to_string(i);
+    }
+    std::vector<Slice> slices(num);
+    for (size_t i = 0; i < num; ++i) {
+        slices[i] = Slice(val[i].data(), val[i].size());
+    }
+
+    uint8_t gram_size = 5;
+    uint16_t bf_size = 65535;
+
+    auto st = test_ngram_bloom_filter_index_reader_writer<FieldType::OLAP_FIELD_TYPE_VARCHAR>(
+            "bloom_filter_ngram_varchar", slices.data(), num, gram_size, bf_size);
+    EXPECT_TRUE(st.ok());
+    st = test_ngram_bloom_filter_index_reader_writer<FieldType::OLAP_FIELD_TYPE_CHAR>(
+            "bloom_filter_ngram_char", slices.data(), num, gram_size, bf_size);
+    EXPECT_TRUE(st.ok());
+    st = test_ngram_bloom_filter_index_reader_writer<FieldType::OLAP_FIELD_TYPE_STRING>(
+            "bloom_filter_ngram_string", slices.data(), num, gram_size, bf_size);
+    EXPECT_TRUE(st.ok());
+    st = test_ngram_bloom_filter_index_reader_writer<FieldType::OLAP_FIELD_TYPE_INT>(
+            "bloom_filter_ngram_string", slices.data(), num, gram_size, bf_size);
+    EXPECT_FALSE(st.ok());
+    EXPECT_EQ(st.code(), TStatusCode::NOT_IMPLEMENTED_ERROR);
+}
+void test_ngram_bloom_filter_with_size(uint16_t bf_size) {
+    const auto* type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>();
+    ColumnIndexMetaPB meta;
+
+    BloomFilterOptions bf_options;
+    size_t num = 1500;
+    std::vector<std::string> val(num);
+    for (size_t i = 0; i < num; ++i) {
+        val[i] = "ngram" + std::to_string(i);
+    }
+    std::vector<Slice> slices(num);
+    for (size_t i = 0; i < num; ++i) {
+        slices[i] = Slice(val[i].data(), val[i].size());
+    }
+    size_t total_pages = (num + 1024 - 1) / 1024;
+    uint8_t gram_size = 5;
+
+    std::unique_ptr<BloomFilterIndexWriter> bf_index_writer;
+    auto st = NGramBloomFilterIndexWriterImpl::create(bf_options, type_info, gram_size, bf_size,
+                                                      &bf_index_writer);
+    EXPECT_TRUE(st.ok());
+
+    std::string file_name = "bloom_filter_ngram_varchar_size_" + std::to_string(bf_size);
+    st = write_ngram_bloom_filter_index_file<FieldType::OLAP_FIELD_TYPE_VARCHAR>(
+            file_name, slices.data(), num, type_info, bf_index_writer.get(), &meta);
+    EXPECT_TRUE(st.ok());
+    EXPECT_EQ(bf_index_writer->size(), static_cast<uint64_t>(bf_size) * total_pages);
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_ngram_bloom_filter_size) {
+    std::vector<uint16_t> bf_sizes = {1024, 2048, 4096, 8192, 16384, 32768, 65535};
+    for (uint16_t bf_size : bf_sizes) {
+        test_ngram_bloom_filter_with_size(bf_size);
+    }
+}
+
+TEST_F(BloomFilterIndexReaderWriterTest, test_unsupported_type) {
+    auto type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_FLOAT>();
+    BloomFilterOptions bf_options;
+    std::unique_ptr<BloomFilterIndexWriter> bloom_filter_index_writer;
+    auto st = BloomFilterIndexWriter::create(bf_options, type_info, &bloom_filter_index_writer);
+    EXPECT_FALSE(st.ok());
+    EXPECT_EQ(st.code(), TStatusCode::NOT_IMPLEMENTED_ERROR);
+}
+
+// Test function for verifying Bloom Filter FPP
+void test_bloom_filter_fpp(double expected_fpp) {
+    size_t n = 10000;  // Number of elements to insert into the Bloom Filter
+    size_t m = 100000; // Number of non-existent elements to test for false positives
+
+    // Generate and insert elements into the Bloom Filter index
+    std::vector<int64_t> insert_values;
+    for (size_t i = 0; i < n; ++i) {
+        int64_t val = static_cast<int64_t>(i);
+        insert_values.push_back(val);
+    }
+
+    // Write the Bloom Filter index to file
+    std::string file_name = "bloom_filter_fpp_test";
+    ColumnIndexMetaPB index_meta;
+    Status st = write_bloom_filter_index_file<FieldType::OLAP_FIELD_TYPE_BIGINT>(
+            file_name, insert_values.data(), n, 0, &index_meta, false, expected_fpp);
+    EXPECT_TRUE(st.ok());
+
+    // Read the Bloom Filter index
+    BloomFilterIndexReader* reader = nullptr;
+    std::unique_ptr<BloomFilterIndexIterator> iter;
+    get_bloom_filter_reader_iter(file_name, index_meta, &reader, &iter);
+
+    // Read the Bloom Filter (only one page since we flushed once)
+    std::unique_ptr<BloomFilter> bf;
+    st = iter->read_bloom_filter(0, &bf);
+    EXPECT_TRUE(st.ok());
+
+    // Generate non-existent elements for testing false positive rate
+    std::unordered_set<int64_t> inserted_elements(insert_values.begin(), insert_values.end());
+    std::unordered_set<int64_t> non_exist_elements;
+    std::vector<int64_t> test_values;
+    size_t max_value = n + m * 10; // Ensure test values are not in the inserted range
+    boost::mt19937_64 rng(12345);  // Seed the random number generator for reproducibility
+    std::uniform_int_distribution<int64_t> dist(static_cast<int64_t>(n + 1),
+                                                static_cast<int64_t>(max_value));
+    while (non_exist_elements.size() < m) {
+        int64_t val = dist(rng);
+        if (inserted_elements.find(val) == inserted_elements.end()) {
+            non_exist_elements.insert(val);
+            test_values.push_back(val);
+        }
+    }
+
+    // Test non-existent elements and count false positives
+    size_t fp_count = 0;
+    for (const auto& val : test_values) {
+        if (bf->test_bytes(reinterpret_cast<const char*>(&val), sizeof(int64_t))) {
+            fp_count++;
+        }
+    }
+
+    // Compute actual false positive probability
+    double actual_fpp = static_cast<double>(fp_count) / static_cast<double>(m);
+    std::cout << "Expected FPP: " << expected_fpp << ", Actual FPP: " << actual_fpp << std::endl;
+
+    // Verify that actual FPP is within the allowable error range
+    EXPECT_LE(actual_fpp, expected_fpp);
+
+    delete reader;
+}
+
+// Test case to run FPP tests with multiple expected FPP values
+TEST_F(BloomFilterIndexReaderWriterTest, test_bloom_filter_fpp_multiple) {
+    std::vector<double> fpp_values = {0.01, 0.02, 0.05};
+    for (double fpp : fpp_values) {
+        test_bloom_filter_fpp(fpp);
+    }
+}
 } // namespace segment_v2
 } // namespace doris

From 334b3435bc998e5e2405d8fe7b25ba16d0267f03 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Mon, 2 Dec 2024 13:05:44 +0530
Subject: [PATCH 111/399] [Enhancement] (nereids)implement
 CreateEncryptKeyCommand in nereids (#44807)

Issue Number: close #42591
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../doris/catalog/EncryptKeyHelper.java       | 10 ++-
 .../nereids/parser/LogicalPlanBuilder.java    |  9 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/CreateEncryptkeyCommand.java     | 73 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../test_nereids_encrypt_test.groovy          |  2 +-
 7 files changed, 98 insertions(+), 4 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateEncryptkeyCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index a372a993ac6f22..5200f714a090ce 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -188,6 +188,7 @@ supportedCreateStatement
         USING LEFT_PAREN booleanExpression RIGHT_PAREN                    #createRowPolicy
     | CREATE SQL_BLOCK_RULE (IF NOT EXISTS)?
         name=identifier properties=propertyClause?                        #createSqlBlockRule
+    | CREATE ENCRYPTKEY (IF NOT EXISTS)? multipartIdentifier AS STRING_LITERAL  #createEncryptkey
     ;
 
 supportedAlterStatement
@@ -767,7 +768,6 @@ unsupportedCreateStatement
         (CONDITIONS LEFT_PAREN workloadPolicyConditions RIGHT_PAREN)?
         (ACTIONS LEFT_PAREN workloadPolicyActions RIGHT_PAREN)?
         properties=propertyClause?                                              #createWorkloadPolicy
-    | CREATE ENCRYPTKEY (IF NOT EXISTS)? multipartIdentifier AS STRING_LITERAL  #createEncryptkey
     | CREATE STORAGE POLICY (IF NOT EXISTS)?
         name=identifier properties=propertyClause?                              #createStoragePolicy
     | BUILD INDEX name=identifier ON tableName=multipartIdentifier
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EncryptKeyHelper.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EncryptKeyHelper.java
index 2e30f0ba86f326..934dd22f3a5b89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EncryptKeyHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EncryptKeyHelper.java
@@ -32,10 +32,16 @@ public class EncryptKeyHelper {
 
     public static void createEncryptKey(CreateEncryptKeyStmt stmt) throws UserException {
         EncryptKeyName name = stmt.getEncryptKeyName();
-        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(name.getDb());
-        db.addEncryptKey(stmt.getEncryptKey(), stmt.isIfNotExists());
+        createEncryptKey(name.getDb(), stmt.getEncryptKey(), stmt.isIfNotExists());
+    }
+
+    public static void createEncryptKey(String dbName, EncryptKey encryptKey,
+                                        boolean isIfNotExists) throws UserException {
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
+        db.addEncryptKey(encryptKey, isIfNotExists);
     }
 
+
     public static void replayCreateEncryptKey(EncryptKey encryptKey) throws MetaNotFoundException {
         String dbName = encryptKey.getEncryptKeyName().getDb();
         Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(dbName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index dfe879e7d44dad..f94284145d5320 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -89,6 +89,7 @@
 import org.apache.doris.nereids.DorisParser.ComplexColTypeListContext;
 import org.apache.doris.nereids.DorisParser.ComplexDataTypeContext;
 import org.apache.doris.nereids.DorisParser.ConstantContext;
+import org.apache.doris.nereids.DorisParser.CreateEncryptkeyContext;
 import org.apache.doris.nereids.DorisParser.CreateMTMVContext;
 import org.apache.doris.nereids.DorisParser.CreateProcedureContext;
 import org.apache.doris.nereids.DorisParser.CreateRoutineLoadContext;
@@ -476,6 +477,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.Constraint;
+import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
@@ -4749,6 +4751,13 @@ public LogicalPlan visitDropRole(DropRoleContext ctx) {
         return new DropRoleCommand(ctx.name.getText(), ctx.EXISTS() != null);
     }
 
+    @Override
+    public LogicalPlan visitCreateEncryptkey(CreateEncryptkeyContext ctx) {
+        List<String> nameParts = visitMultipartIdentifier(ctx.multipartIdentifier());
+        return new CreateEncryptkeyCommand(new EncryptKeyName(nameParts), ctx.EXISTS() != null,
+                                            stripQuotes(ctx.STRING_LITERAL().getText()));
+    }
+
     @Override
     public LogicalPlan visitDropEncryptkey(DropEncryptkeyContext ctx) {
         List<String> nameParts = visitMultipartIdentifier(ctx.name);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 14a1b13a4667df..d7331d5a65d738 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -237,5 +237,6 @@ public enum PlanType {
     RECOVER_TABLE_COMMAND,
     RECOVER_PARTITION_COMMAND,
     REPLAY_COMMAND,
+    CREATE_ENCRYPTKEY_COMMAND,
     CREATE_ROUTINE_LOAD_COMMAND
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateEncryptkeyCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateEncryptkeyCommand.java
new file mode 100644
index 00000000000000..f8fc7c3cc12c5e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateEncryptkeyCommand.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.EncryptKeyName;
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.EncryptKey;
+import org.apache.doris.catalog.EncryptKeyHelper;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+/** CreateEncryptkeyCommand */
+public class CreateEncryptkeyCommand extends Command implements ForwardWithSync {
+    private final boolean ifNotExists;
+    private final EncryptKeyName encryptKeyName;
+    private final String keyString;
+
+    public CreateEncryptkeyCommand(EncryptKeyName encryptKeyName, boolean ifNotExists, String keyString) {
+        super(PlanType.CREATE_ENCRYPTKEY_COMMAND);
+        this.ifNotExists = ifNotExists;
+        this.encryptKeyName = encryptKeyName;
+        this.keyString = keyString;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check operation privilege
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        encryptKeyName.analyze(ctx);
+        if (Strings.isNullOrEmpty(keyString)) {
+            throw new AnalysisException("keyString can not be null or empty string.");
+        }
+        EncryptKeyHelper.createEncryptKey(encryptKeyName.getDb(),
+                                            new EncryptKey(encryptKeyName, keyString), ifNotExists);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCreateEncryptKeyCommand(this, context);
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.CREATE;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index d386d097a93b49..033290d37132e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -33,6 +33,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CancelWarmUpJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
@@ -175,6 +176,10 @@ default R visitExportCommand(ExportCommand exportCommand, C context) {
         return visitCommand(exportCommand, context);
     }
 
+    default R visitCreateEncryptKeyCommand(CreateEncryptkeyCommand createEncryptKeyCommand, C context) {
+        return visitCommand(createEncryptKeyCommand, context);
+    }
+
     default R visitCreateTableCommand(CreateTableCommand createTableCommand, C context) {
         return visitCommand(createTableCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy b/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy
index 2fab616580faaa..8954090bb4c31d 100644
--- a/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy
+++ b/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy
@@ -20,7 +20,7 @@ suite("test_nereids_encrypt_test") {
     sql """ create database IF NOT EXISTS ${dbName}; """
     sql """ use ${dbName}; """
     checkNereidsExecute("drop encryptkey if exists ${encryptkeyName}")    
-    sql """CREATE ENCRYPTKEY ${encryptkeyName} AS "ABCD123456789";"""
+    checkNereidsExecute("""CREATE ENCRYPTKEY ${encryptkeyName} AS "ABCD123456789";""")
     qt_check_encrypt_1("SHOW ENCRYPTKEYS FROM ${dbName}")
     checkNereidsExecute("drop encryptkey ${encryptkeyName}")
     qt_check_encrypt_2("SHOW ENCRYPTKEYS FROM ${dbName}")    

From 54fe0b463a9dd29a562539bdc351929aebb0b064 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Mon, 2 Dec 2024 13:06:03 +0530
Subject: [PATCH 112/399] [Enhancement] (nereids)implement CreateRoleCommand in
 nereids (#44814)

Issue Number: close #42597
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../apache/doris/mysql/privilege/Auth.java    |  4 +
 .../nereids/parser/LogicalPlanBuilder.java    |  9 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/CreateRoleCommand.java     | 75 +++++++++++++++++++
 .../trees/plans/commands/DropRoleCommand.java |  4 -
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../ddl/account/test_nereids_role.groovy      |  4 +-
 .../ddl/alter/test_nereids_role.groovy        | 66 ----------------
 9 files changed, 97 insertions(+), 73 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateRoleCommand.java
 delete mode 100644 regression-test/suites/nereids_p0/ddl/alter/test_nereids_role.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 5200f714a090ce..3a377ba05bc707 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -181,6 +181,7 @@ supportedCreateStatement
     | CREATE (EXTERNAL)? TABLE (IF NOT EXISTS)? name=multipartIdentifier
         LIKE existedTable=multipartIdentifier
         (WITH ROLLUP (rollupNames=identifierList)?)?                      #createTableLike
+    | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?    #createRole        
     | CREATE ROW POLICY (IF NOT EXISTS)? name=identifier
         ON table=multipartIdentifier
         AS type=(RESTRICTIVE | PERMISSIVE)
@@ -751,7 +752,6 @@ unsupportedCreateStatement
         (SUPERUSER | DEFAULT ROLE role=STRING_LITERAL)?
         passwordOption (COMMENT STRING_LITERAL)?                                #createUser
     | CREATE (READ ONLY)? REPOSITORY name=identifier WITH storageBackend        #createRepository
-    | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?    #createRole
     | CREATE FILE name=STRING_LITERAL
         ((FROM | IN) database=identifier)? properties=propertyClause            #createFile
     | CREATE INDEX (IF NOT EXISTS)? name=identifier
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
index f1277a1cd0a3b5..8a13c6dd125548 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
@@ -1019,6 +1019,10 @@ public void createRole(CreateRoleStmt stmt) throws DdlException {
         createRoleInternal(stmt.getRole(), stmt.isSetIfNotExists(), stmt.getComment(), false);
     }
 
+    public void createRole(String role, boolean ignoreIfExists, String comment) throws DdlException {
+        createRoleInternal(role, ignoreIfExists, comment, false);
+    }
+
     public void alterRole(AlterRoleStmt stmt) throws DdlException {
         alterRoleInternal(stmt.getRole(), stmt.getComment(), false);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index f94284145d5320..2d55baffb1ebd9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -92,6 +92,7 @@
 import org.apache.doris.nereids.DorisParser.CreateEncryptkeyContext;
 import org.apache.doris.nereids.DorisParser.CreateMTMVContext;
 import org.apache.doris.nereids.DorisParser.CreateProcedureContext;
+import org.apache.doris.nereids.DorisParser.CreateRoleContext;
 import org.apache.doris.nereids.DorisParser.CreateRoutineLoadContext;
 import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext;
 import org.apache.doris.nereids.DorisParser.CreateSqlBlockRuleContext;
@@ -482,6 +483,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableLikeCommand;
@@ -4671,6 +4673,13 @@ public LogicalPlan visitAlterRole(AlterRoleContext ctx) {
         return new AlterRoleCommand(ctx.role.getText(), comment);
     }
 
+    @Override
+    public LogicalPlan visitCreateRole(CreateRoleContext ctx) {
+        String comment = ctx.STRING_LITERAL() == null ? "" : LogicalPlanBuilderAssistant.escapeBackSlash(
+                ctx.STRING_LITERAL().getText().substring(1, ctx.STRING_LITERAL().getText().length() - 1));
+        return new CreateRoleCommand(ctx.EXISTS() != null, ctx.name.getText(), comment);
+    }
+
     @Override
     public LogicalPlan visitShowFrontends(ShowFrontendsContext ctx) {
         String detail = (ctx.name != null) ? ctx.name.getText() : null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index d7331d5a65d738..0cc0e5594075b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -171,6 +171,7 @@ public enum PlanType {
     SHOW_CREATE_PROCEDURE_COMMAND,
     CREATE_VIEW_COMMAND,
     CLEAN_ALL_PROFILE_COMMAND,
+    CREATE_ROLE_COMMAND,
     ALTER_ROLE_COMMAND,
     ALTER_VIEW_COMMAND,
     ALTER_STORAGE_VAULT,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateRoleCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateRoleCommand.java
new file mode 100644
index 00000000000000..1596993ba669f2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateRoleCommand.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+/**
+ * Create role command
+ */
+public class CreateRoleCommand extends Command implements ForwardWithSync {
+    private boolean ifNotExists;
+    private String role;
+    private String comment;
+
+    /**
+     * ctor of this command.
+     */
+    public CreateRoleCommand(boolean ifNotExists, String role, String comment) {
+        super(PlanType.CREATE_ROLE_COMMAND);
+        this.ifNotExists = ifNotExists;
+        this.role = role;
+        this.comment = Strings.nullToEmpty(comment);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCreateRoleCommand(this, context);
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        if (Config.access_controller_type.equalsIgnoreCase("ranger-doris")) {
+            throw new AnalysisException("Create role is prohibited when Ranger is enabled.");
+        }
+        FeNameFormat.checkRoleName(role, false /* can not be admin */, "Can not create role");
+        // check if current user has GRANT priv on GLOBAL level.
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE ROLE");
+        }
+        Env.getCurrentEnv().getAuth().createRole(role, ifNotExists, comment);
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.CREATE;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropRoleCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropRoleCommand.java
index 2f7858aa6ae180..5b74d32cf6a4df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropRoleCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropRoleCommand.java
@@ -29,14 +29,10 @@
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.StmtExecutor;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
 /**
  * drop roles command
  */
 public class DropRoleCommand extends DropCommand {
-    public static final Logger LOG = LogManager.getLogger(DropRoleCommand.class);
     private final boolean ifExists;
     private final String role;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 033290d37132e8..d1d424738f4924 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -38,6 +38,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableLikeCommand;
@@ -486,6 +487,10 @@ default R visitCreateSqlBlockRuleCommand(CreateSqlBlockRuleCommand dropRoleComma
         return visitCommand(dropRoleCommand, context);
     }
 
+    default R visitCreateRoleCommand(CreateRoleCommand createRoleCommand, C context) {
+        return visitCommand(createRoleCommand, context);
+    }
+
     default R visitDropRoleCommand(DropRoleCommand dropRoleCommand, C context) {
         return visitCommand(dropRoleCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/ddl/account/test_nereids_role.groovy b/regression-test/suites/nereids_p0/ddl/account/test_nereids_role.groovy
index 4e2cd4ca6375d2..a3bee25cf5822a 100644
--- a/regression-test/suites/nereids_p0/ddl/account/test_nereids_role.groovy
+++ b/regression-test/suites/nereids_p0/ddl/account/test_nereids_role.groovy
@@ -19,7 +19,7 @@ import org.junit.Assert;
 
 suite("test_nereids_role") {
     def role= 'nereids_account_role_test'
-    def user = 'acount_role_user_test'
+    def user = 'nereids_acount_role_user_test'
     def dbName = 'nereids_account_role_test_db'
     def pwd = 'C123_567p'
 
@@ -28,7 +28,7 @@ suite("test_nereids_role") {
     sql """DROP DATABASE IF EXISTS ${dbName}"""
     sql """CREATE DATABASE ${dbName}"""
 
-    sql """CREATE ROLE ${role}"""
+    checkNereidsExecute("CREATE ROLE ${role}")
     sql """GRANT SELECT_PRIV ON ${context.config.defaultDb} TO ROLE '${role}'"""
     sql """GRANT SELECT_PRIV ON ${dbName} TO ROLE '${role}'"""
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}' DEFAULT ROLE '${role}'"""
diff --git a/regression-test/suites/nereids_p0/ddl/alter/test_nereids_role.groovy b/regression-test/suites/nereids_p0/ddl/alter/test_nereids_role.groovy
deleted file mode 100644
index f23982188269b7..00000000000000
--- a/regression-test/suites/nereids_p0/ddl/alter/test_nereids_role.groovy
+++ /dev/null
@@ -1,66 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import org.junit.Assert;
-
-suite("test_nereids_role_x", "account") {
-    def role= 'nereids_account_role_test_x'
-    def user = 'nereids_acount_role_user_test_x'
-    def dbName = 'nereids_account_role_test_db_x'
-    def pwd = 'C123_567p'
-
-    try_sql("DROP ROLE ${role}")
-    try_sql("DROP USER ${user}")
-    sql """DROP DATABASE IF EXISTS ${dbName}"""
-    sql """CREATE DATABASE IF NOT EXISTS ${dbName}"""
-    sql """CREATE ROLE ${role}"""
-    sql """GRANT SELECT_PRIV ON internal.${context.config.defaultDb}.* TO ROLE '${role}'"""
-    sql """GRANT SELECT_PRIV ON internal.${dbName}.* TO ROLE '${role}'"""
-    sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}' DEFAULT ROLE '${role}'"""
-    def result1 = connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
-        sql "show databases like '${dbName}'"
-    }
-    assertEquals(result1.size(), 1)
-
-    sql """REVOKE SELECT_PRIV ON ${dbName} FROM ROLE '${role}'"""
-    def result2 = connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
-        sql "show databases like '${dbName}'"
-    }
-    assertEquals(result2.size(), 0)
-
-    sql """DROP USER ${user}"""
-    sql """DROP ROLE ${role}"""
-    sql """DROP DATABASE ${dbName}"""
-
-    // test comment
-    // create role with comment
-    sql """CREATE ROLE ${role} comment 'account_p0_account_role_test_comment_create'"""
-    def roles_create = sql """show roles"""
-    logger.info("roles_create: " + roles_create.toString())
-    assertTrue(roles_create.toString().contains("account_p0_account_role_test_comment_create"))
-    // alter role with comment
-    checkNereidsExecute("ALTER ROLE ${role} comment 'account_p0_account_role_test_comment_alter';");
-    def roles_alter = sql """show roles"""
-    logger.info("roles_alter: " + roles_alter.toString())
-    assertTrue(roles_alter.toString().contains("account_p0_account_role_test_comment_alter"))
-    // drop role
-    checkNereidsExecute("""DROP ROLE ${role}""")
-    def roles_drop = sql """show roles"""
-    logger.info("roles_drop: " + roles_drop.toString())
-    assertFalse(roles_drop.toString().contains("account_p0_account_role_test_comment_alter"))
-}
-

From 53ea2a0a1d18900a3ad5bca8d1b1d5fe1a98111c Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Mon, 2 Dec 2024 16:40:38 +0800
Subject: [PATCH 113/399] [fix](coordinator) fix NereidsCoordinator can not
 interrupt query in fe (#44795)

fix NereidsCoordinator can not interrupt query in fe, the QueryProcessor
should set the status to failed
---
 .../apache/doris/qe/AbstractJobProcessor.java | 118 ++++++++++++++++++
 .../org/apache/doris/qe/JobProcessor.java     |   5 +-
 .../apache/doris/qe/NereidsCoordinator.java   |   5 +-
 .../doris/qe/runtime/LoadProcessor.java       |  74 ++---------
 .../qe/runtime/PipelineExecutionTask.java     |   2 +-
 .../runtime/PipelineExecutionTaskBuilder.java |   2 +-
 .../doris/qe/runtime/QueryProcessor.java      |  18 ++-
 7 files changed, 142 insertions(+), 82 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java
new file mode 100644
index 00000000000000..2858de25d578fc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AbstractJobProcessor.java
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.qe;
+
+import org.apache.doris.common.Status;
+import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.qe.runtime.BackendFragmentId;
+import org.apache.doris.qe.runtime.MultiFragmentsPipelineTask;
+import org.apache.doris.qe.runtime.PipelineExecutionTask;
+import org.apache.doris.qe.runtime.SingleFragmentPipelineTask;
+import org.apache.doris.thrift.TReportExecStatusParams;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Optional;
+
+/** AbstractJobProcessor */
+public abstract class AbstractJobProcessor implements JobProcessor {
+    private final Logger logger = LogManager.getLogger(getClass());
+
+    protected final CoordinatorContext coordinatorContext;
+    protected volatile Optional<PipelineExecutionTask> executionTask;
+    protected volatile Optional<Map<BackendFragmentId, SingleFragmentPipelineTask>> backendFragmentTasks;
+
+    public AbstractJobProcessor(CoordinatorContext coordinatorContext) {
+        this.coordinatorContext = Objects.requireNonNull(coordinatorContext, "coordinatorContext can not be null");
+        this.executionTask = Optional.empty();
+        this.backendFragmentTasks = Optional.empty();
+    }
+
+    protected abstract void doProcessReportExecStatus(
+            TReportExecStatusParams params, SingleFragmentPipelineTask fragmentTask);
+
+    @Override
+    public final void setPipelineExecutionTask(PipelineExecutionTask pipelineExecutionTask) {
+        Preconditions.checkArgument(pipelineExecutionTask != null, "sqlPipelineTask can not be null");
+
+        this.executionTask = Optional.of(pipelineExecutionTask);
+        Map<BackendFragmentId, SingleFragmentPipelineTask> backendFragmentTasks
+                = buildBackendFragmentTasks(pipelineExecutionTask);
+        this.backendFragmentTasks = Optional.of(backendFragmentTasks);
+
+        afterSetPipelineExecutionTask(pipelineExecutionTask);
+    }
+
+    protected void afterSetPipelineExecutionTask(PipelineExecutionTask pipelineExecutionTask) {}
+
+    @Override
+    public final void updateFragmentExecStatus(TReportExecStatusParams params) {
+        SingleFragmentPipelineTask fragmentTask = backendFragmentTasks.get().get(
+                new BackendFragmentId(params.getBackendId(), params.getFragmentId()));
+        if (fragmentTask == null || !fragmentTask.processReportExecStatus(params)) {
+            return;
+        }
+
+        TUniqueId queryId = coordinatorContext.queryId;
+        Status status = new Status(params.status);
+        // for now, abort the query if we see any error except if the error is cancelled
+        // and returned_all_results_ is true.
+        // (UpdateStatus() initiates cancellation, if it hasn't already been initiated)
+        if (!status.ok()) {
+            if (coordinatorContext.isEos() && status.isCancelled()) {
+                logger.warn("Query {} has returned all results, fragment_id={} instance_id={}, be={}"
+                                + " is reporting failed status {}",
+                        DebugUtil.printId(queryId), params.getFragmentId(),
+                        DebugUtil.printId(params.getFragmentInstanceId()),
+                        params.getBackendId(),
+                        status.toString());
+            } else {
+                logger.warn("one instance report fail, query_id={} fragment_id={} instance_id={}, be={},"
+                                + " error message: {}",
+                        DebugUtil.printId(queryId), params.getFragmentId(),
+                        DebugUtil.printId(params.getFragmentInstanceId()),
+                        params.getBackendId(), status.toString());
+                coordinatorContext.updateStatusIfOk(status);
+            }
+        }
+        doProcessReportExecStatus(params, fragmentTask);
+    }
+
+    private Map<BackendFragmentId, SingleFragmentPipelineTask> buildBackendFragmentTasks(
+            PipelineExecutionTask executionTask) {
+        ImmutableMap.Builder<BackendFragmentId, SingleFragmentPipelineTask> backendFragmentTasks
+                = ImmutableMap.builder();
+        for (Entry<Long, MultiFragmentsPipelineTask> backendTask : executionTask.getChildrenTasks().entrySet()) {
+            Long backendId = backendTask.getKey();
+            for (Entry<Integer, SingleFragmentPipelineTask> fragmentIdToTask : backendTask.getValue()
+                    .getChildrenTasks().entrySet()) {
+                Integer fragmentId = fragmentIdToTask.getKey();
+                SingleFragmentPipelineTask fragmentTask = fragmentIdToTask.getValue();
+                backendFragmentTasks.put(new BackendFragmentId(backendId, fragmentId), fragmentTask);
+            }
+        }
+        return backendFragmentTasks.build();
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java
index ede218848c7221..7e4042dde3c3bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java
@@ -19,9 +19,12 @@
 
 import org.apache.doris.common.Status;
 import org.apache.doris.qe.runtime.PipelineExecutionTask;
+import org.apache.doris.thrift.TReportExecStatusParams;
 
 public interface JobProcessor {
-    void setSqlPipelineTask(PipelineExecutionTask pipelineExecutionTask);
+    void setPipelineExecutionTask(PipelineExecutionTask pipelineExecutionTask);
 
     void cancel(Status cancelReason);
+
+    void updateFragmentExecStatus(TReportExecStatusParams params);
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
index a9d6becc7fafe3..a6f24806ed74aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
@@ -233,10 +233,7 @@ public boolean isDone() {
 
     @Override
     public void updateFragmentExecStatus(TReportExecStatusParams params) {
-        JobProcessor jobProcessor = coordinatorContext.getJobProcessor();
-        if (jobProcessor instanceof LoadProcessor) {
-            coordinatorContext.asLoadProcessor().updateFragmentExecStatus(params);
-        }
+        coordinatorContext.getJobProcessor().updateFragmentExecStatus(params);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java
index 3a448521fca0bf..fb32919d834a54 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java
@@ -24,46 +24,39 @@
 import org.apache.doris.datasource.hive.HMSTransaction;
 import org.apache.doris.datasource.iceberg.IcebergTransaction;
 import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.qe.AbstractJobProcessor;
 import org.apache.doris.qe.CoordinatorContext;
-import org.apache.doris.qe.JobProcessor;
 import org.apache.doris.qe.LoadContext;
 import org.apache.doris.thrift.TFragmentInstanceReport;
 import org.apache.doris.thrift.TReportExecStatusParams;
 import org.apache.doris.thrift.TStatusCode;
 import org.apache.doris.thrift.TUniqueId;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
-public class LoadProcessor implements JobProcessor {
+public class LoadProcessor extends AbstractJobProcessor {
     private static final Logger LOG = LogManager.getLogger(LoadProcessor.class);
 
-    public final CoordinatorContext coordinatorContext;
     public final LoadContext loadContext;
     public final long jobId;
 
     // this latch is used to wait finish for load, for example, insert into statement
     // MarkedCountDownLatch:
     //  key: fragmentId, value: backendId
-    private volatile Optional<PipelineExecutionTask> executionTask;
     private volatile Optional<MarkedCountDownLatch<Integer, Long>> latch;
-    private volatile Optional<Map<BackendFragmentId, SingleFragmentPipelineTask>> backendFragmentTasks;
     private volatile List<SingleFragmentPipelineTask> topFragmentTasks;
 
     public LoadProcessor(CoordinatorContext coordinatorContext, long jobId) {
-        this.coordinatorContext = Objects.requireNonNull(coordinatorContext, "coordinatorContext can not be null");
+        super(coordinatorContext);
+
         this.loadContext = new LoadContext();
-        this.executionTask = Optional.empty();
         this.latch = Optional.empty();
         this.backendFragmentTasks = Optional.empty();
 
@@ -87,14 +80,8 @@ public LoadProcessor(CoordinatorContext coordinatorContext, long jobId) {
     }
 
     @Override
-    public void setSqlPipelineTask(PipelineExecutionTask pipelineExecutionTask) {
-        Preconditions.checkArgument(pipelineExecutionTask != null, "sqlPipelineTask can not be null");
-
-        this.executionTask = Optional.of(pipelineExecutionTask);
-        Map<BackendFragmentId, SingleFragmentPipelineTask> backendFragmentTasks
-                = buildBackendFragmentTasks(pipelineExecutionTask);
-        this.backendFragmentTasks = Optional.of(backendFragmentTasks);
-
+    protected void afterSetPipelineExecutionTask(PipelineExecutionTask pipelineExecutionTask) {
+        Map<BackendFragmentId, SingleFragmentPipelineTask> backendFragmentTasks = this.backendFragmentTasks.get();
         MarkedCountDownLatch<Integer, Long> latch = new MarkedCountDownLatch<>(backendFragmentTasks.size());
         for (BackendFragmentId backendFragmentId : backendFragmentTasks.keySet()) {
             latch.addMark(backendFragmentId.fragmentId, backendFragmentId.backendId);
@@ -168,34 +155,9 @@ public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
         return latch.get().await(timeout, unit);
     }
 
-    public void updateFragmentExecStatus(TReportExecStatusParams params) {
-        SingleFragmentPipelineTask fragmentTask = backendFragmentTasks.get().get(
-                new BackendFragmentId(params.getBackendId(), params.getFragmentId()));
-        if (fragmentTask == null || !fragmentTask.processReportExecStatus(params)) {
-            return;
-        }
-        TUniqueId queryId = coordinatorContext.queryId;
-        Status status = new Status(params.status);
-        // for now, abort the query if we see any error except if the error is cancelled
-        // and returned_all_results_ is true.
-        // (UpdateStatus() initiates cancellation, if it hasn't already been initiated)
-        if (!status.ok()) {
-            if (coordinatorContext.isEos() && status.isCancelled()) {
-                LOG.warn("Query {} has returned all results, fragment_id={} instance_id={}, be={}"
-                                + " is reporting failed status {}",
-                        DebugUtil.printId(queryId), params.getFragmentId(),
-                        DebugUtil.printId(params.getFragmentInstanceId()),
-                        params.getBackendId(),
-                        status.toString());
-            } else {
-                LOG.warn("one instance report fail, query_id={} fragment_id={} instance_id={}, be={},"
-                                + " error message: {}",
-                        DebugUtil.printId(queryId), params.getFragmentId(),
-                        DebugUtil.printId(params.getFragmentInstanceId()),
-                        params.getBackendId(), status.toString());
-                coordinatorContext.updateStatusIfOk(status);
-            }
-        }
+
+    @Override
+    protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleFragmentPipelineTask fragmentTask) {
         LoadContext loadContext = coordinatorContext.asLoadProcessor().loadContext;
         if (params.isSetDeltaUrls()) {
             loadContext.updateDeltaUrls(params.getDeltaUrls());
@@ -234,7 +196,7 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) {
         if (fragmentTask.isDone()) {
             if (LOG.isDebugEnabled()) {
                 LOG.debug("Query {} fragment {} is marked done",
-                        DebugUtil.printId(queryId), params.getFragmentId());
+                        DebugUtil.printId(coordinatorContext.queryId), params.getFragmentId());
             }
             latch.get().markedCountDown(params.getFragmentId(), params.getBackendId());
         }
@@ -258,22 +220,6 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) {
         }
     }
 
-    private Map<BackendFragmentId, SingleFragmentPipelineTask> buildBackendFragmentTasks(
-            PipelineExecutionTask executionTask) {
-        ImmutableMap.Builder<BackendFragmentId, SingleFragmentPipelineTask> backendFragmentTasks
-                = ImmutableMap.builder();
-        for (Entry<Long, MultiFragmentsPipelineTask> backendTask : executionTask.getChildrenTasks().entrySet()) {
-            Long backendId = backendTask.getKey();
-            for (Entry<Integer, SingleFragmentPipelineTask> fragmentIdToTask : backendTask.getValue()
-                    .getChildrenTasks().entrySet()) {
-                Integer fragmentId = fragmentIdToTask.getKey();
-                SingleFragmentPipelineTask fragmentTask = fragmentIdToTask.getValue();
-                backendFragmentTasks.put(new BackendFragmentId(backendId, fragmentId), fragmentTask);
-            }
-        }
-        return backendFragmentTasks.build();
-    }
-
     /*
      * Check the state of backends in needCheckBackendExecStates.
      * return true if all of them are OK. Otherwise, return false.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java
index 8c1b9714c35db8..ae87d59d075d12 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java
@@ -102,7 +102,7 @@ public void execute() throws Exception {
 
     @Override
     public String toString() {
-        return "SqlPipelineTask(\n"
+        return "PipelineExecutionTask(\n"
                 + childrenTasks.allTasks()
                     .stream()
                     .map(multiFragmentsPipelineTask -> "  " + multiFragmentsPipelineTask)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java
index fd00bf0e3e8536..0da6f4a5fe2e43 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java
@@ -61,7 +61,7 @@ private PipelineExecutionTask buildTask(CoordinatorContext coordinatorContext,
                 backendServiceProxy,
                 buildMultiFragmentTasks(coordinatorContext, backendServiceProxy, workerToFragmentsParam)
         );
-        coordinatorContext.getJobProcessor().setSqlPipelineTask(pipelineExecutionTask);
+        coordinatorContext.getJobProcessor().setPipelineExecutionTask(pipelineExecutionTask);
         return pipelineExecutionTask;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java
index 2ec38e8cc8e3ea..a5a5100faece1a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java
@@ -25,13 +25,14 @@
 import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob;
 import org.apache.doris.planner.DataSink;
 import org.apache.doris.planner.ResultSink;
+import org.apache.doris.qe.AbstractJobProcessor;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.CoordinatorContext;
-import org.apache.doris.qe.JobProcessor;
 import org.apache.doris.qe.ResultReceiver;
 import org.apache.doris.qe.RowBatch;
 import org.apache.doris.rpc.RpcException;
 import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TReportExecStatusParams;
 import org.apache.doris.thrift.TStatusCode;
 
 import com.google.common.base.Strings;
@@ -44,24 +45,21 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.concurrent.CopyOnWriteArrayList;
 
-public class QueryProcessor implements JobProcessor {
+public class QueryProcessor extends AbstractJobProcessor {
     private static final Logger LOG = LogManager.getLogger(QueryProcessor.class);
 
     // constant fields
     private final long limitRows;
 
     // mutable field
-    private Optional<PipelineExecutionTask> sqlPipelineTask;
-    private final CoordinatorContext coordinatorContext;
     private final List<ResultReceiver> runningReceivers;
     private int receiverOffset;
     private long numReceivedRows;
 
     public QueryProcessor(CoordinatorContext coordinatorContext, List<ResultReceiver> runningReceivers) {
-        this.coordinatorContext = Objects.requireNonNull(coordinatorContext, "coordinatorContext can not be null");
+        super(coordinatorContext);
         this.runningReceivers = new CopyOnWriteArrayList<>(
                 Objects.requireNonNull(runningReceivers, "runningReceivers can not be null")
         );
@@ -69,8 +67,6 @@ public QueryProcessor(CoordinatorContext coordinatorContext, List<ResultReceiver
         this.limitRows = coordinatorContext.fragments.get(coordinatorContext.fragments.size() - 1)
                 .getPlanRoot()
                 .getLimit();
-
-        this.sqlPipelineTask = Optional.empty();
     }
 
     public static QueryProcessor build(CoordinatorContext coordinatorContext) {
@@ -109,8 +105,8 @@ public static QueryProcessor build(CoordinatorContext coordinatorContext) {
     }
 
     @Override
-    public void setSqlPipelineTask(PipelineExecutionTask pipelineExecutionTask) {
-        this.sqlPipelineTask = Optional.ofNullable(pipelineExecutionTask);
+    protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleFragmentPipelineTask fragmentTask) {
+
     }
 
     public boolean isEos() {
@@ -178,7 +174,7 @@ public void cancel(Status cancelReason) {
             receiver.cancel(cancelReason);
         }
 
-        this.sqlPipelineTask.ifPresent(sqlPipelineTask -> {
+        this.executionTask.ifPresent(sqlPipelineTask -> {
             for (MultiFragmentsPipelineTask fragmentsTask : sqlPipelineTask.getChildrenTasks().values()) {
                 fragmentsTask.cancelExecute(cancelReason);
             }

From 2cde7b0839d1607ef6dd619b7d9db95bc4c211ee Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Mon, 2 Dec 2024 16:41:17 +0800
Subject: [PATCH 114/399] [fix](nereids) fix NereidsCoordinator compute wrong
 result when exists CTE (#44753)

fix NereidsCoordinator compute wrong result when exists CTE, introduced
by #41730
---
 .../doris/qe/runtime/ThriftPlansBuilder.java  | 11 +++-
 .../distribute/test_multicast_sink.groovy     | 60 +++++++++++++++++++
 2 files changed, 68 insertions(+), 3 deletions(-)
 create mode 100644 regression-test/suites/nereids_syntax_p0/distribute/test_multicast_sink.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
index a02ee90e901cd5..54bc0b24d3e821 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
@@ -240,14 +240,19 @@ private static Map<Integer, Integer> computeExchangeSenderNum(PipelineDistribute
         return senderNum;
     }
 
-    private static void setMultiCastDestinationThrift(PipelineDistributedPlan fragmentPlan) {
+    private static void setMultiCastDestinationThriftIfNotSet(PipelineDistributedPlan fragmentPlan) {
         MultiCastDataSink multiCastDataSink = (MultiCastDataSink) fragmentPlan.getFragmentJob().getFragment().getSink();
         List<List<TPlanFragmentDestination>> destinationList = multiCastDataSink.getDestinations();
 
         List<DataStreamSink> dataStreamSinks = multiCastDataSink.getDataStreamSinks();
         for (int i = 0; i < dataStreamSinks.size(); i++) {
-            DataStreamSink realSink = dataStreamSinks.get(i);
             List<TPlanFragmentDestination> destinations = destinationList.get(i);
+            if (!destinations.isEmpty()) {
+                // we should only set destination only once,
+                // because all backends share the same MultiCastDataSink object
+                continue;
+            }
+            DataStreamSink realSink = dataStreamSinks.get(i);
             for (Entry<DataSink, List<AssignedJob>> kv : fragmentPlan.getDestinations().entrySet()) {
                 DataSink sink = kv.getKey();
                 if (sink == realSink) {
@@ -318,7 +323,7 @@ private static TPipelineFragmentParams fragmentToThriftIfAbsent(
             List<TPlanFragmentDestination> nonMultiCastDestinations;
             if (fragment.getSink() instanceof MultiCastDataSink) {
                 nonMultiCastDestinations = Lists.newArrayList();
-                setMultiCastDestinationThrift(fragmentPlan);
+                setMultiCastDestinationThriftIfNotSet(fragmentPlan);
             } else {
                 nonMultiCastDestinations = nonMultiCastDestinationToThrift(fragmentPlan);
             }
diff --git a/regression-test/suites/nereids_syntax_p0/distribute/test_multicast_sink.groovy b/regression-test/suites/nereids_syntax_p0/distribute/test_multicast_sink.groovy
new file mode 100644
index 00000000000000..eeeaad06d5ee22
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/distribute/test_multicast_sink.groovy
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_multicast_sink") {
+    multi_sql """
+            drop table if exists table_1_undef_partitions2_keys3_properties4_distributed_by5;
+            CREATE TABLE `table_1_undef_partitions2_keys3_properties4_distributed_by5` (
+              `col_int_undef_signed` int NULL,
+              `col_int_undef_signed_not_null` int NOT NULL,
+              `col_date_undef_signed` date NULL,
+              `col_date_undef_signed_not_null` date NOT NULL,
+              `col_varchar_10__undef_signed` varchar(10) NULL,
+              `col_varchar_10__undef_signed_not_null` varchar(10) NOT NULL,
+              `col_varchar_1024__undef_signed` varchar(1024) NULL,
+              `col_varchar_1024__undef_signed_not_null` varchar(1024) NOT NULL,
+              `pk` int NULL
+            ) ENGINE=OLAP
+            DUPLICATE KEY(`col_int_undef_signed`, `col_int_undef_signed_not_null`, `col_date_undef_signed`)
+            DISTRIBUTED BY HASH(`pk`) BUCKETS 10
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1"
+            );
+            insert into table_1_undef_partitions2_keys3_properties4_distributed_by5 values(3, 6, '2023-12-17', '2023-12-17', 'ok', 'v', 'want', 'z', 0);
+            set enable_nereids_distribute_planner=true;
+            set parallel_pipeline_task_num = 1;
+            """
+
+    for (def i in 0..<100) {
+        test {
+            sql """
+            WITH cte1 AS(
+                SELECT t1.`pk`
+                FROM table_1_undef_partitions2_keys3_properties4_distributed_by5 AS t1
+                ORDER BY t1.pk
+            )
+            SELECT cte1.`pk` AS pk1
+            FROM cte1
+            LEFT OUTER JOIN cte1 AS alias1
+            ON cte1 . `pk` = alias1 . `pk`
+            WHERE cte1.`pk` < 3
+            LIMIT 66666666
+            """
+            result([[0]])
+        }
+    }
+}

From 30b4df9d4f6d3c1f719775b7104c6105af95872d Mon Sep 17 00:00:00 2001
From: xyf <116467462+xyfsjq@users.noreply.github.com>
Date: Mon, 2 Dec 2024 17:30:37 +0800
Subject: [PATCH 115/399] [fix](profile) Change the check of whether the
 username is admin and root to check whether the user specifically corresponds
 to admin permissions (#41714)

---
 .../rest/manager/QueryProfileAction.java      |  2 +-
 .../test_http_permissions_check_auth.groovy   | 66 +++++++++++++++++++
 2 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/suites/http_auth_check/test_http_permissions_check_auth.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java
index 8c3dfa73aea759..0886edb56fb2f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java
@@ -419,7 +419,7 @@ private ResponseEntity getProfileFromAllFrontends(HttpServletRequest request, St
 
     private void checkAuthByUserAndQueryId(String queryId) throws AuthenticationException {
         String user = ConnectContext.get().getCurrentUserIdentity().getQualifiedUser();
-        if (!user.equalsIgnoreCase(Auth.ADMIN_USER) && !user.equalsIgnoreCase(Auth.ROOT_USER)) {
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
             ProfileManager.getInstance().checkAuthByUserAndQueryId(user, queryId);
         }
     }
diff --git a/regression-test/suites/http_auth_check/test_http_permissions_check_auth.groovy b/regression-test/suites/http_auth_check/test_http_permissions_check_auth.groovy
new file mode 100644
index 00000000000000..36c6241a244db7
--- /dev/null
+++ b/regression-test/suites/http_auth_check/test_http_permissions_check_auth.groovy
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_http_permissions_check_auth","p0,auth") {
+    String suiteName = "test_http_permissions_check_auth"
+    String tableName = "${suiteName}_table"
+    String user = "${suiteName}_user"
+    String pwd = 'C123_567p'
+    try_sql("DROP USER ${user}")
+    sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
+    sql """drop table if exists `${tableName}`"""
+    sql """
+        CREATE TABLE `${tableName}` (
+          `k1` int,
+          `k2` int
+        ) ENGINE=OLAP
+        DISTRIBUTED BY random BUCKETS auto
+        PROPERTIES ('replication_num' = '1') ;
+        """
+    sql """insert into ${tableName} values(1,1)"""
+    sql """set session_context = 'trace_id:mmn9';"""
+    sql """select * from ${tableName};"""
+
+    def get_queryid_by_traceid = { check_func ->
+        httpTest {
+            basicAuthorization "${user}","${pwd}"
+            endpoint "${context.config.feHttpAddress}"
+            uri "/rest/v2/manager/query/trace_id/mmn9"
+            op "get"
+            check check_func
+        }
+    }
+
+    get_queryid_by_traceid.call() {
+        respCode, body ->
+            log.info("body:${body}")
+            assertTrue("${body}".contains("Bad Request"))
+    }
+
+    sql """grant 'admin' to ${user}"""
+
+    get_queryid_by_traceid.call() {
+        respCode, body ->
+            log.info("body:${body}")
+            assertTrue("${body}".contains("success"))
+    }
+
+    sql """drop table if exists `${tableName}`"""
+    try_sql("DROP USER ${user}")
+}

From 0995618169d807d815d0abb023ff22eb7b3ce656 Mon Sep 17 00:00:00 2001
From: xzj7019 <xiongzhongjian@selectdb.com>
Date: Mon, 2 Dec 2024 17:44:17 +0800
Subject: [PATCH 116/399] [opt](nereids) remove enable nereids rules usage
 (#44769)

Related PR: (@35925)

Problem Summary:
(1) Remove enable_nereids_rules usage, and replaced with use_cbo_rule
hint.
(2) Current case with use_cbo_rule which has deleted the
enable_nereids_rules can't really tracking the rule's impact since the
enable_nereids_rules related checking can't be passed. This pr activates
these part of plan change.
---
 .../rules/rewrite/PushDownAggThroughJoin.java |  10 -
 .../PushDownAggThroughJoinOneSide.java        |  10 -
 .../rewrite/PushDownDistinctThroughJoin.java  |   7 -
 .../eager_aggregate/basic.out                 |  46 +-
 .../eager_aggregate/basic_one_side.out        |  46 +-
 .../push_down_count_through_join_one_side.out | 472 ++++++++++++------
 .../push_down_max_through_join.out            | 168 ++++---
 .../push_down_min_through_join.out            | 166 +++---
 .../push_down_sum_through_join_one_side.out   | 254 +++++++---
 .../push_down_distinct_through_join.out       |  11 +-
 ...sh_down_count_through_join_one_side.groovy |   1 -
 ...sh_down_limit_distinct_through_join.groovy |   1 -
 .../constraints/query23.groovy                |   1 -
 .../tpcds_sf100/constraints/query23.groovy    |   1 -
 14 files changed, 780 insertions(+), 414 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoin.java
index f003d2ac2cc918..665e3294a93a72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoin.java
@@ -85,11 +85,6 @@ public List<Rule> buildRules() {
                                     );
                         })
                         .thenApply(ctx -> {
-                            Set<Integer> enableNereidsRules = ctx.cascadesContext.getConnectContext()
-                                    .getSessionVariable().getEnableNereidsRules();
-                            if (!enableNereidsRules.contains(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN.type())) {
-                                return null;
-                            }
                             LogicalAggregate<LogicalJoin<Plan, Plan>> agg = ctx.root;
                             return pushAgg(agg, agg.child(), ImmutableList.of());
                         })
@@ -109,11 +104,6 @@ public List<Rule> buildRules() {
                                     );
                         })
                         .thenApply(ctx -> {
-                            Set<Integer> enableNereidsRules = ctx.cascadesContext.getConnectContext()
-                                    .getSessionVariable().getEnableNereidsRules();
-                            if (!enableNereidsRules.contains(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN.type())) {
-                                return null;
-                            }
                             LogicalAggregate<LogicalProject<LogicalJoin<Plan, Plan>>> agg = ctx.root;
                             return pushAgg(agg, agg.child().child(), agg.child().getProjects());
                         })
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOneSide.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOneSide.java
index c5d3d0fb49a0a5..9319b09cc30470 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOneSide.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOneSide.java
@@ -79,11 +79,6 @@ public List<Rule> buildRules() {
                                             && (f.children().isEmpty() || f.child(0) instanceof Slot)));
                         })
                         .thenApply(ctx -> {
-                            Set<Integer> enableNereidsRules = ctx.cascadesContext.getConnectContext()
-                                    .getSessionVariable().getEnableNereidsRules();
-                            if (!enableNereidsRules.contains(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE.type())) {
-                                return null;
-                            }
                             LogicalAggregate<LogicalJoin<Plan, Plan>> agg = ctx.root;
                             return pushMinMaxSumCount(agg, agg.child(), ImmutableList.of());
                         })
@@ -101,11 +96,6 @@ public List<Rule> buildRules() {
                                             && (f.children().isEmpty() || f.child(0) instanceof Slot));
                         })
                         .thenApply(ctx -> {
-                            Set<Integer> enableNereidsRules = ctx.cascadesContext.getConnectContext()
-                                    .getSessionVariable().getEnableNereidsRules();
-                            if (!enableNereidsRules.contains(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE.type())) {
-                                return null;
-                            }
                             LogicalAggregate<LogicalProject<LogicalJoin<Plan, Plan>>> agg = ctx.root;
                             return pushMinMaxSumCount(agg, agg.child().child(), agg.child().getProjects());
                         })
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoin.java
index fa705a08d8db8a..533452c2f28f61 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoin.java
@@ -18,7 +18,6 @@
 package org.apache.doris.nereids.rules.rewrite;
 
 import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.rules.RuleType;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.algebra.Relation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
@@ -30,7 +29,6 @@
 
 import com.google.common.collect.ImmutableList;
 
-import java.util.Set;
 import java.util.function.Function;
 
 /**
@@ -39,11 +37,6 @@
 public class PushDownDistinctThroughJoin extends DefaultPlanRewriter<JobContext> implements CustomRewriter {
     @Override
     public Plan rewriteRoot(Plan plan, JobContext context) {
-        Set<Integer> enableNereidsRules = context.getCascadesContext().getConnectContext()
-                .getSessionVariable().getEnableNereidsRules();
-        if (!enableNereidsRules.contains(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN.type())) {
-            return null;
-        }
         return plan.accept(this, context);
     }
 
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/basic.out b/regression-test/data/nereids_rules_p0/eager_aggregate/basic.out
index ba18189efcad82..19b1b3cdb85590 100644
--- a/regression-test/data/nereids_rules_p0/eager_aggregate/basic.out
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/basic.out
@@ -40,14 +40,17 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=()
---------filter((a.event_id = 'ad_click'))
-----------PhysicalOlapScan[com_dd_library]
---------filter((cast(experiment_id as DOUBLE) = 37.0))
-----------PhysicalOlapScan[shunt_log_com_dd_library]
+--------hashAgg[LOCAL]
+----------hashAgg[LOCAL]
+------------filter((a.event_id = 'ad_click'))
+--------------PhysicalOlapScan[com_dd_library]
+--------hashAgg[LOCAL]
+----------filter((cast(experiment_id as DOUBLE) = 37.0))
+------------PhysicalOlapScan[shunt_log_com_dd_library]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed: use_push_down_agg_through_join
 SyntaxError:
 
 -- !with_hint_2 --
@@ -55,13 +58,16 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=()
---------PhysicalOlapScan[com_dd_library]
---------filter((cast(experiment_id as DOUBLE) = 73.0))
-----------PhysicalOlapScan[shunt_log_com_dd_library]
+--------hashAgg[LOCAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[com_dd_library]
+--------hashAgg[LOCAL]
+----------filter((cast(experiment_id as DOUBLE) = 73.0))
+------------PhysicalOlapScan[shunt_log_com_dd_library]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed: use_push_down_agg_through_join
 SyntaxError:
 
 -- !with_hint_3 --
@@ -69,13 +75,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=()
---------PhysicalOlapScan[com_dd_library]
+--------hashAgg[LOCAL]
+----------PhysicalOlapScan[com_dd_library]
 --------filter((cast(experiment_id as DOUBLE) = 73.0))
 ----------PhysicalOlapScan[shunt_log_com_dd_library]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed: use_push_down_agg_through_join
 SyntaxError:
 
 -- !with_hint_4 --
@@ -83,11 +90,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=()
---------PhysicalOlapScan[com_dd_library]
---------PhysicalOlapScan[shunt_log_com_dd_library]
+--------hashAgg[LOCAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[com_dd_library]
+--------hashAgg[LOCAL]
+----------PhysicalOlapScan[shunt_log_com_dd_library]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed: use_push_down_agg_through_join
 SyntaxError:
 
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/basic_one_side.out b/regression-test/data/nereids_rules_p0/eager_aggregate/basic_one_side.out
index aaf6afeca1e9f6..66f1729e2e3816 100644
--- a/regression-test/data/nereids_rules_p0/eager_aggregate/basic_one_side.out
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/basic_one_side.out
@@ -40,14 +40,17 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=()
---------filter((a.event_id = 'ad_click'))
-----------PhysicalOlapScan[com_dd_library_one_side]
---------filter((cast(experiment_id as DOUBLE) = 37.0))
-----------PhysicalOlapScan[shunt_log_com_dd_library_one_side]
+--------hashAgg[LOCAL]
+----------hashAgg[LOCAL]
+------------filter((a.event_id = 'ad_click'))
+--------------PhysicalOlapScan[com_dd_library_one_side]
+--------hashAgg[LOCAL]
+----------filter((cast(experiment_id as DOUBLE) = 37.0))
+------------PhysicalOlapScan[shunt_log_com_dd_library_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_2 --
@@ -55,13 +58,16 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=()
---------PhysicalOlapScan[com_dd_library_one_side]
---------filter((cast(experiment_id as DOUBLE) = 73.0))
-----------PhysicalOlapScan[shunt_log_com_dd_library_one_side]
+--------hashAgg[LOCAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[com_dd_library_one_side]
+--------hashAgg[LOCAL]
+----------filter((cast(experiment_id as DOUBLE) = 73.0))
+------------PhysicalOlapScan[shunt_log_com_dd_library_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_3 --
@@ -69,13 +75,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=()
---------PhysicalOlapScan[com_dd_library_one_side]
+--------hashAgg[LOCAL]
+----------PhysicalOlapScan[com_dd_library_one_side]
 --------filter((cast(experiment_id as DOUBLE) = 73.0))
 ----------PhysicalOlapScan[shunt_log_com_dd_library_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_4 --
@@ -83,11 +90,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=()
---------PhysicalOlapScan[com_dd_library_one_side]
---------PhysicalOlapScan[shunt_log_com_dd_library_one_side]
+--------hashAgg[LOCAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[com_dd_library_one_side]
+--------hashAgg[LOCAL]
+----------PhysicalOlapScan[shunt_log_com_dd_library_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.out
index 8267eb3e38ff91..251bc580d3279d 100644
--- a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.out
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.out
@@ -404,12 +404,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_join --
@@ -456,12 +462,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_anti_join --
@@ -482,12 +494,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.name < t2.name))
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 --------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_aggregate --
@@ -508,13 +522,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------filter((count_t_one_side.score > 10))
-----------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((count_t_one_side.score > 10))
+------------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_outer_join --
@@ -535,13 +555,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------filter((count_t_one_side.score > 10))
-----------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((count_t_one_side.score > 10))
+------------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_having --
@@ -550,12 +576,18 @@ PhysicalResultSink
 ----hashAgg[GLOBAL]
 ------hashAgg[LOCAL]
 --------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----------PhysicalOlapScan[count_t_one_side]
-----------PhysicalOlapScan[count_t_one_side]
+----------hashAgg[GLOBAL]
+------------hashAgg[LOCAL]
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalOlapScan[count_t_one_side]
+----------hashAgg[GLOBAL]
+------------hashAgg[LOCAL]
+--------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_mixed_aggregates --
@@ -563,12 +595,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 --------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multi_table_join --
@@ -576,14 +610,26 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.name = t3.name)) otherCondition=()
---------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----------PhysicalOlapScan[count_t_one_side]
-----------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
+------------------hashAgg[GLOBAL]
+--------------------hashAgg[LOCAL]
+----------------------hashAgg[GLOBAL]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalOlapScan[count_t_one_side]
+------------------hashAgg[GLOBAL]
+--------------------hashAgg[LOCAL]
+----------------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_order_by --
@@ -593,12 +639,18 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[count_t_one_side]
-------------PhysicalOlapScan[count_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------hashAgg[GLOBAL]
+------------------hashAgg[LOCAL]
+--------------------PhysicalOlapScan[count_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multiple_equal_conditions --
@@ -606,12 +658,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_with_aggregate --
@@ -619,12 +677,16 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_selection --
@@ -632,12 +694,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_selection_with_aggregate --
@@ -645,12 +713,20 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_where_clause --
@@ -658,13 +734,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------filter((t1.score > 50))
-----------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((t1.score > 50))
+------------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_varied_aggregates --
@@ -687,12 +769,18 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[count_t_one_side]
-------------PhysicalOlapScan[count_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------hashAgg[GLOBAL]
+------------------hashAgg[LOCAL]
+--------------------PhysicalOlapScan[count_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_alias_multiple_equal_conditions --
@@ -700,12 +788,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1_alias.id = t2_alias.id) and (t1_alias.name = t2_alias.name)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_complex_join_condition --
@@ -713,12 +807,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.score = t2.score)) otherCondition=(( not (name = name)))
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 --------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_function_processed_columns --
@@ -739,14 +835,20 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------filter((count_t_one_side.id < 100))
-----------PhysicalOlapScan[count_t_one_side]
---------filter((count_t_one_side.score > 20) and (t1.id < 100))
-----------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((count_t_one_side.id < 100))
+--------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((count_t_one_side.score > 20) and (t1.id < 100))
+------------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_basic --
@@ -754,12 +856,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_join --
@@ -806,12 +914,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_anti_join --
@@ -832,12 +946,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.name < t2.name))
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 --------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_aggregate --
@@ -845,12 +961,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_subquery --
@@ -858,13 +980,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------filter((count_t_one_side.score > 10))
-----------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((count_t_one_side.score > 10))
+------------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_outer_join --
@@ -885,13 +1013,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------filter((count_t_one_side.score > 10))
-----------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((count_t_one_side.score > 10))
+------------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_having --
@@ -900,12 +1034,18 @@ PhysicalResultSink
 ----hashAgg[GLOBAL]
 ------hashAgg[LOCAL]
 --------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----------PhysicalOlapScan[count_t_one_side]
-----------PhysicalOlapScan[count_t_one_side]
+----------hashAgg[GLOBAL]
+------------hashAgg[LOCAL]
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalOlapScan[count_t_one_side]
+----------hashAgg[GLOBAL]
+------------hashAgg[LOCAL]
+--------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multi_table_join --
@@ -913,14 +1053,26 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.name = t3.name)) otherCondition=()
---------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----------PhysicalOlapScan[count_t_one_side]
-----------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
+------------------hashAgg[GLOBAL]
+--------------------hashAgg[LOCAL]
+----------------------hashAgg[GLOBAL]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalOlapScan[count_t_one_side]
+------------------hashAgg[GLOBAL]
+--------------------hashAgg[LOCAL]
+----------------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_order_by --
@@ -930,12 +1082,18 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[count_t_one_side]
-------------PhysicalOlapScan[count_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------hashAgg[GLOBAL]
+------------------hashAgg[LOCAL]
+--------------------PhysicalOlapScan[count_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multiple_equal_conditions --
@@ -943,12 +1101,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_selection --
@@ -956,12 +1120,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_where_clause --
@@ -969,13 +1139,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[count_t_one_side]
---------filter((t1.score > 50))
-----------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((t1.score > 50))
+------------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_varied_aggregates --
@@ -998,12 +1174,18 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[count_t_one_side]
-------------PhysicalOlapScan[count_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------hashAgg[GLOBAL]
+------------------hashAgg[LOCAL]
+--------------------PhysicalOlapScan[count_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_complex_join_condition --
@@ -1011,12 +1193,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.score = t2.score)) otherCondition=(( not (name = name)))
---------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[count_t_one_side]
 --------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_nested_queries --
@@ -1024,14 +1208,20 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------filter((count_t_one_side.id < 100))
-----------PhysicalOlapScan[count_t_one_side]
---------filter((count_t_one_side.score > 20) and (t1.id < 100))
-----------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((count_t_one_side.id < 100))
+--------------PhysicalOlapScan[count_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((count_t_one_side.score > 20) and (t1.id < 100))
+------------------PhysicalOlapScan[count_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !shape --
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out
index 79b4ed890ded95..7cf9813e5b7181 100644
--- a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out
@@ -224,12 +224,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 --------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_join --
@@ -276,12 +278,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 --------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_anti_join --
@@ -302,12 +306,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.name < t2.name))
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 --------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_aggregate --
@@ -329,12 +335,14 @@ PhysicalResultSink
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[max_t]
---------filter((max_t.score > 10))
-----------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((max_t.score > 10))
+--------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_outer_join --
@@ -356,12 +364,14 @@ PhysicalResultSink
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[max_t]
---------filter((max_t.score > 10))
-----------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((max_t.score > 10))
+--------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_having --
@@ -370,12 +380,14 @@ PhysicalResultSink
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[max_t]
---------filter((t1.score > 100))
-----------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((t1.score > 100))
+--------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_mixed_aggregates --
@@ -383,12 +395,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 --------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multi_table_join --
@@ -396,14 +410,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.name = t3.name)) otherCondition=()
---------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----------PhysicalOlapScan[max_t]
-----------PhysicalOlapScan[max_t]
 --------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalOlapScan[max_t]
+--------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_order_by --
@@ -413,12 +431,14 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[max_t]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[max_t]
 ------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multiple_equal_conditions --
@@ -426,12 +446,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 --------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_with_aggregate --
@@ -439,12 +461,16 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[max_t]
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_selection --
@@ -452,12 +478,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 --------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_selection_with_aggregate --
@@ -465,12 +493,16 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[max_t]
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_where_clause --
@@ -479,12 +511,14 @@ PhysicalResultSink
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[max_t]
---------filter((t1.score > 50))
-----------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((t1.score > 50))
+--------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_varied_aggregates --
@@ -507,12 +541,14 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[max_t]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[max_t]
 ------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_alias_multiple_equal_conditions --
@@ -520,12 +556,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1_alias.id = t2_alias.id) and (t1_alias.name = t2_alias.name)) otherCondition=()
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 --------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_complex_join_condition --
@@ -533,12 +571,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.score = t2.score)) otherCondition=(( not (name = name)))
---------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[max_t]
 --------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_function_processed_columns --
@@ -561,11 +601,13 @@ PhysicalResultSink
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------filter((max_t.id < 100))
 ----------PhysicalOlapScan[max_t]
---------filter((max_t.score > 20) and (t1.id < 100))
-----------PhysicalOlapScan[max_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((max_t.score > 20) and (t1.id < 100))
+--------------PhysicalOlapScan[max_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_through_join.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_through_join.out
index d4efe7df093503..e435bcd4efd335 100644
--- a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_through_join.out
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_through_join.out
@@ -224,12 +224,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 --------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_join --
@@ -276,12 +278,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 --------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_anti_join --
@@ -302,12 +306,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.name < t2.name))
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 --------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_aggregate --
@@ -329,12 +335,14 @@ PhysicalResultSink
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[min_t]
---------filter((min_t.score > 10))
-----------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((min_t.score > 10))
+--------------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_outer_join --
@@ -356,12 +364,14 @@ PhysicalResultSink
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[min_t]
---------filter((min_t.score > 10))
-----------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((min_t.score > 10))
+--------------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_having --
@@ -370,12 +380,14 @@ PhysicalResultSink
 ----hashAgg[GLOBAL]
 ------hashAgg[LOCAL]
 --------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----------PhysicalOlapScan[min_t]
+----------hashAgg[GLOBAL]
+------------hashAgg[LOCAL]
+--------------PhysicalOlapScan[min_t]
 ----------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_mixed_aggregates --
@@ -383,12 +395,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 --------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multi_table_join --
@@ -396,14 +410,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.name = t3.name)) otherCondition=()
---------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----------PhysicalOlapScan[min_t]
-----------PhysicalOlapScan[min_t]
 --------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalOlapScan[min_t]
+--------------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_order_by --
@@ -413,12 +431,14 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[min_t]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[min_t]
 ------------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multiple_equal_conditions --
@@ -426,12 +446,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 --------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_with_aggregate --
@@ -439,12 +461,16 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[min_t]
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_selection --
@@ -452,12 +478,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 --------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_selection_with_aggregate --
@@ -465,12 +493,16 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[min_t]
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_where_clause --
@@ -479,12 +511,14 @@ PhysicalResultSink
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------PhysicalOlapScan[min_t]
---------filter((t1.score > 50))
-----------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((t1.score > 50))
+--------------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_varied_aggregates --
@@ -507,12 +541,14 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[min_t]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[min_t]
 ------------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_alias_multiple_equal_conditions --
@@ -520,12 +556,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1_alias.id = t2_alias.id) and (t1_alias.name = t2_alias.name)) otherCondition=()
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 --------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_complex_join_condition --
@@ -533,12 +571,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.score = t2.score)) otherCondition=(( not (name = name)))
---------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[min_t]
 --------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_function_processed_columns --
@@ -561,11 +601,13 @@ PhysicalResultSink
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
 --------filter((min_t.id < 100))
 ----------PhysicalOlapScan[min_t]
---------filter((min_t.score > 20) and (t1.id < 100))
-----------PhysicalOlapScan[min_t]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((min_t.score > 20) and (t1.id < 100))
+--------------PhysicalOlapScan[min_t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_sum_through_join_one_side.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_sum_through_join_one_side.out
index 5013dc7dbdfb93..3a945f58e29e3c 100644
--- a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_sum_through_join_one_side.out
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_sum_through_join_one_side.out
@@ -224,12 +224,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_join --
@@ -276,12 +282,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_left_anti_join --
@@ -302,12 +314,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.name < t2.name))
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
 --------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_aggregate --
@@ -328,13 +342,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------filter((sum_t_one_side.score > 10))
-----------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((sum_t_one_side.score > 10))
+------------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_outer_join --
@@ -355,13 +375,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------filter((sum_t_one_side.score > 10))
-----------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((sum_t_one_side.score > 10))
+------------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_having --
@@ -370,12 +396,18 @@ PhysicalResultSink
 ----hashAgg[GLOBAL]
 ------hashAgg[LOCAL]
 --------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----------PhysicalOlapScan[sum_t_one_side]
-----------PhysicalOlapScan[sum_t_one_side]
+----------hashAgg[GLOBAL]
+------------hashAgg[LOCAL]
+--------------PhysicalOlapScan[sum_t_one_side]
+----------hashAgg[GLOBAL]
+------------hashAgg[LOCAL]
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_mixed_aggregates --
@@ -383,12 +415,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
 --------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multi_table_join --
@@ -396,14 +430,26 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.name = t3.name)) otherCondition=()
---------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----------PhysicalOlapScan[sum_t_one_side]
-----------PhysicalOlapScan[sum_t_one_side]
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
+------------------hashAgg[GLOBAL]
+--------------------hashAgg[LOCAL]
+----------------------PhysicalOlapScan[sum_t_one_side]
+------------------hashAgg[GLOBAL]
+--------------------hashAgg[LOCAL]
+----------------------hashAgg[GLOBAL]
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_order_by --
@@ -413,12 +459,18 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[sum_t_one_side]
-------------PhysicalOlapScan[sum_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[sum_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------hashAgg[GLOBAL]
+------------------hashAgg[LOCAL]
+--------------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_multiple_equal_conditions --
@@ -426,12 +478,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_with_aggregate --
@@ -439,12 +497,16 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_selection --
@@ -452,12 +514,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_selection_with_aggregate --
@@ -465,12 +533,20 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.name = t2.name)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_with_where_clause --
@@ -478,13 +554,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------filter((t1.score > 50))
-----------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((t1.score > 50))
+------------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_varied_aggregates --
@@ -507,12 +589,18 @@ PhysicalResultSink
 ------hashAgg[GLOBAL]
 --------hashAgg[LOCAL]
 ----------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-------------PhysicalOlapScan[sum_t_one_side]
-------------PhysicalOlapScan[sum_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[sum_t_one_side]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------hashAgg[GLOBAL]
+------------------hashAgg[LOCAL]
+--------------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_alias_multiple_equal_conditions --
@@ -520,12 +608,18 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1_alias.id = t2_alias.id) and (t1_alias.name = t2_alias.name)) otherCondition=()
---------PhysicalOlapScan[sum_t_one_side]
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_complex_join_condition --
@@ -533,12 +627,14 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.score = t2.score)) otherCondition=(( not (name = name)))
---------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------PhysicalOlapScan[sum_t_one_side]
 --------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
 -- !with_hint_groupby_pushdown_function_processed_columns --
@@ -559,13 +655,19 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----hashAgg[LOCAL]
 ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------filter((sum_t_one_side.id < 100))
-----------PhysicalOlapScan[sum_t_one_side]
---------filter((sum_t_one_side.score > 20) and (t1.id < 100))
-----------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------filter((sum_t_one_side.id < 100))
+--------------PhysicalOlapScan[sum_t_one_side]
+--------hashAgg[GLOBAL]
+----------hashAgg[LOCAL]
+------------hashAgg[GLOBAL]
+--------------hashAgg[LOCAL]
+----------------filter((sum_t_one_side.score > 20) and (t1.id < 100))
+------------------PhysicalOlapScan[sum_t_one_side]
 
 Hint log:
-Used:
-UnUsed: use_push_down_agg_through_join_one_side
+Used: use_push_down_agg_through_join_one_side
+UnUsed:
 SyntaxError:
 
diff --git a/regression-test/data/nereids_rules_p0/push_down_distinct_through_join/push_down_distinct_through_join.out b/regression-test/data/nereids_rules_p0/push_down_distinct_through_join/push_down_distinct_through_join.out
index 65baed4f1194a0..4c5c60acd31a8e 100644
--- a/regression-test/data/nereids_rules_p0/push_down_distinct_through_join/push_down_distinct_through_join.out
+++ b/regression-test/data/nereids_rules_p0/push_down_distinct_through_join/push_down_distinct_through_join.out
@@ -15,14 +15,15 @@ SyntaxError:
 PhysicalResultSink
 --hashAgg[LOCAL]
 ----hashJoin[INNER_JOIN] hashCondition=((t1.id = t3.id)) otherCondition=()
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
---------PhysicalOlapScan[t]
---------PhysicalOlapScan[t]
+------hashAgg[LOCAL]
+--------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
+----------PhysicalOlapScan[t]
+----------PhysicalOlapScan[t]
 ------PhysicalOlapScan[t]
 
 Hint log:
-Used:
-UnUsed: use_push_down_distinct_through_join
+Used: use_push_down_distinct_through_join
+UnUsed:
 SyntaxError:
 
 -- !basic_sql --
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy
index e551fa04c9110a..031430a48265ae 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy
@@ -453,7 +453,6 @@ suite("push_down_count_through_join_one_side") {
     insert into dwd_tracking_sensor_init_tmp_ymd values(1, '2024-08-19', 'click'), (1, '2024-08-19', 'click');
     """
     sql """ 
-    set ENABLE_NEREIDS_RULES = "PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE";
     set disable_join_reorder=true;
     """
 
diff --git a/regression-test/suites/nereids_rules_p0/push_down_limit_distinct/push_down_limit_distinct_through_join.groovy b/regression-test/suites/nereids_rules_p0/push_down_limit_distinct/push_down_limit_distinct_through_join.groovy
index f53b100c29aff2..22a59dd1cbdf46 100644
--- a/regression-test/suites/nereids_rules_p0/push_down_limit_distinct/push_down_limit_distinct_through_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/push_down_limit_distinct/push_down_limit_distinct_through_join.groovy
@@ -21,7 +21,6 @@ suite("push_down_limit_distinct_through_join") {
       SET enable_fallback_to_original_planner=false;
       set runtime_filter_mode=OFF;
       SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject';
-      SET ENABLE_NEREIDS_RULES=push_down_limit_distinct_through_join;
       set push_topn_to_agg = true;
       SET disable_join_reorder=true;
       set topn_opt_limit_threshold=1024;
diff --git a/regression-test/suites/nereids_tpcds_shape_sf100_p0/constraints/query23.groovy b/regression-test/suites/nereids_tpcds_shape_sf100_p0/constraints/query23.groovy
index 96e49d61bfa89f..9010034d49273e 100644
--- a/regression-test/suites/nereids_tpcds_shape_sf100_p0/constraints/query23.groovy
+++ b/regression-test/suites/nereids_tpcds_shape_sf100_p0/constraints/query23.groovy
@@ -37,7 +37,6 @@ suite("query23") {
     set disable_nereids_rules='PRUNE_EMPTY_PARTITION';
     set push_topn_to_agg = true;
     set topn_opt_limit_threshold=1024;
-    set enable_nereids_rules='ELIMINATE_GROUP_BY_KEY';
     """
 
     def ds = """with frequent_ss_items as 
diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/constraints/query23.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/constraints/query23.groovy
index b17e9c6d3d0e7f..82db5a725ce44a 100644
--- a/regression-test/suites/new_shapes_p0/tpcds_sf100/constraints/query23.groovy
+++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/constraints/query23.groovy
@@ -40,7 +40,6 @@ suite("query23") {
     set disable_nereids_rules='PRUNE_EMPTY_PARTITION';
     set push_topn_to_agg = true;
     set topn_opt_limit_threshold=1024;
-    set enable_nereids_rules='ELIMINATE_GROUP_BY_KEY';
     """
 
     def ds = """with frequent_ss_items as 

From 6f87e35bced5be8f6aec0f6b4a8eb84b496c2efa Mon Sep 17 00:00:00 2001
From: seawinde <wusi@selectdb.com>
Date: Mon, 2 Dec 2024 19:14:32 +0800
Subject: [PATCH 117/399] [fix](mtmv) Fix data wrong when query with table
 operator such as TABLESAMPLE or tablet and so on (#43030)

Related PR: #28064

Materialized view is as following:

        CREATE MATERIALIZED VIEW mv1
        BUILD IMMEDIATE REFRESH AUTO ON MANUAL
        DISTRIBUTED BY RANDOM BUCKETS 2
        PROPERTIES ('replication_num' = '1')
        AS
       select * from orders

If run query as following, should rewrite fail by materialized view
above to make sure data correctness

select * from orders TABLET(110);
select * from orders index query_index_test;
select * from orders TABLESAMPLE(20 percent);
select * from orders_partition PARTITION (day_2);

At before, this would rewrite by materialized view succesfully and the
result data is wrong, This pr fix this.
---
 .../org/apache/doris/mtmv/MTMVPlanUtil.java   |   6 +
 .../nereids/rules/analysis/BindRelation.java  |   4 +
 .../mv/InitMaterializationContextHook.java    |   5 +
 .../exploration/mv/MaterializedViewUtils.java |  20 +-
 .../rules/exploration/mv/StructInfo.java      |   7 +
 .../rules/rewrite/PruneEmptyPartition.java    |   7 +-
 .../rules/rewrite/PruneOlapScanTablet.java    |   6 +-
 .../plans/commands/info/CreateMTMVInfo.java   |   3 +
 .../trees/plans/logical/LogicalOlapScan.java  |  55 +++--
 .../rewrite/PruneOlapScanTabletTest.java      |  21 +-
 .../doris/nereids/sqltest/SqlTestBase.java    |   3 +
 .../doris/regression/suite/Suite.groovy       |  16 ++
 .../is_in_debug_mode/is_in_debug_mode.groovy  | 156 ++++++++++++++
 .../with_table_operator.groovy                | 195 ++++++++++++++++++
 14 files changed, 482 insertions(+), 22 deletions(-)
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
index c0cd47bd5a0f2f..576e87b44f8d3e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
@@ -58,6 +58,12 @@ public static ConnectContext createMTMVContext(MTMV mtmv) {
         ctx.setCurrentUserIdentity(UserIdentity.ADMIN);
         ctx.getState().reset();
         ctx.setThreadLocalInfo();
+        // Debug session variable should be disabled when refreshed
+        ctx.getSessionVariable().skipDeletePredicate = false;
+        ctx.getSessionVariable().skipDeleteBitmap = false;
+        ctx.getSessionVariable().skipDeleteSign = false;
+        ctx.getSessionVariable().skipStorageEngineMerge = false;
+        ctx.getSessionVariable().showHiddenColumns = false;
         ctx.getSessionVariable().allowModifyMaterializedViewData = true;
         // Disable add default limit rule to avoid refresh data wrong
         ctx.getSessionVariable().setDisableNereidsRules(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index cba3afca6f08d3..c62dda5a539df7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -243,6 +243,10 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
                     unboundRelation.getTableSample());
             }
         }
+        if (!tabletIds.isEmpty()) {
+            // This tabletIds is set manually, so need to set specifiedTabletIds
+            scan = scan.withManuallySpecifiedTabletIds(tabletIds);
+        }
         if (needGenerateLogicalAggForRandomDistAggTable(scan)) {
             // it's a random distribution agg table
             // add agg on olap scan
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 2e8baecf165639..f9ea00e178b5b7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -79,6 +79,11 @@ public void initMaterializationContext(CascadesContext cascadesContext) {
      * @param cascadesContext current cascadesContext in the planner
      */
     protected void doInitMaterializationContext(CascadesContext cascadesContext) {
+        if (cascadesContext.getConnectContext().getSessionVariable().isInDebugMode()) {
+            LOG.info(String.format("MaterializationContext init return because is in debug mode, current queryId is %s",
+                    cascadesContext.getConnectContext().getQueryIdentifier()));
+            return;
+        }
         // Only collect the table or mv which query use directly, to avoid useless mv partition in rewrite
         TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false);
         try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 484abd11f01e72..ee4b002007ea05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -347,7 +347,15 @@ public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResu
                 planner.getCascadesContext().getMemo().getRoot().getStatistics(), null);
     }
 
-    private static final class TableQueryOperatorChecker extends DefaultPlanVisitor<Boolean, Void> {
+    /**
+     * Check the query if Contains query operator
+     * Such sql as following should return true
+     * select * from orders TABLET(10098) because TABLET(10098) should return true
+     * select * from orders_partition PARTITION (day_2) because PARTITION (day_2)
+     * select * from orders index query_index_test because index query_index_test
+     * select * from orders TABLESAMPLE(20 percent) because TABLESAMPLE(20 percent)
+     * */
+    public static final class TableQueryOperatorChecker extends DefaultPlanVisitor<Boolean, Void> {
         public static final TableQueryOperatorChecker INSTANCE = new TableQueryOperatorChecker();
 
         @Override
@@ -358,12 +366,20 @@ public Boolean visitLogicalRelation(LogicalRelation relation, Void context) {
             if (relation instanceof LogicalOlapScan) {
                 LogicalOlapScan logicalOlapScan = (LogicalOlapScan) relation;
                 if (logicalOlapScan.getTableSample().isPresent()) {
+                    // Contain sample, select * from orders TABLESAMPLE(20 percent)
                     return true;
                 }
-                if (!logicalOlapScan.getSelectedTabletIds().isEmpty()) {
+                if (!logicalOlapScan.getManuallySpecifiedTabletIds().isEmpty()) {
+                    // Contain tablets, select * from orders TABLET(10098) because TABLET(10098)
                     return true;
                 }
                 if (!logicalOlapScan.getManuallySpecifiedPartitions().isEmpty()) {
+                    // Contain specified partitions, select * from orders_partition PARTITION (day_2)
+                    return true;
+                }
+                if (logicalOlapScan.getSelectedIndexId() != logicalOlapScan.getTable().getBaseIndexId()) {
+                    // Contains select index or use sync mv in rbo rewrite
+                    // select * from orders index query_index_test
                     return true;
                 }
             }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 526ec7030d2db5..5a84ab787d7bd5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -28,6 +28,7 @@
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
 import org.apache.doris.nereids.memo.Group;
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.TableQueryOperatorChecker;
 import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
 import org.apache.doris.nereids.trees.copier.DeepCopierContext;
 import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier;
@@ -36,6 +37,7 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.plans.AbstractPlan;
 import org.apache.doris.nereids.trees.plans.GroupPlan;
 import org.apache.doris.nereids.trees.plans.JoinType;
 import org.apache.doris.nereids.trees.plans.ObjectId;
@@ -323,6 +325,11 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable
                 cascadesContext);
         valid = valid
                 && hyperGraph.getNodes().stream().allMatch(n -> ((StructInfoNode) n).getExpressions() != null);
+        // if relationList has any relation which contains table operator,
+        // such as query with sample, index, table, is invalid
+        boolean invalid = relationList.stream().anyMatch(relation ->
+                ((AbstractPlan) relation).accept(TableQueryOperatorChecker.INSTANCE, null));
+        valid = valid && !invalid;
         // collect predicate from top plan which not in hyper graph
         Set<Expression> topPlanPredicates = new LinkedHashSet<>();
         topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
index 5465211c73e793..c7b8f452afbe66 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
@@ -36,11 +36,16 @@ public Rule build() {
         return logicalOlapScan().thenApply(ctx -> {
             LogicalOlapScan scan = ctx.root;
             OlapTable table = scan.getTable();
-            List<Long> ids = table.selectNonEmptyPartitionIds(scan.getSelectedPartitionIds());
+            List<Long> partitionIdsToPrune = scan.getSelectedPartitionIds();
+            List<Long> ids = table.selectNonEmptyPartitionIds(partitionIdsToPrune);
             if (ids.isEmpty()) {
                 return new LogicalEmptyRelation(ConnectContext.get().getStatementContext().getNextRelationId(),
                         scan.getOutput());
             }
+            if (partitionIdsToPrune.equals(ids)) {
+                // Not Prune actually, return directly
+                return null;
+            }
             return scan.withSelectedPartitionIds(ids);
         }).toRule(RuleType.PRUNE_EMPTY_PARTITION);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
index 4b86b7a3706f0e..fc212550b6f364 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
@@ -53,7 +53,7 @@ public Rule build() {
             LogicalOlapScan olapScan = filter.child();
             OlapTable table = olapScan.getTable();
             Builder<Long> selectedTabletIdsBuilder = ImmutableList.builder();
-            if (olapScan.getSelectedTabletIds().isEmpty()) {
+            if (olapScan.getManuallySpecifiedTabletIds().isEmpty()) {
                 for (Long id : olapScan.getSelectedPartitionIds()) {
                     Partition partition = table.getPartition(id);
                     MaterializedIndex index = partition.getIndex(olapScan.getSelectedIndexId());
@@ -64,10 +64,10 @@ public Rule build() {
                                     partition.getDistributionInfo()));
                 }
             } else {
-                selectedTabletIdsBuilder.addAll(olapScan.getSelectedTabletIds());
+                selectedTabletIdsBuilder.addAll(olapScan.getManuallySpecifiedTabletIds());
             }
             List<Long> selectedTabletIds = selectedTabletIdsBuilder.build();
-            if (new HashSet<>(selectedTabletIds).equals(new HashSet<>(olapScan.getSelectedTabletIds()))) {
+            if (new HashSet<>(selectedTabletIds).equals(new HashSet<>(olapScan.getManuallySpecifiedTabletIds()))) {
                 return null;
             }
             return filter.withChildren(olapScan.withSelectedTabletIds(selectedTabletIds));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index 5c6c71c92e8111..f9e298fd9d40d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -157,6 +157,9 @@ public void analyze(ConnectContext ctx) throws Exception {
         if (!InternalCatalog.INTERNAL_CATALOG_NAME.equals(mvName.getCtl())) {
             throw new AnalysisException("Only support creating asynchronous materialized views in internal catalog");
         }
+        if (ctx.getSessionVariable().isInDebugMode()) {
+            throw new AnalysisException("Create materialized view fail, because is in debug mode");
+        }
         try {
             FeNameFormat.checkTableName(mvName.getTbl());
         } catch (org.apache.doris.common.AnalysisException e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 90ceb24231340c..2216e58c4fa3b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -99,6 +99,12 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
      */
     private final List<Long> selectedTabletIds;
 
+    /**
+     * Selected tablet ids to read data from, this would be set if user query with tablets manually
+     * Such as select * from  orders TABLET(100);
+     */
+    private final List<Long> manuallySpecifiedTabletIds;
+
     ///////////////////////////////////////////////////////////////////////////
     // Members for partition ids.
     ///////////////////////////////////////////////////////////////////////////
@@ -127,12 +133,16 @@ public LogicalOlapScan(RelationId id, OlapTable table) {
         this(id, table, ImmutableList.of());
     }
 
+    /**
+     * LogicalOlapScan construct method
+     */
     public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier) {
         this(id, table, qualifier, Optional.empty(), Optional.empty(),
                 table.getPartitionIds(), false,
                 ImmutableList.of(),
                 -1, false, PreAggStatus.unset(), ImmutableList.of(), ImmutableList.of(),
-                Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of());
+                Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of(),
+                ImmutableList.of());
     }
 
     public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds,
@@ -140,7 +150,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, L
         this(id, table, qualifier, Optional.empty(), Optional.empty(),
                 table.getPartitionIds(), false, tabletIds,
                 -1, false, PreAggStatus.unset(), ImmutableList.of(), hints, Maps.newHashMap(),
-                tableSample, false, ImmutableMap.of());
+                tableSample, false, ImmutableMap.of(), ImmutableList.of());
     }
 
     public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> specifiedPartitions,
@@ -149,7 +159,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, L
                 // must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1'
                 specifiedPartitions, false, tabletIds,
                 -1, false, PreAggStatus.unset(), specifiedPartitions, hints, Maps.newHashMap(),
-                tableSample, false, ImmutableMap.of());
+                tableSample, false, ImmutableMap.of(), ImmutableList.of());
     }
 
     public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds,
@@ -158,7 +168,8 @@ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, L
         this(id, table, qualifier, Optional.empty(), Optional.empty(),
                 selectedPartitionIds, false, tabletIds,
                 selectedIndexId, true, preAggStatus,
-                specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of());
+                specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of(),
+                ImmutableList.of());
     }
 
     /**
@@ -171,7 +182,7 @@ public LogicalOlapScan(RelationId id, Table table, List<String> qualifier,
             PreAggStatus preAggStatus, List<Long> specifiedPartitions,
             List<String> hints, Map<Pair<Long, String>, Slot> cacheSlotWithSlotName,
             Optional<TableSample> tableSample, boolean directMvScan,
-            Map<String, Set<List<String>>> colToSubPathsMap) {
+            Map<String, Set<List<String>>> colToSubPathsMap, List<Long> specifiedTabletIds) {
         super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier,
                 groupExpression, logicalProperties);
         Preconditions.checkArgument(selectedPartitionIds != null,
@@ -182,6 +193,7 @@ public LogicalOlapScan(RelationId id, Table table, List<String> qualifier,
         this.indexSelected = indexSelected;
         this.preAggStatus = preAggStatus;
         this.manuallySpecifiedPartitions = ImmutableList.copyOf(specifiedPartitions);
+        this.manuallySpecifiedTabletIds = ImmutableList.copyOf(specifiedTabletIds);
 
         if (selectedPartitionIds.isEmpty()) {
             this.selectedPartitionIds = ImmutableList.of();
@@ -240,6 +252,7 @@ public boolean equals(Object o) {
                 && partitionPruned == that.partitionPruned && Objects.equals(preAggStatus, that.preAggStatus)
                 && Objects.equals(selectedTabletIds, that.selectedTabletIds)
                 && Objects.equals(manuallySpecifiedPartitions, that.manuallySpecifiedPartitions)
+                && Objects.equals(manuallySpecifiedTabletIds, that.manuallySpecifiedTabletIds)
                 && Objects.equals(selectedPartitionIds, that.selectedPartitionIds)
                 && Objects.equals(hints, that.hints)
                 && Objects.equals(tableSample, that.tableSample);
@@ -248,8 +261,8 @@ public boolean equals(Object o) {
     @Override
     public int hashCode() {
         return Objects.hash(super.hashCode(), selectedIndexId, indexSelected, preAggStatus, cacheSlotWithSlotName,
-                selectedTabletIds, partitionPruned, manuallySpecifiedPartitions, selectedPartitionIds, hints,
-                tableSample);
+                selectedTabletIds, partitionPruned, manuallySpecifiedTabletIds, manuallySpecifiedPartitions,
+                selectedPartitionIds, hints, tableSample);
     }
 
     @Override
@@ -258,7 +271,7 @@ public LogicalOlapScan withGroupExpression(Optional<GroupExpression> groupExpres
                 groupExpression, Optional.of(getLogicalProperties()),
                 selectedPartitionIds, partitionPruned, selectedTabletIds,
                 selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
-                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap);
+                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds);
     }
 
     @Override
@@ -267,7 +280,7 @@ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpr
         return new LogicalOlapScan(relationId, (Table) table, qualifier, groupExpression, logicalProperties,
                 selectedPartitionIds, partitionPruned, selectedTabletIds,
                 selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
-                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap);
+                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds);
     }
 
     public LogicalOlapScan withSelectedPartitionIds(List<Long> selectedPartitionIds) {
@@ -275,7 +288,7 @@ public LogicalOlapScan withSelectedPartitionIds(List<Long> selectedPartitionIds)
                 Optional.empty(), Optional.of(getLogicalProperties()),
                 selectedPartitionIds, true, selectedTabletIds,
                 selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
-                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap);
+                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds);
     }
 
     public LogicalOlapScan withMaterializedIndexSelected(long indexId) {
@@ -283,7 +296,7 @@ public LogicalOlapScan withMaterializedIndexSelected(long indexId) {
                 Optional.empty(), Optional.of(getLogicalProperties()),
                 selectedPartitionIds, partitionPruned, selectedTabletIds,
                 indexId, true, PreAggStatus.unset(), manuallySpecifiedPartitions, hints, cacheSlotWithSlotName,
-                tableSample, directMvScan, colToSubPathsMap);
+                tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds);
     }
 
     public LogicalOlapScan withSelectedTabletIds(List<Long> selectedTabletIds) {
@@ -291,7 +304,7 @@ public LogicalOlapScan withSelectedTabletIds(List<Long> selectedTabletIds) {
                 Optional.empty(), Optional.of(getLogicalProperties()),
                 selectedPartitionIds, partitionPruned, selectedTabletIds,
                 selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
-                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap);
+                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds);
     }
 
     public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) {
@@ -299,7 +312,7 @@ public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) {
                 Optional.empty(), Optional.of(getLogicalProperties()),
                 selectedPartitionIds, partitionPruned, selectedTabletIds,
                 selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
-                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap);
+                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds);
     }
 
     public LogicalOlapScan withColToSubPathsMap(Map<String, Set<List<String>>> colToSubPathsMap) {
@@ -307,7 +320,15 @@ public LogicalOlapScan withColToSubPathsMap(Map<String, Set<List<String>>> colTo
                 Optional.empty(), Optional.empty(),
                 selectedPartitionIds, partitionPruned, selectedTabletIds,
                 selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
-                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap);
+                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds);
+    }
+
+    public LogicalOlapScan withManuallySpecifiedTabletIds(List<Long> manuallySpecifiedTabletIds) {
+        return new LogicalOlapScan(relationId, (Table) table, qualifier,
+                Optional.empty(), Optional.of(getLogicalProperties()),
+                selectedPartitionIds, partitionPruned, selectedTabletIds,
+                selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
+                hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds);
     }
 
     @Override
@@ -317,7 +338,7 @@ public LogicalOlapScan withRelationId(RelationId relationId) {
                 Optional.empty(), Optional.empty(),
                 selectedPartitionIds, false, selectedTabletIds,
                 selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
-                hints, Maps.newHashMap(), tableSample, directMvScan, colToSubPathsMap);
+                hints, Maps.newHashMap(), tableSample, directMvScan, colToSubPathsMap, selectedTabletIds);
     }
 
     @Override
@@ -333,6 +354,10 @@ public List<Long> getSelectedTabletIds() {
         return selectedTabletIds;
     }
 
+    public List<Long> getManuallySpecifiedTabletIds() {
+        return manuallySpecifiedTabletIds;
+    }
+
     @Override
     public long getSelectedIndexId() {
         return selectedIndexId;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java
index e7940cdfb9b519..385657cd3f229e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java
@@ -28,6 +28,7 @@
 import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.Partition;
 import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
 import org.apache.doris.nereids.trees.expressions.InPredicate;
@@ -51,8 +52,9 @@
 import org.junit.jupiter.api.Test;
 
 import java.util.List;
+import java.util.Objects;
 
-class PruneOlapScanTabletTest implements MemoPatternMatchSupported {
+class PruneOlapScanTabletTest extends SqlTestBase implements MemoPatternMatchSupported {
 
     @Test
     void testPruneOlapScanTablet(@Mocked OlapTable olapTable,
@@ -154,4 +156,21 @@ void testPruneOlapScanTablet(@Mocked OlapTable olapTable,
                         )
                 );
     }
+
+    @Test
+    void testPruneOlapScanTabletWithManually() {
+        String sql = "select * from T4 TABLET(110) where id > 8";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .applyTopDown(new PruneOlapScanTablet())
+                .matches(
+                        logicalFilter(
+                                logicalOlapScan().when(s ->
+                                        Objects.equals(s.getSelectedTabletIds(), Lists.newArrayList(110L))
+                                                && Objects.equals(s.getManuallySpecifiedTabletIds(),
+                                                Lists.newArrayList(110L))
+                                )
+                        )
+                );
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
index f5b301dcc49b25..9ebbc22a85c826 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
@@ -87,6 +87,9 @@ protected void runBeforeAll() throws Exception {
                         + "    score bigint\n"
                         + ")\n"
                         + "DUPLICATE KEY(id)\n"
+                        + "AUTO PARTITION BY LIST(`id`)\n"
+                        + "(\n"
+                        + ")\n"
                         + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
                         + "PROPERTIES (\n"
                         + "  \"replication_num\" = \"1\"\n"
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 62e6032578fd6a..744edfe54cbc24 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1876,6 +1876,22 @@ class Suite implements GroovyInterceptable {
         sql "analyze table ${db}.${mv_name} with sync;"
     }
 
+    def create_async_partition_mv = { db, mv_name, mv_sql, partition_col ->
+
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${db}.${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${db}.${mv_name} 
+        BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL 
+        PARTITION BY ${partition_col} 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1')  
+        AS ${mv_sql}
+        """
+        def job_name = getJobName(db, mv_name);
+        waitingMTMVTaskFinished(job_name)
+        sql "analyze table ${db}.${mv_name} with sync;"
+    }
+
     // mv not part in rewrite process
     void mv_not_part_in(query_sql, mv_name, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) {
         logger.info("query_sql = " + query_sql + ", mv_names = " + mv_name + ", sync_cbo_rewrite = " + sync_cbo_rewrite)
diff --git a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
new file mode 100644
index 00000000000000..15d93e32f65dc2
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert
+
+suite("is_in_debug_mode") {
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    sql "set runtime_filter_mode=OFF";
+    sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+
+    sql """
+    drop table if exists orders;
+    """
+
+    sql """
+     CREATE TABLE IF NOT EXISTS orders  (
+      o_orderkey       INTEGER NOT NULL,
+      o_custkey        INTEGER NOT NULL,
+      o_orderstatus    CHAR(1) NOT NULL,
+      o_totalprice     DECIMALV3(15,2) NOT NULL,
+      o_orderdate      DATE NOT NULL,
+      o_orderpriority  CHAR(15) NOT NULL,  
+      o_clerk          CHAR(15) NOT NULL, 
+      o_shippriority   INTEGER NOT NULL,
+      O_COMMENT        VARCHAR(79) NOT NULL
+    )
+    UNIQUE KEY(o_orderkey, o_custkey)
+    DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    );
+    """
+
+    sql """
+    insert into orders values
+    (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); 
+    """
+
+    create_async_mv(db, "basic_mv", """
+    select * from orders where o_orderkey > 1;
+    """)
+
+    sql """set skip_delete_sign = true;"""
+    mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv")
+    try {
+        sql """
+        CREATE MATERIALIZED VIEW test_create_mv
+        BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS select * from orders where o_orderkey > 2;
+        """
+    } catch (Exception e) {
+        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+    }
+    sql """set skip_delete_sign = false;"""
+
+
+    sql """set skip_storage_engine_merge = true;"""
+    mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv")
+    try {
+        sql """
+        CREATE MATERIALIZED VIEW test_create_mv
+        BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS select * from orders where o_orderkey > 2;
+        """
+    } catch (Exception e) {
+        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+    }
+    sql """set skip_storage_engine_merge = false;"""
+
+
+    sql """set skip_delete_bitmap = true;"""
+    mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv")
+    try {
+        sql """
+        CREATE MATERIALIZED VIEW test_create_mv
+        BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS select * from orders where o_orderkey > 2;
+        """
+    } catch (Exception e) {
+        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+    }
+    sql """set skip_delete_bitmap = false;"""
+
+
+    sql """set skip_delete_predicate = true;"""
+    mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv")
+    try {
+        sql """
+        CREATE MATERIALIZED VIEW test_create_mv
+        BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS select * from orders where o_orderkey > 2;
+        """
+    } catch (Exception e) {
+        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+    }
+    sql """set skip_delete_predicate = false;"""
+
+
+    sql """set show_hidden_columns = true;"""
+    mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv")
+    try {
+        sql """
+        CREATE MATERIALIZED VIEW test_create_mv
+        BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS select * from orders where o_orderkey > 2;
+        """
+    } catch (Exception e) {
+        Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+    }
+    sql """set show_hidden_columns = false;"""
+
+    sql """drop materialized view if exists basic_mv"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy b/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy
new file mode 100644
index 00000000000000..5ba2c05053e2a2
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy
@@ -0,0 +1,195 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("with_table_operator") {
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    sql "set runtime_filter_mode=OFF";
+    sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+
+    sql """
+    drop table if exists orders;
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS orders  (
+      o_orderkey       INTEGER NOT NULL,
+      o_custkey        INTEGER NOT NULL,
+      o_orderstatus    CHAR(1) NOT NULL,
+      o_totalprice     DECIMALV3(15,2) NOT NULL,
+      o_orderdate      DATE NOT NULL,
+      o_orderpriority  CHAR(15) NOT NULL,  
+      o_clerk          CHAR(15) NOT NULL, 
+      o_shippriority   INTEGER NOT NULL,
+      O_COMMENT        VARCHAR(79) NOT NULL
+    )
+    DUPLICATE KEY(o_orderkey, o_custkey)
+    DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    );
+    """
+
+    sql """
+    insert into orders values
+    (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); 
+    """
+
+    sql """
+    drop table if exists orders_partition;
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS orders_partition  (
+      o_orderkey       INTEGER NOT NULL,
+      o_custkey        INTEGER NOT NULL,
+      o_orderstatus    CHAR(1) NOT NULL,
+      o_totalprice     DECIMALV3(15,2) NOT NULL,
+      o_orderdate      DATE NOT NULL,
+      o_orderpriority  CHAR(15) NOT NULL,  
+      o_clerk          CHAR(15) NOT NULL, 
+      o_shippriority   INTEGER NOT NULL,
+      O_COMMENT        VARCHAR(79) NOT NULL
+    )
+    DUPLICATE KEY(o_orderkey, o_custkey)
+    PARTITION BY RANGE(o_orderdate) (
+    PARTITION `day_2` VALUES LESS THAN ('2023-12-9'),
+    PARTITION `day_3` VALUES LESS THAN ("2023-12-11"),
+    PARTITION `day_4` VALUES LESS THAN ("2023-12-30")
+    )
+    DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    );
+    """
+
+    sql """
+    insert into orders_partition values
+    (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); 
+    """
+
+    create_async_mv(db, "complete_agg_mv", """
+    select count(*),o_orderdate
+    from orders
+    group by o_orderdate;
+    """)
+
+    create_async_mv(db, "complete_mv", """
+    select * from orders;
+    """)
+
+    create_async_partition_mv(db, "partition_agg_mv", """
+    select count(*),o_orderdate
+    from orders_partition
+    group by o_orderdate;
+    """, "(o_orderdate)")
+
+    create_async_partition_mv(db, "partition_mv", """
+    select * from orders_partition;
+    """, "(o_orderdate)")
+
+    // query rewrite with tablet should fail
+    mv_rewrite_all_fail("""
+    select count(*),o_orderdate
+    from orders TABLET(110)
+    group by o_orderdate;
+    """, ["complete_mv", "complete_agg_mv"])
+
+    mv_rewrite_all_fail("""select * from orders TABLET(110);""", ["complete_mv", "complete_agg_mv"])
+
+    // query rewrite with index should fail
+    createMV("""
+    create materialized view query_index_test 
+    as
+    select o_orderdate, count(*)
+    from orders
+    group by o_orderdate;
+    """)
+    mv_rewrite_all_fail("""
+    select * from orders index query_index_test;
+    """, ["complete_mv", "complete_agg_mv"])
+
+    // query rewrite with sample should fail
+    mv_rewrite_all_fail("""
+    select count(*),o_orderdate
+    from orders TABLESAMPLE(20 percent)
+    group by o_orderdate;
+    """, ["complete_mv", "complete_agg_mv"])
+
+    mv_rewrite_all_fail("""select * from orders TABLESAMPLE(20 percent);""", ["complete_mv", "complete_agg_mv"])
+
+    // query rewrite with partition should fail
+    mv_rewrite_all_fail("""
+    select count(*),o_orderdate
+    from orders_partition PARTITION (day_2)
+    group by o_orderdate;
+    """, ["partition_mv", "partition_agg_mv"])
+
+    mv_rewrite_all_fail("""select * from orders_partition PARTITION (day_2);""", ["partition_mv", "partition_agg_mv"])
+
+
+    sql """
+    DROP MATERIALIZED VIEW IF EXISTS complete_agg_mv;
+    """
+    sql """
+    DROP MATERIALIZED VIEW IF EXISTS partition_agg_mv;
+    """
+    sql """
+    DROP MATERIALIZED VIEW IF EXISTS complete_mv;
+    """
+    sql """
+    DROP MATERIALIZED VIEW IF EXISTS partition_mv;
+    """
+    sql """
+    DROP MATERIALIZED VIEW IF EXISTS query_index_test on orders;
+    """
+}
+

From ce9c7174fb5ea04781d042d72b69a677793bb965 Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Mon, 2 Dec 2024 19:45:27 +0800
Subject: [PATCH 118/399] [fix](auth) Prohibit deleting admin user (#44751)

---
 .../apache/doris/analysis/DropUserStmt.java   |  4 +-
 .../apache/doris/analysis/UserIdentity.java   |  4 ++
 .../trees/plans/commands/DropUserCommand.java |  4 +-
 .../suites/account_p0/test_system_user.groovy | 39 +++++++++++++++++++
 4 files changed, 47 insertions(+), 4 deletions(-)
 create mode 100644 regression-test/suites/account_p0/test_system_user.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java
index f9bb218744fc85..5f9872e42ad7cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java
@@ -63,8 +63,8 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
 
         userIdent.analyze();
 
-        if (userIdent.isRootUser()) {
-            ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, "Can not drop root user");
+        if (userIdent.isSystemUser()) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, "Can not drop system user");
         }
 
         // only user with GLOBAL level's GRANT_PRIV can drop user.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java
index 122f28ca45092b..6898915dab73ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java
@@ -189,6 +189,10 @@ public boolean isAdminUser() {
         return user.equals(Auth.ADMIN_USER);
     }
 
+    public boolean isSystemUser() {
+        return isRootUser() || isAdminUser();
+    }
+
     public TUserIdentity toThrift() {
         Preconditions.checkState(isAnalyzed);
         TUserIdentity tUserIdent = new TUserIdentity();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropUserCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropUserCommand.java
index 094ce06c36143e..5c4fb407e40213 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropUserCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropUserCommand.java
@@ -55,8 +55,8 @@ public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
 
         userIdent.analyze();
 
-        if (userIdent.isRootUser()) {
-            ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, "Can not drop root user");
+        if (userIdent.isSystemUser()) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, "Can not drop system user");
         }
 
         // only user with GLOBAL level's GRANT_PRIV can drop user.
diff --git a/regression-test/suites/account_p0/test_system_user.groovy b/regression-test/suites/account_p0/test_system_user.groovy
new file mode 100644
index 00000000000000..1805f1669ea570
--- /dev/null
+++ b/regression-test/suites/account_p0/test_system_user.groovy
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_system_user") {
+    test {
+          sql """
+              create user `root`;
+          """
+          exception "root"
+    }
+    test {
+          sql """
+              drop user `root`;
+          """
+          exception "system"
+    }
+    test {
+          sql """
+              drop user `admin`;
+          """
+          exception "system"
+    }
+}

From 70b0a08815b1ea4e39a8e39f8f067ea27c795abb Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Mon, 2 Dec 2024 20:04:14 +0800
Subject: [PATCH 119/399] [fix](nereids) fix merge_percentile_to_array when has
 same agg function (#44783)

Related PR: #34313

Problem Summary
The original PR did not handle the following scenario:
```sql
SELECT SUM(a), PERCENTILE(pk, 0.1) AS c1, PERCENTILE(pk, 0.1) AS c2, PERCENTILE(pk, 0.4) AS c3 FROM test_merge_percentile;
```
In this case, the aggregate outputs include two identical functions
(PERCENTILE(pk, 0.1)). When constructing the LogicalProject, a map was
used where the key is the child of an Alias and the value is the Alias
itself. However, this approach loses information when two Aliases share
the same child.
This PR modifies the map structure to use the child of an Alias as the
key and a list of Alias objects as the value. This ensures that all
Alias instances with the same child are preserved, resolving the issue
of lost information in such cases.
---
 .../rules/rewrite/MergePercentileToArray.java | 26 +++++++++----------
 .../merge_percentile_to_array.out             | 12 +++++++++
 .../merge_percentile_to_array.groovy          |  4 +++
 3 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java
index f92ad84bde8525..fe81adf13bf29d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java
@@ -152,10 +152,10 @@ private Plan doMerge(LogicalAggregate<Plan> aggregate) {
                 (List<Expression>) (List) newPercentileArrays);
         ImmutableList.Builder<NamedExpression> newProjectOutputExpressions = ImmutableList.builder();
         newProjectOutputExpressions.addAll((List<NamedExpression>) (List) notChangeForProject);
-        Map<Expression, Alias> existsAliasMap = Maps.newHashMap();
+        Map<Expression, List<Alias>> existsAliasMap = Maps.newHashMap();
         // existsAliasMap is used to keep upper plan refer the same expr
         for (Alias alias : existsAliases) {
-            existsAliasMap.put(alias.child(), alias);
+            existsAliasMap.computeIfAbsent(alias.child(), k -> new ArrayList<>()).add(alias);
         }
         Map<DistinctAndExpr, Slot> slotMap = Maps.newHashMap();
         // slotMap is used to find the correspondence
@@ -169,20 +169,22 @@ private Plan doMerge(LogicalAggregate<Plan> aggregate) {
         for (Map.Entry<DistinctAndExpr, List<AggregateFunction>> entry : funcMap.entrySet()) {
             for (int i = 0; i < entry.getValue().size(); i++) {
                 AggregateFunction aggFunc = entry.getValue().get(i);
-                Alias originAlias = existsAliasMap.get(aggFunc);
-                DistinctAndExpr distinctAndExpr = new DistinctAndExpr(aggFunc.child(0), aggFunc.isDistinct());
-                Alias newAlias = new Alias(originAlias.getExprId(), new ElementAt(slotMap.get(distinctAndExpr),
-                        new IntegerLiteral(i + 1)), originAlias.getName());
-                newProjectOutputExpressions.add(newAlias);
+                List<Alias> originAliases = existsAliasMap.get(aggFunc);
+                for (Alias originAlias : originAliases) {
+                    DistinctAndExpr distinctAndExpr = new DistinctAndExpr(aggFunc.child(0), aggFunc.isDistinct());
+                    Alias newAlias = new Alias(originAlias.getExprId(), new ElementAt(slotMap.get(distinctAndExpr),
+                            new IntegerLiteral(i + 1)), originAlias.getName());
+                    newProjectOutputExpressions.add(newAlias);
+                }
             }
         }
         newProjectOutputExpressions.addAll(groupBySlots);
-        return new LogicalProject(newProjectOutputExpressions.build(), newAggregate);
+        return new LogicalProject<>(newProjectOutputExpressions.build(), newAggregate);
     }
 
     private static class DistinctAndExpr {
-        private Expression expression;
-        private boolean isDistinct;
+        private final Expression expression;
+        private final boolean isDistinct;
 
         public DistinctAndExpr(Expression expression, boolean isDistinct) {
             this.expression = expression;
@@ -193,10 +195,6 @@ public Expression getExpression() {
             return expression;
         }
 
-        public boolean isDistinct() {
-            return isDistinct;
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) {
diff --git a/regression-test/data/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.out b/regression-test/data/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.out
index b495302e80d3c8..1b2f876cfba50a 100644
--- a/regression-test/data/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.out
+++ b/regression-test/data/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.out
@@ -41,3 +41,15 @@
 7.0	\N	\N
 7.0	7.0	7
 
+-- !same_percentile --
+52	1.0	1.0	2.0
+
+-- !same_percentile_group_by --
+\N	6.0	6.0	6.0
+2	3.0	3.0	3.0
+25	3.0	3.0	3.0
+4	2.0	2.0	2.0
+5	1.0	1.0	1.6
+7	6.0	6.0	6.0
+9	1.2	1.2	1.8
+
diff --git a/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy b/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy
index 2071d75ae85d4e..5bb13c6336c264 100644
--- a/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy
+++ b/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy
@@ -57,4 +57,8 @@ suite("merge_percentile_to_array") {
             percentile(abs(a), 0.55) as c2 from test_merge_percentile group by a) t;      
     """
 
+    order_qt_same_percentile """select sum(a),percentile(pk, 0.1) as c1 , percentile(pk, 0.1) as c2 ,
+            percentile(pk, 0.4) as c2 from test_merge_percentile;"""
+    order_qt_same_percentile_group_by """select sum(a),percentile(pk, 0.1) as c1 , percentile(pk, 0.1) as c2 ,
+            percentile(pk, 0.4) as c2 from test_merge_percentile group by a;"""
 }
\ No newline at end of file

From 21e1d6d8a7ccb08848e7be61ae574c7ec4b27e69 Mon Sep 17 00:00:00 2001
From: Mingxi <71588583+joker-star-l@users.noreply.github.com>
Date: Mon, 2 Dec 2024 20:27:10 +0800
Subject: [PATCH 120/399] [fix](logstash) remove ShortNameResolver to solve
 thread race problem (#44598)

remove ShortNameResolver to solve thread race problem
---
 .../logstash/lib/logstash/outputs/doris.rb    | 50 +---------------
 .../lib/logstash/util/shortname_resolver.rb   | 58 -------------------
 .../logstash/logstash-output-doris.gemspec    |  1 -
 3 files changed, 2 insertions(+), 107 deletions(-)
 delete mode 100644 extension/logstash/lib/logstash/util/shortname_resolver.rb

diff --git a/extension/logstash/lib/logstash/outputs/doris.rb b/extension/logstash/lib/logstash/outputs/doris.rb
index 02e7591b0a354b..21d3ee6e752b08 100644
--- a/extension/logstash/lib/logstash/outputs/doris.rb
+++ b/extension/logstash/lib/logstash/outputs/doris.rb
@@ -21,7 +21,6 @@
 require "logstash/outputs/base"
 require "logstash/namespace"
 require "logstash/json"
-require "logstash/util/shortname_resolver"
 require 'logstash/util/formater'
 require "uri"
 require "securerandom"
@@ -67,8 +66,6 @@ class LogStash::Outputs::Doris < LogStash::Outputs::Base
 
    config :save_file, :validate => :string, :default => "failed.data"
 
-   config :host_resolve_ttl_sec, :validate => :number, :default => 120
-
    config :max_retries, :validate => :number, :default => -1
 
    config :log_request, :validate => :boolean, :default => true
@@ -92,10 +89,6 @@ def print_plugin_info()
    def register
       @http_query = "/api/#{@db}/#{@table}/_stream_load"
 
-      @hostnames_pool =
-      parse_http_hosts(@http_hosts,
-      ShortNameResolver.new(ttl: @host_resolve_ttl_sec, logger: @logger))
-
       @request_headers = make_request_headers
       @logger.info("request headers: ", @request_headers)
 
@@ -141,39 +134,6 @@ def register
       print_plugin_info()
    end # def register
 
-   private
-
-   def parse_http_hosts(hosts, resolver)
-      ip_re = /^[\d]+\.[\d]+\.[\d]+\.[\d]+$/
-
-      lambda {
-         hosts.flat_map { |h|
-            scheme = URI(h).scheme
-            host = URI(h).host
-            port = URI(h).port
-            path = URI(h).path
-
-            if ip_re !~ host
-               resolver.get_addresses(host).map { |ip|
-                  "#{scheme}://#{ip}:#{port}#{path}"
-               }
-            else
-               [h]
-            end
-         }
-      }
-   end
-
-   private
-
-   def get_host_addresses()
-      begin
-         @hostnames_pool.call
-      rescue Exception => ex
-         @logger.error('Error while resolving host', :error => ex.to_s)
-      end
-   end
-
    def multi_receive(events)
       return if events.empty?
       send_events(events)
@@ -191,8 +151,6 @@ def send_events(events)
       # @logger.info("get event num: #{event_num}")
       @logger.debug("get documents: #{documents}")
 
-      hosts = get_host_addresses()
-
       http_headers = @request_headers.dup
       if !@group_commit
          # only set label if group_commit is off_mode or not set, since lable can not be used with group_commit
@@ -202,7 +160,7 @@ def send_events(events)
       req_count = 0
       sleep_for = 1
       while true
-         response = make_request(documents, http_headers, hosts, @http_query, hosts.sample)
+         response = make_request(documents, http_headers, @http_query, @http_hosts.sample)
 
          req_count += 1
          response_json = {}
@@ -246,11 +204,7 @@ def send_events(events)
    end
 
    private
-   def make_request(documents, http_headers, hosts, query, host = "")
-      if host == ""
-         host = hosts.pop
-      end
-
+   def make_request(documents, http_headers, query, host)
       url = host + query
 
       if @log_request or @logger.debug?
diff --git a/extension/logstash/lib/logstash/util/shortname_resolver.rb b/extension/logstash/lib/logstash/util/shortname_resolver.rb
deleted file mode 100644
index 1437ccba007988..00000000000000
--- a/extension/logstash/lib/logstash/util/shortname_resolver.rb
+++ /dev/null
@@ -1,58 +0,0 @@
-=begin
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
-=end
-require 'resolv'
-require 'mini_cache'
-
-class ShortNameResolver
-  def initialize(ttl:, logger:)
-    @ttl = ttl
-    @store = MiniCache::Store.new
-    @logger = logger
-  end
-
-  private
-  def resolve_cached(shortname)
-    @store.get_or_set(shortname) do
-      addresses = resolve(shortname)
-      raise "Bad shortname '#{shortname}'" if addresses.empty?
-      MiniCache::Data.new(addresses, expires_in: @ttl)
-    end
-  end
-
-  private
-  def resolve(shortname)
-    addresses = Resolv::DNS.open do |dns|
-      dns.getaddresses(shortname).map { |r| r.to_s }
-    end
-
-    @logger.info("Resolved shortname '#{shortname}' to addresses #{addresses}")
-
-    return addresses
-  end
-
-  public
-  def get_address(shortname)
-    return resolve_cached(shortname).sample
-  end
-
-  public
-  def get_addresses(shortname)
-    return resolve_cached(shortname)
-  end
-end
diff --git a/extension/logstash/logstash-output-doris.gemspec b/extension/logstash/logstash-output-doris.gemspec
index 689b93503f6a8b..30341b83156ed4 100644
--- a/extension/logstash/logstash-output-doris.gemspec
+++ b/extension/logstash/logstash-output-doris.gemspec
@@ -38,7 +38,6 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'mini_cache', ">= 1.0.0", "< 2.0.0"
   s.add_runtime_dependency "rest-client", '~> 2.1'
 
   s.add_development_dependency 'logstash-devutils', '~> 1.3'

From 303e27597e3f6c2ee82033ab90593ae8d2599628 Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Mon, 2 Dec 2024 21:42:18 +0800
Subject: [PATCH 121/399] [Fix](TPartitionVersionInfo) Fix duplicate
 `TPartitionVersionInfo` in `PublishVersionTask.partitionVersionInfos`
 (#44846)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What problem does this PR solve?

Problem Summary: When FE handles BEs' tablet report and finds that there
exists some expired txns on BE, it will generate publish version task.
`TPartitionVersionInfo` with same values may be added to
`transactionsToPublish` under same txn id many times when partitions
involved in this failed txn involves many tablets on this BE. Because it
uses `ArrayListMultimap`, these duplicate values may occupy a lot of
memories when the number of tablets is large.

### Release note

Fixed the issue of FE's memory occupation growing too fast in cases of
persistent load and clone failures on merge-on-write tables.

修复了在merge-on-write表上有持续的导入失败和clone失败的情况下，FE使用内存增长过快的问题。
---
 .../apache/doris/catalog/TabletInvertedIndex.java   | 13 +++++++------
 .../java/org/apache/doris/master/ReportHandler.java |  9 +++++----
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
index 4a421dc7b2b1ef..a51d1f55014fcc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
@@ -37,13 +37,14 @@
 import org.apache.doris.transaction.TransactionStatus;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.HashBasedTable;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Ordering;
+import com.google.common.collect.SetMultimap;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Table;
 import com.google.common.collect.TreeMultimap;
@@ -135,7 +136,7 @@ public void tabletReport(long backendId, Map<Long, TTablet> backendTablets,
                              Set<Long> tabletFoundInMeta,
                              ListMultimap<TStorageMedium, Long> tabletMigrationMap,
                              Map<Long, Long> partitionVersionSyncMap,
-                             Map<Long, ListMultimap<Long, TPartitionVersionInfo>> transactionsToPublish,
+                             Map<Long, SetMultimap<Long, TPartitionVersionInfo>> transactionsToPublish,
                              ListMultimap<Long, Long> transactionsToClear,
                              ListMultimap<Long, Long> tabletRecoveryMap,
                              List<TTabletMetaInfo> tabletToUpdate,
@@ -314,7 +315,7 @@ && isLocal(tabletMeta.getStorageMedium())) {
     }
 
     private void handleBackendTransactions(long backendId, List<Long> transactionIds, long tabletId,
-            TabletMeta tabletMeta, Map<Long, ListMultimap<Long, TPartitionVersionInfo>> transactionsToPublish,
+            TabletMeta tabletMeta, Map<Long, SetMultimap<Long, TPartitionVersionInfo>> transactionsToPublish,
             ListMultimap<Long, Long> transactionsToClear) {
         GlobalTransactionMgrIface transactionMgr = Env.getCurrentGlobalTransactionMgr();
         long partitionId = tabletMeta.getPartitionId();
@@ -376,15 +377,15 @@ private TPartitionVersionInfo generatePartitionVersionInfoWhenReport(Transaction
     }
 
     private void publishPartition(TransactionState transactionState, long transactionId, TabletMeta tabletMeta,
-            long partitionId, Map<Long, ListMultimap<Long, TPartitionVersionInfo>> transactionsToPublish) {
+            long partitionId, Map<Long, SetMultimap<Long, TPartitionVersionInfo>> transactionsToPublish) {
         TPartitionVersionInfo versionInfo = generatePartitionVersionInfoWhenReport(transactionState,
                 transactionId, tabletMeta, partitionId);
         if (versionInfo != null) {
             synchronized (transactionsToPublish) {
-                ListMultimap<Long, TPartitionVersionInfo> map = transactionsToPublish.get(
+                SetMultimap<Long, TPartitionVersionInfo> map = transactionsToPublish.get(
                         transactionState.getDbId());
                 if (map == null) {
-                    map = ArrayListMultimap.create();
+                    map = LinkedHashMultimap.create();
                     transactionsToPublish.put(transactionState.getDbId(), map);
                 }
                 map.put(transactionId, versionInfo);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index c5c72eae3c5a8b..06047e2cf16682 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -95,6 +95,7 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Queues;
+import com.google.common.collect.SetMultimap;
 import com.google.common.collect.Sets;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -503,7 +504,7 @@ public void tabletReport(long backendId, Map<Long, TTablet> backendTablets,
         Map<Long, Long> partitionVersionSyncMap = Maps.newConcurrentMap();
 
         // dbid -> txn id -> [partition info]
-        Map<Long, ListMultimap<Long, TPartitionVersionInfo>> transactionsToPublish = Maps.newHashMap();
+        Map<Long, SetMultimap<Long, TPartitionVersionInfo>> transactionsToPublish = Maps.newHashMap();
         ListMultimap<Long, Long> transactionsToClear = LinkedListMultimap.create();
 
         // db id -> tablet id
@@ -1148,14 +1149,14 @@ private static void handleMigration(ListMultimap<TStorageMedium, Long> tabletMet
     }
 
     private static void handleRepublishVersionInfo(
-            Map<Long, ListMultimap<Long, TPartitionVersionInfo>> transactionsToPublish, long backendId) {
+            Map<Long, SetMultimap<Long, TPartitionVersionInfo>> transactionsToPublish, long backendId) {
         AgentBatchTask batchTask = new AgentBatchTask();
         long createPublishVersionTaskTime = System.currentTimeMillis();
         for (Long dbId : transactionsToPublish.keySet()) {
-            ListMultimap<Long, TPartitionVersionInfo> map = transactionsToPublish.get(dbId);
+            SetMultimap<Long, TPartitionVersionInfo> map = transactionsToPublish.get(dbId);
             for (long txnId : map.keySet()) {
                 PublishVersionTask task = new PublishVersionTask(backendId, txnId, dbId,
-                        map.get(txnId), createPublishVersionTaskTime);
+                        Lists.newArrayList(map.get(txnId)), createPublishVersionTaskTime);
                 batchTask.addTask(task);
                 // add to AgentTaskQueue for handling finish report.
                 AgentTaskQueue.addTask(task);

From ee3f09d00d70daa8689a1d855f9dc0e433e658f5 Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Tue, 3 Dec 2024 10:49:41 +0800
Subject: [PATCH 122/399] [opt](ms) print segment key bounds size when
 prepare/commit/update rowset meta (#44715)

---
 cloud/src/meta-service/meta_service.cpp      | 32 ++++++++++++++++++--
 cloud/src/meta-service/meta_service_helper.h |  1 +
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index ddf4ee3889c00e..9da5750d8d83f9 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -1033,14 +1033,18 @@ void MetaServiceImpl::prepare_rowset(::google::protobuf::RpcController* controll
     prepare_rowset.SerializeToString(&val);
     DCHECK_GT(prepare_rowset.expiration(), 0);
     txn->put(prepare_rs_key, val);
+    std::size_t segment_key_bounds_bytes = get_segments_key_bounds_bytes(rowset_meta);
     LOG(INFO) << "put prepare_rs_key " << hex(prepare_rs_key) << " value_size " << val.size()
-              << " txn_id " << request->txn_id();
+              << " txn_id " << request->txn_id() << " segment_key_bounds_bytes "
+              << segment_key_bounds_bytes;
     err = txn->commit();
     if (err != TxnErrorCode::TXN_OK) {
         if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE) {
             LOG(WARNING) << "failed to prepare rowset, err=value too large"
                          << ", txn_id=" << request->txn_id() << ", tablet_id=" << tablet_id
                          << ", rowset_id=" << rowset_id
+                         << ", rowset_meta_bytes=" << rowset_meta.ByteSizeLong()
+                         << ", segment_key_bounds_bytes=" << segment_key_bounds_bytes
                          << ", rowset_meta=" << rowset_meta.ShortDebugString();
         }
         code = cast_as<ErrCategory::COMMIT>(err);
@@ -1167,15 +1171,18 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
     DCHECK_GT(rowset_meta.txn_expiration(), 0);
     auto tmp_rs_val = rowset_meta.SerializeAsString();
     txn->put(tmp_rs_key, tmp_rs_val);
+    std::size_t segment_key_bounds_bytes = get_segments_key_bounds_bytes(rowset_meta);
     LOG(INFO) << "put tmp_rs_key " << hex(tmp_rs_key) << " delete recycle_rs_key "
               << hex(recycle_rs_key) << " value_size " << tmp_rs_val.size() << " txn_id "
-              << request->txn_id();
+              << request->txn_id() << " segment_key_bounds_bytes " << segment_key_bounds_bytes;
     err = txn->commit();
     if (err != TxnErrorCode::TXN_OK) {
         if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE) {
             LOG(WARNING) << "failed to commit rowset, err=value too large"
                          << ", txn_id=" << request->txn_id() << ", tablet_id=" << tablet_id
                          << ", rowset_id=" << rowset_id
+                         << ", rowset_meta_bytes=" << rowset_meta.ByteSizeLong()
+                         << ", segment_key_bounds_bytes=" << segment_key_bounds_bytes
                          << ", rowset_meta=" << rowset_meta.ShortDebugString();
         }
         code = cast_as<ErrCategory::COMMIT>(err);
@@ -1267,10 +1274,21 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr
     }
 
     txn->put(update_key, update_val);
+    std::size_t segment_key_bounds_bytes = get_segments_key_bounds_bytes(rowset_meta);
     LOG(INFO) << "xxx put "
-              << "update_rowset_key " << hex(update_key) << " value_size " << update_val.size();
+              << "update_rowset_key " << hex(update_key) << " value_size " << update_val.size()
+              << " segment_key_bounds_bytes " << segment_key_bounds_bytes;
     err = txn->commit();
     if (err != TxnErrorCode::TXN_OK) {
+        if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE) {
+            const auto& rowset_id = rowset_meta.rowset_id_v2();
+            LOG(WARNING) << "failed to update tmp rowset, err=value too large"
+                         << ", txn_id=" << request->txn_id() << ", tablet_id=" << tablet_id
+                         << ", rowset_id=" << rowset_id
+                         << ", rowset_meta_bytes=" << rowset_meta.ByteSizeLong()
+                         << ", segment_key_bounds_bytes=" << segment_key_bounds_bytes
+                         << ", rowset_meta=" << rowset_meta.ShortDebugString();
+        }
         code = cast_as<ErrCategory::COMMIT>(err);
         ss << "failed to update rowset meta, err=" << err;
         msg = ss.str();
@@ -2372,4 +2390,12 @@ MetaServiceResponseStatus MetaServiceImpl::fix_tablet_stats(std::string cloud_un
     return st;
 }
 
+std::size_t get_segments_key_bounds_bytes(const doris::RowsetMetaCloudPB& rowset_meta) {
+    size_t ret {0};
+    for (const auto& key_bounds : rowset_meta.segments_key_bounds()) {
+        ret += key_bounds.ByteSizeLong();
+    }
+    return ret;
+}
+
 } // namespace doris::cloud
diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h
index 4ef98ea2fb7b0c..5355ac0eb61ed2 100644
--- a/cloud/src/meta-service/meta_service_helper.h
+++ b/cloud/src/meta-service/meta_service_helper.h
@@ -248,4 +248,5 @@ void get_tablet_idx(MetaServiceCode& code, std::string& msg, Transaction* txn,
 bool is_dropped_tablet(Transaction* txn, const std::string& instance_id, int64_t index_id,
                        int64_t partition_id);
 
+std::size_t get_segments_key_bounds_bytes(const doris::RowsetMetaCloudPB& rowset_meta);
 } // namespace doris::cloud

From c382dcca72134288b8e785d0c5d8715ce55a0837 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Tue, 3 Dec 2024 10:50:25 +0800
Subject: [PATCH 123/399] [fix](catalog) Fix infinity partition key toSql
 (#44834)

---
 .../main/java/org/apache/doris/catalog/PartitionKey.java    | 3 +--
 .../java/org/apache/doris/catalog/PartitionKeyTest.java     | 6 ++++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
index f9640ccae14023..29bfda8b201d20 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
@@ -373,15 +373,14 @@ public String toSql() {
             if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) {
                 value = expr.toSql();
                 sb.append(value);
-                continue;
             } else {
                 value = "\"" + expr.getRealValue() + "\"";
                 if (expr instanceof DateLiteral) {
                     DateLiteral dateLiteral = (DateLiteral) expr;
                     value = dateLiteral.toSql();
                 }
+                sb.append(value);
             }
-            sb.append(value);
 
             if (keys.size() - 1 != i) {
                 sb.append(", ");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java
index affd5a4cc2c397..c6b45c0b136e61 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java
@@ -268,4 +268,10 @@ public void testSerialization() throws Exception {
         dis.close();
         Files.deleteIfExists(path);
     }
+
+    @Test
+    public void testMaxValueToSql() throws Exception {
+        PartitionKey key = PartitionKey.createInfinityPartitionKey(allColumns, true);
+        Assert.assertEquals("(MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE)", key.toSql());
+    }
 }

From b0f2c17fee2e6b5de0213db5c7b4f5a8b6e740b0 Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Tue, 3 Dec 2024 11:25:39 +0800
Subject: [PATCH 124/399] [fix](cluster key) fix some case (#44881)

---
 .../data/compaction/test_full_compaction.out  |   4 +
 .../compaction/test_full_compaction_ck.out    |  47 +++++
 .../test_generated_column_nereids.out         |  12 +-
 .../org/apache/doris/regression/Config.groovy |  10 +-
 .../compaction/test_full_compaction.groovy    |   5 +
 .../compaction/test_full_compaction_ck.groovy | 189 ++++++++++++++++++
 .../test_generated_column_nereids.groovy      |  22 +-
 7 files changed, 273 insertions(+), 16 deletions(-)
 create mode 100644 regression-test/data/compaction/test_full_compaction_ck.out
 create mode 100644 regression-test/suites/compaction/test_full_compaction_ck.groovy

diff --git a/regression-test/data/compaction/test_full_compaction.out b/regression-test/data/compaction/test_full_compaction.out
index b25fdad93145cc..aaab47b4c3678b 100644
--- a/regression-test/data/compaction/test_full_compaction.out
+++ b/regression-test/data/compaction/test_full_compaction.out
@@ -41,3 +41,7 @@
 2	200
 3	0
 
+-- !select_final2 --
+1	100
+2	200
+
diff --git a/regression-test/data/compaction/test_full_compaction_ck.out b/regression-test/data/compaction/test_full_compaction_ck.out
new file mode 100644
index 00000000000000..f76314e79a7579
--- /dev/null
+++ b/regression-test/data/compaction/test_full_compaction_ck.out
@@ -0,0 +1,47 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !1 --
+1	1
+2	2
+
+-- !2 --
+1	10
+2	20
+
+-- !3 --
+1	100
+2	200
+
+-- !4 --
+1	100
+2	200
+3	300
+
+-- !5 --
+1	100
+2	200
+3	100
+
+-- !6 --
+1	100
+2	200
+
+-- !skip_delete --
+1	1
+1	10
+1	100
+2	2
+2	20
+2	200
+3	100
+3	100
+3	300
+
+-- !select_final --
+1	100
+2	200
+3	100
+
+-- !select_final2 --
+1	100
+2	200
+
diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out
index b7757222ab4991..24f2f19c5769a5 100644
--- a/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out
+++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out
@@ -170,12 +170,6 @@ c	double	No	false	\N	NONE,STORED GENERATED
 b	int	Yes	false	\N	NONE
 d	int	Yes	false	\N	NONE,STORED GENERATED
 
--- !test_update --
-1
-
--- !test_update_generated_column --
-1	20	21
-
 -- !gen_col_unique_key --
 0
 
@@ -221,3 +215,9 @@ d	int	Yes	false	\N	NONE,STORED GENERATED
 -- !agg_replace_null --
 1	2	3	4	13
 
+-- !test_update --
+1
+
+-- !test_update_generated_column --
+1	20	21
+
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 379d177fbc280d..2f632f5c9252a7 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -1028,7 +1028,15 @@ class Config {
             excludeDirectorySet.add("fault_injection_p0/partial_update")
             excludeDirectorySet.add("fault_injection_p0/flexible")
             excludeDirectorySet.add("doc")
-            List<String> excludeCases = ["test_table_properties", "test_default_hll", "test_default_pi", "test_full_compaction", "test_full_compaction_by_table_id", "test_create_table", "txn_insert", "test_update_mow", "test_new_update", "test_update_unique", "test_partial_update_generated_column", "nereids_partial_update_native_insert_stmt", "partial_update", "nereids_update_on_current_timestamp", "update_on_current_timestamp", "test_default_bitmap_empty", "nereids_delete_mow_partial_update", "delete_mow_partial_update", "partial_update_seq_col", "nereids_partial_update_native_insert_stmt_complex", "regression_test_variant_delete_and_update", "test_unique_table_auto_inc_partial_update_correct_stream_load", "test_unique_table_auto_inc", "test_unique_table_auto_inc_partial_update_correct_insert", "test_update_schema_change", "test_partial_update_rowset_not_found_fault_injection"]
+            excludeDirectorySet.add("schema_change_p0/unique_ck")
+            List<String> excludeCases = ["test_table_properties", "test_create_table"
+                , "test_default_hll", "test_default_pi", "test_default_bitmap_empty"
+                // partial update
+                , "txn_insert", "test_update_schema_change", "test_generated_column_update", "test_nested_type_with_rowstore", "test_partial_update_generated_column", "nereids_partial_update_native_insert_stmt"
+                , "partial_update", "nereids_update_on_current_timestamp", "update_on_current_timestamp", "nereids_delete_mow_partial_update", "delete_mow_partial_update", "test_unique_table_auto_inc"
+                , "test_unique_table_auto_inc_partial_update_correct_insert", "partial_update_seq_col", "nereids_partial_update_native_insert_stmt_complex", "regression_test_variant_delete_and_update"
+                , "test_unique_table_auto_inc_partial_update_correct_stream_load", "test_update_mow", "test_new_update", "test_update_unique", "nereids_partial_update_native_insert_seq_col"
+                , "test_partial_update_rowset_not_found_fault_injection"]
             for (def excludeCase in excludeCases) {
                 excludeSuiteWildcard.add(excludeCase)
             }
diff --git a/regression-test/suites/compaction/test_full_compaction.groovy b/regression-test/suites/compaction/test_full_compaction.groovy
index 60f52f6f5a55a0..ed2d2ce5bc5e46 100644
--- a/regression-test/suites/compaction/test_full_compaction.groovy
+++ b/regression-test/suites/compaction/test_full_compaction.groovy
@@ -177,6 +177,11 @@ suite("test_full_compaction") {
         // make sure all hidden data has been deleted
         // (1,100)(2,200)
         qt_select_final """select * from ${tableName} order by user_id"""
+
+        sql "SET skip_delete_predicate = false"
+        sql "SET skip_delete_sign = false"
+        sql "SET skip_delete_bitmap = false"
+        qt_select_final2 """select * from ${tableName} order by user_id"""
     } finally {
         try_sql("DROP TABLE IF EXISTS ${tableName}")
     }
diff --git a/regression-test/suites/compaction/test_full_compaction_ck.groovy b/regression-test/suites/compaction/test_full_compaction_ck.groovy
new file mode 100644
index 00000000000000..8e2f86c596de65
--- /dev/null
+++ b/regression-test/suites/compaction/test_full_compaction_ck.groovy
@@ -0,0 +1,189 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_full_compaction_ck") {
+    def tableName = "test_full_compaction_ck"
+
+    try {
+        String backend_id;
+
+        def backendId_to_backendIP = [:]
+        def backendId_to_backendHttpPort = [:]
+        getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+        backend_id = backendId_to_backendIP.keySet()[0]
+        def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+        logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+        assertEquals(code, 0)
+        def configList = parseJson(out.trim())
+        assert configList instanceof List
+
+        boolean disableAutoCompaction = true
+        for (Object ele in (List) configList) {
+            assert ele instanceof List<String>
+            if (((List<String>) ele)[0] == "disable_auto_compaction") {
+                disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
+            }
+        }
+
+        sql """ DROP TABLE IF EXISTS ${tableName} """
+        sql """
+            CREATE TABLE ${tableName} (
+            `user_id` INT NOT NULL, `value` INT NOT NULL)
+            UNIQUE KEY(`user_id`) 
+            CLUSTER BY(`value`)
+            DISTRIBUTED BY HASH(`user_id`) 
+            BUCKETS 1 
+            PROPERTIES ("replication_allocation" = "tag.location.default: 1",
+            "disable_auto_compaction" = "true",
+            "enable_mow_light_delete" = "false",
+            "enable_unique_key_merge_on_write" = "true");"""
+
+        // version1 (1,1)(2,2)
+        sql """ INSERT INTO ${tableName} VALUES
+            (1,1),(2,2)
+            """
+        qt_1 """select * from ${tableName} order by user_id"""
+
+
+        // version2 (1,10)(2,20)
+        sql """ INSERT INTO ${tableName} VALUES
+            (1,10),(2,20)
+            """
+        qt_2 """select * from ${tableName} order by user_id"""
+
+
+        // version3 (1,100)(2,200)
+        sql """ INSERT INTO ${tableName} VALUES
+            (1,100),(2,200)
+            """
+        qt_3 """select * from ${tableName} order by user_id"""
+
+
+        // version4 (1,100)(2,200)(3,300)
+        sql """ INSERT INTO ${tableName} VALUES
+            (3,300)
+            """
+        qt_4 """select * from ${tableName} order by user_id"""
+
+
+        // version5 (1,100)(2,200)(3,100)
+        sql """update ${tableName} set value = 100 where user_id = 3"""
+        qt_5 """select * from ${tableName} order by user_id"""
+
+
+        // version6 (1,100)(2,200)
+        sql """delete from ${tableName} where user_id = 3"""
+        qt_6 """select * from ${tableName} order by user_id"""
+
+        sql "SET skip_delete_predicate = true"
+        sql "SET skip_delete_sign = true"
+        sql "SET skip_delete_bitmap = true"
+        // show all hidden data
+        // (1,10)(1,100)(2,2)(2,20)(2,200)(3,300)(3,100)
+        qt_skip_delete """select * from ${tableName} order by user_id, value"""
+
+        //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
+        def tablets = sql_return_maparray """ show tablets from ${tableName}; """
+
+        def replicaNum = get_table_replica_num(tableName)
+        logger.info("get table replica num: " + replicaNum)
+        // before full compaction, there are 7 rowsets.
+        int rowsetCount = 0
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            rowsetCount +=((List<String>) tabletJson.rowsets).size()
+        }
+        assert (rowsetCount == 7 * replicaNum)
+
+        // trigger full compactions for all tablets in ${tableName}
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            backend_id = tablet.BackendId
+            times = 1
+
+            do{
+                (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+                ++times
+                sleep(2000)
+            } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10)
+
+            def compactJson = parseJson(out.trim())
+            if (compactJson.status.toLowerCase() == "fail") {
+                assertEquals(disableAutoCompaction, false)
+                logger.info("Compaction was done automatically!")
+            }
+            if (disableAutoCompaction) {
+                assertEquals("success", compactJson.status.toLowerCase())
+            }
+        }
+
+        // wait for full compaction done
+        for (def tablet in tablets) {
+            boolean running = true
+            do {
+                Thread.sleep(1000)
+                String tablet_id = tablet.TabletId
+                backend_id = tablet.BackendId
+                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def compactionStatus = parseJson(out.trim())
+                assertEquals("success", compactionStatus.status.toLowerCase())
+                running = compactionStatus.run_status
+            } while (running)
+        }
+
+        // after full compaction, there is only 1 rowset.
+        
+        rowsetCount = 0
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            rowsetCount +=((List<String>) tabletJson.rowsets).size()
+        }
+        def cloudMode = isCloudMode()
+        if (cloudMode) {
+            assert (rowsetCount == 2)
+        } else {
+            assert (rowsetCount == 1 * replicaNum)
+        }
+
+        // make sure all hidden data has been deleted
+        // (1,100)(2,200)
+        qt_select_final """select * from ${tableName} order by user_id"""
+
+        sql "SET skip_delete_predicate = false"
+        sql "SET skip_delete_sign = false"
+        sql "SET skip_delete_bitmap = false"
+        qt_select_final2 """select * from ${tableName} order by user_id"""
+    } finally {
+        // try_sql("DROP TABLE IF EXISTS ${tableName}")
+    }
+}
diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy
index 36fbf9d80aa663..b22b7d82ae34c8 100644
--- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy
+++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy
@@ -134,15 +134,6 @@ suite("test_generated_column") {
 
     qt_describe "describe gencol_refer_gencol"
 
-    //test update
-    sql "drop table if exists test_gen_col_update"
-    sql """create table test_gen_col_update (a int, b int, c int as (a+b))
-    unique key(a)
-    distributed by hash(a) properties("replication_num"="1")"""
-    sql "insert into test_gen_col_update values(1,3,default)"
-    qt_test_update "update test_gen_col_update set b=20"
-    qt_test_update_generated_column "select * from test_gen_col_update"
-
     // test unique table, generated column is not key
     sql "drop table if exists test_gen_col_unique_key"
     qt_gen_col_unique_key """create table test_gen_col_unique_key(a int,b int,c int generated always as (abs(a+b)) not null)
@@ -233,4 +224,17 @@ suite("test_generated_column") {
         PROPERTIES("replication_num" = "1");"""
         exception "The generated columns can be key columns, or value columns of replace and replace_if_not_null aggregation type."
     }
+
+    //test update
+    sql "drop table if exists test_gen_col_update"
+    sql """create table test_gen_col_update (a int, b int, c int as (a+b))
+    unique key(a)
+    distributed by hash(a) properties("replication_num"="1")"""
+    sql "insert into test_gen_col_update values(1,3,default)"
+    if (!isClusterKeyEnabled()) {
+        qt_test_update "update test_gen_col_update set b=20"
+        qt_test_update_generated_column "select * from test_gen_col_update"
+    } else {
+        // errCode = 2, detailMessage = The value specified for generated column 'c' in table 'test_gen_col_update' is not allowed
+    }
 }
\ No newline at end of file

From 829b4b79d178ad878fbc20f4057b77583ef26af7 Mon Sep 17 00:00:00 2001
From: HappenLee <happenlee@selectdb.com>
Date: Tue, 3 Dec 2024 11:31:14 +0800
Subject: [PATCH 125/399] [Refactor](query) refactor lock in fragment mgr and
 change std::unorder_map to phmap (#44821)

---
 be/src/runtime/fragment_mgr.cpp  | 262 +++++++++++++++----------------
 be/src/runtime/fragment_mgr.h    |  29 ++--
 be/src/runtime/load_channel.cpp  |   3 +-
 be/src/runtime/load_stream.cpp   |   2 +-
 be/src/runtime/runtime_state.cpp |   2 +-
 5 files changed, 140 insertions(+), 158 deletions(-)

diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index 1e72fa756d3dd3..f96e4152500808 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -269,8 +269,11 @@ void FragmentMgr::stop() {
 
     // Only me can delete
     {
-        std::lock_guard<std::mutex> lock(_lock);
+        std::unique_lock lock(_query_ctx_map_mutex);
         _query_ctx_map.clear();
+    }
+    {
+        std::unique_lock lock(_pipeline_map_mutex);
         _pipeline_map.clear();
     }
     _thread_pool->shutdown();
@@ -583,11 +586,7 @@ Status FragmentMgr::start_query_execution(const PExecPlanFragmentStartRequest* r
     TUniqueId query_id;
     query_id.__set_hi(request->query_id().hi());
     query_id.__set_lo(request->query_id().lo());
-    std::shared_ptr<QueryContext> q_ctx = nullptr;
-    {
-        std::lock_guard<std::mutex> lock(_lock);
-        q_ctx = _get_or_erase_query_ctx(query_id);
-    }
+    auto q_ctx = get_query_ctx(query_id);
     if (q_ctx) {
         q_ctx->set_ready_to_execute(Status::OK());
         LOG_INFO("Query {} start execution", print_id(query_id));
@@ -602,114 +601,107 @@ Status FragmentMgr::start_query_execution(const PExecPlanFragmentStartRequest* r
 
 void FragmentMgr::remove_pipeline_context(
         std::shared_ptr<pipeline::PipelineFragmentContext> f_context) {
-    {
-        std::lock_guard<std::mutex> lock(_lock);
-        auto query_id = f_context->get_query_id();
-        int64 now = duration_cast<std::chrono::milliseconds>(
-                            std::chrono::system_clock::now().time_since_epoch())
-                            .count();
-        g_fragment_executing_count << -1;
-        g_fragment_last_active_time.set_value(now);
-        _pipeline_map.erase({query_id, f_context->get_fragment_id()});
-    }
+    auto query_id = f_context->get_query_id();
+    int64 now = duration_cast<std::chrono::milliseconds>(
+                        std::chrono::system_clock::now().time_since_epoch())
+                        .count();
+    g_fragment_executing_count << -1;
+    g_fragment_last_active_time.set_value(now);
+
+    std::unique_lock lock(_pipeline_map_mutex);
+    _pipeline_map.erase({query_id, f_context->get_fragment_id()});
 }
 
-std::shared_ptr<QueryContext> FragmentMgr::_get_or_erase_query_ctx(const TUniqueId& query_id) {
+std::shared_ptr<QueryContext> FragmentMgr::get_query_ctx(const TUniqueId& query_id) {
+    std::shared_lock lock(_query_ctx_map_mutex);
     auto search = _query_ctx_map.find(query_id);
     if (search != _query_ctx_map.end()) {
         if (auto q_ctx = search->second.lock()) {
             return q_ctx;
-        } else {
-            LOG(WARNING) << "Query context (query id = " << print_id(query_id)
-                         << ") has been released.";
-            _query_ctx_map.erase(search);
-            return nullptr;
         }
     }
     return nullptr;
 }
 
-std::shared_ptr<QueryContext> FragmentMgr::get_or_erase_query_ctx_with_lock(
-        const TUniqueId& query_id) {
-    std::unique_lock<std::mutex> lock(_lock);
-    return _get_or_erase_query_ctx(query_id);
-}
-
-template <typename Params>
-Status FragmentMgr::_get_query_ctx(const Params& params, TUniqueId query_id, bool pipeline,
-                                   QuerySource query_source,
-                                   std::shared_ptr<QueryContext>& query_ctx) {
+Status FragmentMgr::_get_or_create_query_ctx(const TPipelineFragmentParams& params,
+                                             TUniqueId query_id, bool pipeline,
+                                             QuerySource query_source,
+                                             std::shared_ptr<QueryContext>& query_ctx) {
     DBUG_EXECUTE_IF("FragmentMgr._get_query_ctx.failed", {
         return Status::InternalError("FragmentMgr._get_query_ctx.failed, query id {}",
                                      print_id(query_id));
     });
+
+    // Find _query_ctx_map, in case some other request has already
+    // create the query fragments context.
+    query_ctx = get_query_ctx(query_id);
     if (params.is_simplified_param) {
         // Get common components from _query_ctx_map
-        std::lock_guard<std::mutex> lock(_lock);
-        if (auto q_ctx = _get_or_erase_query_ctx(query_id)) {
-            query_ctx = q_ctx;
-        } else {
+        if (!query_ctx) {
             return Status::InternalError(
                     "Failed to get query fragments context. Query {} may be timeout or be "
                     "cancelled. host: {}",
                     print_id(query_id), BackendOptions::get_localhost());
         }
     } else {
-        // Find _query_ctx_map, in case some other request has already
-        // create the query fragments context.
-        std::lock_guard<std::mutex> lock(_lock);
-        if (auto q_ctx = _get_or_erase_query_ctx(query_id)) {
-            query_ctx = q_ctx;
-            return Status::OK();
-        }
+        if (!query_ctx) {
+            std::unique_lock lock(_query_ctx_map_mutex);
+            // Only one thread need create query ctx. other thread just get query_ctx in _query_ctx_map.
+            auto search = _query_ctx_map.find(query_id);
+            if (search != _query_ctx_map.end()) {
+                query_ctx = search->second.lock();
+            }
 
-        // First time a fragment of a query arrived. print logs.
-        LOG(INFO) << "query_id: " << print_id(query_id) << ", coord_addr: " << params.coord
-                  << ", total fragment num on current host: " << params.fragment_num_on_host
-                  << ", fe process uuid: " << params.query_options.fe_process_uuid
-                  << ", query type: " << params.query_options.query_type
-                  << ", report audit fe:" << params.current_connect_fe;
-
-        // This may be a first fragment request of the query.
-        // Create the query fragments context.
-        query_ctx = QueryContext::create_shared(query_id, _exec_env, params.query_options,
-                                                params.coord, params.is_nereids,
-                                                params.current_connect_fe, query_source);
-        SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(query_ctx->query_mem_tracker);
-        RETURN_IF_ERROR(DescriptorTbl::create(&(query_ctx->obj_pool), params.desc_tbl,
-                                              &(query_ctx->desc_tbl)));
-        // set file scan range params
-        if (params.__isset.file_scan_params) {
-            query_ctx->file_scan_range_params_map = params.file_scan_params;
-        }
+            if (!query_ctx) {
+                // First time a fragment of a query arrived. print logs.
+                LOG(INFO) << "query_id: " << print_id(query_id) << ", coord_addr: " << params.coord
+                          << ", total fragment num on current host: " << params.fragment_num_on_host
+                          << ", fe process uuid: " << params.query_options.fe_process_uuid
+                          << ", query type: " << params.query_options.query_type
+                          << ", report audit fe:" << params.current_connect_fe;
+
+                // This may be a first fragment request of the query.
+                // Create the query fragments context.
+                query_ctx = QueryContext::create_shared(query_id, _exec_env, params.query_options,
+                                                        params.coord, params.is_nereids,
+                                                        params.current_connect_fe, query_source);
+                SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(query_ctx->query_mem_tracker);
+                RETURN_IF_ERROR(DescriptorTbl::create(&(query_ctx->obj_pool), params.desc_tbl,
+                                                      &(query_ctx->desc_tbl)));
+                // set file scan range params
+                if (params.__isset.file_scan_params) {
+                    query_ctx->file_scan_range_params_map = params.file_scan_params;
+                }
 
-        query_ctx->query_globals = params.query_globals;
+                query_ctx->query_globals = params.query_globals;
 
-        if (params.__isset.resource_info) {
-            query_ctx->user = params.resource_info.user;
-            query_ctx->group = params.resource_info.group;
-            query_ctx->set_rsc_info = true;
-        }
+                if (params.__isset.resource_info) {
+                    query_ctx->user = params.resource_info.user;
+                    query_ctx->group = params.resource_info.group;
+                    query_ctx->set_rsc_info = true;
+                }
 
-        _set_scan_concurrency(params, query_ctx.get());
-
-        if (params.__isset.workload_groups && !params.workload_groups.empty()) {
-            uint64_t tg_id = params.workload_groups[0].id;
-            WorkloadGroupPtr workload_group_ptr =
-                    _exec_env->workload_group_mgr()->get_task_group_by_id(tg_id);
-            if (workload_group_ptr != nullptr) {
-                RETURN_IF_ERROR(workload_group_ptr->add_query(query_id, query_ctx));
-                RETURN_IF_ERROR(query_ctx->set_workload_group(workload_group_ptr));
-                _exec_env->runtime_query_statistics_mgr()->set_workload_group_id(print_id(query_id),
-                                                                                 tg_id);
-            } else {
-                LOG(WARNING) << "Query/load id: " << print_id(query_ctx->query_id())
-                             << "can't find its workload group " << tg_id;
+                _set_scan_concurrency(params, query_ctx.get());
+
+                if (params.__isset.workload_groups && !params.workload_groups.empty()) {
+                    uint64_t tg_id = params.workload_groups[0].id;
+                    WorkloadGroupPtr workload_group_ptr =
+                            _exec_env->workload_group_mgr()->get_task_group_by_id(tg_id);
+                    if (workload_group_ptr != nullptr) {
+                        RETURN_IF_ERROR(workload_group_ptr->add_query(query_id, query_ctx));
+                        RETURN_IF_ERROR(query_ctx->set_workload_group(workload_group_ptr));
+                        _exec_env->runtime_query_statistics_mgr()->set_workload_group_id(
+                                print_id(query_id), tg_id);
+                    } else {
+                        LOG(WARNING) << "Query/load id: " << print_id(query_ctx->query_id())
+                                     << "can't find its workload group " << tg_id;
+                    }
+                }
+                // There is some logic in query ctx's dctor, we could not check if exists and delete the
+                // temp query ctx now. For example, the query id maybe removed from workload group's queryset.
+                _query_ctx_map.insert({query_id, query_ctx});
             }
         }
-        // There is some logic in query ctx's dctor, we could not check if exists and delete the
-        // temp query ctx now. For example, the query id maybe removed from workload group's queryset.
-        _query_ctx_map.insert(std::make_pair(query_ctx->query_id(), query_ctx));
     }
     return Status::OK();
 }
@@ -723,13 +715,13 @@ std::string FragmentMgr::dump_pipeline_tasks(int64_t duration) {
     fmt::memory_buffer debug_string_buffer;
     size_t i = 0;
     {
-        std::lock_guard<std::mutex> lock(_lock);
         fmt::format_to(debug_string_buffer,
                        "{} pipeline fragment contexts are still running! duration_limit={}\n",
                        _pipeline_map.size(), duration);
-
         timespec now;
         clock_gettime(CLOCK_MONOTONIC, &now);
+
+        std::shared_lock lock(_pipeline_map_mutex);
         for (auto& it : _pipeline_map) {
             auto elapsed = it.second->elapsed_time() / 1000000000.0;
             if (elapsed < duration) {
@@ -748,7 +740,7 @@ std::string FragmentMgr::dump_pipeline_tasks(int64_t duration) {
 }
 
 std::string FragmentMgr::dump_pipeline_tasks(TUniqueId& query_id) {
-    if (auto q_ctx = _get_or_erase_query_ctx(query_id)) {
+    if (auto q_ctx = get_query_ctx(query_id)) {
         return q_ctx->print_all_pipeline_context();
     } else {
         return fmt::format(
@@ -767,7 +759,8 @@ Status FragmentMgr::exec_plan_fragment(const TPipelineFragmentParams& params,
              << apache::thrift::ThriftDebugString(params.query_options).c_str();
 
     std::shared_ptr<QueryContext> query_ctx;
-    RETURN_IF_ERROR(_get_query_ctx(params, params.query_id, true, query_source, query_ctx));
+    RETURN_IF_ERROR(
+            _get_or_create_query_ctx(params, params.query_id, true, query_source, query_ctx));
     SCOPED_ATTACH_TASK(query_ctx.get());
     int64_t duration_ns = 0;
     std::shared_ptr<pipeline::PipelineFragmentContext> context =
@@ -800,16 +793,8 @@ Status FragmentMgr::exec_plan_fragment(const TPipelineFragmentParams& params,
     }
 
     {
-        // (query_id, fragment_id) is executed only on one BE, locks _pipeline_map.
-        std::lock_guard<std::mutex> lock(_lock);
         for (const auto& local_param : params.local_params) {
             const TUniqueId& fragment_instance_id = local_param.fragment_instance_id;
-            auto iter = _pipeline_map.find({params.query_id, params.fragment_id});
-            if (iter != _pipeline_map.end()) {
-                return Status::InternalError(
-                        "exec_plan_fragment query_id({}) input duplicated fragment_id({})",
-                        print_id(params.query_id), params.fragment_id);
-            }
             query_ctx->fragment_instance_ids.push_back(fragment_instance_id);
         }
 
@@ -818,7 +803,15 @@ Status FragmentMgr::exec_plan_fragment(const TPipelineFragmentParams& params,
                             .count();
         g_fragment_executing_count << 1;
         g_fragment_last_active_time.set_value(now);
-        // TODO: simplify this mapping
+
+        // (query_id, fragment_id) is executed only on one BE, locks _pipeline_map.
+        std::unique_lock lock(_pipeline_map_mutex);
+        auto iter = _pipeline_map.find({params.query_id, params.fragment_id});
+        if (iter != _pipeline_map.end()) {
+            return Status::InternalError(
+                    "exec_plan_fragment query_id({}) input duplicated fragment_id({})",
+                    print_id(params.query_id), params.fragment_id);
+        }
         _pipeline_map.insert({{params.query_id, params.fragment_id}, context});
     }
 
@@ -848,8 +841,7 @@ void FragmentMgr::cancel_query(const TUniqueId query_id, const Status reason) {
     std::shared_ptr<QueryContext> query_ctx = nullptr;
     std::vector<TUniqueId> all_instance_ids;
     {
-        std::lock_guard<std::mutex> state_lock(_lock);
-        if (auto q_ctx = _get_or_erase_query_ctx(query_id)) {
+        if (auto q_ctx = get_query_ctx(query_id)) {
             query_ctx = q_ctx;
             // Copy instanceids to avoid concurrent modification.
             // And to reduce the scope of lock.
@@ -862,7 +854,7 @@ void FragmentMgr::cancel_query(const TUniqueId query_id, const Status reason) {
     }
     query_ctx->cancel(reason);
     {
-        std::lock_guard<std::mutex> state_lock(_lock);
+        std::unique_lock l(_query_ctx_map_mutex);
         _query_ctx_map.erase(query_id);
     }
     LOG(INFO) << "Query " << print_id(query_id)
@@ -898,7 +890,7 @@ void FragmentMgr::cancel_worker() {
 
         std::vector<std::shared_ptr<pipeline::PipelineFragmentContext>> ctx;
         {
-            std::lock_guard<std::mutex> lock(_lock);
+            std::shared_lock lock(_pipeline_map_mutex);
             ctx.reserve(_pipeline_map.size());
             for (auto& pipeline_itr : _pipeline_map) {
                 ctx.push_back(pipeline_itr.second);
@@ -910,29 +902,34 @@ void FragmentMgr::cancel_worker() {
 
         std::unordered_map<std::shared_ptr<PBackendService_Stub>, BrpcItem> brpc_stub_with_queries;
         {
-            std::lock_guard<std::mutex> lock(_lock);
-            for (auto it = _query_ctx_map.begin(); it != _query_ctx_map.end();) {
-                if (auto q_ctx = it->second.lock()) {
-                    if (q_ctx->is_timeout(now)) {
-                        LOG_WARNING("Query {} is timeout", print_id(it->first));
-                        queries_timeout.push_back(it->first);
-                    } else if (config::enable_brpc_connection_check) {
-                        auto brpc_stubs = q_ctx->get_using_brpc_stubs();
-                        for (auto& item : brpc_stubs) {
-                            if (!brpc_stub_with_queries.contains(item.second)) {
-                                brpc_stub_with_queries.emplace(item.second,
-                                                               BrpcItem {item.first, {q_ctx}});
-                            } else {
-                                brpc_stub_with_queries[item.second].queries.emplace_back(q_ctx);
+            {
+                // TODO: Now only the cancel worker do the GC the _query_ctx_map. each query must
+                // do erase the finish query unless in _query_ctx_map. Rethink the logic is ok
+                std::unique_lock lock(_query_ctx_map_mutex);
+                for (auto it = _query_ctx_map.begin(); it != _query_ctx_map.end();) {
+                    if (auto q_ctx = it->second.lock()) {
+                        if (q_ctx->is_timeout(now)) {
+                            LOG_WARNING("Query {} is timeout", print_id(it->first));
+                            queries_timeout.push_back(it->first);
+                        } else if (config::enable_brpc_connection_check) {
+                            auto brpc_stubs = q_ctx->get_using_brpc_stubs();
+                            for (auto& item : brpc_stubs) {
+                                if (!brpc_stub_with_queries.contains(item.second)) {
+                                    brpc_stub_with_queries.emplace(item.second,
+                                                                   BrpcItem {item.first, {q_ctx}});
+                                } else {
+                                    brpc_stub_with_queries[item.second].queries.emplace_back(q_ctx);
+                                }
                             }
                         }
+                        ++it;
+                    } else {
+                        it = _query_ctx_map.erase(it);
                     }
-                    ++it;
-                } else {
-                    it = _query_ctx_map.erase(it);
                 }
             }
 
+            std::shared_lock lock(_query_ctx_map_mutex);
             // We use a very conservative cancel strategy.
             // 0. If there are no running frontends, do not cancel any queries.
             // 1. If query's process uuid is zero, do not cancel
@@ -1215,7 +1212,7 @@ Status FragmentMgr::apply_filterv2(const PPublishFilterRequestV2* request,
 
     const auto& fragment_ids = request->fragment_ids();
     {
-        std::unique_lock<std::mutex> lock(_lock);
+        std::shared_lock lock(_pipeline_map_mutex);
         for (auto fragment_id : fragment_ids) {
             auto iter =
                     _pipeline_map.find({UniqueId(request->query_id()).to_thrift(), fragment_id});
@@ -1267,8 +1264,7 @@ Status FragmentMgr::send_filter_size(const PSendFilterSizeRequest* request) {
         TUniqueId query_id;
         query_id.__set_hi(queryid.hi);
         query_id.__set_lo(queryid.lo);
-        std::lock_guard<std::mutex> lock(_lock);
-        if (auto q_ctx = _get_or_erase_query_ctx(query_id)) {
+        if (auto q_ctx = get_query_ctx(query_id)) {
             query_ctx = q_ctx;
         } else {
             return Status::EndOfFile(
@@ -1291,8 +1287,7 @@ Status FragmentMgr::sync_filter_size(const PSyncFilterSizeRequest* request) {
         TUniqueId query_id;
         query_id.__set_hi(queryid.hi);
         query_id.__set_lo(queryid.lo);
-        std::lock_guard<std::mutex> lock(_lock);
-        if (auto q_ctx = _get_or_erase_query_ctx(query_id)) {
+        if (auto q_ctx = get_query_ctx(query_id)) {
             query_ctx = q_ctx;
         } else {
             return Status::EndOfFile(
@@ -1312,8 +1307,7 @@ Status FragmentMgr::merge_filter(const PMergeFilterRequest* request,
         TUniqueId query_id;
         query_id.__set_hi(queryid.hi);
         query_id.__set_lo(queryid.lo);
-        std::lock_guard<std::mutex> lock(_lock);
-        if (auto q_ctx = _get_or_erase_query_ctx(query_id)) {
+        if (auto q_ctx = get_query_ctx(query_id)) {
             query_ctx = q_ctx;
         } else {
             return Status::EndOfFile(
@@ -1330,7 +1324,7 @@ Status FragmentMgr::merge_filter(const PMergeFilterRequest* request,
 
 void FragmentMgr::get_runtime_query_info(std::vector<WorkloadQueryInfo>* query_info_list) {
     {
-        std::lock_guard<std::mutex> lock(_lock);
+        std::unique_lock lock(_query_ctx_map_mutex);
         for (auto iter = _query_ctx_map.begin(); iter != _query_ctx_map.end();) {
             if (auto q_ctx = iter->second.lock()) {
                 WorkloadQueryInfo workload_query_info;
@@ -1353,19 +1347,9 @@ Status FragmentMgr::get_realtime_exec_status(const TUniqueId& query_id,
         return Status::InvalidArgument("exes_status is nullptr");
     }
 
-    std::shared_ptr<QueryContext> query_context = nullptr;
-
-    {
-        std::lock_guard<std::mutex> lock(_lock);
-        if (auto q_ctx = _get_or_erase_query_ctx(query_id)) {
-            query_context = q_ctx;
-        } else {
-            return Status::NotFound("Query {} has been released", print_id(query_id));
-        }
-    }
-
+    std::shared_ptr<QueryContext> query_context = get_query_ctx(query_id);
     if (query_context == nullptr) {
-        return Status::NotFound("Query {} not found", print_id(query_id));
+        return Status::NotFound("Query {} not found or released", print_id(query_id));
     }
 
     *exec_status = query_context->get_realtime_exec_status();
diff --git a/be/src/runtime/fragment_mgr.h b/be/src/runtime/fragment_mgr.h
index 0eac0469683961..63d666788d0a5f 100644
--- a/be/src/runtime/fragment_mgr.h
+++ b/be/src/runtime/fragment_mgr.h
@@ -133,7 +133,7 @@ class FragmentMgr : public RestMonitorIface {
     ThreadPool* get_thread_pool() { return _thread_pool.get(); }
 
     int32_t running_query_num() {
-        std::unique_lock<std::mutex> ctx_lock(_lock);
+        std::shared_lock lock(_query_ctx_map_mutex);
         return _query_ctx_map.size();
     }
 
@@ -145,7 +145,7 @@ class FragmentMgr : public RestMonitorIface {
     Status get_realtime_exec_status(const TUniqueId& query_id,
                                     TReportExecStatusParams* exec_status);
 
-    std::shared_ptr<QueryContext> get_or_erase_query_ctx_with_lock(const TUniqueId& query_id);
+    std::shared_ptr<QueryContext> get_query_ctx(const TUniqueId& query_id);
 
 private:
     struct BrpcItem {
@@ -153,14 +153,12 @@ class FragmentMgr : public RestMonitorIface {
         std::vector<std::weak_ptr<QueryContext>> queries;
     };
 
-    std::shared_ptr<QueryContext> _get_or_erase_query_ctx(const TUniqueId& query_id);
-
     template <typename Param>
     void _set_scan_concurrency(const Param& params, QueryContext* query_ctx);
 
-    template <typename Params>
-    Status _get_query_ctx(const Params& params, TUniqueId query_id, bool pipeline,
-                          QuerySource query_type, std::shared_ptr<QueryContext>& query_ctx);
+    Status _get_or_create_query_ctx(const TPipelineFragmentParams& params, TUniqueId query_id,
+                                    bool pipeline, QuerySource query_type,
+                                    std::shared_ptr<QueryContext>& query_ctx);
 
     void _check_brpc_available(const std::shared_ptr<PBackendService_Stub>& brpc_stub,
                                const BrpcItem& brpc_item);
@@ -168,20 +166,21 @@ class FragmentMgr : public RestMonitorIface {
     // This is input params
     ExecEnv* _exec_env = nullptr;
 
+    // The lock protect the `_pipeline_map`
+    std::shared_mutex _pipeline_map_mutex;
+    // (QueryID, FragmentID) -> PipelineFragmentContext
+    phmap::flat_hash_map<std::pair<TUniqueId, int>,
+                         std::shared_ptr<pipeline::PipelineFragmentContext>>
+            _pipeline_map;
+
     // The lock should only be used to protect the structures in fragment manager. Has to be
     // used in a very small scope because it may dead lock. For example, if the _lock is used
     // in prepare stage, the call path is  prepare --> expr prepare --> may call allocator
     // when allocate failed, allocator may call query_is_cancelled, query is callced will also
     // call _lock, so that there is dead lock.
-    std::mutex _lock;
-
-    // (QueryID, FragmentID) -> PipelineFragmentContext
-    std::unordered_map<std::pair<TUniqueId, int>,
-                       std::shared_ptr<pipeline::PipelineFragmentContext>>
-            _pipeline_map;
-
+    std::shared_mutex _query_ctx_map_mutex;
     // query id -> QueryContext
-    std::unordered_map<TUniqueId, std::weak_ptr<QueryContext>> _query_ctx_map;
+    phmap::flat_hash_map<TUniqueId, std::weak_ptr<QueryContext>> _query_ctx_map;
     std::unordered_map<TUniqueId, std::unordered_map<int, int64_t>> _bf_size_map;
 
     CountDownLatch _stop_background_threads_latch;
diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp
index 0cb313747b0373..dd426f1ab81d3e 100644
--- a/be/src/runtime/load_channel.cpp
+++ b/be/src/runtime/load_channel.cpp
@@ -45,8 +45,7 @@ LoadChannel::LoadChannel(const UniqueId& load_id, int64_t timeout_s, bool is_hig
           _backend_id(backend_id),
           _enable_profile(enable_profile) {
     std::shared_ptr<QueryContext> query_context =
-            ExecEnv::GetInstance()->fragment_mgr()->get_or_erase_query_ctx_with_lock(
-                    _load_id.to_thrift());
+            ExecEnv::GetInstance()->fragment_mgr()->get_query_ctx(_load_id.to_thrift());
     std::shared_ptr<MemTrackerLimiter> mem_tracker = nullptr;
     WorkloadGroupPtr wg_ptr = nullptr;
 
diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp
index 752e2ff95b2917..60da45fa685fbf 100644
--- a/be/src/runtime/load_stream.cpp
+++ b/be/src/runtime/load_stream.cpp
@@ -428,7 +428,7 @@ LoadStream::LoadStream(PUniqueId load_id, LoadStreamMgr* load_stream_mgr, bool e
     TUniqueId load_tid = ((UniqueId)load_id).to_thrift();
 #ifndef BE_TEST
     std::shared_ptr<QueryContext> query_context =
-            ExecEnv::GetInstance()->fragment_mgr()->get_or_erase_query_ctx_with_lock(load_tid);
+            ExecEnv::GetInstance()->fragment_mgr()->get_query_ctx(load_tid);
     if (query_context != nullptr) {
         _query_thread_context = {load_tid, query_context->query_mem_tracker,
                                  query_context->workload_group()};
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 344180bad771ac..072c6e87f740a6 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -295,7 +295,7 @@ Status RuntimeState::init(const TUniqueId& fragment_instance_id, const TQueryOpt
 }
 
 std::weak_ptr<QueryContext> RuntimeState::get_query_ctx_weak() {
-    return _exec_env->fragment_mgr()->get_or_erase_query_ctx_with_lock(_query_ctx->query_id());
+    return _exec_env->fragment_mgr()->get_query_ctx(_query_ctx->query_id());
 }
 
 void RuntimeState::init_mem_trackers(const std::string& name, const TUniqueId& id) {

From 3e3948c87fb285a5bf36fdf5d70bdebaeffc33ae Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Tue, 3 Dec 2024 11:33:40 +0800
Subject: [PATCH 126/399] [env](compile)open compile check in  some function
 agg file (#44549)

### What problem does this PR solve?

This is part of the changes because modifying too much at once can
easily cause conflicts with other PRs.
---
 .../aggregate_functions/aggregate_function.h  |  3 +++
 ...gregate_function_approx_count_distinct.cpp |  1 +
 ...aggregate_function_approx_count_distinct.h |  6 ++++--
 .../aggregate_function_avg.cpp                |  1 +
 .../aggregate_function_avg.h                  | 16 +++++++++++----
 .../aggregate_function_avg_weighted.cpp       |  1 +
 .../aggregate_function_avg_weighted.h         |  5 ++++-
 .../aggregate_function_binary.h               |  3 +++
 .../aggregate_function_bit.cpp                |  1 +
 .../aggregate_function_bit.h                  |  4 +++-
 .../aggregate_function_bitmap.cpp             |  7 ++++++-
 .../aggregate_function_bitmap.h               |  4 +++-
 .../aggregate_function_bitmap_agg.cpp         |  7 ++++++-
 .../aggregate_function_bitmap_agg.h           |  4 +++-
 .../aggregate_function_collect.cpp            |  1 +
 .../aggregate_function_collect.h              |  3 +++
 .../aggregate_function_combinator.h           |  3 +++
 .../aggregate_function_corr.cpp               |  1 +
 .../aggregate_function_count.cpp              |  1 +
 .../aggregate_function_count.h                |  3 +++
 .../aggregate_function_count_by_enum.cpp      |  1 +
 .../aggregate_function_count_by_enum.h        |  7 ++++---
 .../aggregate_function_covar.cpp              |  1 +
 .../aggregate_function_covar.h                |  9 +++++----
 .../aggregate_function_distinct.cpp           |  1 +
 .../aggregate_function_distinct.h             |  3 +++
 .../aggregate_function_foreach.cpp            |  1 +
 .../aggregate_function_foreach.h              |  3 +++
 ...gregate_function_group_array_intersect.cpp |  1 +
 ...aggregate_function_group_array_intersect.h |  3 +++
 .../aggregate_function_group_concat.cpp       |  1 +
 .../aggregate_function_group_concat.h         |  3 +++
 .../aggregate_function_histogram.cpp          |  1 +
 .../aggregate_function_histogram.h            |  3 +++
 .../aggregate_function_hll_union_agg.cpp      |  1 +
 .../aggregate_function_hll_union_agg.h        |  6 ++++--
 .../aggregate_function_java_udaf.h            | 18 +++++++++++------
 .../aggregate_function_kurtosis.cpp           |  1 +
 .../aggregate_function_linear_histogram.cpp   |  1 +
 .../aggregate_function_linear_histogram.h     |  6 +++++-
 .../aggregate_function_map.cpp                |  1 +
 .../aggregate_function_map.h                  |  3 +++
 .../aggregate_function_max_by.cpp             |  1 +
 .../aggregate_function_min_by.cpp             |  1 +
 .../aggregate_function_min_max.cpp            |  1 +
 .../aggregate_function_min_max.h              | 14 ++++++++-----
 .../aggregate_function_min_max_by.h           |  3 +++
 .../aggregate_function_null.h                 |  3 +++
 .../aggregate_function_orthogonal_bitmap.cpp  |  1 +
 .../aggregate_function_orthogonal_bitmap.h    |  6 +++++-
 .../aggregate_function_product.h              |  3 +++
 .../aggregate_function_quantile_state.cpp     |  1 +
 .../aggregate_function_quantile_state.h       |  3 +++
 .../aggregate_function_reader.cpp             |  1 +
 .../aggregate_function_reader.h               |  3 +++
 .../aggregate_function_reader_first_last.h    |  4 +++-
 .../aggregate_function_regr_union.cpp         |  1 +
 .../aggregate_function_regr_union.h           | 17 +++++++++-------
 .../aggregate_function_retention.cpp          |  1 +
 .../aggregate_function_retention.h            |  4 +++-
 .../aggregate_function_sequence_match.cpp     |  1 +
 .../aggregate_function_sequence_match.h       |  6 +++++-
 .../aggregate_function_simple_factory.cpp     |  1 +
 .../aggregate_function_simple_factory.h       |  3 +++
 .../aggregate_function_skew.cpp               |  1 +
 .../aggregate_function_sort.cpp               |  1 +
 .../aggregate_function_sort.h                 |  3 +++
 .../aggregate_function_state_merge.h          |  3 +++
 .../aggregate_function_state_union.h          |  3 +++
 .../aggregate_function_statistic.h            | 16 ++++++++++-----
 .../aggregate_function_stddev.cpp             |  1 +
 .../aggregate_function_stddev.h               | 20 ++++++++++---------
 .../aggregate_function_sum.cpp                |  1 +
 .../aggregate_function_sum.h                  |  3 +++
 .../aggregate_function_topn.cpp               |  1 +
 .../aggregate_function_topn.h                 |  3 +++
 .../aggregate_function_uniq.cpp               |  1 +
 .../aggregate_function_uniq.h                 |  3 +++
 ...aggregate_function_uniq_distribute_key.cpp |  1 +
 .../aggregate_function_uniq_distribute_key.h  |  3 +++
 .../aggregate_function_window.cpp             |  1 +
 .../aggregate_function_window.h               |  7 +++++--
 .../aggregate_function_window_funnel.cpp      |  1 +
 .../aggregate_function_window_funnel.h        | 10 ++++++++--
 .../vec/aggregate_functions/factory_helpers.h |  3 +++
 be/src/vec/aggregate_functions/helpers.h      |  3 +++
 be/src/vec/aggregate_functions/moments.h      |  4 +++-
 be/src/vec/common/string_buffer.hpp           |  2 +-
 88 files changed, 260 insertions(+), 64 deletions(-)

diff --git a/be/src/vec/aggregate_functions/aggregate_function.h b/be/src/vec/aggregate_functions/aggregate_function.h
index 32fc9d5efce771..e0ec2bef62fc2a 100644
--- a/be/src/vec/aggregate_functions/aggregate_function.h
+++ b/be/src/vec/aggregate_functions/aggregate_function.h
@@ -36,6 +36,7 @@
 #include "vec/data_types/data_type_string.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 class Arena;
 class IColumn;
@@ -598,3 +599,5 @@ class AggregateFunctionGuard {
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.cpp b/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.cpp
index 18662bf66cf38c..8bf6c32c0872de 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.cpp
@@ -29,6 +29,7 @@
 #include "vec/functions/function.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 AggregateFunctionPtr create_aggregate_function_approx_count_distinct(
         const std::string& name, const DataTypes& argument_types, const bool result_is_nullable,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.h b/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.h
index d267499e059818..3ef22be9fca74c 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.h
@@ -38,6 +38,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -64,8 +65,7 @@ struct AggregateFunctionApproxCountDistinctData {
     void write(BufferWritable& buf) const {
         std::string result;
         result.resize(hll_data.max_serialized_size());
-        int size = hll_data.serialize((uint8_t*)result.data());
-        result.resize(size);
+        result.resize(hll_data.serialize((uint8_t*)result.data()));
         write_binary(result, buf);
     }
 
@@ -136,3 +136,5 @@ class AggregateFunctionApproxCountDistinct final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.cpp b/be/src/vec/aggregate_functions/aggregate_function_avg.cpp
index 6a6711f90f983e..6109f0b0c601cd 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_avg.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_avg.cpp
@@ -25,6 +25,7 @@
 #include "vec/core/field.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 struct Avg {
diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.h b/be/src/vec/aggregate_functions/aggregate_function_avg.h
index 62fbb8078ea949..8b24db692aef05 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_avg.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_avg.h
@@ -41,6 +41,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -72,7 +73,8 @@ struct AggregateFunctionAvgData {
     ResultT result() const {
         if constexpr (std::is_floating_point_v<ResultT>) {
             if constexpr (std::numeric_limits<ResultT>::is_iec559) {
-                return static_cast<ResultT>(sum) / count; /// allow division by zero
+                return static_cast<ResultT>(sum) /
+                       static_cast<ResultT>(count); /// allow division by zero
             }
         }
 
@@ -91,7 +93,7 @@ struct AggregateFunctionAvgData {
             if constexpr (IsDecimal256<T>) {
                 return static_cast<ResultT>(sum / T(count));
             } else {
-                return static_cast<ResultT>(sum) / count;
+                return static_cast<ResultT>(sum) / static_cast<ResultT>(count);
             }
         }
     }
@@ -124,7 +126,11 @@ class AggregateFunctionAvg final
             IsDecimalV2<T>, ColumnDecimal<Decimal128V2>,
             std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<typename Data::ResultType>,
                                ColumnFloat64>>;
+    // The result calculated by PercentileApprox is an approximate value,
+    // so the underlying storage uses float. The following calls will involve
+    // an implicit cast to float.
 
+    using DataType = typename Data::ResultType;
     /// ctor for native types
     AggregateFunctionAvg(const DataTypes& argument_types_)
             : IAggregateFunctionDataHelper<Data, AggregateFunctionAvg<T, Data>>(argument_types_),
@@ -148,9 +154,9 @@ class AggregateFunctionAvg final
         const auto& column =
                 assert_cast<const ColVecType&, TypeCheckOnRelease::DISABLE>(*columns[0]);
         if constexpr (IsDecimalNumber<T>) {
-            this->data(place).sum += column.get_data()[row_num].value;
+            this->data(place).sum += (DataType)column.get_data()[row_num].value;
         } else {
-            this->data(place).sum += column.get_data()[row_num];
+            this->data(place).sum += (DataType)column.get_data()[row_num];
         }
         ++this->data(place).count;
     }
@@ -282,3 +288,5 @@ class AggregateFunctionAvg final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.cpp b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.cpp
index fc5df5303fd15d..70a707b02e992b 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.cpp
@@ -21,6 +21,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 void register_aggregate_function_avg_weighted(AggregateFunctionSimpleFactory& factory) {
     factory.register_function_both("avg_weighted",
                                    creator_with_type::creator<AggregateFunctionAvgWeight>);
diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h
index b59a3dccf0cea8..d1a5921b45039f 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h
@@ -35,6 +35,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -57,7 +58,7 @@ struct AggregateFunctionAvgWeightedData {
             DecimalV2Value value = binary_cast<Int128, DecimalV2Value>(data_val);
             data_sum = data_sum + (double(value) * weight_val);
         } else {
-            data_sum = data_sum + (data_val * weight_val);
+            data_sum = data_sum + (double(data_val) * weight_val);
         }
         weight_sum = weight_sum + weight_val;
     }
@@ -138,3 +139,5 @@ class AggregateFunctionAvgWeight final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_binary.h b/be/src/vec/aggregate_functions/aggregate_function_binary.h
index 9fba9d11a1013a..fd5fc55d253661 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_binary.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_binary.h
@@ -36,6 +36,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T1, typename T2, template <typename> typename Moments>
 struct StatFunc {
@@ -127,3 +128,5 @@ AggregateFunctionPtr create_with_two_basic_numeric_types(const DataTypePtr& firs
 }
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_bit.cpp b/be/src/vec/aggregate_functions/aggregate_function_bit.cpp
index 97a6c0e92fa723..981ced1fbd5a46 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_bit.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_bit.cpp
@@ -24,6 +24,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 void register_aggregate_function_bit(AggregateFunctionSimpleFactory& factory) {
     factory.register_function_both(
diff --git a/be/src/vec/aggregate_functions/aggregate_function_bit.h b/be/src/vec/aggregate_functions/aggregate_function_bit.h
index 1ab01b03ceea38..d9760fdd30080b 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_bit.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_bit.h
@@ -30,6 +30,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -142,4 +143,5 @@ class AggregateFunctionBitwise final
     }
 };
 
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp b/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp
index e9c86d4b9556da..47ddf2d81b6a71 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp
@@ -23,6 +23,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <bool nullable, template <bool, typename> class AggregateFunctionTemplate>
 AggregateFunctionPtr create_with_int_data_type(const DataTypes& argument_type) {
@@ -33,7 +34,11 @@ AggregateFunctionPtr create_with_int_data_type(const DataTypes& argument_type) {
         return std::make_shared<AggregateFunctionTemplate<nullable, ColumnVector<TYPE>>>( \
                 argument_type);                                                           \
     }
-    FOR_INTEGER_TYPES(DISPATCH)
+    // Keep consistent with the FE definition; the function does not have an int128 type.
+    DISPATCH(Int8)
+    DISPATCH(Int16)
+    DISPATCH(Int32)
+    DISPATCH(Int64)
 #undef DISPATCH
     LOG(WARNING) << "with unknowed type, failed in create_with_int_data_type bitmap_union_int"
                  << " and type is: " << argument_type[0]->get_name();
diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h
index b0619a63e1ffe8..fb17b0a80be092 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_bitmap.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h
@@ -38,6 +38,7 @@
 #include "vec/data_types/data_type_number.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -432,4 +433,5 @@ AggregateFunctionPtr create_aggregate_function_bitmap_union(const std::string& n
                                                             const DataTypes& argument_types,
                                                             const bool result_is_nullable);
 
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap_agg.cpp b/be/src/vec/aggregate_functions/aggregate_function_bitmap_agg.cpp
index 0b95ddfd46f0d5..2a2c86303f3000 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_bitmap_agg.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap_agg.cpp
@@ -23,6 +23,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <bool nullable>
 AggregateFunctionPtr create_with_int_data_type(const DataTypes& argument_types) {
@@ -32,7 +33,11 @@ AggregateFunctionPtr create_with_int_data_type(const DataTypes& argument_types)
     if (which.idx == TypeIndex::TYPE) {                                                      \
         return std::make_shared<AggregateFunctionBitmapAgg<nullable, TYPE>>(argument_types); \
     }
-    FOR_INTEGER_TYPES(DISPATCH)
+    // Keep consistent with the FE definition; the function does not have an int128 type.
+    DISPATCH(Int8)
+    DISPATCH(Int16)
+    DISPATCH(Int32)
+    DISPATCH(Int64)
 #undef DISPATCH
     LOG(WARNING) << "with unknown type, failed in create_with_int_data_type bitmap_union_int"
                  << " and type is: " << argument_types[0]->get_name();
diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap_agg.h b/be/src/vec/aggregate_functions/aggregate_function_bitmap_agg.h
index 5747faf1b8e8c1..bff32aa606ccd2 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_bitmap_agg.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap_agg.h
@@ -31,6 +31,7 @@
 #include "vec/data_types/data_type_bitmap.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -226,4 +227,5 @@ class AggregateFunctionBitmapAgg final
     }
 };
 
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.cpp b/be/src/vec/aggregate_functions/aggregate_function_collect.cpp
index d726b7c6355318..a4853ff98659a0 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_collect.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_collect.cpp
@@ -26,6 +26,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T, typename HasLimit, typename ShowNull>
 AggregateFunctionPtr do_create_agg_function_collect(bool distinct, const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.h b/be/src/vec/aggregate_functions/aggregate_function_collect.h
index da310c6e0cc4c2..2d18a56313f3f9 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_collect.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_collect.h
@@ -46,6 +46,7 @@
 #include "vec/io/var_int.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 } // namespace vectorized
@@ -836,3 +837,5 @@ class AggregateFunctionCollect
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_combinator.h b/be/src/vec/aggregate_functions/aggregate_function_combinator.h
index 1593d74ed4e59d..0908ac8d0278f1 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_combinator.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_combinator.h
@@ -26,6 +26,7 @@
 #include "vec/data_types/data_type.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 /** Aggregate function combinator allows to take one aggregate function
   *  and transform it to another aggregate function.
@@ -69,3 +70,5 @@ class IAggregateFunctionCombinator {
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_corr.cpp b/be/src/vec/aggregate_functions/aggregate_function_corr.cpp
index cdaab6e086f4a5..e0a51ca6629a06 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_corr.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_corr.cpp
@@ -21,6 +21,7 @@
 #include "vec/core/types.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 struct CorrMoment {
diff --git a/be/src/vec/aggregate_functions/aggregate_function_count.cpp b/be/src/vec/aggregate_functions/aggregate_function_count.cpp
index 5cfe5af41982f6..72d12cf65fe9d0 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_count.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_count.cpp
@@ -26,6 +26,7 @@
 #include "vec/aggregate_functions/factory_helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 AggregateFunctionPtr create_aggregate_function_count(const std::string& name,
                                                      const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_count.h b/be/src/vec/aggregate_functions/aggregate_function_count.h
index 7b54d074683b04..630994a7967957 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_count.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_count.h
@@ -41,6 +41,7 @@
 #include "vec/io/var_int.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -321,3 +322,5 @@ class AggregateFunctionCountNotNullUnary final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_count_by_enum.cpp b/be/src/vec/aggregate_functions/aggregate_function_count_by_enum.cpp
index 093b31d57db554..20235d9e2ef2e9 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_count_by_enum.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_count_by_enum.cpp
@@ -26,6 +26,7 @@
 #include "vec/core/types.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 AggregateFunctionPtr create_aggregate_function_count_by_enum(const std::string& name,
                                                              const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_count_by_enum.h b/be/src/vec/aggregate_functions/aggregate_function_count_by_enum.h
index 1f5093de68263e..543ae55f872da6 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_count_by_enum.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_count_by_enum.h
@@ -32,6 +32,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 struct CountByEnumData {
     std::unordered_map<std::string, uint64_t> cbe;
@@ -46,8 +47,7 @@ void build_json_from_vec(rapidjson::StringBuffer& buffer,
     doc.SetArray();
     rapidjson::Document::AllocatorType& allocator = doc.GetAllocator();
 
-    int vec_size_number = data_vec.size();
-    for (int idx = 0; idx < vec_size_number; ++idx) {
+    for (size_t idx = 0; idx < data_vec.size(); ++idx) {
         rapidjson::Value obj(rapidjson::kObjectType);
 
         rapidjson::Value obj_cbe(rapidjson::kObjectType);
@@ -239,4 +239,5 @@ class AggregateFunctionCountByEnum final
     size_t arg_count;
 };
 
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_covar.cpp b/be/src/vec/aggregate_functions/aggregate_function_covar.cpp
index 4c5fe1321952d6..d9c091fb601868 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_covar.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_covar.cpp
@@ -28,6 +28,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <template <typename> class Function, template <typename> class Data>
 AggregateFunctionPtr create_function_single_value(const String& name,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_covar.h b/be/src/vec/aggregate_functions/aggregate_function_covar.h
index b01d1ba11c80df..56dd443f050e6e 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_covar.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_covar.h
@@ -75,11 +75,12 @@ struct BaseData {
         if (count == 1) {
             return 0.0;
         }
-        return sum_xy / count - sum_x * sum_y / (count * count);
+        return sum_xy / (double)count - sum_x * sum_y / ((double)count * (double)count);
     }
 
     double get_samp_result() const {
-        return sum_xy / (count - 1) - sum_x * sum_y / (count * (count - 1));
+        return sum_xy / double(count - 1) -
+               sum_x * sum_y / ((double)(count) * ((double)(count - 1)));
     }
 
     void merge(const BaseData& rhs) {
@@ -95,10 +96,10 @@ struct BaseData {
     void add(const IColumn* column_x, const IColumn* column_y, size_t row_num) {
         const auto& sources_x =
                 assert_cast<const ColumnVector<T>&, TypeCheckOnRelease::DISABLE>(*column_x);
-        double source_data_x = sources_x.get_data()[row_num];
+        double source_data_x = double(sources_x.get_data()[row_num]);
         const auto& sources_y =
                 assert_cast<const ColumnVector<T>&, TypeCheckOnRelease::DISABLE>(*column_y);
-        double source_data_y = sources_y.get_data()[row_num];
+        double source_data_y = double(sources_y.get_data()[row_num]);
 
         sum_x += source_data_x;
         sum_y += source_data_y;
diff --git a/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp b/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp
index fce58b38688b28..566d74dc4d537e 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp
@@ -28,6 +28,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 struct Reducer {
diff --git a/be/src/vec/aggregate_functions/aggregate_function_distinct.h b/be/src/vec/aggregate_functions/aggregate_function_distinct.h
index 64f60319aac545..46450394627474 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_distinct.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_distinct.h
@@ -42,6 +42,7 @@
 #include "vec/io/var_int.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -347,3 +348,5 @@ class AggregateFunctionDistinct
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_foreach.cpp b/be/src/vec/aggregate_functions/aggregate_function_foreach.cpp
index c1cbcc89996caf..4dba4dd52ebdf3 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_foreach.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_foreach.cpp
@@ -32,6 +32,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 void register_aggregate_function_combinator_foreach(AggregateFunctionSimpleFactory& factory) {
     AggregateFunctionCreator creator =
diff --git a/be/src/vec/aggregate_functions/aggregate_function_foreach.h b/be/src/vec/aggregate_functions/aggregate_function_foreach.h
index 7f746e53daac70..9a4801c43bdf2b 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_foreach.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_foreach.h
@@ -31,6 +31,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 struct AggregateFunctionForEachData {
     size_t dynamic_array_size = 0;
@@ -259,3 +260,5 @@ class AggregateFunctionForEach : public IAggregateFunctionDataHelper<AggregateFu
     }
 };
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.cpp b/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.cpp
index 24faf58b2e1ff9..395f1ea1e4ceaf 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.cpp
@@ -21,6 +21,7 @@
 #include "vec/aggregate_functions/aggregate_function_group_array_intersect.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 IAggregateFunction* create_with_extra_types(const DataTypePtr& nested_type,
                                             const DataTypes& argument_types) {
diff --git a/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h b/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
index fd6076686acb65..dfa9e7e37e5397 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
@@ -37,6 +37,7 @@
 #include "vec/io/var_int.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 class Arena;
 class BufferReadable;
 class BufferWritable;
@@ -541,3 +542,5 @@ class AggregateFunctionGroupArrayIntersectGeneric
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_group_concat.cpp b/be/src/vec/aggregate_functions/aggregate_function_group_concat.cpp
index 286795ea2ba70c..b5ee82875f3f80 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_group_concat.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_group_concat.cpp
@@ -23,6 +23,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 const std::string AggregateFunctionGroupConcatImplStr::separator = ",";
 
diff --git a/be/src/vec/aggregate_functions/aggregate_function_group_concat.h b/be/src/vec/aggregate_functions/aggregate_function_group_concat.h
index a0cac9ab78016d..5f598e411b28ab 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_group_concat.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_group_concat.h
@@ -32,6 +32,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -169,3 +170,5 @@ class AggregateFunctionGroupConcat final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_histogram.cpp b/be/src/vec/aggregate_functions/aggregate_function_histogram.cpp
index fb2fa9c2513ec0..9df12c4c55142d 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_histogram.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_histogram.cpp
@@ -28,6 +28,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 AggregateFunctionPtr create_agg_function_histogram(const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_histogram.h b/be/src/vec/aggregate_functions/aggregate_function_histogram.h
index 1d2c5725ed370f..aae5ab00ce34f0 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_histogram.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_histogram.h
@@ -44,6 +44,7 @@
 #include "vec/utils/histogram_helpers.hpp"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -243,3 +244,5 @@ class AggregateFunctionHistogram final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp
index 4f2232d00d5f1e..42399a77f9b100 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp
@@ -23,6 +23,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 void register_aggregate_function_HLL_union_agg(AggregateFunctionSimpleFactory& factory) {
     factory.register_function_both(
diff --git a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h
index 44835194eb4b88..40e964eba782d1 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h
@@ -38,6 +38,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -58,8 +59,7 @@ struct AggregateFunctionHLLData {
 
     void write(BufferWritable& buf) const {
         std::string result(dst_hll.max_serialized_size(), '0');
-        int size = dst_hll.serialize((uint8_t*)result.c_str());
-        result.resize(size);
+        result.resize(dst_hll.serialize((uint8_t*)result.c_str()));
         write_binary(result, buf);
     }
 
@@ -149,3 +149,5 @@ AggregateFunctionPtr create_aggregate_function_HLL(const std::string& name,
                                                    const bool result_is_nullable);
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
index d16da1a34e66e3..c4c050f807954a 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
@@ -23,6 +23,7 @@
 #include <cstdint>
 #include <memory>
 
+#include "common/cast_set.h"
 #include "common/compiler_util.h"
 #include "common/exception.h"
 #include "common/logging.h"
@@ -41,6 +42,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 const char* UDAF_EXECUTOR_CLASS = "org/apache/doris/udf/UdafExecutor";
 const char* UDAF_EXECUTOR_CTOR_SIGNATURE = "([B)V";
@@ -57,7 +59,7 @@ const char* UDAF_EXECUTOR_RESET_SIGNATURE = "(J)V";
 struct AggregateJavaUdafData {
 public:
     AggregateJavaUdafData() = default;
-    AggregateJavaUdafData(int64_t num_args) { argument_size = num_args; }
+    AggregateJavaUdafData(int64_t num_args) { cast_set(argument_size, num_args); }
 
     ~AggregateJavaUdafData() = default;
 
@@ -115,8 +117,8 @@ struct AggregateJavaUdafData {
     }
 
     Status add(int64_t places_address, bool is_single_place, const IColumn** columns,
-               int row_num_start, int row_num_end, const DataTypes& argument_types,
-               int place_offset) {
+               int64_t row_num_start, int64_t row_num_end, const DataTypes& argument_types,
+               int64_t place_offset) {
         JNIEnv* env = nullptr;
         RETURN_NOT_OK_STATUS_WITH_WARN(JniUtil::GetJNIEnv(&env), "Java-Udaf add function");
 
@@ -134,8 +136,10 @@ struct AggregateJavaUdafData {
                 {"columns_types", input_table_schema.second}};
         jobject input_map = JniUtil::convert_to_java_map(env, input_params);
         // invoke add batch
-        env->CallObjectMethod(executor_obj, executor_add_batch_id, is_single_place, row_num_start,
-                              row_num_end, places_address, place_offset, input_map);
+        // Keep consistent with the function signature of executor_add_batch_id.
+        env->CallObjectMethod(executor_obj, executor_add_batch_id, is_single_place,
+                              cast_set<int>(row_num_start), cast_set<int>(row_num_end),
+                              places_address, cast_set<int>(place_offset), input_map);
         env->DeleteLocalRef(input_map);
         return JniUtil::GetJniExceptionMsg(env);
     }
@@ -144,7 +148,7 @@ struct AggregateJavaUdafData {
         JNIEnv* env = nullptr;
         RETURN_NOT_OK_STATUS_WITH_WARN(JniUtil::GetJNIEnv(&env), "Java-Udaf merge function");
         serialize_data = rhs.serialize_data;
-        long len = serialize_data.length();
+        jsize len = cast_set<jsize>(serialize_data.length()); // jsize needs to be used.
         jbyteArray arr = env->NewByteArray(len);
         env->SetByteArrayRegion(arr, 0, len, reinterpret_cast<jbyte*>(serialize_data.data()));
         env->CallNonvirtualVoidMethod(executor_obj, executor_cl, executor_merge_id, place, arr);
@@ -417,3 +421,5 @@ class AggregateJavaUdaf final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_kurtosis.cpp b/be/src/vec/aggregate_functions/aggregate_function_kurtosis.cpp
index a763721f3f4061..777bb4393cc770 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_kurtosis.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_kurtosis.cpp
@@ -25,6 +25,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 AggregateFunctionPtr type_dispatch_for_aggregate_function_kurt(const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_linear_histogram.cpp b/be/src/vec/aggregate_functions/aggregate_function_linear_histogram.cpp
index 683cf1a18f78ba..e61a97aca9f6c0 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_linear_histogram.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_linear_histogram.cpp
@@ -20,6 +20,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 const std::string AggregateFunctionLinearHistogramConsts::NAME = "linear_histogram";
 
diff --git a/be/src/vec/aggregate_functions/aggregate_function_linear_histogram.h b/be/src/vec/aggregate_functions/aggregate_function_linear_histogram.h
index 173324b9463750..f4d4b5455f1f83 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_linear_histogram.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_linear_histogram.h
@@ -35,6 +35,7 @@
 // TODO: support foreach
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 struct AggregateFunctionLinearHistogramData {
@@ -70,7 +71,8 @@ struct AggregateFunctionLinearHistogramData {
         double val = 0;
         if constexpr (IsDecimalNumber<T>) {
             using NativeType = typename T::NativeType;
-            val = static_cast<double>(value.value) / decimal_scale_multiplier<NativeType>(scale);
+            val = static_cast<double>(value.value) /
+                  static_cast<double>(decimal_scale_multiplier<NativeType>(scale));
         } else {
             val = static_cast<double>(value);
         }
@@ -255,3 +257,5 @@ class AggregateFunctionLinearHistogram final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_map.cpp b/be/src/vec/aggregate_functions/aggregate_function_map.cpp
index f289d885f48f52..627a6019080a03 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_map.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_map.cpp
@@ -21,6 +21,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename K>
 AggregateFunctionPtr create_agg_function_map_agg(const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_map.h b/be/src/vec/aggregate_functions/aggregate_function_map.h
index 3ec25cdc706152..17bc54f7499adb 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_map.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_map.h
@@ -33,6 +33,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename K>
 struct AggregateFunctionMapAggData {
@@ -345,3 +346,5 @@ class AggregateFunctionMapAgg final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_max_by.cpp b/be/src/vec/aggregate_functions/aggregate_function_max_by.cpp
index 35b4a4400da0d8..50e611b11d8b3e 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_max_by.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_max_by.cpp
@@ -21,6 +21,7 @@
 #include "vec/aggregate_functions/aggregate_function_simple_factory.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 void register_aggregate_function_max_by(AggregateFunctionSimpleFactory& factory) {
     factory.register_function_both(
diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_by.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_by.cpp
index 6e1d96ddf5d9c5..0af34292bbb2eb 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_min_by.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_min_by.cpp
@@ -21,6 +21,7 @@
 #include "vec/aggregate_functions/aggregate_function_simple_factory.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 void register_aggregate_function_min_by(AggregateFunctionSimpleFactory& factory) {
     factory.register_function_both(
diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
index c1a72fd52bdd76..e6e558b7fdc2db 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
@@ -26,6 +26,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 /// min, max, any
 template <template <typename> class Data>
 AggregateFunctionPtr create_aggregate_function_single_value(const String& name,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.h b/be/src/vec/aggregate_functions/aggregate_function_min_max.h
index efc2854ff149c8..4ced874e36d488 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_min_max.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.h
@@ -26,6 +26,7 @@
 #include <memory>
 #include <vector>
 
+#include "common/cast_set.h"
 #include "common/logging.h"
 #include "vec/aggregate_functions/aggregate_function.h"
 #include "vec/columns/column.h"
@@ -41,6 +42,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 template <typename T>
@@ -302,7 +304,8 @@ struct SingleValueDataDecimal {
 struct SingleValueDataString {
 private:
     using Self = SingleValueDataString;
-
+    // This function uses int32 for storage, which triggers a 64-bit to 32-bit conversion warning.
+    // However, considering compatibility with future upgrades, no changes will be made here.
     Int32 size = -1;    /// -1 indicates that there is no value.
     Int32 capacity = 0; /// power of two or zero
     std::unique_ptr<char[]> large_data;
@@ -364,7 +367,7 @@ struct SingleValueDataString {
                 }
             } else {
                 if (capacity < rhs_size) {
-                    capacity = round_up_to_power_of_two_or_zero(rhs_size);
+                    capacity = (Int32)round_up_to_power_of_two_or_zero(rhs_size);
                     large_data.reset(new char[capacity]);
                 }
 
@@ -381,8 +384,7 @@ struct SingleValueDataString {
 
     /// Assuming to.has()
     void change_impl(StringRef value, Arena*) {
-        Int32 value_size = value.size;
-
+        Int32 value_size = cast_set<Int32>(value.size);
         if (value_size <= MAX_SMALL_STRING_SIZE) {
             /// Don't free large_data here.
             size = value_size;
@@ -393,7 +395,7 @@ struct SingleValueDataString {
         } else {
             if (capacity < value_size) {
                 /// Don't free large_data here.
-                capacity = round_up_to_power_of_two_or_zero(value_size);
+                capacity = (Int32)round_up_to_power_of_two_or_zero(value_size);
                 large_data.reset(new char[capacity]);
             }
 
@@ -717,3 +719,5 @@ AggregateFunctionPtr create_aggregate_function_single_value(const String& name,
                                                             const bool result_is_nullable,
                                                             const AggregateFunctionAttr& attr = {});
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max_by.h b/be/src/vec/aggregate_functions/aggregate_function_min_max_by.h
index 5c73ac9aa67cbe..3b8d182ff46862 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_min_max_by.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_min_max_by.h
@@ -30,6 +30,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 /// For bitmap value
 struct BitmapValueData {
@@ -295,3 +296,5 @@ AggregateFunctionPtr create_aggregate_function_min_max_by(const String& name,
 }
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_null.h b/be/src/vec/aggregate_functions/aggregate_function_null.h
index 014a3e9c603b88..b3fa3b8230dc52 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_null.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_null.h
@@ -31,6 +31,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename NestFunction, bool result_is_nullable, typename Derived>
 class AggregateFunctionNullBaseInline : public IAggregateFunctionHelper<Derived> {
@@ -334,3 +335,5 @@ class AggregateFunctionNullVariadicInline final
             is_nullable; /// Plain array is better than std::vector due to one indirection less.
 };
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp
index fe41aba2f0bf8d..94f942ec6443ac 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp
@@ -26,6 +26,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 struct StringRef;
 } // namespace doris
 
diff --git a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h
index a81bdcddaa3310..ed4b9675a76e6b 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h
@@ -39,6 +39,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 class Arena;
 class BufferReadable;
 class BufferWritable;
@@ -379,7 +380,8 @@ class AggFunctionOrthBitmapFunc final
 
     AggFunctionOrthBitmapFunc(const DataTypes& argument_types_)
             : IAggregateFunctionDataHelper<Impl, AggFunctionOrthBitmapFunc<Impl>>(argument_types_),
-              _argument_size(argument_types_.size()) {}
+              // The number of arguments will not exceed the size of an int
+              _argument_size(int(argument_types_.size())) {}
 
     DataTypePtr get_return_type() const override { return Impl::get_return_type(); }
 
@@ -413,3 +415,5 @@ class AggFunctionOrthBitmapFunc final
     int _argument_size;
 };
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_product.h b/be/src/vec/aggregate_functions/aggregate_function_product.h
index 82f765a909d42a..bbec428b25acef 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_product.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_product.h
@@ -31,6 +31,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 
 template <typename T>
@@ -174,3 +175,5 @@ class AggregateFunctionProduct final
 
 } // namespace vectorized
 } // namespace doris
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_quantile_state.cpp b/be/src/vec/aggregate_functions/aggregate_function_quantile_state.cpp
index 128edc59915525..1af3bfb3529f13 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_quantile_state.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_quantile_state.cpp
@@ -23,6 +23,7 @@
 #include "vec/data_types/data_type.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 AggregateFunctionPtr create_aggregate_function_quantile_state_union(
         const std::string& name, const DataTypes& argument_types, const bool result_is_nullable,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_quantile_state.h b/be/src/vec/aggregate_functions/aggregate_function_quantile_state.h
index c48ac920919cf6..7da2118b5b0d0d 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_quantile_state.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_quantile_state.h
@@ -32,6 +32,7 @@
 #include "vec/data_types/data_type_quantilestate.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -157,3 +158,5 @@ AggregateFunctionPtr create_aggregate_function_quantile_state_union(
         const AggregateFunctionAttr& attr);
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_reader.cpp b/be/src/vec/aggregate_functions/aggregate_function_reader.cpp
index 6f27a5dd128241..737f66832dab61 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_reader.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_reader.cpp
@@ -30,6 +30,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 // auto spread at nullable condition, null value do not participate aggregate
 void register_aggregate_function_reader_load(AggregateFunctionSimpleFactory& factory) {
diff --git a/be/src/vec/aggregate_functions/aggregate_function_reader.h b/be/src/vec/aggregate_functions/aggregate_function_reader.h
index ef35ea16fd41a3..7aa6f00ee81a65 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_reader.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_reader.h
@@ -18,6 +18,7 @@
 #pragma once
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class AggregateFunctionSimpleFactory;
 } // namespace vectorized
@@ -33,3 +34,5 @@ void register_aggregate_function_reader_load(AggregateFunctionSimpleFactory& fac
 void register_aggregate_function_replace_reader_load(AggregateFunctionSimpleFactory& factory);
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h b/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
index 066ef2a257914c..2657feb9380acd 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
@@ -30,6 +30,7 @@
 #include "vec/functions/function.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename ColVecType, bool arg_is_nullable>
 struct Value {
@@ -309,4 +310,5 @@ CREATE_READER_FUNCTION_WITH_NAME_AND_DATA(create_aggregate_function_last_non_nul
                                           ReaderFunctionLastNonNullData);
 #undef CREATE_READER_FUNCTION_WITH_NAME_AND_DATA
 
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git a/be/src/vec/aggregate_functions/aggregate_function_regr_union.cpp b/be/src/vec/aggregate_functions/aggregate_function_regr_union.cpp
index c20b5977f2198b..8f17456ad2e5c3 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_regr_union.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_regr_union.cpp
@@ -26,6 +26,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T, template <typename> class StatFunctionTemplate>
 AggregateFunctionPtr type_dispatch_for_aggregate_function_regr(const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_regr_union.h b/be/src/vec/aggregate_functions/aggregate_function_regr_union.h
index a95daaf0d840cd..b16763745a48f1 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_regr_union.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_regr_union.h
@@ -36,6 +36,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 struct AggregateFunctionRegrData {
@@ -82,19 +83,19 @@ struct AggregateFunctionRegrData {
     }
 
     void add(T value_y, T value_x) {
-        sum_x += value_x;
-        sum_y += value_y;
-        sum_of_x_mul_y += value_x * value_y;
-        sum_of_x_squared += value_x * value_x;
+        sum_x += (double)value_x;
+        sum_y += (double)value_y;
+        sum_of_x_mul_y += (double)value_x * (double)value_y;
+        sum_of_x_squared += (double)value_x * (double)value_x;
         count += 1;
     }
 
     Float64 get_slope() const {
-        Float64 denominator = count * sum_of_x_squared - sum_x * sum_x;
+        Float64 denominator = (double)count * sum_of_x_squared - sum_x * sum_x;
         if (count < 2 || denominator == 0.0) {
             return std::numeric_limits<Float64>::quiet_NaN();
         }
-        Float64 slope = (count * sum_of_x_mul_y - sum_x * sum_y) / denominator;
+        Float64 slope = ((double)count * sum_of_x_mul_y - sum_x * sum_y) / denominator;
         return slope;
     }
 };
@@ -115,7 +116,7 @@ struct RegrInterceptFunc : AggregateFunctionRegrData<T> {
         if (std::isnan(slope)) {
             return slope;
         } else {
-            Float64 intercept = (this->sum_y - slope * this->sum_x) / this->count;
+            Float64 intercept = (this->sum_y - slope * this->sum_x) / (double)this->count;
             return intercept;
         }
     }
@@ -214,3 +215,5 @@ class AggregateFunctionRegrSimple
     }
 };
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_retention.cpp b/be/src/vec/aggregate_functions/aggregate_function_retention.cpp
index 38dd8f9de6667d..ba45950f55510e 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_retention.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_retention.cpp
@@ -21,6 +21,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 void register_aggregate_function_retention(AggregateFunctionSimpleFactory& factory) {
     factory.register_function_both("retention",
                                    creator_without_type::creator<AggregateFunctionRetention>);
diff --git a/be/src/vec/aggregate_functions/aggregate_function_retention.h b/be/src/vec/aggregate_functions/aggregate_function_retention.h
index f6bf03282e5711..3318ac96ef36cf 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_retention.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_retention.h
@@ -43,6 +43,7 @@
 #include "vec/io/var_int.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -166,4 +167,5 @@ class AggregateFunctionRetention
         to_arr.get_offsets().push_back(to_nested_col.size());
     }
 };
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.cpp b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.cpp
index c49ee021dbfae7..f9e901b34daf63 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.cpp
@@ -26,6 +26,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <template <typename, typename> typename AggregateFunction>
 AggregateFunctionPtr create_aggregate_function_sequence_base(const std::string& name,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h
index 586003043311c4..827e506a3b0ed9 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h
@@ -51,6 +51,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -144,7 +145,8 @@ struct AggregateFunctionSequenceMatchData final {
             write_binary(events.second.to_ulong(), buf);
         }
 
-        UInt32 conditions_met_value = conditions_met.to_ulong();
+        // This is std::bitset<32>, which will not exceed 32 bits.
+        UInt32 conditions_met_value = (UInt32)conditions_met.to_ulong();
         write_binary(conditions_met_value, buf);
 
         write_binary(pattern, buf);
@@ -754,3 +756,5 @@ class AggregateFunctionSequenceCount final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
index 90bdab70522cf1..6869263882344e 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
@@ -25,6 +25,7 @@
 #include "vec/aggregate_functions/aggregate_function_reader.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 void register_aggregate_function_combinator_distinct(AggregateFunctionSimpleFactory& factory);
 void register_aggregate_function_combinator_foreach(AggregateFunctionSimpleFactory& factory);
diff --git a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h
index aa33e7289df01e..842170c18eba45 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h
@@ -36,6 +36,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 using DataTypePtr = std::shared_ptr<const IDataType>;
 using DataTypes = std::vector<DataTypePtr>;
 using AggregateFunctionCreator = std::function<AggregateFunctionPtr(
@@ -184,3 +185,5 @@ class AggregateFunctionSimpleFactory {
     static AggregateFunctionSimpleFactory& instance();
 };
 }; // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_skew.cpp b/be/src/vec/aggregate_functions/aggregate_function_skew.cpp
index af2eb443eb0603..993a558d290569 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_skew.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_skew.cpp
@@ -25,6 +25,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <typename T>
 AggregateFunctionPtr type_dispatch_for_aggregate_function_skew(const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sort.cpp b/be/src/vec/aggregate_functions/aggregate_function_sort.cpp
index c80d36aec58f33..99d93127f0c29f 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sort.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_sort.cpp
@@ -22,6 +22,7 @@
 #include "vec/aggregate_functions/aggregate_function_simple_factory.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 AggregateFunctionPtr transform_to_sort_agg_function(const AggregateFunctionPtr& nested_function,
                                                     const DataTypes& arguments,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sort.h b/be/src/vec/aggregate_functions/aggregate_function_sort.h
index 981580f8e6ac38..a0a05cc7ddd397 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sort.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_sort.h
@@ -41,6 +41,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -204,3 +205,5 @@ AggregateFunctionPtr transform_to_sort_agg_function(const AggregateFunctionPtr&
                                                     const SortDescription& sort_desc,
                                                     RuntimeState* state);
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_state_merge.h b/be/src/vec/aggregate_functions/aggregate_function_state_merge.h
index d16c823b0292b7..649873373472c6 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_state_merge.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_state_merge.h
@@ -20,6 +20,7 @@
 #include "vec/aggregate_functions/aggregate_function_state_union.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 const static std::string AGG_MERGE_SUFFIX = "_merge";
 
 class AggregateStateMerge : public AggregateStateUnion {
@@ -55,3 +56,5 @@ class AggregateStateMerge : public AggregateStateUnion {
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_state_union.h b/be/src/vec/aggregate_functions/aggregate_function_state_union.h
index 6ff900c90d38ef..c69a3b9c1577e6 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_state_union.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_state_union.h
@@ -23,6 +23,7 @@
 #include "vec/data_types/data_type_agg_state.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 const static std::string AGG_UNION_SUFFIX = "_union";
 
 class AggregateStateUnion : public IAggregateFunctionHelper<AggregateStateUnion> {
@@ -112,3 +113,5 @@ class AggregateStateUnion : public IAggregateFunctionHelper<AggregateStateUnion>
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_statistic.h b/be/src/vec/aggregate_functions/aggregate_function_statistic.h
index a1fd4395eb848a..9c3ff9cbf73210 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_statistic.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_statistic.h
@@ -35,6 +35,7 @@
 #include "vec/data_types/data_type_number.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 enum class STATISTICS_FUNCTION_KIND : uint8_t { SKEW_POP, KURT_POP };
 
@@ -53,6 +54,7 @@ template <typename T, std::size_t _level>
 struct StatFuncOneArg {
     using Type = T;
     using Data = VarMoments<Float64, _level>;
+    using DataType = Float64;
 };
 
 template <typename StatFunc, bool NullableInput>
@@ -63,6 +65,7 @@ class AggregateFunctionVarianceSimple
 public:
     using InputCol = ColumnVector<typename StatFunc::Type>;
     using ResultCol = ColumnVector<Float64>;
+    using InputType = typename StatFunc::DataType;
 
     explicit AggregateFunctionVarianceSimple(STATISTICS_FUNCTION_KIND kind_,
                                              const DataTypes& argument_types_)
@@ -88,14 +91,16 @@ class AggregateFunctionVarianceSimple
             if (column_with_nullable.is_null_at(row_num)) {
                 return;
             } else {
-                this->data(place).add(assert_cast<const InputCol&, TypeCheckOnRelease::DISABLE>(
-                                              column_with_nullable.get_nested_column())
-                                              .get_data()[row_num]);
+                this->data(place).add(
+                        (InputType)assert_cast<const InputCol&, TypeCheckOnRelease::DISABLE>(
+                                column_with_nullable.get_nested_column())
+                                .get_data()[row_num]);
             }
 
         } else {
             this->data(place).add(
-                    assert_cast<const InputCol&, TypeCheckOnRelease::DISABLE>(*columns[0])
+                    (InputType)assert_cast<const InputCol&, TypeCheckOnRelease::DISABLE>(
+                            *columns[0])
                             .get_data()[row_num]);
         }
     }
@@ -160,4 +165,5 @@ class AggregateFunctionVarianceSimple
     STATISTICS_FUNCTION_KIND kind;
 };
 
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp b/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp
index 5a76c3b836d3c1..6c463b8417e0bf 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp
@@ -28,6 +28,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <template <typename> class Function, typename Name,
           template <typename, typename, bool> class Data, bool is_stddev>
diff --git a/be/src/vec/aggregate_functions/aggregate_function_stddev.h b/be/src/vec/aggregate_functions/aggregate_function_stddev.h
index 9b3cd190991fea..d42adc28d73fc5 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_stddev.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_stddev.h
@@ -34,7 +34,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
-
+#include "common/compile_check_begin.h"
 class Arena;
 class BufferReadable;
 class BufferWritable;
@@ -88,12 +88,12 @@ struct BaseData {
         if (count == 1) {
             return 0.0;
         }
-        double res = m2 / count;
+        double res = m2 / (double)count;
         return get_result(res);
     }
 
     double get_samp_result() const {
-        double res = m2 / (count - 1);
+        double res = m2 / double(count - 1);
         return get_result(res);
     }
 
@@ -102,21 +102,21 @@ struct BaseData {
             return;
         }
         double delta = mean - rhs.mean;
-        double sum_count = count + rhs.count;
-        mean = rhs.mean + delta * count / sum_count;
-        m2 = rhs.m2 + m2 + (delta * delta) * rhs.count * count / sum_count;
+        double sum_count = double(count + rhs.count);
+        mean = rhs.mean + delta * (double)count / sum_count;
+        m2 = rhs.m2 + m2 + (delta * delta) * (double)rhs.count * (double)count / sum_count;
         count = int64_t(sum_count);
     }
 
     void add(const IColumn* column, size_t row_num) {
         const auto& sources =
                 assert_cast<const ColumnVector<T>&, TypeCheckOnRelease::DISABLE>(*column);
-        double source_data = sources.get_data()[row_num];
+        double source_data = (double)sources.get_data()[row_num];
 
         double delta = source_data - mean;
-        double r = delta / (1 + count);
+        double r = delta / double(1 + count);
         mean += r;
-        m2 += count * delta * r;
+        m2 += (double)count * delta * r;
         count += 1;
     }
 
@@ -217,3 +217,5 @@ class AggregateFunctionSampVariance
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sum.cpp b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp
index 91063c22dc6376..3ad9efa121c6a4 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sum.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp
@@ -24,6 +24,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 void register_aggregate_function_sum(AggregateFunctionSimpleFactory& factory) {
     AggregateFunctionCreator creator = [&](const std::string& name, const DataTypes& types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sum.h b/be/src/vec/aggregate_functions/aggregate_function_sum.h
index 13fb3864bd1aaf..68fb1806724e98 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sum.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_sum.h
@@ -37,6 +37,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -248,3 +249,5 @@ template <typename T>
 using AggregateFunctionSumSimpleReader = typename SumSimple<T, false>::Function;
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_topn.cpp b/be/src/vec/aggregate_functions/aggregate_function_topn.cpp
index 799d8fe1c754fb..ea7a33969b9a88 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_topn.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_topn.cpp
@@ -25,6 +25,7 @@
 #include "vec/data_types/data_type.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 AggregateFunctionPtr create_aggregate_function_topn(const std::string& name,
                                                     const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_topn.h b/be/src/vec/aggregate_functions/aggregate_function_topn.h
index 1c0fba5099fb17..29eee8eee55805 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_topn.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_topn.h
@@ -50,6 +50,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -385,3 +386,5 @@ class AggregateFunctionTopNArray final : public AggregateFunctionTopNBase<Impl,
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp b/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp
index 25231025416b83..df0f2321947311 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp
@@ -30,6 +30,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <template <typename> class Data>
 AggregateFunctionPtr create_aggregate_function_uniq(const std::string& name,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq.h b/be/src/vec/aggregate_functions/aggregate_function_uniq.h
index a3bdad635057fd..59f477363f706c 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_uniq.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_uniq.h
@@ -44,6 +44,7 @@
 #include "vec/io/var_int.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -232,3 +233,5 @@ class AggregateFunctionUniq final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.cpp b/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.cpp
index c89c8aa14f01eb..6a7c6b8f3b0a88 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.cpp
@@ -24,6 +24,7 @@
 #include "vec/aggregate_functions/helpers.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <template <typename> class Data>
 AggregateFunctionPtr create_aggregate_function_uniq(const std::string& name,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.h b/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.h
index 90d137c62384f6..ee1ae91ab3e85b 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.h
@@ -42,6 +42,7 @@
 #include "vec/io/var_int.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -258,3 +259,5 @@ class AggregateFunctionUniqDistributeKey final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.cpp b/be/src/vec/aggregate_functions/aggregate_function_window.cpp
index 9da838a6b9067c..fcb0072abe8d6f 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_window.cpp
@@ -30,6 +30,7 @@
 #include "vec/utils/template_helpers.hpp"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 template <template <typename> class AggregateFunctionTemplate,
           template <typename ColVecType, bool, bool> class Data,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.h b/be/src/vec/aggregate_functions/aggregate_function_window.h
index cb038fe31168b9..0011ae3aba96b3 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_window.h
@@ -42,6 +42,7 @@
 #include "vec/data_types/data_type_number.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 class Arena;
 class BufferReadable;
 class BufferWritable;
@@ -181,7 +182,7 @@ class WindowFunctionPercentRank final
         : public IAggregateFunctionDataHelper<PercentRankData, WindowFunctionPercentRank> {
 private:
     static double _cal_percent(int64 rank, int64 total_rows) {
-        return total_rows <= 1 ? 0.0 : (rank - 1) * 1.0 / (total_rows - 1);
+        return total_rows <= 1 ? 0.0 : double(rank - 1) * 1.0 / double(total_rows - 1);
     }
 
 public:
@@ -270,7 +271,7 @@ class WindowFunctionCumeDist final
     }
 
     void insert_result_into(ConstAggregateDataPtr place, IColumn& to) const override {
-        auto cume_dist = data(place).numerator * 1.0 / data(place).denominator;
+        auto cume_dist = (double)data(place).numerator * 1.0 / (double)data(place).denominator;
         assert_cast<ColumnFloat64&>(to).get_data().push_back(cume_dist);
     }
 
@@ -580,3 +581,5 @@ class WindowFunctionData final
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp
index f95dccd547eb5f..606df4d2c4facf 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp
@@ -29,6 +29,7 @@
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 AggregateFunctionPtr create_aggregate_function_window_funnel(const std::string& name,
                                                              const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h
index 84222f0d01ba0c..efd51444c56560 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h
@@ -32,6 +32,7 @@
 #include <utility>
 
 #include "agent/be_exec_version_manager.h"
+#include "common/cast_set.h"
 #include "common/compiler_util.h"
 #include "common/exception.h"
 #include "util/binary_cast.hpp"
@@ -49,6 +50,7 @@
 #include "vec/runtime/vdatetime_value.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class Arena;
 class BufferReadable;
@@ -349,7 +351,9 @@ struct WindowFunnelState {
         in.read(buff.data(), data_bytes);
 
         PBlock pblock;
-        if (!pblock.ParseFromArray(buff.data(), data_bytes)) {
+        // It is preferable to change data_bytes to int type here,
+        // but due to compatibility issues, no changes will be made.
+        if (!pblock.ParseFromArray(buff.data(), (int)data_bytes)) {
             throw doris::Exception(ErrorCode::INTERNAL_ERROR,
                                    "Failed to parse window_funnel data to block");
         }
@@ -376,7 +380,7 @@ class AggregateFunctionWindowFunnel
 
     void create(AggregateDataPtr __restrict place) const override {
         auto data = new (place) WindowFunnelState<TYPE_INDEX, NativeType>(
-                IAggregateFunction::get_argument_types().size() - 3);
+                cast_set<int>(IAggregateFunction::get_argument_types().size() - 3));
         /// support window funnel mode from 2.0. See `BeExecVersionManager::max_be_exec_version`
         data->enable_mode = version >= 3;
     }
@@ -424,3 +428,5 @@ class AggregateFunctionWindowFunnel
 };
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/factory_helpers.h b/be/src/vec/aggregate_functions/factory_helpers.h
index 553f05c3665a5b..d5a4a022176304 100644
--- a/be/src/vec/aggregate_functions/factory_helpers.h
+++ b/be/src/vec/aggregate_functions/factory_helpers.h
@@ -25,6 +25,7 @@
 #include "vec/data_types/data_type.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 inline void assert_unary(const std::string& name, const DataTypes& argument_types) {
     CHECK_EQ(argument_types.size(), 1)
@@ -57,3 +58,5 @@ void assert_arity_at_most(const std::string& name, const DataTypes& argument_typ
 }
 
 } // namespace doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/helpers.h b/be/src/vec/aggregate_functions/helpers.h
index 34b7e76c2eaad2..75ef07bae646b8 100644
--- a/be/src/vec/aggregate_functions/helpers.h
+++ b/be/src/vec/aggregate_functions/helpers.h
@@ -99,6 +99,7 @@
     } while (false)
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 struct creator_without_type {
     template <bool multi_arguments, bool f, typename T>
@@ -260,3 +261,5 @@ using creator_with_decimal_type = creator_with_type_base<false, false, true>;
 using creator_with_type = creator_with_type_base<true, true, true>;
 
 } // namespace  doris::vectorized
+
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/moments.h b/be/src/vec/aggregate_functions/moments.h
index d9db12774709bc..0628b3116e2120 100644
--- a/be/src/vec/aggregate_functions/moments.h
+++ b/be/src/vec/aggregate_functions/moments.h
@@ -24,6 +24,7 @@
 #include "vec/io/io_helper.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 class BufferReadable;
 class BufferWritable;
@@ -111,4 +112,5 @@ struct VarMoments {
     }
 };
 
-} // namespace doris::vectorized
\ No newline at end of file
+} // namespace doris::vectorized
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/common/string_buffer.hpp b/be/src/vec/common/string_buffer.hpp
index 8dca6f057a26d3..769b2e93d7d9dd 100644
--- a/be/src/vec/common/string_buffer.hpp
+++ b/be/src/vec/common/string_buffer.hpp
@@ -76,7 +76,7 @@ class BufferReadable {
         return ref;
     }
 
-    void read(char* data, int len) {
+    void read(char* data, size_t len) {
         memcpy(data, _data, len);
         _data += len;
     }

From 7ee8681695195d748a15c0fd4cae3be0b1649704 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Tue, 3 Dec 2024 09:20:10 +0530
Subject: [PATCH 127/399] [Enhancement] (nereids)implement CreateFileCommand in
 nereids (#44806)

Issue Number: close #42598
---
 .../org/apache/doris/nereids/DorisParser.g4   |   4 +-
 .../doris/common/util/SmallFileMgr.java       |   6 +
 .../nereids/parser/LogicalPlanBuilder.java    |  13 ++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../plans/commands/CreateFileCommand.java     | 126 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../auth_call/test_ddl_file_auth.groovy       |   4 +-
 7 files changed, 155 insertions(+), 4 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFileCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 3a377ba05bc707..cc5c223f9ec0dd 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -178,6 +178,8 @@ supportedCreateStatement
     | CREATE (OR REPLACE)? VIEW (IF NOT EXISTS)? name=multipartIdentifier
         (LEFT_PAREN cols=simpleColumnDefs RIGHT_PAREN)?
         (COMMENT STRING_LITERAL)? AS query                                #createView
+    | CREATE FILE name=STRING_LITERAL
+        ((FROM | IN) database=identifier)? properties=propertyClause            #createFile        
     | CREATE (EXTERNAL)? TABLE (IF NOT EXISTS)? name=multipartIdentifier
         LIKE existedTable=multipartIdentifier
         (WITH ROLLUP (rollupNames=identifierList)?)?                      #createTableLike
@@ -752,8 +754,6 @@ unsupportedCreateStatement
         (SUPERUSER | DEFAULT ROLE role=STRING_LITERAL)?
         passwordOption (COMMENT STRING_LITERAL)?                                #createUser
     | CREATE (READ ONLY)? REPOSITORY name=identifier WITH storageBackend        #createRepository
-    | CREATE FILE name=STRING_LITERAL
-        ((FROM | IN) database=identifier)? properties=propertyClause            #createFile
     | CREATE INDEX (IF NOT EXISTS)? name=identifier
         ON tableName=multipartIdentifier identifierList
         (USING (BITMAP | NGRAM_BF | INVERTED))?
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/SmallFileMgr.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/SmallFileMgr.java
index 0fc63a8e327476..5d01eb56033e1d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/SmallFileMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/SmallFileMgr.java
@@ -177,6 +177,12 @@ public void createFile(CreateFileStmt stmt) throws DdlException {
                 stmt.getDownloadUrl(), stmt.getChecksum(), stmt.isSaveContent());
     }
 
+    public void createFile(String dbName, String catalog, String fileName, String downloadUrl, String md5sum,
+                            boolean saveContent) throws DdlException {
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
+        downloadAndAddFile(db.getId(), catalog, fileName, downloadUrl, md5sum, saveContent);
+    }
+
     public void dropFile(DropFileStmt stmt) throws DdlException {
         String dbName = stmt.getDbName();
         Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 2d55baffb1ebd9..7ec8ce62f3001e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -90,6 +90,7 @@
 import org.apache.doris.nereids.DorisParser.ComplexDataTypeContext;
 import org.apache.doris.nereids.DorisParser.ConstantContext;
 import org.apache.doris.nereids.DorisParser.CreateEncryptkeyContext;
+import org.apache.doris.nereids.DorisParser.CreateFileContext;
 import org.apache.doris.nereids.DorisParser.CreateMTMVContext;
 import org.apache.doris.nereids.DorisParser.CreateProcedureContext;
 import org.apache.doris.nereids.DorisParser.CreateRoleContext;
@@ -479,6 +480,7 @@
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.Constraint;
 import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
@@ -4680,6 +4682,17 @@ public LogicalPlan visitCreateRole(CreateRoleContext ctx) {
         return new CreateRoleCommand(ctx.EXISTS() != null, ctx.name.getText(), comment);
     }
 
+    @Override
+    public LogicalPlan visitCreateFile(CreateFileContext ctx) {
+        String dbName = null;
+        if (ctx.database != null) {
+            dbName = ctx.database.getText();
+        }
+        Map<String, String> properties = ctx.propertyClause() != null
+                                    ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+        return new CreateFileCommand(stripQuotes(ctx.name.getText()), dbName, properties);
+    }
+
     @Override
     public LogicalPlan visitShowFrontends(ShowFrontendsContext ctx) {
         String detail = (ctx.name != null) ? ctx.name.getText() : null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 0cc0e5594075b4..8061309e8b137c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -239,5 +239,6 @@ public enum PlanType {
     RECOVER_PARTITION_COMMAND,
     REPLAY_COMMAND,
     CREATE_ENCRYPTKEY_COMMAND,
+    CREATE_FILE_COMMAND,
     CREATE_ROUTINE_LOAD_COMMAND
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFileCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFileCommand.java
new file mode 100644
index 00000000000000..c8ea3cc161567d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFileCommand.java
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Map;
+import java.util.Optional;
+
+/** CreateFileCommand */
+public class CreateFileCommand extends Command implements ForwardWithSync {
+    public static final String PROP_CATALOG_DEFAULT = "DEFAULT";
+    private static final String PROP_CATALOG = "catalog";
+    private static final String PROP_URL = "url";
+    private static final String PROP_MD5 = "md5";
+    private static final String PROP_SAVE_CONTENT = "save_content";
+
+    private static final ImmutableSet<String> PROPERTIES_SET = new ImmutableSet.Builder<String>()
+            .add(PROP_CATALOG).add(PROP_URL).add(PROP_MD5).build();
+    private final String fileName;
+    private String dbName;
+    private final Map<String, String> properties;
+
+    // analyze properties will fill these.
+    // needed item set after analyzed
+    private String catalogName;
+    private String downloadUrl;
+    private String checksum;
+    // if saveContent is true, the file content will be saved in FE memory, so the file size will be limited
+    // by the configuration. If it is false, only URL will be saved.
+    private boolean saveContent = true;
+
+    public CreateFileCommand(String fileName, String dbName, Map<String, String> properties) {
+        super(PlanType.CREATE_FILE_COMMAND);
+        this.fileName = fileName;
+        this.dbName = dbName;
+        this.properties = properties;
+    }
+
+    private void analyzeProperties() throws Exception {
+        Optional<String> optional = properties.keySet().stream().filter(
+                entity -> !PROPERTIES_SET.contains(entity)).findFirst();
+        if (optional.isPresent()) {
+            throw new AnalysisException(optional.get() + " is invalid property");
+        }
+
+        catalogName = properties.get(PROP_CATALOG);
+        if (Strings.isNullOrEmpty(catalogName)) {
+            catalogName = PROP_CATALOG_DEFAULT;
+        }
+
+        downloadUrl = properties.get(PROP_URL);
+        if (Strings.isNullOrEmpty(downloadUrl)) {
+            throw new AnalysisException("download url is missing");
+        }
+
+        if (properties.containsKey(PROP_MD5)) {
+            checksum = properties.get(PROP_MD5);
+        }
+
+        if (properties.containsKey(PROP_SAVE_CONTENT)) {
+            throw new AnalysisException("'save_content' property is not supported yet");
+        }
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check operation privilege
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        if (dbName == null) {
+            dbName = ctx.getDatabase();
+        }
+
+        if (Strings.isNullOrEmpty(dbName)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
+        }
+
+        if (Strings.isNullOrEmpty(fileName)) {
+            throw new AnalysisException("File name is not specified");
+        }
+
+        analyzeProperties();
+        Env.getCurrentEnv().getSmallFileMgr().createFile(dbName, catalogName, fileName,
+                                                    downloadUrl, checksum, saveContent);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCreateFileCommand(this, context);
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.CREATE;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index d1d424738f4924..fe9d27d9f303a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -34,6 +34,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
@@ -193,6 +194,10 @@ default R visitCreateJobCommand(CreateJobCommand createJobCommand, C context) {
         return visitCommand(createJobCommand, context);
     }
 
+    default R visitCreateFileCommand(CreateFileCommand createFileCommand, C context) {
+        return visitCommand(createFileCommand, context);
+    }
+
     default R visitAlterMTMVCommand(AlterMTMVCommand alterMTMVCommand, C context) {
         return visitCommand(alterMTMVCommand, context);
     }
diff --git a/regression-test/suites/auth_call/test_ddl_file_auth.groovy b/regression-test/suites/auth_call/test_ddl_file_auth.groovy
index 53cab5b3687ba5..73aa9204b54c02 100644
--- a/regression-test/suites/auth_call/test_ddl_file_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_file_auth.groovy
@@ -69,12 +69,12 @@ suite("test_ddl_file_auth","p0,auth_call") {
 
     sql """grant admin_priv on *.*.* to ${user}"""
     connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
-        sql """CREATE FILE "${fileName}" IN ${dbName}
+        checkNereidsExecute("""CREATE FILE "${fileName}" IN ${dbName}
                 PROPERTIES
                 (
                     "url" = "${dataFilePath}",
                     "catalog" = "internal"
-                );"""
+                );""")
         sql """use ${dbName}"""
         checkNereidsExecute("SHOW FILE;")
         checkNereidsExecute("SHOW FILE FROM ${dbName};")

From 702ca90e58260de6aee876ac2ae2ce461057dc4f Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Tue, 3 Dec 2024 11:50:54 +0800
Subject: [PATCH 128/399] [fix](ip) fix ip nullable param without check
 (#44700)

if we use ipv6_cidr_to_range function with nullable func which with
invalid ipv6 will make be core
```
mysql> select id, ipv6_cidr_to_range(nullable(''), 32) from fn_test_ip_nullable order by id;
```
---
 be/src/vec/functions/function_ip.h            |  74 +++----
 .../scalar_function/IP.out                    | 204 ++++++++++++++++++
 .../scalar_function/IP.groovy                 |  23 +-
 .../test_ipv6_cidr_to_range_function.groovy   |  12 +-
 4 files changed, 256 insertions(+), 57 deletions(-)

diff --git a/be/src/vec/functions/function_ip.h b/be/src/vec/functions/function_ip.h
index 724121ce57c109..928ce4369726b2 100644
--- a/be/src/vec/functions/function_ip.h
+++ b/be/src/vec/functions/function_ip.h
@@ -871,6 +871,11 @@ class FunctionIPv4CIDRToRange : public IFunction {
     }
 };
 
+/**
+ * this function accepts two arguments: an IPv6 address and a CIDR mask
+ *  IPv6 address can be either ipv6 type or string type as ipv6 string address
+ *  FE: PropagateNullable is used to handle nullable columns
+ */
 class FunctionIPv6CIDRToRange : public IFunction {
 public:
     static constexpr auto name = "ipv6_cidr_to_range";
@@ -902,12 +907,14 @@ class FunctionIPv6CIDRToRange : public IFunction {
 
         if (addr_type.is_ipv6()) {
             const auto* ipv6_addr_column = assert_cast<const ColumnIPv6*>(addr_column.get());
-            col_res = execute_impl<ColumnIPv6>(*ipv6_addr_column, *cidr_col, input_rows_count,
-                                               add_col_const, col_const);
+            col_res = execute_impl(*ipv6_addr_column, *cidr_col, input_rows_count, add_col_const,
+                                   col_const);
         } else if (addr_type.is_string()) {
-            const auto* str_addr_column = assert_cast<const ColumnString*>(addr_column.get());
-            col_res = execute_impl<ColumnString>(*str_addr_column, *cidr_col, input_rows_count,
-                                                 add_col_const, col_const);
+            ColumnPtr col_ipv6 =
+                    convert_to_ipv6<IPConvertExceptionMode::Throw>(addr_column, nullptr);
+            const auto* ipv6_addr_column = assert_cast<const ColumnIPv6*>(col_ipv6.get());
+            col_res = execute_impl(*ipv6_addr_column, *cidr_col, input_rows_count, add_col_const,
+                                   col_const);
         } else {
             return Status::RuntimeError(
                     "Illegal column {} of argument of function {}, Expected IPv6 or String",
@@ -918,8 +925,7 @@ class FunctionIPv6CIDRToRange : public IFunction {
         return Status::OK();
     }
 
-    template <typename FromColumn>
-    static ColumnPtr execute_impl(const FromColumn& from_column, const ColumnInt16& cidr_column,
+    static ColumnPtr execute_impl(const ColumnIPv6& from_column, const ColumnInt16& cidr_column,
                                   size_t input_rows_count, bool is_addr_const = false,
                                   bool is_cidr_const = false) {
         auto col_res_lower_range = ColumnIPv6::create(input_rows_count, 0);
@@ -936,20 +942,10 @@ class FunctionIPv6CIDRToRange : public IFunction {
                     throw Exception(ErrorCode::INVALID_ARGUMENT, "Illegal cidr value '{}'",
                                     std::to_string(cidr));
                 }
-                if constexpr (std::is_same_v<FromColumn, ColumnString>) {
-                    // 16 bytes ipv6 string is stored in big-endian byte order
-                    // so transfer to little-endian firstly
-                    auto* src_data = const_cast<char*>(from_column.get_data_at(0).data);
-                    std::reverse(src_data, src_data + IPV6_BINARY_LENGTH);
-                    apply_cidr_mask(src_data, reinterpret_cast<char*>(&vec_res_lower_range[i]),
-                                    reinterpret_cast<char*>(&vec_res_upper_range[i]),
-                                    cast_set<UInt8>(cidr));
-                } else {
-                    apply_cidr_mask(from_column.get_data_at(0).data,
-                                    reinterpret_cast<char*>(&vec_res_lower_range[i]),
-                                    reinterpret_cast<char*>(&vec_res_upper_range[i]),
-                                    cast_set<UInt8>(cidr));
-                }
+                apply_cidr_mask(from_column.get_data_at(0).data,
+                                reinterpret_cast<char*>(&vec_res_lower_range[i]),
+                                reinterpret_cast<char*>(&vec_res_upper_range[i]),
+                                cast_set<UInt8>(cidr));
             }
         } else if (is_cidr_const) {
             auto cidr = cidr_column.get_int(0);
@@ -958,20 +954,10 @@ class FunctionIPv6CIDRToRange : public IFunction {
                                 std::to_string(cidr));
             }
             for (size_t i = 0; i < input_rows_count; ++i) {
-                if constexpr (std::is_same_v<FromColumn, ColumnString>) {
-                    // 16 bytes ipv6 string is stored in big-endian byte order
-                    // so transfer to little-endian firstly
-                    auto* src_data = const_cast<char*>(from_column.get_data_at(i).data);
-                    std::reverse(src_data, src_data + IPV6_BINARY_LENGTH);
-                    apply_cidr_mask(src_data, reinterpret_cast<char*>(&vec_res_lower_range[i]),
-                                    reinterpret_cast<char*>(&vec_res_upper_range[i]),
-                                    cast_set<UInt8>(cidr));
-                } else {
-                    apply_cidr_mask(from_column.get_data_at(i).data,
-                                    reinterpret_cast<char*>(&vec_res_lower_range[i]),
-                                    reinterpret_cast<char*>(&vec_res_upper_range[i]),
-                                    cast_set<UInt8>(cidr));
-                }
+                apply_cidr_mask(from_column.get_data_at(i).data,
+                                reinterpret_cast<char*>(&vec_res_lower_range[i]),
+                                reinterpret_cast<char*>(&vec_res_upper_range[i]),
+                                cast_set<UInt8>(cidr));
             }
         } else {
             for (size_t i = 0; i < input_rows_count; ++i) {
@@ -980,20 +966,10 @@ class FunctionIPv6CIDRToRange : public IFunction {
                     throw Exception(ErrorCode::INVALID_ARGUMENT, "Illegal cidr value '{}'",
                                     std::to_string(cidr));
                 }
-                if constexpr (std::is_same_v<FromColumn, ColumnString>) {
-                    // 16 bytes ipv6 string is stored in big-endian byte order
-                    // so transfer to little-endian firstly
-                    auto* src_data = const_cast<char*>(from_column.get_data_at(i).data);
-                    std::reverse(src_data, src_data + IPV6_BINARY_LENGTH);
-                    apply_cidr_mask(src_data, reinterpret_cast<char*>(&vec_res_lower_range[i]),
-                                    reinterpret_cast<char*>(&vec_res_upper_range[i]),
-                                    cast_set<UInt8>(cidr));
-                } else {
-                    apply_cidr_mask(from_column.get_data_at(i).data,
-                                    reinterpret_cast<char*>(&vec_res_lower_range[i]),
-                                    reinterpret_cast<char*>(&vec_res_upper_range[i]),
-                                    cast_set<UInt8>(cidr));
-                }
+                apply_cidr_mask(from_column.get_data_at(i).data,
+                                reinterpret_cast<char*>(&vec_res_lower_range[i]),
+                                reinterpret_cast<char*>(&vec_res_upper_range[i]),
+                                cast_set<UInt8>(cidr));
             }
         }
         return ColumnStruct::create(
diff --git a/regression-test/data/nereids_function_p0/scalar_function/IP.out b/regression-test/data/nereids_function_p0/scalar_function/IP.out
index 5b44453ceb6232..af146b66223378 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/IP.out
+++ b/regression-test/data/nereids_function_p0/scalar_function/IP.out
@@ -410,6 +410,108 @@
 99	{"min":"224.0.0.0", "max":"224.0.255.255"}
 100	{"min":"224.0.0.0", "max":"224.0.255.255"}
 
+-- !sql_cidr_ipv6_nullable_ --
+1	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+2	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+3	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+4	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+5	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+6	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+7	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+8	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+9	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+10	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+11	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+12	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+13	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+14	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+15	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+16	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+17	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+18	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+19	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+20	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+21	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+22	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+23	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+24	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+25	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+26	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+27	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+28	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+29	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+30	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+31	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+32	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+33	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+34	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+35	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+36	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+37	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+38	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+39	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+40	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+41	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+42	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+43	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+44	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+45	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+46	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+47	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+48	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+49	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+50	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+51	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+52	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+53	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+54	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+55	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+56	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+57	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+58	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+59	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+60	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+61	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+62	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+63	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+64	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+65	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+66	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+67	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+68	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+69	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+70	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+71	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+72	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+73	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+74	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+75	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+76	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+77	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+78	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+79	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+80	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+81	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+82	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+83	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+84	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+85	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+86	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+87	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+88	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+89	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+90	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+91	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+92	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+93	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+94	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+95	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+96	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+97	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+98	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+99	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+100	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+
 -- !sql_num2string_ipv6 --
 1	::1
 2	fc00::
@@ -6121,6 +6223,108 @@
 99	{"min":"224.0.0.0", "max":"224.0.255.255"}
 100	{"min":"224.0.0.0", "max":"224.0.255.255"}
 
+-- !sql_not_null_cidr_ipv6_nullable_ --
+1	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+2	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+3	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+4	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+5	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+6	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+7	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+8	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+9	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+10	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+11	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+12	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+13	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+14	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+15	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+16	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+17	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+18	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+19	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+20	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+21	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+22	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+23	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+24	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+25	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+26	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+27	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+28	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+29	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+30	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+31	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+32	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+33	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+34	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+35	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+36	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+37	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+38	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+39	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+40	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+41	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+42	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+43	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+44	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+45	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+46	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+47	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+48	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+49	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+50	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+51	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+52	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+53	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+54	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+55	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+56	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+57	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+58	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+59	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+60	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+61	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+62	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+63	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+64	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+65	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+66	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+67	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+68	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+69	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+70	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+71	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+72	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+73	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+74	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+75	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+76	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+77	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+78	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+79	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+80	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+81	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+82	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+83	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+84	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+85	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+86	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+87	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+88	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+89	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+90	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+91	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+92	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+93	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+94	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+95	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+96	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+97	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+98	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+99	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+100	{"min":"::", "max":"::ffff:ffff:ffff:ffff:ffff:ffff"}
+
 -- !sql_not_null_ipv6_string_to_num --
 1	00000000000000000000000000000001
 2	FC000000000000000000000000000000
diff --git a/regression-test/suites/nereids_function_p0/scalar_function/IP.groovy b/regression-test/suites/nereids_function_p0/scalar_function/IP.groovy
index 9cbdccbe78840b..a1c1d00caa29fe 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/IP.groovy
+++ b/regression-test/suites/nereids_function_p0/scalar_function/IP.groovy
@@ -30,7 +30,16 @@ suite("nereids_scalar_fn_IP") {
     qt_sql_cidr_ipv6_all """ select id, ipv6_cidr_to_range(ip6, 16) from fn_test_ip_nullable order by id; """
     qt_sql_cidr_ipv4_all """ select id, ipv4_cidr_to_range(ip4, 16) from fn_test_ip_nullable order by id; """
 
-
+    // test nullable param
+    qt_sql_cidr_ipv6_nullable_ "select id, ipv6_cidr_to_range(to_ipv6('::'), 32) from fn_test_ip_nullable order by id;"	
+    test {
+   	sql "select id, ipv6_cidr_to_range(nullable(''), 32) from fn_test_ip_nullable order by id" 
+	exception "Invalid IPv6 value"
+    }
+    test {
+        sql "select id, ipv6_cidr_to_range(nullable('abc'), 32) from fn_test_ip_not_nullable order by id"
+        exception "Invalid IPv6 value"
+    }
     // test IPV4_STRING_TO_NUM/IPV6_STRING_TO_NUM (we have null value in ip4 and ip6 column in fn_test_ip_nullable table)
     test {
         sql 'select id, ipv6_string_to_num(ip6) from fn_test_ip_nullable order by id'
@@ -153,7 +162,17 @@ suite("nereids_scalar_fn_IP") {
     qt_sql_not_null_cidr_ipv6_all """ select id, ipv6_cidr_to_range(ip6, 16) from fn_test_ip_not_nullable order by id; """
     qt_sql_not_null_cidr_ipv4_all """ select id, ipv4_cidr_to_range(ip4, 16) from fn_test_ip_not_nullable order by id; """
 
+    // test nullable param
+    qt_sql_not_null_cidr_ipv6_nullable_ "select id, ipv6_cidr_to_range(to_ipv6('::'), 32) from fn_test_ip_nullable order by id;"
+    test {
+        sql "select id, ipv6_cidr_to_range(nullable(''), 32) from fn_test_ip_not_nullable order by id"
+        exception "Invalid IPv6 value"
+    }
 
+    test {
+        sql "select id, ipv6_cidr_to_range(nullable('abc'), 32) from fn_test_ip_not_nullable order by id"
+        exception "Invalid IPv6 value"
+    }
     // test IPV4_STRING_TO_NUM/IPV6_STRING_TO_NUM
     qt_sql_not_null_ipv6_string_to_num 'select id, hex(ipv6_string_to_num(ip6)) from fn_test_ip_not_nullable order by id'
 
@@ -269,4 +288,4 @@ suite("nereids_scalar_fn_IP") {
     qt_sql_not_null_to_ipv4_or_null "select id, to_ipv4_or_null(ip6) from fn_test_ip_not_nullable order by id"
     qt_sql_not_null_to_ipv4_or_null_str "select id, to_ipv4_or_null(ip6_str) from fn_test_ip_not_nullable order by id"
 
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/query_p0/sql_functions/ip_functions/test_ipv6_cidr_to_range_function.groovy b/regression-test/suites/query_p0/sql_functions/ip_functions/test_ipv6_cidr_to_range_function.groovy
index 41432c986fec49..0a8ba107013b4e 100644
--- a/regression-test/suites/query_p0/sql_functions/ip_functions/test_ipv6_cidr_to_range_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/ip_functions/test_ipv6_cidr_to_range_function.groovy
@@ -91,13 +91,13 @@ suite("test_ipv6_cidr_to_range_function") {
         (9, 'ffff:0000:0000:0000:0000:0000:0000:0000', NULL)
         """
 
-    qt_sql "select id, struct_element(ipv6_cidr_to_range(ipv6_string_to_num_or_null(addr), cidr), 'min') as min_range, struct_element(ipv6_cidr_to_range(ipv6_string_to_num_or_null(addr), cidr), 'max') as max_range from test_str_cidr_to_range_function order by id"
+    qt_sql "select id, struct_element(ipv6_cidr_to_range(ipv6_num_to_string(ipv6_string_to_num_or_null(addr)), cidr), 'min') as min_range, struct_element(ipv6_cidr_to_range(ipv6_num_to_string(ipv6_string_to_num_or_null(addr)), cidr), 'max') as max_range from test_str_cidr_to_range_function order by id"
 
     sql """ DROP TABLE IF EXISTS test_str_cidr_to_range_function """
 
-    qt_sql "select ipv6_cidr_to_range(ipv6_string_to_num('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 0)"
-    qt_sql "select ipv6_cidr_to_range(ipv6_string_to_num('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 128)"
-    qt_sql "select ipv6_cidr_to_range(ipv6_string_to_num('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'), 64)"
-    qt_sql "select ipv6_cidr_to_range(ipv6_string_to_num('0000:0000:0000:0000:0000:0000:0000:0000'), 8)"
-    qt_sql "select ipv6_cidr_to_range(ipv6_string_to_num('ffff:0000:0000:0000:0000:0000:0000:0000'), 4)"
+    qt_sql "select ipv6_cidr_to_range(ipv6_num_to_string(ipv6_string_to_num('2001:0db8:0000:85a3:0000:0000:ac1f:8001')), 0)"
+    qt_sql "select ipv6_cidr_to_range(ipv6_num_to_string(ipv6_string_to_num('2001:0db8:0000:85a3:0000:0000:ac1f:8001')), 128)"
+    qt_sql "select ipv6_cidr_to_range(ipv6_num_to_string(ipv6_string_to_num('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')), 64)"
+    qt_sql "select ipv6_cidr_to_range(ipv6_num_to_string(ipv6_string_to_num('0000:0000:0000:0000:0000:0000:0000:0000')), 8)"
+    qt_sql "select ipv6_cidr_to_range(ipv6_num_to_string(ipv6_string_to_num('ffff:0000:0000:0000:0000:0000:0000:0000')), 4)"
 }

From 3faf5672f5b40f4859fa66620194a3b81d1a6f79 Mon Sep 17 00:00:00 2001
From: peterylh <peterylh@163.com>
Date: Tue, 3 Dec 2024 11:55:40 +0800
Subject: [PATCH 129/399] [refactor](agg) use unique_ptr in  agg function
 (#44858)

---
 be/src/vec/aggregate_functions/helpers.h | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/be/src/vec/aggregate_functions/helpers.h b/be/src/vec/aggregate_functions/helpers.h
index 75ef07bae646b8..ad21c2ca6c8d40 100644
--- a/be/src/vec/aggregate_functions/helpers.h
+++ b/be/src/vec/aggregate_functions/helpers.h
@@ -117,20 +117,21 @@ struct creator_without_type {
     template <typename AggregateFunctionTemplate, typename... TArgs>
     static AggregateFunctionPtr create(const DataTypes& argument_types_,
                                        const bool result_is_nullable, TArgs&&... args) {
-        IAggregateFunction* result(new AggregateFunctionTemplate(std::forward<TArgs>(args)...,
-                                                                 remove_nullable(argument_types_)));
+        std::unique_ptr<IAggregateFunction> result(std::make_unique<AggregateFunctionTemplate>(
+                std::forward<TArgs>(args)..., remove_nullable(argument_types_)));
         if (have_nullable(argument_types_)) {
             std::visit(
                     [&](auto multi_arguments, auto result_is_nullable) {
-                        result = new NullableT<multi_arguments, result_is_nullable,
-                                               AggregateFunctionTemplate>(result, argument_types_);
+                        result.reset(new NullableT<multi_arguments, result_is_nullable,
+                                                   AggregateFunctionTemplate>(result.release(),
+                                                                              argument_types_));
                     },
                     make_bool_variant(argument_types_.size() > 1),
                     make_bool_variant(result_is_nullable));
         }
 
         CHECK_AGG_FUNCTION_SERIALIZED_TYPE(AggregateFunctionTemplate);
-        return AggregateFunctionPtr(result);
+        return AggregateFunctionPtr(result.release());
     }
 
     /// AggregateFunctionTemplate will handle the nullable arguments, no need to use
@@ -139,10 +140,10 @@ struct creator_without_type {
     static AggregateFunctionPtr create_ignore_nullable(const DataTypes& argument_types_,
                                                        const bool /*result_is_nullable*/,
                                                        TArgs&&... args) {
-        IAggregateFunction* result(
-                new AggregateFunctionTemplate(std::forward<TArgs>(args)..., argument_types_));
+        std::unique_ptr<IAggregateFunction> result = std::make_unique<AggregateFunctionTemplate>(
+                std::forward<TArgs>(args)..., argument_types_);
         CHECK_AGG_FUNCTION_SERIALIZED_TYPE(AggregateFunctionTemplate);
-        return AggregateFunctionPtr(result);
+        return AggregateFunctionPtr(result.release());
     }
 };
 

From ffd25c43e451c048f1c1eb0ae87dd520e8d10926 Mon Sep 17 00:00:00 2001
From: zhannngchen <zhangchen@selectdb.com>
Date: Tue, 3 Dec 2024 12:50:12 +0800
Subject: [PATCH 130/399] [fix](unique key) getEnableUniqueKeyMergeOnWrite
 should check keys type (#44692)

Problem Summary:

Some user can create Duplicate keys table with
`enable_unique_key_merge_on_write=true` in version 1.2, when they
upgrade to version 2.0, some logic in BE treat such table as an unique
key mow table, make a wrong query plan, which cause BE core.

In the method `OlapTable::getEnableUniqueKeyMergeOnWrite()`, we should
check if the table is a unique key table.
---
 .../src/main/java/org/apache/doris/catalog/OlapTable.java      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 50b1dc6ab3954f..e1bc647d7cec5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2884,6 +2884,9 @@ public boolean getEnableUniqueKeyMergeOnWrite() {
         if (tableProperty == null) {
             return false;
         }
+        if (getKeysType() != KeysType.UNIQUE_KEYS) {
+            return false;
+        }
         return tableProperty.getEnableUniqueKeyMergeOnWrite();
     }
 

From 43390acc7b08d8ec6f129d035aad20d24a4a8980 Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Tue, 3 Dec 2024 11:46:56 +0530
Subject: [PATCH 131/399] [Enhancement] (nereids)implement
 showEncryptKeysCommand in nereids (#44717)

Issue Number: close #42774
---
 .../org/apache/doris/nereids/DorisParser.g4   |   3 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  17 +++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../commands/ShowEncryptKeysCommand.java      | 134 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../test_nereids_encrypt_test.groovy          |   4 +-
 6 files changed, 162 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowEncryptKeysCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index cc5c223f9ec0dd..eac4c661de920d 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -235,6 +235,8 @@ supportedShowStatement
         ((FROM | IN) database=identifier)?                                          #showView
     | SHOW PLUGINS                                                                  #showPlugins    
     | SHOW REPOSITORIES                                                             #showRepositories
+    | SHOW ENCRYPTKEYS ((FROM | IN) database=multipartIdentifier)?
+        (LIKE STRING_LITERAL)?                                                      #showEncryptKeys    
     | SHOW BRIEF? CREATE TABLE name=multipartIdentifier                             #showCreateTable
     | SHOW FULL? PROCESSLIST                                                        #showProcessList
     | SHOW ROLES                                                                    #showRoles        
@@ -352,7 +354,6 @@ unsupportedShowStatement
     | SHOW TRANSACTION ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showTransaction
     | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL                                 #showQueryProfile
     | SHOW CACHE HOTSPOT tablePath=STRING_LITERAL                                   #showCacheHotSpot
-    | SHOW ENCRYPTKEYS ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showEncryptKeys
     | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)?                     #showSyncJob
     | SHOW TABLE CREATION ((FROM | IN) database=multipartIdentifier)? wildWhere?    #showTableCreation
     | SHOW CATALOG RECYCLE BIN wildWhere?                                           #showCatalogRecycleBin
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 7ec8ce62f3001e..401d18775af106 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -243,6 +243,7 @@
 import org.apache.doris.nereids.DorisParser.ShowDeleteContext;
 import org.apache.doris.nereids.DorisParser.ShowDiagnoseTabletContext;
 import org.apache.doris.nereids.DorisParser.ShowDynamicPartitionContext;
+import org.apache.doris.nereids.DorisParser.ShowEncryptKeysContext;
 import org.apache.doris.nereids.DorisParser.ShowEventsContext;
 import org.apache.doris.nereids.DorisParser.ShowFrontendsContext;
 import org.apache.doris.nereids.DorisParser.ShowGrantsContext;
@@ -538,6 +539,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowEncryptKeysCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowEventsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowFrontendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowGrantsCommand;
@@ -1903,6 +1905,21 @@ public static String stripQuotes(String str) {
         return str;
     }
 
+    @Override
+    public LogicalPlan visitShowEncryptKeys(ShowEncryptKeysContext ctx) {
+        String dbName = null;
+        if (ctx.database != null) {
+            List<String> nameParts = visitMultipartIdentifier(ctx.database);
+            dbName = nameParts.get(0); // only one entry possible
+        }
+
+        String likeString = null;
+        if (ctx.LIKE() != null) {
+            likeString = stripQuotes(ctx.STRING_LITERAL().getText());
+        }
+        return new ShowEncryptKeysCommand(dbName, likeString);
+    }
+
     @Override
     public LogicalPlan visitAliasedQuery(AliasedQueryContext ctx) {
         if (ctx.tableAlias().getText().equals("")) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 8061309e8b137c..48e64acaab6377 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -210,6 +210,7 @@ public enum PlanType {
     SHOW_DELETE_COMMAND,
     SHOW_DIAGNOSE_TABLET_COMMAND,
     SHOW_DYNAMIC_PARTITION_COMMAND,
+    SHOW_ENCRYPT_KEYS_COMMAND,
     SHOW_EVENTS_COMMAND,
     SHOW_FRONTENDS_COMMAND,
     SHOW_GRANTS_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowEncryptKeysCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowEncryptKeysCommand.java
new file mode 100644
index 00000000000000..c72d0437bf55d8
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowEncryptKeysCommand.java
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.EncryptKey;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.ListComparator;
+import org.apache.doris.common.util.OrderByPair;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Represents the command for SHOW ENCRYPTKEYS.
+ */
+public class ShowEncryptKeysCommand extends ShowCommand {
+    private static final ShowResultSetMetaData META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("EncryptKey Name", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("EncryptKey String", ScalarType.createVarchar(1024)))
+                    .build();
+
+    private String dbName;
+    private final String likeString;
+
+    public ShowEncryptKeysCommand(String databaseName, String likeString) {
+        super(PlanType.SHOW_ENCRYPT_KEYS_COMMAND);
+        this.dbName = databaseName;
+        this.likeString = likeString;
+    }
+
+    private void validate(ConnectContext ctx) throws AnalysisException {
+        if (Strings.isNullOrEmpty(dbName)) {
+            dbName = ctx.getDatabase();
+            if (Strings.isNullOrEmpty(dbName)) {
+                ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
+            }
+        }
+
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR,
+                    PrivPredicate.ADMIN.getPrivs().toString());
+        }
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        Util.prohibitExternalCatalog(ctx.getDefaultCatalog(), this.getClass().getSimpleName());
+        DatabaseIf db = ctx.getCurrentCatalog().getDbOrAnalysisException(dbName);
+        List<List<String>> resultRowSet = Lists.newArrayList();
+        if (db instanceof Database) {
+            List<EncryptKey> encryptKeys = ((Database) db).getEncryptKeys();
+            List<List<Comparable>> rowSet = Lists.newArrayList();
+            for (EncryptKey encryptKey : encryptKeys) {
+                List<Comparable> row = encryptKey.getInfo();
+                // like predicate
+                if (likeString == null || like(encryptKey.getEncryptKeyName().getKeyName())) {
+                    rowSet.add(row);
+                }
+
+            }
+
+            // sort function rows by first column asc
+            ListComparator<List<Comparable>> comparator = null;
+            OrderByPair orderByPair = new OrderByPair(0, false);
+            comparator = new ListComparator<>(orderByPair);
+            Collections.sort(rowSet, comparator);
+
+            Set<String> encryptKeyNameSet = new HashSet<>();
+            for (List<Comparable> row : rowSet) {
+                List<String> resultRow = Lists.newArrayList();
+                for (Comparable column : row) {
+                    resultRow.add(column.toString());
+                }
+                resultRowSet.add(resultRow);
+                encryptKeyNameSet.add(resultRow.get(0));
+            }
+        }
+
+        return new ShowResultSet(META_DATA, resultRowSet);
+    }
+
+    private boolean like(String str) {
+        str = str.toLowerCase();
+        return str.matches(likeString.replace(".", "\\.").replace("?", ".").replace("%", ".*").toLowerCase());
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowEncryptKeysCommand(this, context);
+    }
+
+    public ShowResultSetMetaData getMetaData() {
+        return META_DATA;
+    }
+}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index fe9d27d9f303a1..8561c934ec976f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -90,6 +90,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowEncryptKeysCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowEventsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowFrontendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowGrantsCommand;
@@ -533,6 +534,10 @@ default R visitShowTableIdCommand(ShowTableIdCommand showTableIdCommand, C conte
         return visitCommand(showTableIdCommand, context);
     }
 
+    default R visitShowEncryptKeysCommand(ShowEncryptKeysCommand showEncryptKeysCommand, C context) {
+        return visitCommand(showEncryptKeysCommand, context);
+    }
+
     default R visitSyncCommand(SyncCommand syncCommand, C context) {
         return visitCommand(syncCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy b/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy
index 8954090bb4c31d..bd984e4164c37b 100644
--- a/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy
+++ b/regression-test/suites/nereids_p0/test_nereids_encrypt_test.groovy
@@ -21,9 +21,11 @@ suite("test_nereids_encrypt_test") {
     sql """ use ${dbName}; """
     checkNereidsExecute("drop encryptkey if exists ${encryptkeyName}")    
     checkNereidsExecute("""CREATE ENCRYPTKEY ${encryptkeyName} AS "ABCD123456789";""")
+    checkNereidsExecute("SHOW ENCRYPTKEYS FROM ${dbName}")
     qt_check_encrypt_1("SHOW ENCRYPTKEYS FROM ${dbName}")
     checkNereidsExecute("drop encryptkey ${encryptkeyName}")
+    checkNereidsExecute("SHOW ENCRYPTKEYS FROM ${dbName}")
     qt_check_encrypt_2("SHOW ENCRYPTKEYS FROM ${dbName}")    
     checkNereidsExecute("drop encryptkey if exists ${encryptkeyName}")
     qt_check_encrypt_3("SHOW ENCRYPTKEYS FROM ${dbName}")        
-}
\ No newline at end of file
+}

From 8b94b9e5d4231bf12364c408555aeb9bc649d2bb Mon Sep 17 00:00:00 2001
From: seawinde <wusi@selectdb.com>
Date: Tue, 3 Dec 2024 14:21:20 +0800
Subject: [PATCH 132/399] [fix](mtmv) Fix filter position different but same
 causing rewritten by materialized view fail (#44575)

Such as mv def is as following and query is the same
this is the same filter `l_orderkey is null or l_orderkey <> 1` but they
are in the different position. this would cause rewrite fail, this pr
fix this.
```sql
select
  o_custkey,
  o_orderdate,
  o_shippriority,
  o_comment,
  o_orderkey,
  orders.public_col as col1,
  l_orderkey,
  l_partkey,
  l_suppkey,
  lineitem.public_col as col2,
  ps_partkey,
  ps_suppkey,
  partsupp.public_col as col3,
  partsupp.public_col * 2 as col4,
  o_orderkey + l_orderkey + ps_partkey * 2,
  sum(
    o_orderkey + l_orderkey + ps_partkey * 2
  ),
  count() as count_all
from
  (
    select
      o_custkey,
      o_orderdate,
      o_shippriority,
      o_comment,
      o_orderkey,
      orders.public_col as public_col
    from
      orders
  ) orders
  left join (
    select
      l_orderkey,
      l_partkey,
      l_suppkey,
      lineitem.public_col as public_col
    from
      lineitem
    where
      l_orderkey is null
      or l_orderkey <> 1
  ) lineitem on l_orderkey = o_orderkey
  inner join (
    select
      ps_partkey,
      ps_suppkey,
      partsupp.public_col as public_col
    from
      partsupp
  ) partsupp on ps_partkey = o_orderkey
  and ps_suppkey = o_custkey
where
  l_orderkey is null
  or l_orderkey <> 1
group by
  1,
  2,
  3,
  4,
  5,
  6,
  7,
  8,
  9,
  10,
  11,
  12,
  13,
  14;
```
Related PR: #34753
Fix filter position different but same causing rewritten by materialized
view fail
---
 .../jobs/joinorder/hypergraph/edge/Edge.java  |   7 +-
 .../exploration/mv/HyperGraphComparator.java  |  73 ++++++-
 .../aggregate_without_roll_up.out             |   8 +
 .../aggregate_without_roll_up.groovy          | 191 ++++++++++++++++--
 4 files changed, 249 insertions(+), 30 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
index 35694169498870..7698d881c661aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
@@ -42,7 +42,9 @@ public abstract class Edge {
     // added by the graph simplifier.
     private final long leftRequiredNodes;
     private final long rightRequiredNodes;
+    // The nodes needed which to prevent wrong association or l-association
     private long leftExtendedNodes;
+    // The nodes needed which to prevent wrong association or r-association
     private long rightExtendedNodes;
 
     // record the left child edges and right child edges in origin plan tree
@@ -53,8 +55,11 @@ public abstract class Edge {
     private final BitSet curOperatorEdges = new BitSet();
     // record all sub nodes behind in this operator. It's T function in paper
     private final long subTreeNodes;
-
+    // The edges which prevents association or l-association when join edge
+    // and prevents push down or pull up when filter edge in the left of edge
     private final Set<JoinEdge> leftRejectEdges;
+    // The edges which prevents association or r-association
+    // and prevents push down or pull up when filter edge in the right of edge
     private final Set<JoinEdge> rightRejectEdges;
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
index 868f97949c0705..d4594583c314c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
@@ -35,12 +35,16 @@
 import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.JoinUtils;
 
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -411,25 +415,72 @@ private Map<Edge, Edge> constructQueryToViewJoinMapWithExpr() {
         return edgeMap;
     }
 
+    // Such as the filter as following, their expression is same, but should be different filter edge
+    // Only construct edge that can mapping, the edges which can not mapping would be handled by buildComparisonRes
+    //     LogicalJoin[569]
+    //       |--LogicalProject[567]
+    //       |  +--LogicalFilter[566] ( predicates=(l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1))) )
+    //       |     +--LogicalJoin[565]
+    //       |        |--LogicalProject[562]
+    //       |        |  +--LogicalOlapScan
+    //       |        +--LogicalProject[564]
+    //       |           +--LogicalFilter[563] ( predicates=(l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1))))
+    //       |              +--LogicalOlapScan
+    //       +--LogicalProject[568]
+    //         +--LogicalOlapScan
     private Map<Edge, Edge> constructQueryToViewFilterMapWithExpr() {
-        Map<Expression, Edge> viewExprToEdge = getViewFilterEdges().stream()
+        Multimap<Expression, Edge> viewExprToEdge = HashMultimap.create();
+        getViewFilterEdges().stream()
                 .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
-                .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
-        Map<Expression, Edge> queryExprToEdge = getQueryFilterEdges().stream()
+                .forEach(pair -> viewExprToEdge.put(pair.key(), pair.value()));
+
+        Multimap<Expression, Edge> queryExprToEdge = HashMultimap.create();
+        getQueryFilterEdges().stream()
                 .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
-                .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
+                .forEach(pair -> queryExprToEdge.put(pair.key(), pair.value()));
 
-        HashMap<Edge, Edge> edgeMap = new HashMap<>();
-        for (Entry<Expression, Edge> entry : queryExprToEdge.entrySet()) {
-            if (edgeMap.containsKey(entry.getValue())) {
+        HashMap<Edge, Edge> queryToViewEdgeMap = new HashMap<>();
+        for (Entry<Expression, Collection<Edge>> entry : queryExprToEdge.asMap().entrySet()) {
+            Expression queryExprViewBased = logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey());
+            if (queryExprViewBased == null) {
                 continue;
             }
-            Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey());
-            if (viewExprToEdge.containsKey(viewExpr)) {
-                edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
+            Collection<Edge> viewEdges = viewExprToEdge.get(queryExprViewBased);
+            if (viewEdges.isEmpty()) {
+                continue;
+            }
+            for (Edge queryEdge : entry.getValue()) {
+                for (Edge viewEdge : viewEdges) {
+                    if (!isSubTreeNodesEquals(queryEdge, viewEdge, logicalCompatibilityContext)) {
+                        // Such as query filter edge is <{1} --FILTER-- {}> but view filter edge is
+                        // <{0, 1} --FILTER-- {}>, though they are all
+                        // l_orderkey#10 IS NULL OR ( not (l_orderkey#10 = 1)) but they are different actually
+                        continue;
+                    }
+                    queryToViewEdgeMap.put(queryEdge, viewEdge);
+                }
             }
         }
-        return edgeMap;
+        return queryToViewEdgeMap;
+    }
+
+    private static boolean isSubTreeNodesEquals(Edge queryEdge, Edge viewEdge,
+            LogicalCompatibilityContext logicalCompatibilityContext) {
+        if (!(queryEdge instanceof FilterEdge) || !(viewEdge instanceof FilterEdge)) {
+            return false;
+        }
+        // subTreeNodes should be equal
+        BiMap<Integer, Integer> queryToViewNodeIdMapping =
+                logicalCompatibilityContext.getQueryToViewNodeIDMapping();
+        List<Integer> queryNodeIndexViewBasedList = new ArrayList<>();
+        for (int queryNodeIndex : LongBitmap.getIterator(queryEdge.getSubTreeNodes())) {
+            Integer queryNodeIndexViewBased = queryToViewNodeIdMapping.get(queryNodeIndex);
+            if (queryNodeIndexViewBased == null) {
+                return false;
+            }
+            queryNodeIndexViewBasedList.add(queryNodeIndexViewBased);
+        }
+        return LongBitmap.newBitmap(queryNodeIndexViewBasedList) == viewEdge.getSubTreeNodes();
     }
 
     private void refreshViewEdges() {
diff --git a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
index 3b9c3a1219ea15..5c9df6b7f92256 100644
--- a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
+++ b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
@@ -315,3 +315,11 @@ a	3	3	a,a,a	4.0	yy	3	1
 a	4	2	a,a	4.0	yy	2	1
 c	3	6	c,c,c	5.333333333333333	mi	3	2
 
+-- !query28_0_before --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
+-- !query28_0_after --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
index 92f54f3dd456e8..3af41bda69065f 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
@@ -36,7 +36,8 @@ suite("aggregate_without_roll_up") {
       o_orderpriority  CHAR(15) NOT NULL,  
       o_clerk          CHAR(15) NOT NULL, 
       o_shippriority   INTEGER NOT NULL,
-      o_comment        VARCHAR(79) NOT NULL
+      o_comment        VARCHAR(79) NOT NULL,
+      public_col       INT NULL
     )
     DUPLICATE KEY(o_orderkey, o_custkey)
     DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
@@ -66,7 +67,8 @@ suite("aggregate_without_roll_up") {
       l_receiptdate DATE NOT NULL,
       l_shipinstruct CHAR(25) NOT NULL,
       l_shipmode     CHAR(10) NOT NULL,
-      l_comment      VARCHAR(44) NOT NULL
+      l_comment      VARCHAR(44) NOT NULL,
+      public_col       INT NULL
     )
     DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber)
     DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3
@@ -85,7 +87,8 @@ suite("aggregate_without_roll_up") {
       ps_suppkey     INTEGER NOT NULL,
       ps_availqty    INTEGER NOT NULL,
       ps_supplycost  DECIMALV3(15,2)  NOT NULL,
-      ps_comment     VARCHAR(199) NOT NULL 
+      ps_comment     VARCHAR(199) NOT NULL,
+      public_col       INT NULL 
     )
     DUPLICATE KEY(ps_partkey, ps_suppkey)
     DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3
@@ -96,29 +99,29 @@ suite("aggregate_without_roll_up") {
 
     sql """
     insert into lineitem values
-    (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
-    (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
-    (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
-    (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
-    (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');
+    (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy', 1),
+    (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy', null),
+    (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy', 2),
+    (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy', null),
+    (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', 3);
     """
 
     sql """
     insert into orders values
-    (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
-    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
-    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
-    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
-    (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
-    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
-    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
-    (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi');  
+    (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy', 1),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy', null),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy', 2),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy', null),
+    (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy', 3),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm', null),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi', 4),
+    (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi', null);  
     """
 
     sql """
     insert into partsupp values
-    (2, 3, 9, 10.01, 'supply1'),
-    (2, 3, 10, 11.01, 'supply2');
+    (2, 3, 9, 10.01, 'supply1', 1),
+    (2, 3, 10, 11.01, 'supply2', null);
     """
 
     sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
@@ -1362,4 +1365,156 @@ suite("aggregate_without_roll_up") {
     """
     async_mv_rewrite_fail(db, mv27_0, query27_0, "mv27_0")
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv27_0"""
+
+
+   // query and mv has the same filter but position is different, should rewrite successfully
+    def mv28_0 = """
+    select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey, 
+      orders.public_col as col1, 
+      l_orderkey, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey + l_orderkey + ps_partkey * 2, 
+      sum(
+        o_orderkey + l_orderkey + ps_partkey * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          l_orderkey is null 
+          or l_orderkey <> 8
+      ) lineitem on l_orderkey = o_orderkey 
+      inner join (
+        select 
+          ps_partkey, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on ps_partkey = o_orderkey 
+    where 
+      l_orderkey is null 
+      or l_orderkey <> 8
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    def query28_0 = """
+    select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey, 
+      orders.public_col as col1, 
+      l_orderkey, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey + l_orderkey + ps_partkey * 2, 
+      sum(
+        o_orderkey + l_orderkey + ps_partkey * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          l_orderkey is null 
+          or l_orderkey <> 8
+      ) lineitem on l_orderkey = o_orderkey 
+      inner join (
+        select 
+          ps_partkey, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on ps_partkey = o_orderkey 
+    where 
+      l_orderkey is null 
+      or l_orderkey <> 8 
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    order_qt_query28_0_before "${query28_0}"
+    async_mv_rewrite_success(db, mv28_0, query28_0, "mv28_0")
+    order_qt_query28_0_after "${query28_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv28_0"""
+
 }

From 4ce20d129f3a8b14c8c23aa45d76ebdbabe281ea Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Tue, 3 Dec 2024 14:42:16 +0800
Subject: [PATCH 133/399] [improve](ut) update data type ut and take datatype
 ip for example (#44642)

update data type ut and take datatype ip for example
---
 .../data_types/common_data_type_serder_test.h | 366 ++++++++++++++++++
 .../vec/data_types/common_data_type_test.h    | 228 +++++++++++
 be/test/vec/data_types/data_type_ip_test.cpp  | 252 ++++++++++++
 3 files changed, 846 insertions(+)
 create mode 100644 be/test/vec/data_types/common_data_type_serder_test.h
 create mode 100644 be/test/vec/data_types/common_data_type_test.h
 create mode 100644 be/test/vec/data_types/data_type_ip_test.cpp

diff --git a/be/test/vec/data_types/common_data_type_serder_test.h b/be/test/vec/data_types/common_data_type_serder_test.h
new file mode 100644
index 00000000000000..46206d5ed7eb15
--- /dev/null
+++ b/be/test/vec/data_types/common_data_type_serder_test.h
@@ -0,0 +1,366 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <arrow/record_batch.h>
+#include <gen_cpp/data.pb.h>
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+
+#include "olap/schema.h"
+#include "runtime/descriptors.cpp"
+#include "runtime/descriptors.h"
+#include "util/arrow/block_convertor.h"
+#include "util/arrow/row_batch.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/columns/columns_number.h"
+#include "vec/core/field.h"
+#include "vec/core/sort_block.h"
+#include "vec/core/sort_description.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_map.h"
+#include "vec/utils/arrow_column_to_doris_column.h"
+
+// this test is gonna to be a data type serialize and deserialize functions
+// such as
+// 1. standard hive text ser-deserialize
+//  deserialize_one_cell_from_hive_text (IColumn &column, Slice &slice, const FormatOptions &options, int hive_text_complex_type_delimiter_level=1) const
+//  deserialize_column_from_hive_text_vector (IColumn &column, std::vector< Slice > &slices, int *num_deserialized, const FormatOptions &options, int hive_text_complex_type_delimiter_level=1) const
+//  serialize_one_cell_to_hive_text (const IColumn &column, int row_num, BufferWritable &bw, FormatOptions &options, int hive_text_complex_type_delimiter_level=1) const
+// 2. json format ser-deserialize which used in table not in doris database
+//  serialize_one_cell_to_json (const IColumn &column, int row_num, BufferWritable &bw, FormatOptions &options) const =0
+//  serialize_column_to_json (const IColumn &column, int start_idx, int end_idx, BufferWritable &bw, FormatOptions &options) const =0
+//  deserialize_one_cell_from_json (IColumn &column, Slice &slice, const FormatOptions &options) const =0
+//  deserialize_column_from_json_vector (IColumn &column, std::vector< Slice > &slices, int *num_deserialized, const FormatOptions &options) const =0
+//  deserialize_column_from_fixed_json (IColumn &column, Slice &slice, int rows, int *num_deserialized, const FormatOptions &options) const
+//  insert_column_last_value_multiple_times (IColumn &column, int times) const
+// 3. fe|be protobuffer ser-deserialize
+//  write_column_to_pb (const IColumn &column, PValues &result, int start, int end) const =0
+//  read_column_from_pb (IColumn &column, const PValues &arg) const =0
+// 4. jsonb ser-deserialize which used in row-store situation
+//  write_one_cell_to_jsonb (const IColumn &column, JsonbWriter &result, Arena *mem_pool, int32_t col_id, int row_num) const =0
+//  read_one_cell_from_jsonb (IColumn &column, const JsonbValue *arg) const =0
+// 5. mysql text ser-deserialize
+//  write_column_to_mysql (const IColumn &column, MysqlRowBuffer< false > &row_buffer, int row_idx, bool col_const, const FormatOptions &options) const =0
+//  write_column_to_mysql (const IColumn &column, MysqlRowBuffer< true > &row_buffer, int row_idx, bool col_const, const FormatOptions &options) const =0
+// 6. arrow ser-deserialize which used in spark-flink connector
+//  write_column_to_arrow (const IColumn &column, const NullMap *null_map, arrow::ArrayBuilder *array_builder, int start, int end, const cctz::time_zone &ctz) const =0
+//  read_column_from_arrow (IColumn &column, const arrow::Array *arrow_array, int start, int end, const cctz::time_zone &ctz) const =0
+// 7. rapidjson ser-deserialize
+//  write_one_cell_to_json (const IColumn &column, rapidjson::Value &result, rapidjson::Document::AllocatorType &allocator, Arena &mem_pool, int row_num) const
+//  read_one_cell_from_json (IColumn &column, const rapidjson::Value &result) const
+//  convert_field_to_rapidjson (const vectorized::Field &field, rapidjson::Value &target, rapidjson::Document::AllocatorType &allocator)
+//  convert_array_to_rapidjson (const vectorized::Array &array, rapidjson::Value &target, rapidjson::Document::AllocatorType &allocator)
+
+namespace doris::vectorized {
+
+class CommonDataTypeSerdeTest : public ::testing::Test {
+public:
+    ////==================================================================================================================
+    // this is common function to check data in column against expected results according different function in assert function
+    // which can be used in all column test
+    // such as run regress tests
+    //  step1. we can set gen_check_data_in_assert to true, then we will generate a file for check data, otherwise we will read the file to check data
+    //  step2. we should write assert callback function to check data
+    static void check_data(
+            MutableColumns& columns, DataTypeSerDeSPtrs serders, char col_spliter,
+            std::set<int> idxes, const std::string& column_data_file,
+            std::function<void(MutableColumns& load_cols, DataTypeSerDeSPtrs serders)>
+                    assert_callback,
+            bool is_hive_format = false, DataTypes dataTypes = {}) {
+        ASSERT_EQ(serders.size(), columns.size());
+        // Step 1: Insert data from `column_data_file` into the column and check result with `check_data_file`
+        // Load column data and expected data from CSV files
+        std::vector<std::vector<std::string>> res;
+        struct stat buff;
+        if (stat(column_data_file.c_str(), &buff) == 0) {
+            if (S_ISREG(buff.st_mode)) {
+                // file
+                if (is_hive_format) {
+                    load_data_and_assert_from_csv<true, true>(serders, columns, column_data_file,
+                                                              col_spliter, idxes);
+                } else {
+                    load_data_and_assert_from_csv<false, true>(serders, columns, column_data_file,
+                                                               col_spliter, idxes);
+                }
+            } else if (S_ISDIR(buff.st_mode)) {
+                // dir
+                std::filesystem::path fs_path(column_data_file);
+                for (const auto& entry : std::filesystem::directory_iterator(fs_path)) {
+                    std::string file_path = entry.path().string();
+                    std::cout << "load data from file: " << file_path << std::endl;
+                    if (is_hive_format) {
+                        load_data_and_assert_from_csv<true, true>(serders, columns, file_path,
+                                                                  col_spliter, idxes);
+                    } else {
+                        load_data_and_assert_from_csv<false, true>(serders, columns, file_path,
+                                                                   col_spliter, idxes);
+                    }
+                }
+            }
+        }
+
+        // Step 2: Validate the data in `column` matches `expected_data`
+        assert_callback(columns, serders);
+    }
+
+    // Helper function to load data from CSV, with index which splited by spliter and load to columns
+    template <bool is_hive_format, bool generate_res_file>
+    static void load_data_and_assert_from_csv(const DataTypeSerDeSPtrs serders,
+                                              MutableColumns& columns, const std::string& file_path,
+                                              const char spliter = ';',
+                                              const std::set<int> idxes = {0}) {
+        ASSERT_EQ(serders.size(), columns.size())
+                << "serder size: " << serders.size() << " column size: " << columns.size();
+        ASSERT_EQ(serders.size(), idxes.size())
+                << "serder size: " << serders.size() << " idxes size: " << idxes.size();
+        std::ifstream file(file_path);
+        if (!file) {
+            throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "can not open the file: {} ",
+                                   file_path);
+        }
+
+        std::string line;
+        DataTypeSerDe::FormatOptions options;
+        std::vector<std::vector<std::string>> res;
+        MutableColumns assert_str_cols(columns.size());
+        for (size_t i = 0; i < columns.size(); ++i) {
+            assert_str_cols[i] = ColumnString::create();
+        }
+
+        while (std::getline(file, line)) {
+            std::stringstream lineStream(line);
+            //            std::cout << "whole : " << lineStream.str() << std::endl;
+            std::string value;
+            int l_idx = 0;
+            int c_idx = 0;
+            std::vector<string> row;
+            while (std::getline(lineStream, value, spliter)) {
+                if (idxes.contains(l_idx)) {
+                    // load csv data
+                    Slice string_slice(value.data(), value.size());
+                    std::cout << "origin : " << string_slice << std::endl;
+                    Status st;
+                    // deserialize data
+                    if constexpr (is_hive_format) {
+                        st = serders[c_idx]->deserialize_one_cell_from_hive_text(
+                                *columns[c_idx], string_slice, options);
+                    } else {
+                        st = serders[c_idx]->deserialize_one_cell_from_json(*columns[c_idx],
+                                                                            string_slice, options);
+                    }
+                    if (!st.ok()) {
+                        // deserialize if happen error now we do not insert any value for input column
+                        // so we push a default value to column for row alignment
+                        columns[c_idx]->insert_default();
+                        std::cout << "error in deserialize but continue: " << st.to_string()
+                                  << std::endl;
+                    }
+                    // serialize data
+                    size_t row_num = columns[c_idx]->size() - 1;
+                    assert_str_cols[c_idx]->reserve(columns[c_idx]->size());
+                    VectorBufferWriter bw(assert_cast<ColumnString&>(*assert_str_cols[c_idx]));
+                    if constexpr (is_hive_format) {
+                        st = serders[c_idx]->serialize_one_cell_to_hive_text(*columns[c_idx],
+                                                                             row_num, bw, options);
+                        EXPECT_TRUE(st.ok()) << st.to_string();
+                    } else {
+                        st = serders[c_idx]->serialize_one_cell_to_json(*columns[c_idx], row_num,
+                                                                        bw, options);
+                        EXPECT_TRUE(st.ok()) << st.to_string();
+                    }
+                    bw.commit();
+                    // assert data : origin data and serialized data should be equal or generated
+                    // file to check data
+                    size_t assert_size = assert_str_cols[c_idx]->size();
+                    if constexpr (!generate_res_file) {
+                        EXPECT_EQ(assert_str_cols[c_idx]->get_data_at(assert_size - 1).to_string(),
+                                  string_slice.to_string())
+                                << "column: " << columns[c_idx]->get_name() << " row: " << row_num
+                                << " is_hive_format: " << is_hive_format;
+                    }
+                    ++c_idx;
+                }
+                res.push_back(row);
+                ++l_idx;
+            }
+        }
+
+        if (generate_res_file) {
+            // generate res
+            auto pos = file_path.find_last_of(".");
+            string hive_format = is_hive_format ? "_hive" : "";
+            std::string res_file = file_path.substr(0, pos) + hive_format + "_serde_res.csv";
+            std::ofstream res_f(res_file);
+            if (!res_f.is_open()) {
+                throw std::ios_base::failure("Failed to open file." + res_file);
+            }
+            for (size_t r = 0; r < assert_str_cols[0]->size(); ++r) {
+                for (size_t c = 0; c < assert_str_cols.size(); ++c) {
+                    std::cout << assert_str_cols[c]->get_data_at(r).to_string() << spliter
+                              << std::endl;
+                    res_f << assert_str_cols[c]->get_data_at(r).to_string() << spliter;
+                }
+                res_f << std::endl;
+            }
+            res_f.close();
+            std::cout << "generate res file: " << res_file << std::endl;
+        }
+    }
+
+    // standard hive text ser-deserialize assert function
+    static void assert_pb_format(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& col = load_cols[i];
+            std::cout << " now we are testing column : " << col->get_name() << std::endl;
+            // serialize to pb
+            PValues pv = PValues();
+            Status st = serders[i]->write_column_to_pb(*col, pv, 0, col->size());
+            if (!st.ok()) {
+                std::cerr << "write_column_to_pb error: " << st.msg() << std::endl;
+                continue;
+            }
+            // deserialize from pb
+            auto except_column = col->clone_empty();
+            st = serders[i]->read_column_from_pb(*except_column, pv);
+            EXPECT_TRUE(st.ok()) << st.to_string();
+            // check pb value from expected column
+            PValues as_pv = PValues();
+            st = serders[i]->write_column_to_pb(*except_column, as_pv, 0, except_column->size());
+            EXPECT_TRUE(st.ok()) << st.to_string();
+            EXPECT_EQ(pv.bytes_value_size(), as_pv.bytes_value_size());
+            // check column value
+            for (size_t j = 0; j < col->size(); ++j) {
+                auto cell = col->operator[](j);
+                auto except_cell = except_column->operator[](j);
+                EXPECT_EQ(cell, except_cell) << "column: " << col->get_name() << " row: " << j;
+            }
+        }
+    }
+
+    // actually this is block_to_jsonb and jsonb_to_block test
+    static void assert_jsonb_format(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        Arena pool;
+        auto jsonb_column = ColumnString::create(); // jsonb column
+        jsonb_column->reserve(load_cols[0]->size());
+        MutableColumns assert_cols;
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            assert_cols.push_back(load_cols[i]->assume_mutable());
+        }
+        for (size_t r = 0; r < load_cols[0]->size(); ++r) {
+            JsonbWriterT<JsonbOutStream> jw;
+            jw.writeStartObject();
+            // serialize to jsonb
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                auto& col = load_cols[i];
+                serders[i]->write_one_cell_to_jsonb(*col, jw, &pool, i, r);
+            }
+            jw.writeEndObject();
+            jsonb_column->insert_data(jw.getOutput()->getBuffer(), jw.getOutput()->getSize());
+        }
+        // deserialize jsonb column to assert column
+        EXPECT_EQ(jsonb_column->size(), load_cols[0]->size());
+        for (size_t r = 0; r < jsonb_column->size(); ++r) {
+            StringRef jsonb_data = jsonb_column->get_data_at(r);
+            auto pdoc = JsonbDocument::createDocument(jsonb_data.data, jsonb_data.size);
+            JsonbDocument& doc = *pdoc;
+            size_t cIdx = 0;
+            for (auto it = doc->begin(); it != doc->end(); ++it) {
+                serders[cIdx]->read_one_cell_from_jsonb(*assert_cols[cIdx], it->value());
+                ++cIdx;
+            }
+        }
+        // check column value
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& col = load_cols[i];
+            auto& assert_col = assert_cols[i];
+            for (size_t j = 0; j < col->size(); ++j) {
+                auto cell = col->operator[](j);
+                auto assert_cell = assert_col->operator[](j);
+                EXPECT_EQ(cell, assert_cell) << "column: " << col->get_name() << " row: " << j;
+            }
+        }
+    }
+
+    // assert mysql text format, now we just simple assert not to fatal or exception here
+    static void assert_mysql_format(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        MysqlRowBuffer<false> row_buffer;
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& col = load_cols[i];
+            for (size_t j = 0; j < col->size(); ++j) {
+                Status st;
+                EXPECT_NO_FATAL_FAILURE(
+                        st = serders[i]->write_column_to_mysql(*col, row_buffer, j, false, {}));
+                EXPECT_TRUE(st.ok()) << st.to_string();
+            }
+        }
+    }
+
+    // assert arrow serialize
+    static void assert_arrow_format(MutableColumns& load_cols, DataTypeSerDeSPtrs serders,
+                                    DataTypes types) {
+        // make a block to write to arrow
+        auto block = std::make_shared<Block>();
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& col = load_cols[i];
+            block->insert(ColumnWithTypeAndName(std::move(col), types[i], types[i]->get_name()));
+        }
+        // print block
+        std::cout << "block: " << block->dump_structure() << std::endl;
+        std::shared_ptr<arrow::Schema> block_arrow_schema;
+        EXPECT_EQ(get_arrow_schema_from_block(*block, &block_arrow_schema, "UTC"), Status::OK());
+        // convert block to arrow
+        std::shared_ptr<arrow::RecordBatch> result;
+        cctz::time_zone _timezone_obj; //default UTC
+        Status stt = convert_to_arrow_batch(*block, block_arrow_schema,
+                                            arrow::default_memory_pool(), &result, _timezone_obj);
+        EXPECT_EQ(Status::OK(), stt) << "convert block to arrow failed" << stt.to_string();
+
+        // deserialize arrow to block
+        auto assert_block = block->clone_empty();
+        auto rows = block->rows();
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto array = result->column(i);
+            auto& column_with_type_and_name = assert_block.get_by_position(i);
+            auto ret = arrow_column_to_doris_column(
+                    array.get(), 0, column_with_type_and_name.column,
+                    column_with_type_and_name.type, rows, _timezone_obj);
+            // do check data
+            EXPECT_EQ(Status::OK(), ret) << "convert arrow to block failed" << ret.to_string();
+            auto& col = block->get_by_position(i).column;
+            auto& assert_col = column_with_type_and_name.column;
+            for (size_t j = 0; j < col->size(); ++j) {
+                auto cell = col->operator[](j);
+                auto assert_cell = assert_col->operator[](j);
+                EXPECT_EQ(cell, assert_cell) << "column: " << col->get_name() << " row: " << j;
+            }
+        }
+    }
+
+    // assert rapidjson format
+    // now rapidjson write_one_cell_to_json and read_one_cell_from_json only used in column_object
+    // can just be replaced by jsonb format
+};
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/data_types/common_data_type_test.h b/be/test/vec/data_types/common_data_type_test.h
new file mode 100644
index 00000000000000..0480137ac7f0e4
--- /dev/null
+++ b/be/test/vec/data_types/common_data_type_test.h
@@ -0,0 +1,228 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gen_cpp/data.pb.h>
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+
+#include "olap/schema.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/columns/columns_number.h"
+#include "vec/core/field.h"
+#include "vec/core/sort_block.h"
+#include "vec/core/sort_description.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_map.h"
+
+// this test is gonna to be a data type test template for all DataType which should make ut test to coverage the function defined
+// for example DataTypeIPv4 should test this function:
+// 1. datatype meta info:
+//         get_type_id, get_type_as_type_descriptor, get_storage_field_type, have_subtypes, get_pdata_type (const IDataType *data_type), to_pb_column_meta (PColumnMeta *col_meta)
+//         get_family_name, get_is_parametric, should_align_right_in_pretty_formats
+//         text_can_contain_only_valid_utf8
+//         have_maximum_size_of_value, get_maximum_size_of_value_in_memory, get_size_of_value_in_memory
+//         get_precision, get_scale
+//         get_field
+//         is_null_literal, is_value_represented_by_number, is_value_represented_by_integer, is_value_represented_by_unsigned_integer, is_value_unambiguously_represented_in_contiguous_memory_region, is_value_unambiguously_represented_in_fixed_size_contiguous_memory_region
+// 2. datatype creation with column: create_column, create_column_const (size_t size, const Field &field), create_column_const_with_default_value (size_t size),  get_uncompressed_serialized_bytes (const IColumn &column, int be_exec_version)
+// 3. serde related: get_serde (int nesting_level=1)
+//          to_string (const IColumn &column, size_t row_num, BufferWritable &ostr), to_string (const IColumn &column, size_t row_num), to_string_batch (const IColumn &column, ColumnString &column_to), from_string (ReadBuffer &rb, IColumn *column)
+//          this two function should move to DataTypeSerDe and only used in Block
+//          serialize (const IColumn &column, char *buf, int be_exec_version), deserialize (const char *buf, MutableColumnPtr *column, int be_exec_version)
+// 4. compare: equals (const IDataType &rhs), is_comparable
+
+namespace doris::vectorized {
+
+static bool gen_check_data_in_assert = true;
+
+class CommonDataTypeTest : public ::testing::Test {
+protected:
+    // Helper function to load data from CSV, with index which splited by spliter and load to columns
+    void load_data_from_csv(const DataTypeSerDeSPtrs serders, MutableColumns& columns,
+                            const std::string& file_path, const char spliter = ';',
+                            const std::set<int> idxes = {0}) {
+        ASSERT_EQ(serders.size(), columns.size())
+                << "serder size: " << serders.size() << " column size: " << columns.size();
+        ASSERT_EQ(serders.size(), idxes.size())
+                << "serder size: " << serders.size() << " idxes size: " << idxes.size();
+        std::ifstream file(file_path);
+        if (!file) {
+            throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "can not open the file: {} ",
+                                   file_path);
+        }
+
+        std::string line;
+        DataTypeSerDe::FormatOptions options;
+        while (std::getline(file, line)) {
+            std::stringstream lineStream(line);
+            //            std::cout << "whole : " << lineStream.str() << std::endl;
+            std::string value;
+            int l_idx = 0;
+            int c_idx = 0;
+            while (std::getline(lineStream, value, spliter)) {
+                if (idxes.contains(l_idx)) {
+                    Slice string_slice(value.data(), value.size());
+                    std::cout << string_slice << std::endl;
+                    if (auto st = serders[c_idx]->deserialize_one_cell_from_json(
+                                *columns[c_idx], string_slice, options);
+                        !st.ok()) {
+                        //                        std::cout << "error in deserialize but continue: " << st.to_string()
+                        //                                  << std::endl;
+                    }
+                    ++c_idx;
+                }
+                ++l_idx;
+            }
+        }
+    }
+
+public:
+    // we make meta info a default value, so assert should change the struct value to the right value
+    struct DataTypeMetaInfo {
+        TypeIndex type_id = TypeIndex::Nothing;
+        TypeDescriptor* type_as_type_descriptor = nullptr;
+        std::string family_name = "";
+        bool has_subtypes = false;
+        doris::FieldType storage_field_type = doris::FieldType::OLAP_FIELD_TYPE_UNKNOWN;
+        bool should_align_right_in_pretty_formats = false;
+        bool text_can_contain_only_valid_utf8 = false;
+        bool have_maximum_size_of_value = false;
+        size_t size_of_value_in_memory = -1;
+        size_t precision = -1;
+        size_t scale = -1;
+        bool is_null_literal = true;
+        bool is_value_represented_by_number = false;
+        bool is_value_represented_by_unsigned_integer = false;
+        PColumnMeta* pColumnMeta = nullptr;
+        DataTypeSerDeSPtr serde = nullptr;
+        //        bool is_value_unambiguously_represented_in_contiguous_memory_region = false;
+    };
+    void SetUp() override {}
+
+    // meta info assert is simple and can be used for all DataType
+    void meta_info_assert(DataTypePtr& data_type, DataTypeMetaInfo& meta_info) {
+        ASSERT_NE(data_type->get_serde(1), nullptr);
+        ASSERT_EQ(IDataType::get_pdata_type(data_type.get()), meta_info.pColumnMeta->type());
+        ASSERT_EQ(data_type->get_type_id(), meta_info.type_id);
+        ASSERT_EQ(data_type->get_type_as_type_descriptor(), *meta_info.type_as_type_descriptor);
+        ASSERT_EQ(data_type->get_family_name(), meta_info.family_name);
+        ASSERT_EQ(data_type->have_subtypes(), meta_info.has_subtypes);
+        ASSERT_EQ(data_type->get_storage_field_type(), meta_info.storage_field_type);
+        ASSERT_EQ(data_type->should_align_right_in_pretty_formats(),
+                  meta_info.should_align_right_in_pretty_formats);
+        ASSERT_EQ(data_type->text_can_contain_only_valid_utf8(),
+                  meta_info.text_can_contain_only_valid_utf8);
+        ASSERT_EQ(data_type->have_maximum_size_of_value(), meta_info.have_maximum_size_of_value);
+        ASSERT_EQ(data_type->get_size_of_value_in_memory(), meta_info.size_of_value_in_memory);
+        if (is_decimal(data_type)) {
+            ASSERT_EQ(data_type->get_precision(), meta_info.precision);
+            ASSERT_EQ(data_type->get_scale(), meta_info.scale);
+        } else {
+            EXPECT_ANY_THROW(EXPECT_FALSE(data_type->get_precision()));
+            EXPECT_THROW(EXPECT_FALSE(data_type->get_scale()), doris::Exception);
+        }
+        ASSERT_EQ(data_type->is_null_literal(), meta_info.is_null_literal);
+        ASSERT_EQ(data_type->is_value_represented_by_number(),
+                  meta_info.is_value_represented_by_number);
+        ASSERT_EQ(data_type->is_value_represented_by_unsigned_integer(),
+                  meta_info.is_value_represented_by_unsigned_integer);
+        //        ASSERT_EQ(data_type->is_value_unambiguously_represented_in_contiguous_memory_region(), meta_info.is_value_unambiguously_represented_in_contiguous_memory_region);
+    }
+
+    // create column assert with default field is simple and can be used for all DataType
+    void create_column_assert(DataTypePtr& data_type, Field& default_field) {
+        auto column = data_type->create_column();
+        ASSERT_EQ(column->size(), 0);
+        ColumnPtr const_col = data_type->create_column_const(10, default_field);
+        ASSERT_EQ(const_col->size(), 10);
+        ColumnPtr default_const_col = data_type->create_column_const_with_default_value(10);
+        ASSERT_EQ(default_const_col->size(), 10);
+        for (int i = 0; i < 10; ++i) {
+            ASSERT_EQ(const_col->operator[](i), default_const_col->operator[](i));
+        }
+        // get_uncompressed_serialized_bytes
+        ASSERT_EQ(data_type->get_uncompressed_serialized_bytes(*column, 0), 4);
+    }
+
+    // get_field assert is simple and can be used for all DataType
+    void get_field_assert(DataTypePtr& data_type, TExprNode& node, Field& assert_field,
+                          bool assert_false = false) {
+        if (assert_false) {
+            EXPECT_ANY_THROW(data_type->get_field(node))
+                    << "get_field_assert: "
+                    << " datatype:" + data_type->get_name() << " node_type:" << node.node_type
+                    << " field: " << assert_field.get_type() << std::endl;
+        } else {
+            Field field = data_type->get_field(node);
+            ASSERT_EQ(field, assert_field)
+                    << "get_field_assert: "
+                    << " datatype:" + data_type->get_name() << " node_type:" << node.node_type
+                    << " field: " << assert_field.get_type() << std::endl;
+        }
+    }
+
+    // to_string | to_string_batch | from_string assert is simple and can be used for all DataType
+    void assert_to_string_from_string_assert(MutableColumnPtr mutableColumn,
+                                             DataTypePtr& data_type) {
+        // to_string_batch | from_string
+        auto col_to = ColumnString::create();
+        data_type->to_string_batch(*mutableColumn, *col_to);
+        ASSERT_EQ(col_to->size(), mutableColumn->size());
+        // from_string assert col_to to assert_column and check same with mutableColumn
+        auto assert_column = data_type->create_column();
+        for (int i = 0; i < col_to->size(); ++i) {
+            std::string s = col_to->get_data_at(i).to_string();
+            ReadBuffer rb(s.data(), s.size());
+            ASSERT_EQ(Status::OK(), data_type->from_string(rb, assert_column.get()));
+            ASSERT_EQ(assert_column->operator[](i), mutableColumn->operator[](i));
+        }
+        // to_string | from_string
+        auto ser_col = ColumnString::create();
+        ser_col->reserve(mutableColumn->size());
+        VectorBufferWriter buffer_writer(*ser_col.get());
+        for (int i = 0; i < mutableColumn->size(); ++i) {
+            data_type->to_string(*mutableColumn, i, buffer_writer);
+            buffer_writer.commit();
+        }
+        // check ser_col to assert_column and check same with mutableColumn
+        auto assert_column_1 = data_type->create_column();
+        for (int i = 0; i < ser_col->size(); ++i) {
+            std::string s = ser_col->get_data_at(i).to_string();
+            ReadBuffer rb(s.data(), s.size());
+            ASSERT_EQ(Status::OK(), data_type->from_string(rb, assert_column_1.get()));
+            ASSERT_EQ(assert_column_1->operator[](i), mutableColumn->operator[](i));
+        }
+    }
+
+    // should all datatype is compare?
+    void assert_compare_behavior(DataTypePtr l_dt, DataTypePtr& r_dt) {
+        ASSERT_TRUE(l_dt->is_comparable());
+        ASSERT_TRUE(r_dt->is_comparable());
+        // compare
+        ASSERT_FALSE(l_dt->equals(*r_dt));
+    }
+};
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/data_types/data_type_ip_test.cpp b/be/test/vec/data_types/data_type_ip_test.cpp
new file mode 100644
index 00000000000000..91ae064e4473a3
--- /dev/null
+++ b/be/test/vec/data_types/data_type_ip_test.cpp
@@ -0,0 +1,252 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+
+#include "olap/schema.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/columns/columns_number.h"
+#include "vec/core/field.h"
+#include "vec/core/sort_block.h"
+#include "vec/core/sort_description.h"
+#include "vec/core/types.h"
+#include "vec/data_types/common_data_type_serder_test.h"
+#include "vec/data_types/common_data_type_test.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+
+// this test is gonna to be a data type test template for all DataType which should make ut test to coverage the function defined
+// for example DataTypeIPv4 should test this function:
+// 1. datatype meta info:
+//         get_type_id, get_type_as_type_descriptor, get_storage_field_type, have_subtypes, get_pdata_type (const IDataType *data_type), to_pb_column_meta (PColumnMeta *col_meta)
+//         get_family_name, get_is_parametric, should_align_right_in_pretty_formats
+//         text_can_contain_only_valid_utf8
+//         have_maximum_size_of_value, get_maximum_size_of_value_in_memory, get_size_of_value_in_memory
+//         get_precision, get_scale
+//         is_null_literal, is_value_represented_by_number, is_value_represented_by_integer, is_value_represented_by_unsigned_integer, is_value_unambiguously_represented_in_contiguous_memory_region
+// 2. datatype creation with column : create_column, create_column_const (size_t size, const Field &field), create_column_const_with_default_value (size_t size), get_uncompressed_serialized_bytes (const IColumn &column, int be_exec_version)
+// 3. serde related: get_serde (int nesting_level=1)
+//          to_string (const IColumn &column, size_t row_num, BufferWritable &ostr), to_string (const IColumn &column, size_t row_num), to_string_batch (const IColumn &column, ColumnString &column_to), from_string (ReadBuffer &rb, IColumn *column)
+//          serialize (const IColumn &column, char *buf, int be_exec_version), deserialize (const char *buf, MutableColumnPtr *column, int be_exec_version)
+// 4. compare: equals (const IDataType &rhs), is_comparable
+// 5. others: update_avg_value_size_hint (const IColumn &column, double &avg_value_size_hint)
+
+namespace doris::vectorized {
+
+class DataTypeIPTest : public CommonDataTypeTest {
+protected:
+    void SetUp() override {
+        data_files = {"regression-test/data/nereids_function_p0/fn_test_ip_invalid.csv",
+                      "regression-test/data/nereids_function_p0/fn_test_ip_normal.csv",
+                      "regression-test/data/nereids_function_p0/fn_test_ip_nullable.csv",
+                      "regression-test/data/nereids_function_p0/fn_test_ip_special.csv",
+                      "regression-test/data/nereids_function_p0/fn_test_ip_special_no_null.csv"};
+    }
+
+public:
+    DataTypePtr dt_ipv4 =
+            DataTypeFactory::instance().create_data_type(FieldType::OLAP_FIELD_TYPE_IPV4, 0, 0);
+    DataTypePtr dt_ipv6 =
+            DataTypeFactory::instance().create_data_type(FieldType::OLAP_FIELD_TYPE_IPV6, 0, 0);
+    DataTypePtr dt_ipv4_nullable = std::make_shared<vectorized::DataTypeNullable>(dt_ipv4);
+    DataTypePtr dt_ipv6_nullable = std::make_shared<vectorized::DataTypeNullable>(dt_ipv6);
+    // common ip data
+    std::vector<string> data_files;
+};
+
+TEST_F(DataTypeIPTest, MetaInfoTest) {
+    TypeDescriptor ipv4_type_descriptor = {PrimitiveType::TYPE_IPV4};
+    auto col_meta = std::make_shared<PColumnMeta>();
+    col_meta->set_type(PGenericType_TypeId_IPV4);
+    DataTypeMetaInfo ipv4_meta_info_to_assert = {
+            .type_id = TypeIndex::IPv4,
+            .type_as_type_descriptor = &ipv4_type_descriptor,
+            .family_name = "IPv4",
+            .has_subtypes = false,
+            .storage_field_type = doris::FieldType::OLAP_FIELD_TYPE_IPV4,
+            .should_align_right_in_pretty_formats = true,
+            .text_can_contain_only_valid_utf8 = true,
+            .have_maximum_size_of_value = true,
+            .size_of_value_in_memory = sizeof(IPv4),
+            .precision = size_t(-1),
+            .scale = size_t(-1),
+            .is_null_literal = false,
+            .is_value_represented_by_number = true,
+            .is_value_represented_by_unsigned_integer = true,
+            .pColumnMeta = col_meta.get()
+            //                .is_value_unambiguously_represented_in_contiguous_memory_region = true
+    };
+    TypeDescriptor ipv6_type_descriptor = {PrimitiveType::TYPE_IPV6};
+    auto col_meta6 = std::make_shared<PColumnMeta>();
+    col_meta6->set_type(PGenericType_TypeId_IPV6);
+    DataTypeMetaInfo ipv6_meta_info = {
+            .type_id = TypeIndex::IPv6,
+            .type_as_type_descriptor = &ipv6_type_descriptor,
+            .family_name = "IPv6",
+            .has_subtypes = false,
+            .storage_field_type = doris::FieldType::OLAP_FIELD_TYPE_IPV6,
+            .should_align_right_in_pretty_formats = true,
+            .text_can_contain_only_valid_utf8 = true,
+            .have_maximum_size_of_value = true,
+            .size_of_value_in_memory = sizeof(IPv6),
+            .precision = size_t(-1),
+            .scale = size_t(-1),
+            .is_null_literal = false,
+            .is_value_represented_by_number = true,
+            .is_value_represented_by_unsigned_integer = true,
+            .pColumnMeta = col_meta6.get()
+            //                .is_value_unambiguously_represented_in_contiguous_memory_region = true
+    };
+    meta_info_assert(dt_ipv4, ipv4_meta_info_to_assert);
+    meta_info_assert(dt_ipv6, ipv6_meta_info);
+}
+
+TEST_F(DataTypeIPTest, CreateColumnTest) {
+    Field default_field_ipv4 = IPv4(0);
+    Field default_field_ipv6 = IPv6(0);
+    create_column_assert(dt_ipv4, default_field_ipv4);
+    create_column_assert(dt_ipv6, default_field_ipv6);
+}
+
+TEST_F(DataTypeIPTest, GetFieldTest) {
+    auto serde_ipv4 = dt_ipv4->get_serde(1);
+    auto serde_ipv6 = dt_ipv6->get_serde(1);
+    auto column_ipv4 = dt_ipv4->create_column();
+    auto column_ipv6 = dt_ipv6->create_column();
+
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    DataTypeSerDeSPtrs serde = {dt_ipv4->get_serde(), dt_ipv6->get_serde()};
+    CommonDataTypeSerdeTest::load_data_and_assert_from_csv<true, true>(serde, ip_cols,
+                                                                       data_files[1], ';', {1, 2});
+    TExprNode node_ipv4;
+    node_ipv4.node_type = TExprNodeType::IPV4_LITERAL;
+    for (size_t i = 0; i < ip_cols[0]->size(); ++i) {
+        node_ipv4.ipv4_literal.value = ip_cols[0]->get_int(i);
+        Field assert_field;
+        ip_cols[0]->get(i, assert_field);
+        get_field_assert(dt_ipv4, node_ipv4, assert_field);
+    }
+
+    TExprNode node_ipv6;
+    node_ipv6.node_type = TExprNodeType::IPV6_LITERAL;
+    for (size_t i = 0; i < ip_cols[1]->size(); ++i) {
+        node_ipv6.ipv6_literal.value =
+                IPv6Value::to_string(assert_cast<ColumnIPv6&>(*ip_cols[1]).get_data()[i]);
+        Field assert_field;
+        ip_cols[1]->get(i, assert_field);
+        get_field_assert(dt_ipv6, node_ipv6, assert_field);
+    }
+
+    TExprNode invalid_node_ipv6;
+    invalid_node_ipv6.node_type = TExprNodeType::IPV6_LITERAL;
+    // todo.(check) 2001:db8:::1 this is invalid ipv6 value, but it can pass the test
+    std::vector<string> invalid_ipv6 = {"2001:db8::12345",
+                                        "",
+                                        "::fffff:0:0",
+                                        "2001:db8::g123",
+                                        "2001:db8:85a3::8a2e:0370:",
+                                        "2001:0db8:85a3:0000:0000:8a2e:0370:7334:1234",
+                                        "::12345:abcd"};
+    for (auto& ipv6 : invalid_ipv6) {
+        invalid_node_ipv6.ipv6_literal.value = ipv6;
+        Field field;
+        get_field_assert(dt_ipv6, invalid_node_ipv6, field, true);
+    }
+}
+
+TEST_F(DataTypeIPTest, FromAndToStringTest) {
+    auto serde_ipv4 = dt_ipv4->get_serde(1);
+    auto serde_ipv6 = dt_ipv6->get_serde(1);
+    auto column_ipv4 = dt_ipv4->create_column();
+    auto column_ipv6 = dt_ipv6->create_column();
+
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    DataTypeSerDeSPtrs serde = {dt_ipv4->get_serde(), dt_ipv6->get_serde()};
+    load_data_from_csv(serde, ip_cols, data_files[0], ';', {1, 2});
+    // test ipv4
+    assert_to_string_from_string_assert(ip_cols[0]->get_ptr(), dt_ipv4);
+    // test ipv6
+    assert_to_string_from_string_assert(ip_cols[1]->get_ptr(), dt_ipv6);
+}
+
+TEST_F(DataTypeIPTest, CompareTest) {
+    assert_compare_behavior(dt_ipv4, dt_ipv6);
+}
+
+TEST_F(DataTypeIPTest, SerdeHiveTextAndJsonFormatTest) {
+    auto serde_ipv4 = dt_ipv4->get_serde(1);
+    auto serde_ipv6 = dt_ipv6->get_serde(1);
+    auto column_ipv4 = dt_ipv4->create_column();
+    auto column_ipv6 = dt_ipv6->create_column();
+
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    DataTypeSerDeSPtrs serde = {dt_ipv4->get_serde(), dt_ipv6->get_serde()};
+    CommonDataTypeSerdeTest::load_data_and_assert_from_csv<true, true>(serde, ip_cols,
+                                                                       data_files[1], ';', {1, 2});
+    CommonDataTypeSerdeTest::load_data_and_assert_from_csv<false, true>(serde, ip_cols,
+                                                                        data_files[1], ';', {1, 2});
+}
+
+TEST_F(DataTypeIPTest, SerdePbTest) {
+    auto serde_ipv4 = dt_ipv4->get_serde(1);
+    auto serde_ipv6 = dt_ipv6->get_serde(1);
+    auto column_ipv4 = dt_ipv4->create_column();
+    auto column_ipv6 = dt_ipv6->create_column();
+
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    DataTypeSerDeSPtrs serde = {dt_ipv4->get_serde(), dt_ipv6->get_serde()};
+    CommonDataTypeSerdeTest::check_data(ip_cols, serde, ';', {1, 2}, data_files[0],
+                                        CommonDataTypeSerdeTest::assert_pb_format);
+}
+
+TEST_F(DataTypeIPTest, SerdeJsonbTest) {
+    auto serde_ipv4 = dt_ipv4->get_serde(1);
+    auto serde_ipv6 = dt_ipv6->get_serde(1);
+    auto column_ipv4 = dt_ipv4->create_column();
+    auto column_ipv6 = dt_ipv6->create_column();
+
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    DataTypeSerDeSPtrs serde = {dt_ipv4->get_serde(), dt_ipv6->get_serde()};
+    CommonDataTypeSerdeTest::check_data(ip_cols, serde, ';', {1, 2}, data_files[0],
+                                        CommonDataTypeSerdeTest::assert_jsonb_format);
+}
+
+} // namespace doris::vectorized
\ No newline at end of file

From 0d4886331c0e0a07fec043bc3c96ef550ba06de6 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Tue, 3 Dec 2024 15:42:07 +0800
Subject: [PATCH 134/399] [Bug](runtime-filter) fix wrong build_bf_exactly when
 sync filter size disabled (#44716)

### What problem does this PR solve?
fix wrong build_bf_exactly when sync filter size disabled

introduced by #44169
---
 be/src/exprs/runtime_filter.cpp               | 20 +++--------
 be/src/exprs/runtime_filter.h                 |  5 ++-
 be/src/pipeline/exec/hashjoin_build_sink.cpp  |  4 +--
 .../exec/nested_loop_join_build_operator.cpp  |  2 +-
 be/src/runtime/runtime_filter_mgr.cpp         | 22 ++++++------
 be/src/runtime/runtime_filter_mgr.h           |  8 ++---
 be/src/runtime/runtime_state.cpp              | 13 ++++---
 be/src/runtime/runtime_state.h                |  3 +-
 .../translator/RuntimeFilterTranslator.java   |  1 +
 .../apache/doris/planner/RuntimeFilter.java   | 35 +++++++++++++++++--
 gensrc/thrift/PlanNodes.thrift                |  4 ++-
 11 files changed, 66 insertions(+), 51 deletions(-)

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index c983af0fb3ea71..606df9fe980571 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -975,11 +975,10 @@ class RuntimePredicateWrapper {
 
 Status IRuntimeFilter::create(RuntimeFilterParamsContext* state, const TRuntimeFilterDesc* desc,
                               const TQueryOptions* query_options, const RuntimeFilterRole role,
-                              int node_id, std::shared_ptr<IRuntimeFilter>* res,
-                              bool build_bf_exactly) {
+                              int node_id, std::shared_ptr<IRuntimeFilter>* res) {
     *res = std::make_shared<IRuntimeFilter>(state, desc);
     (*res)->set_role(role);
-    return (*res)->init_with_desc(desc, query_options, node_id, build_bf_exactly);
+    return (*res)->init_with_desc(desc, query_options, node_id);
 }
 
 RuntimeFilterContextSPtr& IRuntimeFilter::get_shared_context_ref() {
@@ -1348,7 +1347,7 @@ std::string IRuntimeFilter::formatted_state() const {
 }
 
 Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQueryOptions* options,
-                                      int node_id, bool build_bf_exactly) {
+                                      int node_id) {
     // if node_id == -1 , it shouldn't be a consumer
     DCHECK(node_id >= 0 || (node_id == -1 && !is_consumer()));
 
@@ -1370,21 +1369,10 @@ Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQue
     params.runtime_bloom_filter_max_size = options->__isset.runtime_bloom_filter_max_size
                                                    ? options->runtime_bloom_filter_max_size
                                                    : 0;
-    auto sync_filter_size = desc->__isset.sync_filter_size && desc->sync_filter_size;
-    // We build runtime filter by exact distinct count if all of 3 conditions are met:
-    // 1. Only 1 join key
-    // 2. Bloom filter
-    // 3. Size of all bloom filters will be same (size will be sync or this is a broadcast join).
-    params.build_bf_exactly =
-            build_bf_exactly && (_runtime_filter_type == RuntimeFilterType::BLOOM_FILTER ||
-                                 _runtime_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER);
 
+    params.build_bf_exactly = desc->__isset.build_bf_exactly && desc->build_bf_exactly;
     params.bloom_filter_size_calculated_by_ndv = desc->bloom_filter_size_calculated_by_ndv;
 
-    if (!sync_filter_size) {
-        params.build_bf_exactly &= !_is_broadcast_join;
-    }
-
     if (desc->__isset.bloom_filter_size_bytes) {
         params.bloom_filter_size = desc->bloom_filter_size_bytes;
     }
diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index 63ef3e2dbd4653..49232236179669 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -213,8 +213,7 @@ class IRuntimeFilter {
 
     static Status create(RuntimeFilterParamsContext* state, const TRuntimeFilterDesc* desc,
                          const TQueryOptions* query_options, const RuntimeFilterRole role,
-                         int node_id, std::shared_ptr<IRuntimeFilter>* res,
-                         bool build_bf_exactly = false);
+                         int node_id, std::shared_ptr<IRuntimeFilter>* res);
 
     RuntimeFilterContextSPtr& get_shared_context_ref();
 
@@ -260,7 +259,7 @@ class IRuntimeFilter {
 
     // init filter with desc
     Status init_with_desc(const TRuntimeFilterDesc* desc, const TQueryOptions* options,
-                          int node_id = -1, bool build_bf_exactly = false);
+                          int node_id = -1);
 
     // serialize _wrapper to protobuf
     Status serialize(PMergeFilterRequest* request, void** data, int* len);
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index 864927e1721923..e4c67b24193593 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -91,8 +91,8 @@ Status HashJoinBuildSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo
     RETURN_IF_ERROR(_hash_table_init(state));
     _runtime_filters.resize(p._runtime_filter_descs.size());
     for (size_t i = 0; i < p._runtime_filter_descs.size(); i++) {
-        RETURN_IF_ERROR(state->register_producer_runtime_filter(
-                p._runtime_filter_descs[i], &_runtime_filters[i], _build_expr_ctxs.size() == 1));
+        RETURN_IF_ERROR(state->register_producer_runtime_filter(p._runtime_filter_descs[i],
+                                                                &_runtime_filters[i]));
     }
 
     _runtime_filter_slots =
diff --git a/be/src/pipeline/exec/nested_loop_join_build_operator.cpp b/be/src/pipeline/exec/nested_loop_join_build_operator.cpp
index 9e3e8a08ca83a5..35b9de619f393d 100644
--- a/be/src/pipeline/exec/nested_loop_join_build_operator.cpp
+++ b/be/src/pipeline/exec/nested_loop_join_build_operator.cpp
@@ -67,7 +67,7 @@ Status NestedLoopJoinBuildSinkLocalState::init(RuntimeState* state, LocalSinkSta
     _runtime_filters.resize(p._runtime_filter_descs.size());
     for (size_t i = 0; i < p._runtime_filter_descs.size(); i++) {
         RETURN_IF_ERROR(state->register_producer_runtime_filter(p._runtime_filter_descs[i],
-                                                                &_runtime_filters[i], false));
+                                                                &_runtime_filters[i]));
     }
     return Status::OK();
 }
diff --git a/be/src/runtime/runtime_filter_mgr.cpp b/be/src/runtime/runtime_filter_mgr.cpp
index bb100fcbb42ec5..c16db7c67d3420 100644
--- a/be/src/runtime/runtime_filter_mgr.cpp
+++ b/be/src/runtime/runtime_filter_mgr.cpp
@@ -90,7 +90,7 @@ std::vector<std::shared_ptr<IRuntimeFilter>> RuntimeFilterMgr::get_consume_filte
 Status RuntimeFilterMgr::register_consumer_filter(const TRuntimeFilterDesc& desc,
                                                   const TQueryOptions& options, int node_id,
                                                   std::shared_ptr<IRuntimeFilter>* consumer_filter,
-                                                  bool build_bf_exactly, bool need_local_merge) {
+                                                  bool need_local_merge) {
     SCOPED_CONSUME_MEM_TRACKER(_tracker.get());
     int32_t key = desc.filter_id;
     bool has_exist = false;
@@ -110,7 +110,7 @@ Status RuntimeFilterMgr::register_consumer_filter(const TRuntimeFilterDesc& desc
     if (!has_exist) {
         std::shared_ptr<IRuntimeFilter> filter;
         RETURN_IF_ERROR(IRuntimeFilter::create(_state, &desc, &options, RuntimeFilterRole::CONSUMER,
-                                               node_id, &filter, build_bf_exactly));
+                                               node_id, &filter));
         _consumer_map[key].emplace_back(node_id, filter);
         *consumer_filter = filter;
     } else if (!need_local_merge) {
@@ -122,7 +122,7 @@ Status RuntimeFilterMgr::register_consumer_filter(const TRuntimeFilterDesc& desc
 
 Status RuntimeFilterMgr::register_local_merge_producer_filter(
         const doris::TRuntimeFilterDesc& desc, const doris::TQueryOptions& options,
-        std::shared_ptr<IRuntimeFilter> producer_filter, bool build_bf_exactly) {
+        std::shared_ptr<IRuntimeFilter> producer_filter) {
     DCHECK(_is_global);
     SCOPED_CONSUME_MEM_TRACKER(_tracker.get());
     int32_t key = desc.filter_id;
@@ -143,8 +143,7 @@ Status RuntimeFilterMgr::register_local_merge_producer_filter(
         if (iter->second.filters.empty()) {
             std::shared_ptr<IRuntimeFilter> merge_filter;
             RETURN_IF_ERROR(IRuntimeFilter::create(_state, &desc, &options,
-                                                   RuntimeFilterRole::PRODUCER, -1, &merge_filter,
-                                                   build_bf_exactly));
+                                                   RuntimeFilterRole::PRODUCER, -1, &merge_filter));
             merge_filter->set_ignored();
             iter->second.filters.emplace_back(merge_filter);
         }
@@ -181,10 +180,9 @@ doris::LocalMergeFilters* RuntimeFilterMgr::get_local_merge_producer_filters(int
     return &iter->second;
 }
 
-Status RuntimeFilterMgr::register_producer_filter(const TRuntimeFilterDesc& desc,
-                                                  const TQueryOptions& options,
-                                                  std::shared_ptr<IRuntimeFilter>* producer_filter,
-                                                  bool build_bf_exactly) {
+Status RuntimeFilterMgr::register_producer_filter(
+        const TRuntimeFilterDesc& desc, const TQueryOptions& options,
+        std::shared_ptr<IRuntimeFilter>* producer_filter) {
     DCHECK(!_is_global);
     SCOPED_CONSUME_MEM_TRACKER(_tracker.get());
     int32_t key = desc.filter_id;
@@ -196,7 +194,7 @@ Status RuntimeFilterMgr::register_producer_filter(const TRuntimeFilterDesc& desc
         return Status::InvalidArgument("filter has registed");
     }
     RETURN_IF_ERROR(IRuntimeFilter::create(_state, &desc, &options, RuntimeFilterRole::PRODUCER, -1,
-                                           producer_filter, build_bf_exactly));
+                                           producer_filter));
     _producer_map.emplace(key, *producer_filter);
     return Status::OK();
 }
@@ -233,8 +231,8 @@ Status RuntimeFilterMergeControllerEntity::_init_with_desc(
     cnt_val->filter = cnt_val->pool->add(new IRuntimeFilter(_state, runtime_filter_desc));
 
     auto filter_id = runtime_filter_desc->filter_id;
-    RETURN_IF_ERROR(cnt_val->filter->init_with_desc(&cnt_val->runtime_filter_desc, query_options,
-                                                    -1, false));
+    RETURN_IF_ERROR(
+            cnt_val->filter->init_with_desc(&cnt_val->runtime_filter_desc, query_options, -1));
     cnt_val->filter->set_ignored();
     _filter_map.emplace(filter_id, cnt_val);
     return Status::OK();
diff --git a/be/src/runtime/runtime_filter_mgr.h b/be/src/runtime/runtime_filter_mgr.h
index 0a6f8318feaba0..9f4cf5f4e22a07 100644
--- a/be/src/runtime/runtime_filter_mgr.h
+++ b/be/src/runtime/runtime_filter_mgr.h
@@ -100,19 +100,17 @@ class RuntimeFilterMgr {
     // register filter
     Status register_consumer_filter(const TRuntimeFilterDesc& desc, const TQueryOptions& options,
                                     int node_id, std::shared_ptr<IRuntimeFilter>* consumer_filter,
-                                    bool build_bf_exactly = false, bool need_local_merge = false);
+                                    bool need_local_merge = false);
 
     Status register_local_merge_producer_filter(const TRuntimeFilterDesc& desc,
                                                 const TQueryOptions& options,
-                                                std::shared_ptr<IRuntimeFilter> producer_filter,
-                                                bool build_bf_exactly = false);
+                                                std::shared_ptr<IRuntimeFilter> producer_filter);
 
     Status get_local_merge_producer_filters(int filter_id, LocalMergeFilters** local_merge_filters);
     LocalMergeFilters* get_local_merge_producer_filters(int filter_id);
 
     Status register_producer_filter(const TRuntimeFilterDesc& desc, const TQueryOptions& options,
-                                    std::shared_ptr<IRuntimeFilter>* producer_filter,
-                                    bool build_bf_exactly = false);
+                                    std::shared_ptr<IRuntimeFilter>* producer_filter);
 
     // update filter by remote
     void set_runtime_filter_params(const TRuntimeFilterParams& runtime_filter_params);
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 072c6e87f740a6..579329c1082633 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -516,14 +516,13 @@ RuntimeFilterMgr* RuntimeState::global_runtime_filter_mgr() {
 }
 
 Status RuntimeState::register_producer_runtime_filter(
-        const TRuntimeFilterDesc& desc, std::shared_ptr<IRuntimeFilter>* producer_filter,
-        bool build_bf_exactly) {
+        const TRuntimeFilterDesc& desc, std::shared_ptr<IRuntimeFilter>* producer_filter) {
     // Producers are created by local runtime filter mgr and shared by global runtime filter manager.
     // When RF is published, consumers in both global and local RF mgr will be found.
-    RETURN_IF_ERROR(local_runtime_filter_mgr()->register_producer_filter(
-            desc, query_options(), producer_filter, build_bf_exactly));
+    RETURN_IF_ERROR(local_runtime_filter_mgr()->register_producer_filter(desc, query_options(),
+                                                                         producer_filter));
     RETURN_IF_ERROR(global_runtime_filter_mgr()->register_local_merge_producer_filter(
-            desc, query_options(), *producer_filter, build_bf_exactly));
+            desc, query_options(), *producer_filter));
     return Status::OK();
 }
 
@@ -532,10 +531,10 @@ Status RuntimeState::register_consumer_runtime_filter(
         std::shared_ptr<IRuntimeFilter>* consumer_filter) {
     if (desc.has_remote_targets || need_local_merge) {
         return global_runtime_filter_mgr()->register_consumer_filter(desc, query_options(), node_id,
-                                                                     consumer_filter, false, true);
+                                                                     consumer_filter, true);
     } else {
         return local_runtime_filter_mgr()->register_consumer_filter(desc, query_options(), node_id,
-                                                                    consumer_filter, false, false);
+                                                                    consumer_filter, false);
     }
 }
 
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 0bc81bca4d99a1..ad63510e2af82c 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -561,8 +561,7 @@ class RuntimeState {
     }
 
     Status register_producer_runtime_filter(const doris::TRuntimeFilterDesc& desc,
-                                            std::shared_ptr<IRuntimeFilter>* producer_filter,
-                                            bool build_bf_exactly);
+                                            std::shared_ptr<IRuntimeFilter>* producer_filter);
 
     Status register_consumer_runtime_filter(const doris::TRuntimeFilterDesc& desc,
                                             bool need_local_merge, int node_id,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java
index 3dbd6cfcec7917..07e0af601739b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java
@@ -143,6 +143,7 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, JoinNodeBase node, P
                     targetTupleIdMapList, context.getLimits());
             if (node instanceof HashJoinNode) {
                 origFilter.setIsBroadcast(((HashJoinNode) node).getDistributionMode() == DistributionMode.BROADCAST);
+                origFilter.setSingleEq(((HashJoinNode) node).getEqJoinConjuncts().size());
             } else {
                 // nest loop join
                 origFilter.setIsBroadcast(true);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
index 2f3948aee161b8..80497798083dc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
@@ -112,6 +112,8 @@ public final class RuntimeFilter {
 
     private boolean bloomFilterSizeCalculatedByNdv = false;
 
+    private boolean singleEq = false;
+
     /**
      * Internal representation of a runtime filter target.
      */
@@ -216,9 +218,36 @@ public TRuntimeFilterDesc toThrift() {
         tFilter.setIsBroadcastJoin(isBroadcastJoin);
         tFilter.setHasLocalTargets(hasLocalTargets);
         tFilter.setHasRemoteTargets(hasRemoteTargets);
+
+        boolean hasSerialTargets = false;
         for (RuntimeFilterTarget target : targets) {
             tFilter.putToPlanIdToTargetExpr(target.node.getId().asInt(), target.expr.treeToThrift());
+            hasSerialTargets = hasSerialTargets
+                    || (target.node.isSerialOperator() && target.node.fragment.useSerialSource(ConnectContext.get()));
         }
+
+        boolean enableSyncFilterSize = ConnectContext.get() != null
+                && ConnectContext.get().getSessionVariable().enableSyncRuntimeFilterSize();
+
+        // there are two cases has local exchange between join and scan
+        // 1. hasRemoteTargets is true means join probe side do least once shuffle (has shuffle between join and scan)
+        // 2. hasSerialTargets is true means scan is pooled (has local shuffle between join and scan)
+        boolean needShuffle = hasRemoteTargets || hasSerialTargets;
+
+        // There are two cases where all instances of rf have the same size.
+        // 1. enableSyncFilterSize is true means backends will collect global size and send to every instance
+        // 2. isBroadcastJoin is true means each join node instance have the same full amount of data
+        boolean hasGlobalSize = enableSyncFilterSize || isBroadcastJoin;
+
+        // build runtime filter by exact distinct count if all of 3 conditions are met:
+        // 1. only single eq conjunct
+        // 2. rf type may be bf
+        // 3. each filter only acts on self instance(do not need any shuffle), or size of
+        // all filters will be same
+        boolean buildBfExactly = singleEq && (runtimeFilterType == TRuntimeFilterType.IN_OR_BLOOM
+                || runtimeFilterType == TRuntimeFilterType.BLOOM) && (!needShuffle || hasGlobalSize);
+        tFilter.setBuildBfExactly(buildBfExactly);
+
         tFilter.setType(runtimeFilterType);
         tFilter.setBloomFilterSizeBytes(filterSizeBytes);
         if (runtimeFilterType.equals(TRuntimeFilterType.BITMAP)) {
@@ -239,8 +268,6 @@ public TRuntimeFilterDesc toThrift() {
                 tFilter.setNullAware(false);
             }
         }
-        tFilter.setSyncFilterSize(ConnectContext.get() != null
-                && ConnectContext.get().getSessionVariable().enableSyncRuntimeFilterSize());
         return tFilter;
     }
 
@@ -597,6 +624,10 @@ public void addTarget(RuntimeFilterTarget target) {
         targets.add(target);
     }
 
+    public void setSingleEq(int eqJoinConjunctsNumbers) {
+        singleEq = (eqJoinConjunctsNumbers == 1);
+    }
+
     public void setIsBroadcast(boolean isBroadcast) {
         isBroadcastJoin = isBroadcast;
     }
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 0bbd364fda1c2a..961433ab28d7e4 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -1293,7 +1293,9 @@ struct TRuntimeFilterDesc {
   // true, if join type is null aware like <=>. rf should dispose the case
   15: optional bool null_aware;
 
-  16: optional bool sync_filter_size;
+  16: optional bool sync_filter_size; // Deprecated
+  
+  17: optional bool build_bf_exactly;
 }
 
 

From cdcb9786a4aef1a12d8acadac2a321ef6a287032 Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Tue, 3 Dec 2024 13:58:34 +0530
Subject: [PATCH 135/399] [Enhancement] (nereids)implement
 adminDiagnoseTabletCommand in nereids (#44744)

Issue Number: close #42848
---
 .../main/antlr4/org/apache/doris/nereids/DorisParser.g4    | 2 +-
 .../apache/doris/nereids/parser/LogicalPlanBuilder.java    | 7 +++++++
 .../nereids_p0/show/test_show_diagnose_tablet.groovy       | 2 ++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index eac4c661de920d..b51b342e24b895 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -486,6 +486,7 @@ unsupportedCancelStatement
 
 supportedAdminStatement
     : ADMIN SHOW REPLICA DISTRIBUTION FROM baseTableRef                             #adminShowReplicaDistribution
+    | ADMIN DIAGNOSE TABLET tabletId=INTEGER_VALUE                                  #adminDiagnoseTablet
     ;
 
 supportedRecoverStatement
@@ -514,7 +515,6 @@ unsupportedAdminStatement
         (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?                              #adminCleanTrash
     | ADMIN SET TABLE name=multipartIdentifier
         PARTITION VERSION properties=propertyClause?                                #adminSetPartitionVersion
-    | ADMIN DIAGNOSE TABLET tabletId=INTEGER_VALUE                                  #adminDiagnoseTablet
     | ADMIN SHOW TABLET STORAGE FORMAT VERBOSE?                                     #adminShowTabletStorageFormat
     | ADMIN COPY TABLET tabletId=INTEGER_VALUE properties=propertyClause?           #adminCopyTablet
     | ADMIN SET TABLE name=multipartIdentifier STATUS properties=propertyClause?    #adminSetTableStatus
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 401d18775af106..7804959aa79f9e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -50,6 +50,7 @@
 import org.apache.doris.mtmv.MTMVRefreshTriggerInfo;
 import org.apache.doris.nereids.DorisParser;
 import org.apache.doris.nereids.DorisParser.AddConstraintContext;
+import org.apache.doris.nereids.DorisParser.AdminDiagnoseTabletContext;
 import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext;
 import org.apache.doris.nereids.DorisParser.AggClauseContext;
 import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext;
@@ -4654,6 +4655,12 @@ public LogicalPlan visitShowDiagnoseTablet(ShowDiagnoseTabletContext ctx) {
         return new ShowDiagnoseTabletCommand(tabletId);
     }
 
+    @Override
+    public LogicalPlan visitAdminDiagnoseTablet(AdminDiagnoseTabletContext ctx) {
+        long tabletId = Long.parseLong(ctx.INTEGER_VALUE().getText());
+        return new ShowDiagnoseTabletCommand(tabletId);
+    }
+
     @Override
     public LogicalPlan visitShowCreateTable(ShowCreateTableContext ctx) {
         List<String> nameParts = visitMultipartIdentifier(ctx.name);
diff --git a/regression-test/suites/nereids_p0/show/test_show_diagnose_tablet.groovy b/regression-test/suites/nereids_p0/show/test_show_diagnose_tablet.groovy
index 029fac7428e0bb..62e008412bae3b 100644
--- a/regression-test/suites/nereids_p0/show/test_show_diagnose_tablet.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_diagnose_tablet.groovy
@@ -29,6 +29,8 @@ suite("test_show_diagnose_tablet_nereids", "query,diagnose") {
 
         // Execute the SHOW TABLET DIAGNOSIS command and verify the output
         checkNereidsExecute("SHOW TABLET DIAGNOSIS ${tabletId}")
+        // Execute the SHOW TABLET DIAGNOSIS command and verify the output
+        checkNereidsExecute("ADMIN DIAGNOSE TABLET ${tabletId}")
     } catch (Exception e) {
         log.error("Failed to execute SHOW TABLET DIAGNOSIS command", e)
         throw e

From 1e3c39d5c97f79726e051ee740f05b5d3842de6b Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Tue, 3 Dec 2024 16:49:50 +0800
Subject: [PATCH 136/399] [fix](agg) Fixed a core dump when using the IPv6 type
 in array_agg. (#44877)

### What problem does this PR solve?
https://github.com/apache/doris/pull/40697 that introduced the issue.


```
F20241202 18:56:00.498481 233860 assert_cast.h:57] Bad cast from type:doris::vectorized::ColumnVector<unsigned __int128> to doris::vectorized::ColumnVector<wide::integer<128ul, unsigned int> >
*** Check failure stack trace: ***
    @     0x562c12168fb6  google::LogMessage::SendToLog()
    @     0x562c12165a00  google::LogMessage::Flush()
    @     0x562c121697f9  google::LogMessageFatal::~LogMessageFatal()
    @     0x562c0acb66a6  _ZZ11assert_castIRN5doris10vectorized12ColumnVectorIN4wide7integerILm128EjEEEEL18TypeCheckOnRelease1ERNS1_7IColumnEET_OT1_ENKUlOSB_E_clISA_EES7_SE_
    @     0x562c0acb64e7  assert_cast<>()
    @     0x562c0acb637d  doris::vectorized::AggregateFunctionArrayAggData<>::insert_result_into()
    @     0x562c11f12282  doris::pipeline::AggLocalState::_get_results_without_key()
    @     0x562c11f3a251  std::_Function_handler<>::_M_invoke()
    @     0x562c11f13ee8  doris::pipeline::AggSourceOperatorX::get_block()
    @     0x562c1174e3cf  doris::pipeline::OperatorXBase::get_block_after_projects()
    @     0x562c120ad87b  doris::pipeline::PipelineTask::execute()
    @     0x562c120bb7b2  doris::pipeline::TaskScheduler::_do_work()
    @     0x562c081ef935  doris::ThreadPool::dispatch_thread()
    @     0x562c081e6321  doris::Thread::supervise_thread()
    @     0x7f446f9b01ca  start_thread
    @     0x7f447039fe73  __GI___clone
    @              (nil)  (unknown)
*** Query id: c24a997f61b847d2-adece222f56b3b9e ***
*** is nereids: 1 ***
*** tablet id: 0 ***
*** Aborted at 1733136960 (unix time) try "date -d @1733136960" if you are using GNU date ***
*** Current BE git commitID: 1551ef9e63 ***
*** SIGABRT unknown detail explain (@0x47000039002) received by PID 233474 (TID 233860 OR 0x7f436d1f2700) from PID 233474; stack trace: ***
 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /mnt/disk2/yanxuecheng/doris/be/src/common/signal_handler.h:421
 1# 0x00007F44703B4B50 in /lib64/libc.so.6
 2# gsignal in /lib64/libc.so.6
 3# __GI_abort in /lib64/libc.so.6
 4# 0x0000562C1217388D in /mnt/disk2/yanxuecheng/doris/output/be/lib/doris_be
 5# 0x0000562C12165ECA in /mnt/disk2/yanxuecheng/doris/output/be/lib/doris_be
 6# google::LogMessage::SendToLog() in /mnt/disk2/yanxuecheng/doris/output/be/lib/doris_be
 7# google::LogMessage::Flush() in /mnt/disk2/yanxuecheng/doris/output/be/lib/doris_be
 8# google::LogMessageFatal::~LogMessageFatal() in /mnt/disk2/yanxuecheng/doris/output/be/lib/doris_be
 9# doris::vectorized::ColumnVector<wide::integer<128ul, unsigned int> >& assert_cast<doris::vectorized::ColumnVector<wide::integer<128ul, unsigned int> >&, (TypeCheckOnRelease)1, doris::vectorized::IColumn&>(doris::vectorized::IColumn&)::{lambda(auto:1&&)#1}::operator()<doris::vectorized::IColumn&>(doris::vectorized::IColumn&) const at /mnt/disk2/yanxuecheng/doris/be/src/vec/common/assert_cast.h:57
10# doris::vectorized::ColumnVector<wide::integer<128ul, unsigned int> >& assert_cast<doris::vectorized::ColumnVector<wide::integer<128ul, unsigned int> >&, (TypeCheckOnRelease)1, doris::vectorized::IColumn&>(doris::vectorized::IColumn&) at /mnt/disk2/yanxuecheng/doris/be/src/vec/common/assert_cast.h:72
11# doris::vectorized::AggregateFunctionArrayAggData<wide::integer<128ul, unsigned int> >::insert_result_into(doris::vectorized::IColumn&) const in /mnt/disk2/yanxuecheng/doris/output/be/lib/doris_be
12# doris::pipeline::AggLocalState::_get_results_without_key(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /mnt/disk2/yanxuecheng/doris/be/src/pipeline/exec/aggregation_source_operator.cpp:349
13# std::_Function_handler<doris::Status (doris::RuntimeState*, doris::vectorized::Block*, bool*), std::_Bind_result<doris::Status, doris::Status (doris::pipeline::AggLocalState::*(doris::pipeline::AggLocalState*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>))(doris::RuntimeState*, doris::vectorized::Block*, bool*)> >::_M_invoke(std::_Any_data const&, doris::RuntimeState*&&, doris::vectorized::Block*&&, bool*&&) at /mnt/disk2/yanxuecheng/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/13/../../../../include/c++/13/bits/std_function.h:290
14# doris::pipeline::AggSourceOperatorX::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /mnt/disk2/yanxuecheng/doris/be/src/pipeline/exec/aggregation_source_operator.cpp:439
15# doris::pipeline::OperatorXBase::get_block_after_projects(doris::RuntimeState*, doris::vectorized::Block*, bool*) in /mnt/disk2/yanxuecheng/doris/output/be/lib/doris_be
16# doris::pipeline::PipelineTask::execute(bool*) at /mnt/disk2/yanxuecheng/doris/be/src/pipeline/pipeline_task.cpp:378
17# doris::pipeline::TaskScheduler::_do_work(int) at /mnt/disk2/yanxuecheng/doris/be/src/pipeline/task_scheduler.cpp:138
18# doris::ThreadPool::dispatch_thread() in /mnt/disk2/yanxuecheng/doris/output/be/lib/doris_be
19# doris::Thread::supervise_thread(void*) at /mnt/disk2/yanxuecheng/doris/be/src/util/thread.cpp:499
20# start_thread in /lib64/libpthread.so.0
21# __clone in /lib64/libc.so.6
```
---
 .../aggregate_function_collect.cpp               | 10 ++++++++--
 .../data/query_p0/aggregate/array_agg.out        |  3 +++
 .../suites/query_p0/aggregate/array_agg.groovy   | 16 ++++++++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.cpp b/be/src/vec/aggregate_functions/aggregate_function_collect.cpp
index a4853ff98659a0..15806c739ed58c 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_collect.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_collect.cpp
@@ -73,12 +73,18 @@ AggregateFunctionPtr create_aggregate_function_collect_impl(const std::string& n
     if (which.is_date_or_datetime()) {
         return do_create_agg_function_collect<Int64, HasLimit, ShowNull>(distinct, argument_types,
                                                                          result_is_nullable);
-    } else if (which.is_date_v2() || which.is_ipv4()) {
+    } else if (which.is_date_v2()) {
         return do_create_agg_function_collect<UInt32, HasLimit, ShowNull>(distinct, argument_types,
                                                                           result_is_nullable);
-    } else if (which.is_date_time_v2() || which.is_ipv6()) {
+    } else if (which.is_date_time_v2()) {
         return do_create_agg_function_collect<UInt64, HasLimit, ShowNull>(distinct, argument_types,
                                                                           result_is_nullable);
+    } else if (which.is_ipv6()) {
+        return do_create_agg_function_collect<IPv6, HasLimit, ShowNull>(distinct, argument_types,
+                                                                        result_is_nullable);
+    } else if (which.is_ipv4()) {
+        return do_create_agg_function_collect<IPv4, HasLimit, ShowNull>(distinct, argument_types,
+                                                                        result_is_nullable);
     } else if (which.is_string()) {
         return do_create_agg_function_collect<StringRef, HasLimit, ShowNull>(
                 distinct, argument_types, result_is_nullable);
diff --git a/regression-test/data/query_p0/aggregate/array_agg.out b/regression-test/data/query_p0/aggregate/array_agg.out
index 1fe44df3a144bc..62ffb5fcf475c0 100644
--- a/regression-test/data/query_p0/aggregate/array_agg.out
+++ b/regression-test/data/query_p0/aggregate/array_agg.out
@@ -251,3 +251,6 @@
 8	[{"id":8}, {"id":8}, {"id":8}, {"id":8}]
 9	[{"id":9}, {"id":9}, {"id":9}, {"id":9}]
 
+-- !select --
+[null, "0.0.0.123", "0.0.12.42", "0.119.130.67"]	[null, "::855d", "::0.4.221.183", "::a:7429:d0d6:6e08:9f5f"]
+
diff --git a/regression-test/suites/query_p0/aggregate/array_agg.groovy b/regression-test/suites/query_p0/aggregate/array_agg.groovy
index 217285b572c538..42fb3b131a4f86 100644
--- a/regression-test/suites/query_p0/aggregate/array_agg.groovy
+++ b/regression-test/suites/query_p0/aggregate/array_agg.groovy
@@ -277,8 +277,24 @@ suite("array_agg") {
     order_qt_sql_array_agg_map """ SELECT id, array_agg(km) FROM test_array_agg_complex GROUP BY id ORDER BY id """
     order_qt_sql_array_agg_struct """ SELECT id, array_agg(ks) FROM test_array_agg_complex GROUP BY id ORDER BY id """
 
+
+ sql """ DROP TABLE IF EXISTS test_array_agg_ip;"""
+    sql """
+        CREATE TABLE test_array_agg_ip(
+            k1 BIGINT ,
+            k4 ipv4 ,
+            k6 ipv6 ,
+            s string 
+        ) DISTRIBUTED BY HASH(k1) BUCKETS 1 PROPERTIES("replication_num" = "1");
+    """
+    sql """ insert into test_array_agg_ip values(1,123,34141,"0.0.0.123") , (2,3114,318903,"0.0.0.123") , (3,7832131,192837891738927931231,"2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D"),(4,null,null,"2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D"); """
+
+
+     qt_select """select array_sort(array_agg(k4)),array_sort(array_agg(k6)) from test_array_agg_ip """
+
     sql "DROP TABLE `test_array_agg`"
     sql "DROP TABLE `test_array_agg1`"	
     sql "DROP TABLE `test_array_agg_int`"
     sql "DROP TABLE `test_array_agg_decimal`"
+    sql "DROP TABLE `test_array_agg_ip`"
 }

From 76021471269a8590ef4780995f8159afbee7fa6d Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Tue, 3 Dec 2024 17:50:48 +0800
Subject: [PATCH 137/399] =?UTF-8?q?[Chore](runtime-filter)=20display=20wak?=
 =?UTF-8?q?e=5Fup=5Fby=5Fdownstream/should=5Fbuild=5Fhash=5Ftable=20when?=
 =?UTF-8?q?=20build=5Fsink=E2=80=A6=20(#44892)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

display wake_up_by_downstream/should_build_hash_table when build_sink::close prosess rf meet error
---
 be/src/pipeline/exec/hashjoin_build_sink.cpp | 63 +++++++++++---------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index e4c67b24193593..d6129e5af203d2 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -139,36 +139,43 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
         return Base::close(state, exec_status);
     }
 
-    if (state->get_task()->wake_up_by_downstream()) {
-        if (_should_build_hash_table) {
-            // partitial ignore rf to make global rf work
-            RETURN_IF_ERROR(_runtime_filter_slots->send_filter_size(state, 0, _finish_dependency));
-            RETURN_IF_ERROR(_runtime_filter_slots->ignore_all_filters());
-        } else {
-            // do not publish filter coz local rf not inited and useless
-            return Base::close(state, exec_status);
-        }
-    } else if (_should_build_hash_table) {
-        if (p._shared_hashtable_controller && !p._shared_hash_table_context->complete_build_stage) {
-            return Status::InternalError("close before sink meet eos");
-        }
-        auto* block = _shared_state->build_block.get();
-        uint64_t hash_table_size = block ? block->rows() : 0;
-        {
-            SCOPED_TIMER(_runtime_filter_init_timer);
-            RETURN_IF_ERROR_OR_CATCH_EXCEPTION(
-                    _runtime_filter_slots->init_filters(state, hash_table_size));
-            RETURN_IF_ERROR(_runtime_filter_slots->ignore_filters(state));
-        }
-        if (hash_table_size > 1) {
-            SCOPED_TIMER(_runtime_filter_compute_timer);
-            _runtime_filter_slots->insert(block);
+    try {
+        if (state->get_task()->wake_up_by_downstream()) {
+            if (_should_build_hash_table) {
+                // partitial ignore rf to make global rf work
+                RETURN_IF_ERROR(
+                        _runtime_filter_slots->send_filter_size(state, 0, _finish_dependency));
+                RETURN_IF_ERROR(_runtime_filter_slots->ignore_all_filters());
+            } else {
+                // do not publish filter coz local rf not inited and useless
+                return Base::close(state, exec_status);
+            }
+        } else if (_should_build_hash_table) {
+            if (p._shared_hashtable_controller &&
+                !p._shared_hash_table_context->complete_build_stage) {
+                return Status::InternalError("close before sink meet eos");
+            }
+            auto* block = _shared_state->build_block.get();
+            uint64_t hash_table_size = block ? block->rows() : 0;
+            {
+                SCOPED_TIMER(_runtime_filter_init_timer);
+                RETURN_IF_ERROR(_runtime_filter_slots->init_filters(state, hash_table_size));
+                RETURN_IF_ERROR(_runtime_filter_slots->ignore_filters(state));
+            }
+            if (hash_table_size > 1) {
+                SCOPED_TIMER(_runtime_filter_compute_timer);
+                _runtime_filter_slots->insert(block);
+            }
         }
-    }
 
-    SCOPED_TIMER(_publish_runtime_filter_timer);
-    RETURN_IF_ERROR_OR_CATCH_EXCEPTION(
-            _runtime_filter_slots->publish(state, !_should_build_hash_table));
+        SCOPED_TIMER(_publish_runtime_filter_timer);
+        RETURN_IF_ERROR(_runtime_filter_slots->publish(state, !_should_build_hash_table));
+    } catch (Exception& e) {
+        return Status::InternalError(
+                "rf process meet error: {}, wake_up_by_downstream: {}, should_build_hash_table: {}",
+                e.to_string(), state->get_task()->wake_up_by_downstream(),
+                _should_build_hash_table);
+    }
     return Base::close(state, exec_status);
 }
 

From 76d94938fd14d431245b233f6cbf3f82e89699e1 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <chenkaijie@selectdb.com>
Date: Tue, 3 Dec 2024 18:52:16 +0800
Subject: [PATCH 138/399] [test](move-memtable) fix false positives in sinkv2
 injection tests (#44552)

`test_writer_v2_fault_injection` did not assert Exception being thrown,
which caused false positives.
i.e. Code bugged but test passed.

This PR fixed this problem.
Run this test without #44344, it will now report errors as expected.

```
2024-11-25 17:03:25.463 ERROR [non-concurrent-thread-1] (ScriptContext.groovy:122) - Run test_writer_v2_fault_injection in ./doris/regression-test/suites/fault_injection_p0/test_writer_v2_fault_injection.groovy failed
org.opentest4j.AssertionFailedError: expected Exception 'load timed out before close waiting', actual success ==> expected: <true> but was: <false>
```
---
 .../fault_injection_p0/test_writer_v2_fault_injection.groovy | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/regression-test/suites/fault_injection_p0/test_writer_v2_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_writer_v2_fault_injection.groovy
index f05a084035cb3f..7e71de173e96f9 100644
--- a/regression-test/suites/fault_injection_p0/test_writer_v2_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_writer_v2_fault_injection.groovy
@@ -67,10 +67,11 @@ suite("test_writer_v2_fault_injection", "nonConcurrent") {
         file "baseall.txt"
     }
 
-    def load_with_injection = { injection, error_msg->
+    def load_with_injection = { injection, error_msg, success=false->
         try {
             GetDebugPoint().enableDebugPointForAllBEs(injection)
             sql "insert into test select * from baseall where k1 <= 3"
+            assertTrue(success, String.format("expected Exception '%s', actual success", error_msg))
         } catch(Exception e) {
             logger.info(e.getMessage())
             assertTrue(e.getMessage().contains(error_msg),
@@ -88,7 +89,7 @@ suite("test_writer_v2_fault_injection", "nonConcurrent") {
     // VTabletWriterV2 node_info is null
     load_with_injection("VTabletWriterV2._open_streams_to_backend.node_info_null", "failed to open streams to any BE")
     // VTabletWriterV2 do not get tablet schema on open_streams
-    load_with_injection("VTabletWriterV2._open_streams_to_backend.no_schema_when_open_streams", "success")
+    load_with_injection("VTabletWriterV2._open_streams_to_backend.no_schema_when_open_streams", "success", true)
     // VTabletWriterV2 tablet_location is null
     load_with_injection("VTabletWriterV2._build_tablet_node_mapping.tablet_location_null", "unknown tablet location")
     // VTabletWriterV2 location is null

From 39d1acdc5eb633d7ea2fe5cd4f9d3433ced372a7 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Tue, 3 Dec 2024 16:47:31 +0530
Subject: [PATCH 139/399] [chore][test] add testcase for recycle bin (#44815)

Problem Summary:
add multiple drop and multiple insert overwrite cases for recycle bin
feature.
---
 .../test_create_drop_multiple.out             |  20 ++++
 ...test_insert_overwrite_recover_multiple.out |  23 ++++
 .../test_create_drop_multiple.groovy          | 106 ++++++++++++++++++
 ...t_insert_overwrite_recover_multiple.groovy |  87 ++++++++++++++
 4 files changed, 236 insertions(+)
 create mode 100644 regression-test/data/catalog_recycle_bin_p0/test_create_drop_multiple.out
 create mode 100644 regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.out
 create mode 100644 regression-test/suites/catalog_recycle_bin_p0/test_create_drop_multiple.groovy
 create mode 100644 regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.groovy

diff --git a/regression-test/data/catalog_recycle_bin_p0/test_create_drop_multiple.out b/regression-test/data/catalog_recycle_bin_p0/test_create_drop_multiple.out
new file mode 100644
index 00000000000000..9e3ccdba6fe27a
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/test_create_drop_multiple.out
@@ -0,0 +1,20 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_check_1 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_2 --
+3	a	2024-01-02
+
+-- !select_check_recover_1_value4 --
+4	a	2024-01-02
+
+-- !select_check_recover_2_value3 --
+3	a	2024-01-02
+
+-- !select_check_recover_3_value123 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
diff --git a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.out b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.out
new file mode 100644
index 00000000000000..4930be08bd12fe
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.out
@@ -0,0 +1,23 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_check_1 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
+-- !select_check_2 --
+3	a	2024-01-02
+
+-- !select_check_3 --
+4	a	2024-01-02
+
+-- !select_check_recover_1_value4 --
+4	a	2024-01-02
+
+-- !select_check_recover_2_value3 --
+3	a	2024-01-02
+
+-- !select_check_recover_3_value123 --
+1	a	2022-01-02
+2	a	2023-01-02
+3	a	2024-01-02
+
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_create_drop_multiple.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_create_drop_multiple.groovy
new file mode 100644
index 00000000000000..9decc30c39f2b4
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_create_drop_multiple.groovy
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_create_drop_multiple") {
+    def table = "test_create_drop_multiple"
+
+    // create table and insert data
+    sql """ drop table if exists ${table}"""
+    sql """
+    create table ${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+
+    sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+    sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+    sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+    sql """ SYNC;"""
+
+    qt_select_check_1 """ select * from  ${table} order by id,name,da; """
+
+    sql """ drop table ${table}; """
+
+    sql """
+    create table ${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+    sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+    sql """ SYNC;"""
+    qt_select_check_2 """ select * from  ${table} order by id,name,da; """    
+    sql """ drop table ${table}; """
+    sql """
+    create table ${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+    sql """ insert into ${table} values(4, 'a', '2024-01-02'); """
+    sql """ SYNC;"""
+    sql """ drop table ${table}; """
+    sql """ recover table ${table}; """
+
+    qt_select_check_recover_1_value4 """ select * from  ${table} order by id,name,da; """
+    sql """ drop table ${table} force; """
+    sql """ recover table ${table}; """
+    qt_select_check_recover_2_value3 """ select * from  ${table} order by id,name,da; """
+    sql """ drop table ${table} force; """
+    sql """ recover table ${table}; """
+    qt_select_check_recover_3_value123 """ select * from  ${table} order by id,name,da; """    
+}
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.groovy
new file mode 100644
index 00000000000000..e7f65e82610ffa
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_multiple.groovy
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_insert_overwrite_recover_multiple") {
+    def table = "test_insert_overwrite_recover_multiple"
+
+    // create table and insert data
+    sql """ drop table if exists ${table}"""
+    sql """
+    create table ${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+
+    sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+    sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+    sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+    sql """ SYNC;"""
+
+    qt_select_check_1 """ select * from  ${table} order by id,name,da; """
+
+    sql """ insert overwrite  table ${table} values(3, 'a', '2024-01-02'); """
+
+    qt_select_check_2 """ select * from  ${table} order by id,name,da; """
+
+    sql """ insert overwrite  table ${table} values(4, 'a', '2024-01-02'); """
+
+    qt_select_check_3 """ select * from  ${table} order by id,name,da; """    
+    
+    sql """ ALTER TABLE ${table} DROP PARTITION p3; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p4; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p5; """
+
+    sql """ recover partition p3  from ${table}; """
+    sql """ recover partition p4  from ${table}; """
+    sql """ recover partition p5  from ${table}; """    
+
+    qt_select_check_recover_1_value4 """ select * from  ${table} order by id,name,da; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
+
+    sql """ recover partition p3  from ${table}; """
+    sql """ recover partition p4  from ${table}; """
+    sql """ recover partition p5  from ${table}; """    
+
+    qt_select_check_recover_2_value3 """ select * from  ${table} order by id,name,da; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p3 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p4 force; """
+    sql """ ALTER TABLE ${table} DROP PARTITION p5 force; """
+
+    sql """ recover partition p3  from ${table}; """
+    sql """ recover partition p4  from ${table}; """
+    sql """ recover partition p5  from ${table}; """    
+
+    qt_select_check_recover_3_value123 """ select * from  ${table} order by id,name,da; """    
+
+
+}

From 8bc3c4306a43f7741ab2eda8e4f02c783b2ab063 Mon Sep 17 00:00:00 2001
From: camby <cambyzhu@tencent.com>
Date: Tue, 3 Dec 2024 19:56:29 +0800
Subject: [PATCH 140/399] [fix](ForkJoinPool) we should not new a thread pool
 every call (#44891)

### What problem does this PR solve?

come from: https://github.com/apache/doris/pull/34988

Problem:
Use JDK11 and call `show proc "/cluster_health/tablet_health"`
frequently, make large number of ForkJoinPool thread leak
---
 .../java/org/apache/doris/common/proc/TabletHealthProcDir.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java
index 9e91eb88148d3c..6c36a926d088b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java
@@ -67,7 +67,7 @@ public class TabletHealthProcDir implements ProcDirInterface {
 
     private Env env;
 
-    private ForkJoinPool taskPool = new ForkJoinPool();
+    private static ForkJoinPool taskPool = new ForkJoinPool();
 
     public TabletHealthProcDir(Env env) {
         Preconditions.checkNotNull(env);

From 802ba712ac9a8190539356bbfd294e103199e40b Mon Sep 17 00:00:00 2001
From: hui lai <laihui@selectdb.com>
Date: Tue, 3 Dec 2024 21:34:49 +0800
Subject: [PATCH 141/399] [fix](third party) fix hang when destroy of rdkafka
 instances (#44913)

Related PR: https://github.com/confluentinc/librdkafka/pull/4724
---
 thirdparty/patches/librdkafka-1.9.2.patch | 111 +++++++++++++++++++++-
 1 file changed, 110 insertions(+), 1 deletion(-)

diff --git a/thirdparty/patches/librdkafka-1.9.2.patch b/thirdparty/patches/librdkafka-1.9.2.patch
index b13e740bc5c36a..3caac08f79dacd 100644
--- a/thirdparty/patches/librdkafka-1.9.2.patch
+++ b/thirdparty/patches/librdkafka-1.9.2.patch
@@ -67,7 +67,19 @@
  
 --- src/rdkafka_broker.c
 +++ src/rdkafka_broker.c
-@@ -5461,7 +5461,9 @@ static int rd_kafka_broker_thread_main(void *arg) {
+@@ -3288,6 +3288,11 @@ rd_kafka_broker_op_serve(rd_kafka_broker_t *rkb, rd_kafka_op_t *rko) {
+                                 : (topic_err
+                                        ? topic_err
+                                        : RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION));
++
++                        if (rkb->rkb_rk->rk_type == RD_KAFKA_CONSUMER) {
++                                rd_kafka_toppar_purge_internal_fetch_queue_maybe(
++                                    rktp);
++                        }
+                 }
+ 
+                 rd_kafka_toppar_unlock(rktp);
+@@ -5461,7 +5466,9 @@ static int rd_kafka_broker_thread_main(void *arg) {
   */
  void rd_kafka_broker_destroy_final(rd_kafka_broker_t *rkb) {
  
@@ -78,3 +90,100 @@
          rd_assert(TAILQ_EMPTY(&rkb->rkb_monitors));
          rd_assert(TAILQ_EMPTY(&rkb->rkb_outbufs.rkbq_bufs));
          rd_assert(TAILQ_EMPTY(&rkb->rkb_waitresps.rkbq_bufs));
+--- src/rdkafka_cgrp.c
++++ src/rdkafka_cgrp.c
+@@ -2734,6 +2734,9 @@ static void rd_kafka_cgrp_partition_del(rd_kafka_cgrp_t *rkcg,
+         rd_kafka_toppar_lock(rktp);
+         rd_assert(rktp->rktp_flags & RD_KAFKA_TOPPAR_F_ON_CGRP);
+         rktp->rktp_flags &= ~RD_KAFKA_TOPPAR_F_ON_CGRP;
++
++        rd_kafka_toppar_purge_internal_fetch_queue_maybe(rktp);
++
+         rd_kafka_toppar_unlock(rktp);
+ 
+         rd_list_remove(&rkcg->rkcg_toppars, rktp);
+--- src/rdkafka_partition.c
++++ src/rdkafka_partition.c
+@@ -959,7 +959,71 @@ void rd_kafka_toppar_insert_msgq(rd_kafka_toppar_t *rktp,
+         rd_kafka_toppar_unlock(rktp);
+ }
+ 
++/**
++ * @brief Purge internal fetch queue if toppar is stopped
++ * (RD_KAFKA_TOPPAR_FETCH_STOPPED) and removed from the cluster
++ * (RD_KAFKA_TOPPAR_F_REMOVE). Will be called from different places as it's
++ * removed starting from a metadata response and stopped from a rebalance or a
++ * consumer close.
++ *
++ * @remark Avoids circular dependencies in from `rktp_fetchq` ops to the same
++ * toppar that stop destroying a consumer.
++ *
++ * @locks rd_kafka_toppar_lock() MUST be held
++ */
++void rd_kafka_toppar_purge_internal_fetch_queue_maybe(rd_kafka_toppar_t *rktp) {
++        rd_kafka_q_t *rkq;
++        rkq = rktp->rktp_fetchq;
++        mtx_lock(&rkq->rkq_lock);
++        if (rktp->rktp_flags & RD_KAFKA_TOPPAR_F_REMOVE &&
++            !rktp->rktp_fetchq->rkq_fwdq) {
++                rd_kafka_op_t *rko;
++                int cnt = 0, barrier_cnt = 0, message_cnt = 0, other_cnt = 0;
++
++                /* Partition is being removed from the cluster and it's stopped,
++                 * so rktp->rktp_fetchq->rkq_fwdq is NULL.
++                 * Purge remaining operations in rktp->rktp_fetchq->rkq_q,
++                 * while holding lock, to avoid circular references */
++                rko = TAILQ_FIRST(&rkq->rkq_q);
++                while (rko) {
++                        if (rko->rko_type != RD_KAFKA_OP_BARRIER &&
++                            rko->rko_type != RD_KAFKA_OP_FETCH) {
++                                rd_kafka_log(
++                                    rktp->rktp_rkt->rkt_rk, LOG_WARNING,
++                                    "PARTDEL",
++                                    "Purging toppar fetch queue buffer op"
++                                    "with unexpected type: %s",
++                                    rd_kafka_op2str(rko->rko_type));
++                        }
++
++                        if (rko->rko_type == RD_KAFKA_OP_BARRIER)
++                                barrier_cnt++;
++                        else if (rko->rko_type == RD_KAFKA_OP_FETCH)
++                                message_cnt++;
++                        else
++                                other_cnt++;
+ 
++                        rko = TAILQ_NEXT(rko, rko_link);
++                        cnt++;
++                }
++
++                if (cnt) {
++                        rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, CGRP, "PARTDEL",
++                                     "Purge toppar fetch queue buffer "
++                                     "containing %d op(s) "
++                                     "(%d barrier(s), %d message(s), %d other)"
++                                     " to avoid "
++                                     "circular references",
++                                     cnt, barrier_cnt, message_cnt, other_cnt);
++                        rd_kafka_q_purge0(rktp->rktp_fetchq, rd_false);
++                } else {
++                        rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, CGRP, "PARTDEL",
++                                     "Not purging toppar fetch queue buffer."
++                                     " No ops present in the buffer.");
++                }
++        }
++        mtx_unlock(&rkq->rkq_lock);
++}
+ 
+ /**
+  * Helper method for purging queues when removing a toppar.
+--- src/rdkafka_partition.h
++++ src/rdkafka_partition.h
+@@ -541,6 +541,8 @@ void rd_kafka_toppar_offset_request(rd_kafka_toppar_t *rktp,
+                                     int64_t query_offset,
+                                     int backoff_ms);
+ 
++void rd_kafka_toppar_purge_internal_fetch_queue_maybe(rd_kafka_toppar_t *rktp);
++
+ int rd_kafka_toppar_purge_queues(rd_kafka_toppar_t *rktp,
+                                  int purge_flags,
+                                  rd_bool_t include_xmit_msgq);

From 7a68d8f11cf3b5a991cc30ea1854924cf8f4a2d3 Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Tue, 3 Dec 2024 21:41:17 +0800
Subject: [PATCH 142/399] [Opt](checker) Add checker for delete bitmaps
 (#44154)

1. check every delete bitmap in MS belongs to a visible rowset
2. check that https://github.com/apache/doris/pull/40204 works as
expected
---
 cloud/src/common/bvars.cpp     |   7 +
 cloud/src/common/bvars.h       |   4 +
 cloud/src/common/config.h      |   5 +
 cloud/src/recycler/checker.cpp | 457 ++++++++++++++++++++++++++++++++-
 cloud/src/recycler/checker.h   |  28 ++
 cloud/src/recycler/util.cpp    |  66 +++++
 cloud/src/recycler/util.h      |   5 +
 cloud/test/recycler_test.cpp   | 398 +++++++++++++++++++++++++++-
 8 files changed, 965 insertions(+), 5 deletions(-)

diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp
index 507acb00dff0b0..746f109ac6d7fd 100644
--- a/cloud/src/common/bvars.cpp
+++ b/cloud/src/common/bvars.cpp
@@ -198,3 +198,10 @@ BvarStatusWithTag<long> g_bvar_checker_instance_volume("checker", "instance_volu
 BvarStatusWithTag<long> g_bvar_inverted_checker_num_scanned("checker", "num_inverted_scanned");
 BvarStatusWithTag<long> g_bvar_inverted_checker_num_check_failed("checker",
                                                                  "num_inverted_check_failed");
+
+BvarStatusWithTag<int64_t> g_bvar_inverted_checker_leaked_delete_bitmaps("checker",
+                                                                         "leaked_delete_bitmaps");
+BvarStatusWithTag<int64_t> g_bvar_inverted_checker_abnormal_delete_bitmaps(
+        "checker", "abnormal_delete_bitmaps");
+BvarStatusWithTag<int64_t> g_bvar_inverted_checker_delete_bitmaps_scanned(
+        "checker", "delete_bitmap_keys_scanned");
\ No newline at end of file
diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h
index 373a3a63ff2703..d0ad2e97957ae6 100644
--- a/cloud/src/common/bvars.h
+++ b/cloud/src/common/bvars.h
@@ -250,3 +250,7 @@ extern BvarStatusWithTag<long> g_bvar_checker_last_success_time_ms;
 extern BvarStatusWithTag<long> g_bvar_checker_instance_volume;
 extern BvarStatusWithTag<long> g_bvar_inverted_checker_num_scanned;
 extern BvarStatusWithTag<long> g_bvar_inverted_checker_num_check_failed;
+
+extern BvarStatusWithTag<int64_t> g_bvar_inverted_checker_leaked_delete_bitmaps;
+extern BvarStatusWithTag<int64_t> g_bvar_inverted_checker_abnormal_delete_bitmaps;
+extern BvarStatusWithTag<int64_t> g_bvar_inverted_checker_delete_bitmaps_scanned;
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 43a426a89593d5..ac4403aeb24a3c 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -72,6 +72,11 @@ CONF_Bool(enable_checker, "false");
 CONF_Int32(recycle_pool_parallelism, "40");
 // Currently only used for recycler test
 CONF_Bool(enable_inverted_check, "false");
+// Currently only used for recycler test
+CONF_Bool(enable_delete_bitmap_inverted_check, "false");
+// checks if https://github.com/apache/doris/pull/40204 works as expected
+CONF_Bool(enable_delete_bitmap_storage_optimize_check, "false");
+CONF_mInt64(delete_bitmap_storage_optimize_check_version_gap, "1000");
 // interval for scanning instances to do checks and inspections
 CONF_mInt32(scan_instances_interval_seconds, "60"); // 1min
 // interval for check object
diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp
index 19a10d61c12d35..60b6b7fc5eefea 100644
--- a/cloud/src/recycler/checker.cpp
+++ b/cloud/src/recycler/checker.cpp
@@ -168,17 +168,35 @@ int Checker::start() {
             auto ctime_ms =
                     duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
             g_bvar_checker_enqueue_cost_s.put(instance_id, ctime_ms / 1000 - enqueue_time_s);
-            int ret1 = checker->do_check();
 
-            int ret2 = 0;
+            bool success {true};
+
+            if (int ret = checker->do_check(); ret != 0) {
+                success = false;
+            }
+
             if (config::enable_inverted_check) {
-                ret2 = checker->do_inverted_check();
+                if (int ret = checker->do_inverted_check(); ret != 0) {
+                    success = false;
+                }
+            }
+
+            if (config::enable_delete_bitmap_inverted_check) {
+                if (int ret = checker->do_delete_bitmap_inverted_check(); ret != 0) {
+                    success = false;
+                }
+            }
+
+            if (config::enable_delete_bitmap_storage_optimize_check) {
+                if (int ret = checker->do_delete_bitmap_storage_optimize_check(); ret != 0) {
+                    success = false;
+                }
             }
 
             // If instance checker has been aborted, don't finish this job
             if (!checker->stopped()) {
                 finish_instance_recycle_job(txn_kv_.get(), check_job_key, instance.instance_id(),
-                                            ip_port_, ret1 == 0 && ret2 == 0, ctime_ms);
+                                            ip_port_, success, ctime_ms);
             }
             {
                 std::lock_guard lock(mtx_);
@@ -740,4 +758,435 @@ int InstanceChecker::do_inverted_check() {
     return num_file_leak > 0 ? 1 : check_ret;
 }
 
+int InstanceChecker::traverse_mow_tablet(const std::function<int(int64_t)>& check_func) {
+    std::unique_ptr<RangeGetIterator> it;
+    auto begin = meta_rowset_key({instance_id_, 0, 0});
+    auto end = meta_rowset_key({instance_id_, std::numeric_limits<int64_t>::max(), 0});
+    do {
+        std::unique_ptr<Transaction> txn;
+        TxnErrorCode err = txn_kv_->create_txn(&txn);
+        if (err != TxnErrorCode::TXN_OK) {
+            LOG(WARNING) << "failed to create txn";
+            return -1;
+        }
+        err = txn->get(begin, end, &it, false, 1);
+        if (err != TxnErrorCode::TXN_OK) {
+            LOG(WARNING) << "failed to get rowset kv, err=" << err;
+            return -1;
+        }
+        if (!it->has_next()) {
+            break;
+        }
+        while (it->has_next() && !stopped()) {
+            auto [k, v] = it->next();
+            std::string_view k1 = k;
+            k1.remove_prefix(1);
+            std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
+            decode_key(&k1, &out);
+            // 0x01 "meta" ${instance_id} "rowset" ${tablet_id} ${version} -> RowsetMetaCloudPB
+            auto tablet_id = std::get<int64_t>(std::get<0>(out[3]));
+
+            if (!it->has_next()) {
+                // Update to next smallest key for iteration
+                // scan for next tablet in this instance
+                begin = meta_rowset_key({instance_id_, tablet_id + 1, 0});
+            }
+
+            TabletMetaCloudPB tablet_meta;
+            int ret = get_tablet_meta(txn_kv_.get(), instance_id_, tablet_id, tablet_meta);
+            if (ret < 0) {
+                LOG(WARNING) << fmt::format(
+                        "failed to get_tablet_meta in do_delete_bitmap_integrity_check(), "
+                        "instance_id={}, tablet_id={}",
+                        instance_id_, tablet_id);
+                return ret;
+            }
+
+            if (tablet_meta.enable_unique_key_merge_on_write()) {
+                // only check merge-on-write table
+                int ret = check_func(tablet_id);
+                if (ret < 0) {
+                    // return immediately when encounter unexpected error,
+                    // otherwise, we continue to check the next tablet
+                    return ret;
+                }
+            }
+        }
+    } while (it->more() && !stopped());
+    return 0;
+}
+
+int InstanceChecker::traverse_rowset_delete_bitmaps(
+        int64_t tablet_id, std::string rowset_id,
+        const std::function<int(int64_t, std::string_view, int64_t, int64_t)>& callback) {
+    std::unique_ptr<RangeGetIterator> it;
+    auto begin = meta_delete_bitmap_key({instance_id_, tablet_id, rowset_id, 0, 0});
+    auto end = meta_delete_bitmap_key({instance_id_, tablet_id, rowset_id,
+                                       std::numeric_limits<int64_t>::max(),
+                                       std::numeric_limits<int64_t>::max()});
+    do {
+        std::unique_ptr<Transaction> txn;
+        TxnErrorCode err = txn_kv_->create_txn(&txn);
+        if (err != TxnErrorCode::TXN_OK) {
+            LOG(WARNING) << "failed to create txn";
+            return -1;
+        }
+        err = txn->get(begin, end, &it);
+        if (err != TxnErrorCode::TXN_OK) {
+            LOG(WARNING) << "failed to get rowset kv, err=" << err;
+            return -1;
+        }
+        if (!it->has_next()) {
+            break;
+        }
+        while (it->has_next() && !stopped()) {
+            auto [k, v] = it->next();
+            std::string_view k1 = k;
+            k1.remove_prefix(1);
+            std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
+            decode_key(&k1, &out);
+            // 0x01 "meta" ${instance_id} "delete_bitmap" ${tablet_id} ${rowset_id} ${version} ${segment_id} -> roaringbitmap
+            auto version = std::get<std::int64_t>(std::get<0>(out[5]));
+            auto segment_id = std::get<std::int64_t>(std::get<0>(out[6]));
+
+            int ret = callback(tablet_id, rowset_id, version, segment_id);
+            if (ret != 0) {
+                return ret;
+            }
+
+            if (!it->has_next()) {
+                begin = k;
+                begin.push_back('\x00'); // Update to next smallest key for iteration
+                break;
+            }
+        }
+    } while (it->more() && !stopped());
+
+    return 0;
+}
+
+int InstanceChecker::collect_tablet_rowsets(
+        int64_t tablet_id, const std::function<void(const doris::RowsetMetaCloudPB&)>& collect_cb) {
+    std::unique_ptr<Transaction> txn;
+    TxnErrorCode err = txn_kv_->create_txn(&txn);
+    if (err != TxnErrorCode::TXN_OK) {
+        LOG(WARNING) << "failed to create txn";
+        return -1;
+    }
+    std::unique_ptr<RangeGetIterator> it;
+    auto begin = meta_rowset_key({instance_id_, tablet_id, 0});
+    auto end = meta_rowset_key({instance_id_, tablet_id + 1, 0});
+
+    int64_t rowsets_num {0};
+    do {
+        TxnErrorCode err = txn->get(begin, end, &it);
+        if (err != TxnErrorCode::TXN_OK) {
+            LOG(WARNING) << "failed to get rowset kv, err=" << err;
+            return -1;
+        }
+        if (!it->has_next()) {
+            break;
+        }
+        while (it->has_next() && !stopped()) {
+            auto [k, v] = it->next();
+            doris::RowsetMetaCloudPB rowset;
+            if (!rowset.ParseFromArray(v.data(), v.size())) {
+                LOG(WARNING) << "malformed rowset meta value, key=" << hex(k);
+                return -1;
+            }
+
+            ++rowsets_num;
+            collect_cb(rowset);
+
+            if (!it->has_next()) {
+                begin = k;
+                begin.push_back('\x00'); // Update to next smallest key for iteration
+                break;
+            }
+        }
+    } while (it->more() && !stopped());
+
+    LOG(INFO) << fmt::format(
+            "[delete bitmap checker] successfully collect rowsets for instance_id={}, "
+            "tablet_id={}, rowsets_num={}",
+            instance_id_, tablet_id, rowsets_num);
+    return 0;
+}
+
+int InstanceChecker::do_delete_bitmap_inverted_check() {
+    LOG(INFO) << fmt::format(
+            "[delete bitmap checker] begin to do_delete_bitmap_inverted_check for instance_id={}",
+            instance_id_);
+
+    // number of delete bitmap keys being scanned
+    int64_t total_delete_bitmap_keys {0};
+    // number of delete bitmaps which belongs to non mow tablet
+    int64_t abnormal_delete_bitmaps {0};
+    // number of delete bitmaps which doesn't have corresponding rowset in MS
+    int64_t leaked_delete_bitmaps {0};
+
+    auto start_time = std::chrono::steady_clock::now();
+    std::unique_ptr<int, std::function<void(int*)>> defer_log_statistics((int*)0x01, [&](int*) {
+        g_bvar_inverted_checker_leaked_delete_bitmaps.put(instance_id_, leaked_delete_bitmaps);
+        g_bvar_inverted_checker_abnormal_delete_bitmaps.put(instance_id_, abnormal_delete_bitmaps);
+        g_bvar_inverted_checker_delete_bitmaps_scanned.put(instance_id_, total_delete_bitmap_keys);
+
+        auto cost = std::chrono::duration_cast<std::chrono::milliseconds>(
+                            std::chrono::steady_clock::now() - start_time)
+                            .count();
+        if (leaked_delete_bitmaps > 0 || abnormal_delete_bitmaps > 0) {
+            LOG(WARNING) << fmt::format(
+                    "[delete bitmap check fails] delete bitmap inverted check for instance_id={}, "
+                    "cost={} ms, total_delete_bitmap_keys={}, leaked_delete_bitmaps={}, "
+                    "abnormal_delete_bitmaps={}",
+                    instance_id_, cost, total_delete_bitmap_keys, leaked_delete_bitmaps,
+                    abnormal_delete_bitmaps);
+        } else {
+            LOG(INFO) << fmt::format(
+                    "[delete bitmap checker] delete bitmap inverted check for instance_id={}, "
+                    "passed. cost={} ms, total_delete_bitmap_keys={}",
+                    instance_id_, cost, total_delete_bitmap_keys);
+        }
+    });
+
+    struct TabletsRowsetsCache {
+        int64_t tablet_id {-1};
+        bool enable_merge_on_write {false};
+        std::unordered_set<std::string> rowsets {};
+    } tablet_rowsets_cache {};
+
+    std::unique_ptr<RangeGetIterator> it;
+    auto begin = meta_delete_bitmap_key({instance_id_, 0, "", 0, 0});
+    auto end =
+            meta_delete_bitmap_key({instance_id_, std::numeric_limits<int64_t>::max(), "", 0, 0});
+    do {
+        std::unique_ptr<Transaction> txn;
+        TxnErrorCode err = txn_kv_->create_txn(&txn);
+        if (err != TxnErrorCode::TXN_OK) {
+            LOG(WARNING) << "failed to create txn";
+            return -1;
+        }
+        err = txn->get(begin, end, &it);
+        if (err != TxnErrorCode::TXN_OK) {
+            LOG(WARNING) << "failed to get rowset kv, err=" << err;
+            return -1;
+        }
+        if (!it->has_next()) {
+            break;
+        }
+        while (it->has_next() && !stopped()) {
+            auto [k, v] = it->next();
+            std::string_view k1 = k;
+            k1.remove_prefix(1);
+            std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
+            decode_key(&k1, &out);
+            // 0x01 "meta" ${instance_id} "delete_bitmap" ${tablet_id} ${rowset_id} ${version} ${segment_id} -> roaringbitmap
+            auto tablet_id = std::get<int64_t>(std::get<0>(out[3]));
+            auto rowset_id = std::get<std::string>(std::get<0>(out[4]));
+            auto version = std::get<std::int64_t>(std::get<0>(out[5]));
+            auto segment_id = std::get<std::int64_t>(std::get<0>(out[6]));
+
+            ++total_delete_bitmap_keys;
+
+            if (!it->has_next()) {
+                begin = k;
+                begin.push_back('\x00'); // Update to next smallest key for iteration
+            }
+
+            if (tablet_rowsets_cache.tablet_id == -1 ||
+                tablet_rowsets_cache.tablet_id != tablet_id) {
+                TabletMetaCloudPB tablet_meta;
+                int ret = get_tablet_meta(txn_kv_.get(), instance_id_, tablet_id, tablet_meta);
+                if (ret < 0) {
+                    LOG(WARNING) << fmt::format(
+                            "[delete bitmap checker] failed to get_tablet_meta in "
+                            "do_delete_bitmap_inverted_check(), instance_id={}, tablet_id={}",
+                            instance_id_, tablet_id);
+                    return ret;
+                }
+
+                tablet_rowsets_cache.tablet_id = tablet_id;
+                tablet_rowsets_cache.enable_merge_on_write =
+                        tablet_meta.enable_unique_key_merge_on_write();
+                tablet_rowsets_cache.rowsets.clear();
+
+                if (tablet_rowsets_cache.enable_merge_on_write) {
+                    // only collect rowsets for merge-on-write tablet
+                    auto collect_cb =
+                            [&tablet_rowsets_cache](const doris::RowsetMetaCloudPB& rowset) {
+                                tablet_rowsets_cache.rowsets.insert(rowset.rowset_id_v2());
+                            };
+                    ret = collect_tablet_rowsets(tablet_id, collect_cb);
+                    if (ret < 0) {
+                        return ret;
+                    }
+                }
+            }
+            DCHECK_EQ(tablet_id, tablet_rowsets_cache.tablet_id);
+
+            if (!tablet_rowsets_cache.enable_merge_on_write) {
+                // clang-format off
+                TEST_SYNC_POINT_CALLBACK(
+                        "InstanceChecker::do_delete_bitmap_inverted_check.get_abnormal_delete_bitmap",
+                        &tablet_id, &rowset_id, &version, &segment_id);
+                // clang-format on
+                ++abnormal_delete_bitmaps;
+                // log an error and continue to check the next delete bitmap
+                LOG(WARNING) << fmt::format(
+                        "[delete bitmap check fails] find a delete bitmap belongs to tablet "
+                        "which is not a merge-on-write table! instance_id={}, tablet_id={}, "
+                        "version={}, segment_id={}",
+                        instance_id_, tablet_id, version, segment_id);
+                continue;
+            }
+
+            if (!tablet_rowsets_cache.rowsets.contains(rowset_id)) {
+                TEST_SYNC_POINT_CALLBACK(
+                        "InstanceChecker::do_delete_bitmap_inverted_check.get_leaked_delete_bitmap",
+                        &tablet_id, &rowset_id, &version, &segment_id);
+                ++leaked_delete_bitmaps;
+                // log an error and continue to check the next delete bitmap
+                LOG(WARNING) << fmt::format(
+                        "[delete bitmap check fails] can't find corresponding rowset for delete "
+                        "bitmap instance_id={}, tablet_id={}, rowset_id={}, version={}, "
+                        "segment_id={}",
+                        instance_id_, tablet_id, rowset_id, version, segment_id);
+            }
+        }
+    } while (it->more() && !stopped());
+
+    return (leaked_delete_bitmaps > 0 || abnormal_delete_bitmaps > 0) ? 1 : 0;
+}
+
+int InstanceChecker::check_delete_bitmap_storage_optimize(int64_t tablet_id) {
+    using Version = std::pair<int64_t, int64_t>;
+    struct RowsetDigest {
+        std::string rowset_id;
+        Version version;
+        doris::SegmentsOverlapPB segments_overlap;
+
+        bool operator<(const RowsetDigest& other) const {
+            return version.first < other.version.first;
+        }
+
+        bool produced_by_compaction() const {
+            return (version.first < version.second) ||
+                   ((version.first == version.second) && segments_overlap == NONOVERLAPPING);
+        }
+    };
+
+    // number of rowsets which may have problems
+    int64_t abnormal_rowsets_num {0};
+
+    std::vector<RowsetDigest> tablet_rowsets {};
+    // Get all visible rowsets of this tablet
+    auto collect_cb = [&tablet_rowsets](const doris::RowsetMetaCloudPB& rowset) {
+        if (rowset.start_version() == 0 && rowset.end_version() == 1) {
+            // ignore dummy rowset [0-1]
+            return;
+        }
+        tablet_rowsets.emplace_back(
+                rowset.rowset_id_v2(),
+                std::make_pair<int64_t, int64_t>(rowset.start_version(), rowset.end_version()),
+                rowset.segments_overlap_pb());
+    };
+    if (int ret = collect_tablet_rowsets(tablet_id, collect_cb); ret != 0) {
+        return ret;
+    }
+
+    std::sort(tablet_rowsets.begin(), tablet_rowsets.end());
+
+    // find right-most rowset which is produced by compaction
+    auto it = std::find_if(
+            tablet_rowsets.crbegin(), tablet_rowsets.crend(),
+            [](const RowsetDigest& rowset) { return rowset.produced_by_compaction(); });
+    if (it == tablet_rowsets.crend()) {
+        LOG(INFO) << fmt::format(
+                "[delete bitmap checker] skip to check delete bitmap storage optimize for "
+                "tablet_id={} because it doesn't have compacted rowsets.",
+                tablet_id);
+        return 0;
+    }
+
+    int64_t start_version = it->version.first;
+    int64_t pre_min_version = it->version.second;
+
+    // after BE sweeping stale rowsets, all rowsets in this tablet before
+    // should not have delete bitmaps with versions lower than `pre_min_version`
+    if (config::delete_bitmap_storage_optimize_check_version_gap > 0) {
+        pre_min_version -= config::delete_bitmap_storage_optimize_check_version_gap;
+        if (pre_min_version <= 1) {
+            LOG(INFO) << fmt::format(
+                    "[delete bitmap checker] skip to check delete bitmap storage optimize for "
+                    "tablet_id={} because pre_min_version is too small.",
+                    tablet_id);
+            return 0;
+        }
+    }
+
+    auto check_func = [pre_min_version, instance_id = instance_id_](
+                              int64_t tablet_id, std::string_view rowset_id, int64_t version,
+                              int64_t segment_id) -> int {
+        if (version < pre_min_version) {
+            LOG(WARNING) << fmt::format(
+                    "[delete bitmap check fails] delete bitmap storage optimize check fail for "
+                    "instance_id={}, tablet_id={}, rowset_id={}, found delete bitmap with "
+                    "version={} < pre_min_version={}",
+                    instance_id, tablet_id, rowset_id, version, pre_min_version);
+            return 1;
+        }
+        return 0;
+    };
+
+    for (const auto& rowset : tablet_rowsets) {
+        // check for all rowsets before the max compacted rowset
+        if (rowset.version.second < start_version) {
+            auto rowset_id = rowset.rowset_id;
+            int ret = traverse_rowset_delete_bitmaps(tablet_id, rowset_id, check_func);
+            if (ret < 0) {
+                return ret;
+            }
+
+            if (ret != 0) {
+                ++abnormal_rowsets_num;
+                TEST_SYNC_POINT_CALLBACK(
+                        "InstanceChecker::check_delete_bitmap_storage_optimize.get_abnormal_rowset",
+                        &tablet_id, &rowset_id);
+            }
+        }
+    }
+
+    LOG(INFO) << fmt::format(
+            "[delete bitmap checker] finish check delete bitmap storage optimize for "
+            "instance_id={}, tablet_id={}, rowsets_num={}, abnormal_rowsets_num={}, "
+            "pre_min_version={}",
+            instance_id_, tablet_id, tablet_rowsets.size(), abnormal_rowsets_num, pre_min_version);
+
+    return (abnormal_rowsets_num > 1 ? 1 : 0);
+}
+
+int InstanceChecker::do_delete_bitmap_storage_optimize_check() {
+    int64_t total_tablets_num {0};
+    int64_t failed_tablets_num {0};
+
+    // check that for every visible rowset, there exists at least delete one bitmap in MS
+    int ret = traverse_mow_tablet([&](int64_t tablet_id) {
+        ++total_tablets_num;
+        int res = check_delete_bitmap_storage_optimize(tablet_id);
+        failed_tablets_num += (res != 0);
+        return res;
+    });
+
+    if (ret < 0) {
+        return ret;
+    }
+
+    LOG(INFO) << fmt::format(
+            "[delete bitmap checker] check delete bitmap storage optimize for instance_id={}, "
+            "total_tablets_num={}, failed_tablets_num={}",
+            instance_id_, total_tablets_num, failed_tablets_num);
+
+    return (failed_tablets_num > 0) ? 1 : 0;
+}
+
 } // namespace doris::cloud
diff --git a/cloud/src/recycler/checker.h b/cloud/src/recycler/checker.h
index 03717a69b5ea20..7f87e90f7cb366 100644
--- a/cloud/src/recycler/checker.h
+++ b/cloud/src/recycler/checker.h
@@ -23,6 +23,7 @@
 #include <atomic>
 #include <condition_variable>
 #include <deque>
+#include <functional>
 #include <thread>
 #include <unordered_map>
 #include <unordered_set>
@@ -30,6 +31,10 @@
 #include "recycler/storage_vault_accessor.h"
 #include "recycler/white_black_list.h"
 
+namespace doris {
+class RowsetMetaCloudPB;
+} // namespace doris
+
 namespace doris::cloud {
 class StorageVaultAccessor;
 class InstanceChecker;
@@ -86,6 +91,18 @@ class InstanceChecker {
     // Return 1 if data loss is identified.
     // Return negative if a temporary error occurred during the check process.
     int do_check();
+
+    // Return 0 if success.
+    // Return 1 if delete bitmap leak is identified.
+    // Return negative if a temporary error occurred during the check process.
+    int do_delete_bitmap_inverted_check();
+
+    // checks if https://github.com/apache/doris/pull/40204 works as expected
+    // the stale delete bitmap will be cleared in MS when BE delete expired stale rowsets
+    // NOTE: stale rowsets will be lost after BE restarts, so there may be some stale delete bitmaps
+    // which will not be cleared.
+    int do_delete_bitmap_storage_optimize_check();
+
     // If there are multiple buckets, return the minimum lifecycle; if there are no buckets (i.e.
     // all accessors are HdfsAccessor), return INT64_MAX.
     // Return 0 if success, otherwise error
@@ -100,6 +117,17 @@ class InstanceChecker {
     // returns 0 for success otherwise error
     int init_storage_vault_accessors(const InstanceInfoPB& instance);
 
+    int traverse_mow_tablet(const std::function<int(int64_t)>& check_func);
+    int traverse_rowset_delete_bitmaps(
+            int64_t tablet_id, std::string rowset_id,
+            const std::function<int(int64_t, std::string_view, int64_t, int64_t)>& callback);
+    int collect_tablet_rowsets(
+            int64_t tablet_id,
+            const std::function<void(const doris::RowsetMetaCloudPB&)>& collect_cb);
+    int traverse_delete_bitmaps(const std::function<int(int64_t)>& check_func);
+
+    int check_delete_bitmap_storage_optimize(int64_t tablet_id);
+
     std::atomic_bool stopped_ {false};
     std::shared_ptr<TxnKv> txn_kv_;
     std::string instance_id_;
diff --git a/cloud/src/recycler/util.cpp b/cloud/src/recycler/util.cpp
index d6c63ea752f4f1..6797782d5d665c 100644
--- a/cloud/src/recycler/util.cpp
+++ b/cloud/src/recycler/util.cpp
@@ -23,6 +23,7 @@
 
 #include "common/util.h"
 #include "meta-service/keys.h"
+#include "meta-service/meta_service_schema.h"
 #include "meta-service/txn_kv.h"
 #include "meta-service/txn_kv_error.h"
 
@@ -233,4 +234,69 @@ int lease_instance_recycle_job(TxnKv* txn_kv, std::string_view key, const std::s
     return 0;
 }
 
+int get_tablet_idx(TxnKv* txn_kv, const std::string& instance_id, int64_t tablet_id,
+                   TabletIndexPB& tablet_idx) {
+    std::unique_ptr<Transaction> txn;
+    TxnErrorCode err = txn_kv->create_txn(&txn);
+    if (err != TxnErrorCode::TXN_OK) {
+        LOG(WARNING) << "failed to create txn";
+        return -1;
+    }
+
+    std::string key, val;
+    meta_tablet_idx_key({instance_id, tablet_id}, &key);
+    err = txn->get(key, &val);
+    if (err != TxnErrorCode::TXN_OK) {
+        LOG(WARNING) << fmt::format("failed to get tablet_idx, err={} tablet_id={} key={}", err,
+                                    tablet_id, hex(key));
+        return -1;
+    }
+    if (!tablet_idx.ParseFromString(val)) [[unlikely]] {
+        LOG(WARNING) << fmt::format("malformed tablet index value, tablet_id={} key={}", tablet_id,
+                                    hex(key));
+        return -1;
+    }
+    if (tablet_id != tablet_idx.tablet_id()) [[unlikely]] {
+        LOG(WARNING) << "unexpected error given_tablet_id=" << tablet_id
+                     << " idx_pb_tablet_id=" << tablet_idx.tablet_id() << " key=" << hex(key);
+        return -1;
+    }
+    return 0;
+}
+
+int get_tablet_meta(TxnKv* txn_kv, const std::string& instance_id, int64_t tablet_id,
+                    TabletMetaCloudPB& tablet_meta) {
+    TabletIndexPB tablet_idx;
+    int ret = get_tablet_idx(txn_kv, instance_id, tablet_id, tablet_idx);
+    if (ret < 0) {
+        return ret;
+    }
+
+    std::unique_ptr<Transaction> txn;
+    TxnErrorCode err = txn_kv->create_txn(&txn);
+    if (err != TxnErrorCode::TXN_OK) {
+        LOG(WARNING) << "failed to create txn";
+        return -1;
+    }
+
+    std::string key, val;
+    meta_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(),
+                     tablet_idx.partition_id(), tablet_id},
+                    &key);
+    err = txn->get(key, &val);
+    if (err != TxnErrorCode::TXN_OK) {
+        LOG(WARNING) << fmt::format(
+                "failed to get tablet, err={}, table_id={}, index_id={}, partition_id={}, "
+                "tablet_id={} key={}",
+                err, tablet_idx.table_id(), tablet_idx.index_id(), tablet_idx.partition_id(),
+                tablet_id, hex(key));
+        return -1;
+    }
+    if (!tablet_meta.ParseFromString(val)) [[unlikely]] {
+        LOG(WARNING) << fmt::format("malformed tablet meta, tablet_id={} key={}", tablet_id,
+                                    hex(key));
+        return -1;
+    }
+    return 0;
+}
 } // namespace doris::cloud
diff --git a/cloud/src/recycler/util.h b/cloud/src/recycler/util.h
index b6d4d3299b5e0f..5aa929c23984b1 100644
--- a/cloud/src/recycler/util.h
+++ b/cloud/src/recycler/util.h
@@ -85,4 +85,9 @@ inline std::string tablet_path_prefix(int64_t tablet_id) {
     return fmt::format("data/{}/", tablet_id);
 }
 
+int get_tablet_idx(TxnKv* txn_kv, const std::string& instance_id, int64_t tablet_id,
+                   TabletIndexPB& tablet_idx);
+
+int get_tablet_meta(TxnKv* txn_kv, const std::string& instance_id, int64_t tablet_id,
+                    TabletMetaCloudPB& tablet_meta);
 } // namespace doris::cloud
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index 146873548398bf..feecf9552f933b 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -255,14 +255,63 @@ static int create_committed_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor
     return 0;
 }
 
+static int create_committed_rowset_with_rowset_id(TxnKv* txn_kv, StorageVaultAccessor* accessor,
+                                                  const std::string& resource_id, int64_t tablet_id,
+                                                  int64_t start_version, int64_t end_version,
+                                                  std::string rowset_id, bool segments_overlap,
+                                                  int num_segments) {
+    std::string key;
+    std::string val;
+
+    MetaRowsetKeyInfo key_info {instance_id, tablet_id, end_version};
+    meta_rowset_key(key_info, &key);
+
+    doris::RowsetMetaCloudPB rowset_pb;
+    rowset_pb.set_rowset_id(0); // useless but required
+    rowset_pb.set_rowset_id_v2(rowset_id);
+    rowset_pb.set_num_segments(num_segments);
+    rowset_pb.set_tablet_id(tablet_id);
+    rowset_pb.set_resource_id(resource_id);
+    rowset_pb.set_creation_time(current_time);
+    rowset_pb.set_start_version(start_version);
+    rowset_pb.set_end_version(end_version);
+    rowset_pb.set_segments_overlap_pb(segments_overlap ? OVERLAPPING : NONOVERLAPPING);
+    rowset_pb.SerializeToString(&val);
+
+    std::unique_ptr<Transaction> txn;
+    if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) {
+        return -1;
+    }
+    txn->put(key, val);
+    if (txn->commit() != TxnErrorCode::TXN_OK) {
+        return -1;
+    }
+
+    for (int i = 0; i < num_segments; ++i) {
+        auto path = segment_path(tablet_id, rowset_id, i);
+        accessor->put_file(path, "");
+    }
+    return 0;
+}
+
+static void create_delete_bitmaps(Transaction* txn, int64_t tablet_id, std::string rowset_id,
+                                  int64_t start_version, int64_t end_version) {
+    for (int64_t ver {start_version}; ver <= end_version; ver++) {
+        auto key = meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, ver, 0});
+        std::string val {"test_data"};
+        txn->put(key, val);
+    }
+}
+
 static int create_tablet(TxnKv* txn_kv, int64_t table_id, int64_t index_id, int64_t partition_id,
-                         int64_t tablet_id) {
+                         int64_t tablet_id, bool is_mow = false) {
     std::unique_ptr<Transaction> txn;
     if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) {
         return -1;
     }
     doris::TabletMetaCloudPB tablet_meta;
     tablet_meta.set_tablet_id(tablet_id);
+    tablet_meta.set_enable_unique_key_merge_on_write(is_mow);
     auto val = tablet_meta.SerializeAsString();
     auto key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id});
     txn->put(key, val);
@@ -275,6 +324,7 @@ static int create_tablet(TxnKv* txn_kv, int64_t table_id, int64_t index_id, int6
     TabletIndexPB tablet_idx_pb;
     tablet_idx_pb.set_db_id(db_id);
     tablet_idx_pb.set_table_id(table_id);
+    tablet_idx_pb.set_index_id(index_id);
     tablet_idx_pb.set_partition_id(partition_id);
     tablet_idx_pb.set_tablet_id(tablet_id);
     auto idx_val = tablet_idx_pb.SerializeAsString();
@@ -2576,6 +2626,352 @@ TEST(CheckerTest, do_inspect) {
     }
 }
 
+TEST(CheckerTest, delete_bitmap_inverted_check_normal) {
+    // normal case, all delete bitmaps belong to a rowset
+    auto txn_kv = std::make_shared<MemTxnKv>();
+    ASSERT_EQ(txn_kv->init(), 0);
+
+    InstanceInfoPB instance;
+    instance.set_instance_id(instance_id);
+    auto obj_info = instance.add_obj_info();
+    obj_info->set_id("1");
+
+    InstanceChecker checker(txn_kv, instance_id);
+    ASSERT_EQ(checker.init(instance), 0);
+    auto accessor = checker.accessor_map_.begin()->second;
+
+    std::unique_ptr<Transaction> txn;
+    ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+    constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+    // create some rowsets with delete bitmaps in merge-on-write tablet
+    for (int tablet_id = 600001; tablet_id <= 600010; ++tablet_id) {
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, true));
+        int64_t rowset_start_id = 400;
+        for (int ver = 2; ver <= 10; ++ver) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   ver, ver, rowset_id, false, 1);
+            if (ver >= 5) {
+                auto delete_bitmap_key =
+                        meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, ver, 0});
+                std::string delete_bitmap_val {"test"};
+                txn->put(delete_bitmap_key, delete_bitmap_val);
+            } else {
+                // delete bitmaps may be spilitted into mulitiple KVs if too large
+                auto delete_bitmap_key =
+                        meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, ver, 0});
+                std::string delete_bitmap_val(1000, 'A');
+                cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0, 300);
+            }
+        }
+    }
+
+    // also create some rowsets without delete bitmaps in non merge-on-write tablet
+    for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, false));
+        int64_t rowset_start_id = 500;
+        for (int ver = 2; ver < 10; ++ver) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   ver, ver, rowset_id, false, 1);
+        }
+    }
+
+    ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+    ASSERT_EQ(checker.do_delete_bitmap_inverted_check(), 0);
+}
+
+TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) {
+    // abnormal case, some delete bitmaps arem leaked
+    auto txn_kv = std::make_shared<MemTxnKv>();
+    ASSERT_EQ(txn_kv->init(), 0);
+
+    InstanceInfoPB instance;
+    instance.set_instance_id(instance_id);
+    auto obj_info = instance.add_obj_info();
+    obj_info->set_id("1");
+
+    InstanceChecker checker(txn_kv, instance_id);
+    ASSERT_EQ(checker.init(instance), 0);
+    auto accessor = checker.accessor_map_.begin()->second;
+
+    // tablet_id -> [rowset_id, version, segment_id]
+    std::map<std::int64_t, std::set<std::tuple<std::string, int64_t, int64_t>>>
+            expected_abnormal_delete_bitmaps {}, real_abnormal_delete_bitmaps {};
+    std::map<std::int64_t, std::set<std::tuple<std::string, int64_t, int64_t>>>
+            expected_leaked_delete_bitmaps {}, real_leaked_delete_bitmaps {};
+    auto sp = SyncPoint::get_instance();
+    std::unique_ptr<int, std::function<void(int*)>> defer(
+            (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); });
+    sp->set_call_back(
+            "InstanceChecker::do_delete_bitmap_inverted_check.get_abnormal_delete_bitmap",
+            [&real_abnormal_delete_bitmaps](auto&& args) {
+                int64_t tablet_id = *try_any_cast<int64_t*>(args[0]);
+                std::string rowset_id = *try_any_cast<std::string*>(args[1]);
+                int64_t version = *try_any_cast<int64_t*>(args[2]);
+                int64_t segment_id = *try_any_cast<int64_t*>(args[3]);
+                real_abnormal_delete_bitmaps[tablet_id].insert({rowset_id, version, segment_id});
+            });
+    sp->set_call_back(
+            "InstanceChecker::do_delete_bitmap_inverted_check.get_leaked_delete_bitmap",
+            [&real_leaked_delete_bitmaps](auto&& args) {
+                int64_t tablet_id = *try_any_cast<int64_t*>(args[0]);
+                std::string rowset_id = *try_any_cast<std::string*>(args[1]);
+                int64_t version = *try_any_cast<int64_t*>(args[2]);
+                int64_t segment_id = *try_any_cast<int64_t*>(args[3]);
+                real_leaked_delete_bitmaps[tablet_id].insert({rowset_id, version, segment_id});
+            });
+    sp->enable_processing();
+
+    std::unique_ptr<Transaction> txn;
+    ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+    constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+    // create some rowsets with delete bitmaps in merge-on-write tablet
+    for (int tablet_id = 800001; tablet_id <= 800010; ++tablet_id) {
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, true));
+        int64_t rowset_start_id = 600;
+        for (int ver = 2; ver <= 20; ++ver) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+
+            if (ver >= 10) {
+                // only create rowsets for some versions
+                create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                       ver, ver, rowset_id, false, 1);
+            } else {
+                expected_leaked_delete_bitmaps[tablet_id].insert({rowset_id, ver, 0});
+            }
+
+            if (ver >= 5) {
+                auto delete_bitmap_key =
+                        meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, ver, 0});
+                std::string delete_bitmap_val {"test"};
+                txn->put(delete_bitmap_key, delete_bitmap_val);
+            } else {
+                // delete bitmaps may be spilitted into mulitiple KVs if too large
+                auto delete_bitmap_key =
+                        meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, ver, 0});
+                std::string delete_bitmap_val(1000, 'A');
+                cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0, 300);
+            }
+        }
+    }
+
+    // create some rowsets with delete bitmaps in non merge-on-write tablet
+    for (int tablet_id = 900001; tablet_id <= 900010; ++tablet_id) {
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, false));
+        int64_t rowset_start_id = 700;
+        for (int ver = 2; ver < 6; ++ver) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   ver, ver, rowset_id, false, 1);
+            auto delete_bitmap_key =
+                    meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, ver, 0});
+            std::string delete_bitmap_val {"test2"};
+            txn->put(delete_bitmap_key, delete_bitmap_val);
+
+            expected_abnormal_delete_bitmaps[tablet_id].insert({rowset_id, ver, 0});
+        }
+    }
+
+    // create some rowsets without delete bitmaps in non merge-on-write tablet
+    for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, false));
+        int64_t rowset_start_id = 500;
+        for (int ver = 2; ver < 10; ++ver) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   ver, ver, rowset_id, false, 1);
+        }
+    }
+
+    ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+    ASSERT_EQ(checker.do_delete_bitmap_inverted_check(), 1);
+    ASSERT_EQ(expected_leaked_delete_bitmaps, real_leaked_delete_bitmaps);
+    ASSERT_EQ(expected_abnormal_delete_bitmaps, real_abnormal_delete_bitmaps);
+}
+
+TEST(CheckerTest, delete_bitmap_storage_optimize_check_normal) {
+    config::delete_bitmap_storage_optimize_check_version_gap = 0;
+
+    auto txn_kv = std::make_shared<MemTxnKv>();
+    ASSERT_EQ(txn_kv->init(), 0);
+
+    InstanceInfoPB instance;
+    instance.set_instance_id(instance_id);
+    auto obj_info = instance.add_obj_info();
+    obj_info->set_id("1");
+
+    InstanceChecker checker(txn_kv, instance_id);
+    ASSERT_EQ(checker.init(instance), 0);
+    auto accessor = checker.accessor_map_.begin()->second;
+
+    std::unique_ptr<Transaction> txn;
+    ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+    constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+    int64_t rowset_start_id = 600;
+
+    for (int tablet_id = 800001; tablet_id <= 800005; ++tablet_id) {
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, true));
+        std::vector<std::pair<int64_t, int64_t>> rowset_vers {{2, 2}, {3, 3}, {4, 4}, {5, 5},
+                                                              {6, 7}, {8, 8}, {9, 9}};
+        std::vector<std::pair<int64_t, int64_t>> delete_bitmaps_vers {
+                {7, 9}, {8, 9}, {7, 9}, {7, 9}, {7, 9}, {8, 9}, {9, 9}};
+        std::vector<bool> segments_overlap {true, true, true, true, false, true, true};
+        for (size_t i {0}; i < 7; i++) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   rowset_vers[i].first, rowset_vers[i].second,
+                                                   rowset_id, segments_overlap[i], 1);
+            create_delete_bitmaps(txn.get(), tablet_id, rowset_id, delete_bitmaps_vers[i].first,
+                                  delete_bitmaps_vers[i].second);
+        }
+    }
+
+    for (int tablet_id = 800006; tablet_id <= 800010; ++tablet_id) {
+        // [7-7] cumu compaction output rowset start_version == end_version
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, true));
+        std::vector<std::pair<int64_t, int64_t>> rowset_vers {{2, 2}, {3, 3}, {4, 4}, {5, 5},
+                                                              {6, 6}, {7, 7}, {8, 8}, {9, 9}};
+        std::vector<std::pair<int64_t, int64_t>> delete_bitmaps_vers {
+                {7, 9}, {8, 9}, {7, 9}, {7, 9}, {7, 9}, {7, 9}, {8, 9}, {9, 9}};
+        std::vector<bool> segments_overlap {true, true, false, true, false, true, true, true};
+        for (size_t i {0}; i < 8; i++) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   rowset_vers[i].first, rowset_vers[i].second,
+                                                   rowset_id, segments_overlap[i], 1);
+            create_delete_bitmaps(txn.get(), tablet_id, rowset_id, delete_bitmaps_vers[i].first,
+                                  delete_bitmaps_vers[i].second);
+        }
+    }
+
+    for (int tablet_id = 800011; tablet_id <= 800015; ++tablet_id) {
+        // no rowsets are compacted
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, true));
+        std::vector<std::pair<int64_t, int64_t>> rowset_vers {{2, 2}, {3, 3}, {4, 4}, {5, 5},
+                                                              {6, 6}, {7, 7}, {8, 8}, {9, 9}};
+        std::vector<std::pair<int64_t, int64_t>> delete_bitmaps_vers {
+                {2, 9}, {3, 9}, {4, 9}, {5, 9}, {6, 9}, {7, 9}, {8, 9}, {9, 9}};
+        std::vector<bool> segments_overlap {true, true, true, true, true, true, true, true};
+        for (size_t i {0}; i < 8; i++) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   rowset_vers[i].first, rowset_vers[i].second,
+                                                   rowset_id, segments_overlap[i], 1);
+            create_delete_bitmaps(txn.get(), tablet_id, rowset_id, delete_bitmaps_vers[i].first,
+                                  delete_bitmaps_vers[i].second);
+        }
+    }
+
+    for (int tablet_id = 800016; tablet_id <= 800020; ++tablet_id) {
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, true));
+        std::vector<std::pair<int64_t, int64_t>> rowset_vers {
+                {2, 5}, {6, 6}, {7, 7}, {8, 8}, {9, 9}};
+        std::vector<std::pair<int64_t, int64_t>> delete_bitmaps_vers {
+                {5, 9}, {6, 9}, {7, 9}, {8, 9}, {9, 9}};
+        std::vector<bool> segments_overlap {false, true, true, true, true};
+        for (size_t i {0}; i < 5; i++) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   rowset_vers[i].first, rowset_vers[i].second,
+                                                   rowset_id, segments_overlap[i], 1);
+            create_delete_bitmaps(txn.get(), tablet_id, rowset_id, delete_bitmaps_vers[i].first,
+                                  delete_bitmaps_vers[i].second);
+        }
+    }
+
+    // also create some rowsets without delete bitmaps in non merge-on-write tablet
+    for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, false));
+        int64_t rowset_start_id = 500;
+        for (int ver = 2; ver < 10; ++ver) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   ver, ver, rowset_id, false, 1);
+        }
+    }
+
+    ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+    ASSERT_EQ(checker.do_delete_bitmap_storage_optimize_check(), 0);
+}
+
+TEST(CheckerTest, delete_bitmap_storage_optimize_check_abnormal) {
+    config::delete_bitmap_storage_optimize_check_version_gap = 0;
+    // abnormal case, some rowsets' delete bitmaps are not deleted as expected
+    auto txn_kv = std::make_shared<MemTxnKv>();
+    ASSERT_EQ(txn_kv->init(), 0);
+
+    InstanceInfoPB instance;
+    instance.set_instance_id(instance_id);
+    auto obj_info = instance.add_obj_info();
+    obj_info->set_id("1");
+
+    InstanceChecker checker(txn_kv, instance_id);
+    ASSERT_EQ(checker.init(instance), 0);
+    auto accessor = checker.accessor_map_.begin()->second;
+
+    // tablet_id -> [rowset_id]
+    std::map<std::int64_t, std::set<std::string>> expected_abnormal_rowsets {};
+    std::map<std::int64_t, std::set<std::string>> real_abnormal_rowsets {};
+    auto sp = SyncPoint::get_instance();
+    std::unique_ptr<int, std::function<void(int*)>> defer(
+            (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); });
+    sp->set_call_back("InstanceChecker::check_delete_bitmap_storage_optimize.get_abnormal_rowset",
+                      [&real_abnormal_rowsets](auto&& args) {
+                          int64_t tablet_id = *try_any_cast<int64_t*>(args[0]);
+                          std::string rowset_id = *try_any_cast<std::string*>(args[1]);
+                          real_abnormal_rowsets[tablet_id].insert(rowset_id);
+                      });
+    sp->enable_processing();
+
+    std::unique_ptr<Transaction> txn;
+    ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+    constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+
+    int64_t rowset_start_id = 700;
+    for (int tablet_id = 900001; tablet_id <= 900005; ++tablet_id) {
+        ASSERT_EQ(0,
+                  create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id, true));
+        std::vector<std::pair<int64_t, int64_t>> rowset_vers {{2, 2}, {3, 3}, {4, 4}, {5, 5},
+                                                              {6, 7}, {8, 8}, {9, 9}};
+        std::vector<std::pair<int64_t, int64_t>> delete_bitmaps_vers {
+                {2, 9}, {7, 9}, {4, 9}, {7, 9}, {7, 9}, {8, 9}, {9, 9}};
+        std::vector<bool> segments_overlap {true, true, true, true, false, true, true};
+        for (size_t i {0}; i < 7; i++) {
+            std::string rowset_id = std::to_string(rowset_start_id++);
+            create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id,
+                                                   rowset_vers[i].first, rowset_vers[i].second,
+                                                   rowset_id, segments_overlap[i], 1);
+            create_delete_bitmaps(txn.get(), tablet_id, rowset_id, delete_bitmaps_vers[i].first,
+                                  delete_bitmaps_vers[i].second);
+            if (delete_bitmaps_vers[i].first < 7) {
+                expected_abnormal_rowsets[tablet_id].insert(rowset_id);
+            }
+        }
+    }
+
+    ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+    ASSERT_EQ(checker.do_delete_bitmap_storage_optimize_check(), 1);
+    ASSERT_EQ(expected_abnormal_rowsets, real_abnormal_rowsets);
+}
+
 TEST(RecyclerTest, delete_rowset_data) {
     auto txn_kv = std::make_shared<MemTxnKv>();
     ASSERT_EQ(txn_kv->init(), 0);

From e16dbe5071cc47783a67ee252dccda4916ec61de Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Tue, 3 Dec 2024 22:13:24 +0800
Subject: [PATCH 143/399] [test](case) add partition prune cases (#44722)

---
 .../partition_prune/auto_partition.groovy     |  92 +++++
 .../hive_partition_prune.groovy               | 136 ++++++++
 .../int_as_date_partition_col.groovy          | 203 +++++++++++
 .../multi_column_range_partition.groovy       | 260 ++++++++++++++
 .../one_col_list_partition.groovy             | 249 +++++++++++++
 .../one_col_range_partition.groovy            | 326 ++++++++++++++++++
 .../varchar_as_date_partition_col.groovy      | 251 ++++++++++++++
 7 files changed, 1517 insertions(+)
 create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/auto_partition.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/hive_partition_prune.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/multi_column_range_partition.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/one_col_list_partition.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy

diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/auto_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/auto_partition.groovy
new file mode 100644
index 00000000000000..ee6a40c7a0d9e0
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/auto_partition.groovy
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("auto_partition") {
+    sql "drop table if exists one_col_range_partition_date_func"
+    sql """create table one_col_range_partition_date_func (a int, dt datetime not null, d date, c varchar(100)) duplicate key(a)
+    auto partition by range(date_trunc(dt,'day')) () distributed by hash(a) properties("replication_num"="1");"""
+    sql """INSERT INTO one_col_range_partition_date_func SELECT number,
+    date_add('2020-01-01 00:00:00', interval number hour),
+    cast(date_add('2020-01-01 00:00:00', interval number hour) as date), cast(number as varchar(65533)) FROM numbers('number'='10000');"""
+    sql "drop table if exists one_col_range_partition_date_func_month"
+    sql """create table one_col_range_partition_date_func_month(a int, dt datetime not null, d date, c varchar(100)) duplicate key(a)
+    auto partition by range(date_trunc(dt,'month')) () distributed by hash(a) properties("replication_num"="1");"""
+    sql """INSERT INTO one_col_range_partition_date_func_month SELECT number,
+    date_add('2020-01-01 00:00:00', interval number hour),
+    cast(date_add('2020-01-01 00:00:00', interval number hour) as date), cast(number as varchar(65533)) FROM numbers('number'='10000');"""
+    // auto partition
+    // partition by range(date_trunc(dt, 'day'))
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func WHERE dt<'2020-10-01 00:00:00'"
+        contains("partitions=274/417")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func WHERE dt<='2020-10-01 00:00:00'"
+        contains("partitions=275/417")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func WHERE dt>'2020-10-01 00:00:00'"
+        contains("partitions=143/417")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func WHERE dt>='2020-10-01 00:00:00'"
+        contains("partitions=143/417")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func WHERE dt='2020-10-01 00:00:00'"
+        contains("partitions=1/417")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func WHERE dt<=>'2020-10-01 00:00:00'"
+        contains("partitions=1/417")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func WHERE dt!='2020-10-01 00:00:00'"
+        contains("partitions=417/417")
+    }
+
+    // partition by range(date_trunc(dt, 'month'))
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func_month WHERE dt<'2020-10-01 00:00:00'"
+        contains("partitions=9/14")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func_month WHERE dt<='2020-10-01 00:00:00'"
+        contains("partitions=10/14")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func_month WHERE dt>'2020-10-01 00:00:00'"
+        contains("partitions=5/14")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func_month WHERE dt>='2020-10-01 00:00:00'"
+        contains("partitions=5/14")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func_month WHERE dt='2020-10-01 00:00:00'"
+        contains("partitions=1/14 (p20201001000000)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func_month WHERE dt<=>'2020-10-01 00:00:00'"
+        contains("partitions=1/14 (p20201001000000)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date_func_month WHERE dt!='2020-10-01 00:00:00'"
+        contains("partitions=14/14")
+    }
+}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/hive_partition_prune.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/hive_partition_prune.groovy
new file mode 100644
index 00000000000000..ebf4d35d3a1f02
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/hive_partition_prune.groovy
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("hive_partition_prune") {
+    String enabled = context.config.otherConfigs.get("enableHiveTest")
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+        logger.info("disable Hive test.")
+        return;
+    }
+
+    String extHiveHmsHost = context.config.otherConfigs.get("externalEnvIp")
+    String extHiveHmsPort = context.config.otherConfigs.get("hive3HmsPort")
+    String catalog_name = "test_external_catalog_hive_partition_prune"
+
+    sql """drop catalog if exists ${catalog_name};"""
+    sql """
+        create catalog if not exists ${catalog_name} properties (
+            'type'='hms',
+            'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+        );
+    """
+    sql """switch ${catalog_name};"""
+    sql "drop database if exists test_hive_partition_prune"
+    sql "create database test_hive_partition_prune"
+    sql "use test_hive_partition_prune"
+    sql "drop table if exists test_hive_partition "
+    sql """create table test_hive_partition (a int, b int, p int)
+    engine=hive partition by list(p)()
+    properties("file_format"="parquet","orc.compress"="zstd");"""
+
+    sql "insert into test_hive_partition values(1,3,1),(1,2,2),(1,2,3),(1,2,4),(1,2,5),(1,6,null)"
+
+    explain {
+        sql "select * from test_hive_partition where p=1 or p=2"
+        contains("partition=2/6")
+    }
+
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p in (1,2) and p in (2,3,4)"
+        contains("partition=1/6")
+    }
+
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE (p=1 or p=2) and p in (3,5,2)"
+        contains("partition=1/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE (p=1 xor p=2)"
+        contains("partition=2/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p in (1, 5,6)"
+        contains("partition=2/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p not in (15,6,1, '2021-01-02 00:00:00')"
+        contains("partition=4/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p not in (1, 5,6,null)"
+        contains("partition=0/6")
+    }
+    explain {
+        sql "select * from test_hive_partition where !(p is not null)"
+        contains("partition=1/6")
+    }
+    explain {
+        sql "select * from test_hive_partition where p is null"
+        contains("partition=1/6")
+    }
+    explain {
+        sql "select * from test_hive_partition where p is not null"
+        contains("partition=5/6")
+    }
+    explain {
+        sql "select * from test_hive_partition where not p is null"
+        contains("partition=5/6")
+    }
+    explain {
+        sql "select * from test_hive_partition where !(p is null)"
+        contains("partition=5/6")
+    }
+
+    explain {
+        sql "select * from test_hive_partition where p <=> null"
+        contains("partition=1/6")
+    }
+    explain {
+        sql "select * from test_hive_partition where !(p <=> null)"
+        contains("partition=5/6")
+    }
+
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p<1"
+        contains("partition=0/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p<=2"
+        contains("partition=2/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p>2"
+        contains("partition=3/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p>=3"
+        contains("partition=3/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p=4"
+        contains("partition=1/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p<=>5"
+        contains("partition=1/6")
+    }
+    explain {
+        sql "SELECT * FROM test_hive_partition WHERE p!=5 and p!=6"
+        contains("partition=4/6")
+    }
+    sql "drop table if exists test_hive_partition "
+}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy
new file mode 100644
index 00000000000000..5e65e6445ef5e5
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy
@@ -0,0 +1,203 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("int_as_date_partition_col") {
+    sql "drop table if exists partition_int"
+    sql """CREATE TABLE partition_int(a int, dt int) PARTITION BY range(dt) (
+            partition p20240101 values less than ("20240101"),
+            partition p20240201 values less than ("20240201"),
+            partition p20240301 values less than ("20240301"),
+            partition p20240401 values less than ("20240401"),
+            partition p20240501 values less than ("20240501"),
+            partition p20240601 values less than ("20240601")
+    )
+    distributed BY hash(a)
+    properties("replication_num"="1");"""
+
+    sql """insert into partition_int
+    SELECT number,cast(date_add('2023-11-05', interval number day) as int) FROM numbers('number'='150');"""
+    sql "drop table if exists partition_int_has_null"
+    sql """
+    CREATE TABLE partition_int_has_null(a int, dt int) PARTITION BY range(dt) (
+            partition p20240101 values less than ("20240101"),
+            partition p20240201 values less than ("20240201"),
+            partition p20240301 values less than ("20240301"),
+            partition p20240401 values less than ("20240401"),
+            partition p20240501 values less than ("20240501"),
+            partition p20240601 values less than ("20240601")
+    )
+    distributed BY hash(a) properties("replication_num"="1");"""
+
+    sql """insert into partition_int_has_null
+    SELECT number,cast(date_add('2023-11-05', interval number day) as int) FROM numbers('number'='120');"""
+    
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt<'20240501' and dt>'20240301'"
+        contains("partitions=2/6 (p20240401,p20240501)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt<'20240701' or dt>='20241001'"
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_int WHERE (dt<'20240301' or dt>='20241001') or dt<=>'20240905'"
+        contains("partitions=3/6 (p20240101,p20240201,p20240301)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE (dt<'20240301' and dt>='20240201') or dt<=>'20240905'"
+        contains("partitions=1/6 (p20240301)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_int WHERE (dt<'20240701' xor dt>='20240601')"
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+
+    explain {
+        sql """SELECT count(*) FROM partition_int WHERE
+        (date_trunc(dt,'month')<'2024-8-01' and date_trunc(dt,'month')>'2024-6-01' )"""
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+
+    explain {
+        sql """SELECT count(*) FROM partition_int WHERE date_trunc(dt,'month')<'2024-07-01' ;"""
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+
+    explain {
+        sql """SELECT count(*) FROM partition_int WHERE
+        !(date_trunc(dt,'month')<'2024-8-01' and date_trunc(dt,'month')>'2024-6-01' );"""
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+
+    explain {
+        sql """
+        SELECT count(*) FROM partition_int WHERE
+        !(dt<'20241001'  or date_trunc(dt,'month')<'2024-7-01' and date_trunc(dt,'month')>'2024-6-01' )
+        """
+        contains("VEMPTYSET")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_int WHERE !('2020-09-01' not like '%2020-10-01 00:00:00%')"
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE  !('2020-09-01' not in ('2020-10-01','2020-09-01'))"
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE  !('2020-09-01' in ('2020-10-01','2020-09-01'))"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE !('2020-10-02'>'2020-10-01')"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE !(dt>='20240301' and a<100) "
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE !(a<100 )"
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_int WHERE !(dt>='20240301' or a<100 )"
+        contains("partitions=3/6 (p20240101,p20240201,p20240301)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_int WHERE !(dt>='20240301' or (a<100 and dt<'20240801'))"
+        contains("partitions=3/6 (p20240101,p20240201,p20240301)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt regexp '2024-10-01'"
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt like '20241001'"
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt in  ('20240301', '20240302')"
+        contains("partitions=1/6 (p20240401)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt not in ('20240201', '20240202')"
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+
+    explain {
+        sql "select * from partition_int_has_null where dt is null"
+        contains("partitions=1/6 (p20240101)")
+    }
+    explain {
+        sql "select * from partition_int_has_null where dt is not null"
+        contains("partitions=4/6 (p20240101,p20240201,p20240301,p20240401)")
+    }
+    explain {
+        sql "select * from partition_int_has_null where not dt is null"
+        contains("partitions=4/6 (p20240101,p20240201,p20240301,p20240401)")
+    }
+    explain {
+        sql "select * from partition_int_has_null where !(dt is null)"
+        contains("partitions=4/6 (p20240101,p20240201,p20240301,p20240401)")
+    }
+    explain {
+        sql "select * from partition_int_has_null where dt <=> null"
+        contains("partitions=1/6 (p20240101)")
+    }
+    explain {
+        sql "select * from partition_int_has_null where !(dt <=> null)"
+        contains("partitions=4/6 (p20240101,p20240201,p20240301,p20240401)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt<'20240301'"
+        contains("partitions=3/6 (p20240101,p20240201,p20240301)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt<='20240301'"
+        contains("partitions=4/6 (p20240101,p20240201,p20240301,p20240401)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt>'20240301'"
+        contains("partitions=2/6 (p20240401,p20240501)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt>='20240301'"
+        contains("partitions=2/6 (p20240401,p20240501)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt='20240301'"
+        contains("partitions=1/6 (p20240401)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt<=>'20240302'"
+        contains("partitions=1/6 (p20240401)")
+    }
+    explain {
+        sql "SELECT * FROM partition_int WHERE dt!='20241001'"
+        contains("partitions=5/6 (p20240101,p20240201,p20240301,p20240401,p20240501)")
+    }
+}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/multi_column_range_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/multi_column_range_partition.groovy
new file mode 100644
index 00000000000000..78da2304542bc9
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/multi_column_range_partition.groovy
@@ -0,0 +1,260 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("multi_column_range_partition") {
+    sql "drop table if exists t_multi_column_range_partition"
+    sql """
+    create table t_multi_column_range_partition(a int, dt datetime, v int) partition by range(a, dt)
+    (
+    partition p0 values [(0,'2024-01-01 00:00:00'), (10,'2024-01-10 00:00:00')),
+    partition p10 values [(10,'2024-01-10 00:00:00'), (20,'2024-01-20 00:00:00')),
+    partition p20 values [(20,'2024-01-20 00:00:00'), (30,'2024-01-31 00:00:00')),
+    partition p30 values [(30,'2024-01-31 00:00:00'), (40,'2024-02-10 00:00:00')),
+    partition p40 values [(40,'2024-02-10 00:00:00'), (50,'2024-02-20 00:00:00'))
+    )
+    distributed by hash(a) properties("replication_num"="1"); """
+
+    sql """insert into t_multi_column_range_partition values(0,'2024-01-01 00:00:00',2),(1,'2024-01-01 00:00:00',2),(1,'2025-01-01 00:00:00',2),
+    (10,'2024-01-10 00:00:00',3),(10,'2024-01-11 00:00:00',200),(12,'2021-01-01 00:00:00',2),
+    (25,'2024-01-10 00:00:00',3),(20,'2024-01-11 00:00:00',200),(30,'2021-01-01 00:00:00',2),
+    (40,'2024-01-01 00:00:00',2),(40,'2024-01-31 00:00:00',2),(10,'2024-01-9 00:00:00',1000),(10,'2024-01-10 00:00:00',1000),(10,'2024-01-10 01:00:00',1000),
+    (2,'2023-01-10 01:00:00',1000)
+    """
+
+    sql "drop table if exists t_multi_column_partition_datetime_first"
+    sql """
+    create table t_multi_column_partition_datetime_first(a int, dt datetime, v int not null) partition by range(dt,a)
+    (
+    partition p0 values [('2024-01-01 00:00:00',0), ('2024-01-10 00:00:00',10)),
+    partition p10 values [('2024-01-10 00:00:00',10), ('2024-01-20 00:00:00',20)),
+    partition p20 values [('2024-01-20 00:00:00',20), ('2024-01-31 00:00:00',30)),
+    partition p30 values [('2024-01-31 00:00:00',30), ('2024-02-10 00:00:00',40)),
+    partition p40 values [('2024-02-10 00:00:00',40), ('2024-02-20 00:00:00',50))
+    )
+    distributed by hash(a) properties("replication_num"="1");"""
+
+    sql "insert into t_multi_column_partition_datetime_first values(0,'2024-01-01 00:00:00',2)"
+    sql "insert into t_multi_column_partition_datetime_first values(1,'2024-01-01 00:00:00',2)"
+    sql "insert into t_multi_column_partition_datetime_first values(10,'2024-01-10 00:00:00',3),(10,'2024-01-11 00:00:00',200)"
+    sql "insert into t_multi_column_partition_datetime_first values(25,'2024-01-10 00:00:00',3),(20,'2024-01-11 00:00:00',200)"
+    sql "insert into t_multi_column_partition_datetime_first values(40,'2024-01-01 00:00:00',2),(40,'2024-01-31 00:00:00',2)"
+    sql "insert into t_multi_column_partition_datetime_first values(10,'2024-01-9 00:00:00',1000),(10,'2024-01-10 00:00:00',1000),(10,'2024-01-10 01:00:00',1000)"
+
+    explain {
+        sql "select * from t_multi_column_range_partition where a=10 and date_trunc(dt, 'day') <'2024-01-10'"
+        contains("partitions=1/5 (p0)")
+    }
+    sql "set PARTITION_PRUNING_EXPAND_THRESHOLD=5;"
+    explain {
+        sql "select * from t_multi_column_range_partition where a=10 and date_trunc(dt, 'day') <'2024-01-10'"
+        contains("partitions=2/5 (p0,p10)")
+    }
+    for (int i = 0; i < 2; ++i) {
+        if (i == 1) {
+            sql "set PARTITION_PRUNING_EXPAND_THRESHOLD=5;"
+        } else if (i == 0) {
+            sql "set PARTITION_PRUNING_EXPAND_THRESHOLD=30;"
+        }
+        explain {
+            sql "select * from t_multi_column_range_partition where a=10 and date_trunc(dt, 'day') ='2024-01-10'"
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql "select * from t_multi_column_range_partition where a<19 and (date_trunc(dt, 'day') <'2024-01-20' OR date_trunc(dt, 'day') >'2024-02-10')"
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql "select * from t_multi_column_range_partition where a>1 and a<30 and (date_trunc(dt, 'day') <'2024-01-20' OR date_trunc(dt, 'day') >'2024-02-10')"
+            contains("partitions=3/5 (p0,p10,p20)")
+        }
+        explain {
+            sql "select * from t_multi_column_range_partition where a>1 and a<30 and (date_trunc(dt, 'day') >'2024-01-20' and date_trunc(dt, 'day') <'2024-02-10')"
+            contains("partitions=3/5 (p0,p10,p20)")
+        }
+
+        explain {
+            sql "select * from t_multi_column_range_partition where !(a > 1 and a < 30 and(date_trunc(dt, 'day') > '2024-01-20' and date_trunc(dt, 'day') < '2024-02-10') )"
+            contains("partitions=4/5 (p0,p10,p20,p30)")
+        }
+        explain {
+            sql "select * from t_multi_column_range_partition where !(a >= 10 or date_trunc(dt, 'day') < '2024-01-10' )"
+            contains("partitions=1/5 (p0)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE a>1 and date_trunc(dt, 'day') regexp '2020-10-01 00:00:00'"
+            contains("partitions=4/5 (p0,p10,p20,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE a>1 and !date_trunc(dt, 'day') regexp '2020-10-01 00:00:00'"
+            contains("partitions=4/5 (p0,p10,p20,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE a=1 and !date_trunc(dt, 'day') regexp '2020-10-01 00:00:00'"
+            contains("partitions=1/5 (p0)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE a!=1 and !date_trunc(dt, 'day') like '2020-10-01 00:00:00'"
+            contains("partitions=4/5 (p0,p10,p20,p30)")
+        }
+
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE a in (7,8) and date_trunc(dt, 'day') in ('2024-01-01', '2024-10-02')"
+            contains("partitions=1/5 (p0)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE a not in (7,8) and date_trunc(dt, 'day') not in ('2024-01-01', '2024-10-02')"
+            contains("partitions=4/5 (p0,p10,p20,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE a in (7,8) and date_trunc(dt, 'day') not in ('2024-01-01', '2024-10-02')"
+            contains("partitions=1/5 (p0)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE a in (7,8,15) and date_trunc(dt, 'day') <'2024-01-20'"
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE (a in (7,8,15) or a>=31) and date_trunc(dt, 'day') >'2024-01-20'"
+            contains("partitions=3/5 (p0,p10,p30)")
+        }
+
+        explain {
+            sql "select * from t_multi_column_partition_datetime_first where dt is null"
+            contains("VEMPTYSET")
+        }
+        explain {
+            sql "select * from t_multi_column_partition_datetime_first where date_trunc(dt,'day') is null"
+            contains("partitions=3/5 (p0,p10,p30)")
+        }
+        explain {
+            sql "select * from t_multi_column_partition_datetime_first where date_trunc(dt,'day') is not null"
+            contains("partitions=3/5 (p0,p10,p30)")
+        }
+        explain {
+            sql "select * from t_multi_column_partition_datetime_first where a=1 and date_trunc(dt,'month') is null"
+            contains("partitions=1/5 (p30)")
+        }
+        explain {
+            sql "select * from t_multi_column_partition_datetime_first where a>1 and date_trunc(dt,'month') is null"
+            contains("partitions=1/5 (p30)")
+        }
+        explain {
+            sql "select * from t_multi_column_partition_datetime_first where date_trunc(dt,'month') is null"
+            contains("partitions=1/5 (p30)")
+        }
+        explain {
+            sql "select * from t_multi_column_partition_datetime_first where a=1 and date_trunc(dt,'month') is not null"
+            contains("partitions=3/5 (p0,p10,p30)")
+        }
+
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')<'2024-1-19 00:00:00'"
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')<='2024-1-10 00:00:00'"
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')>'2024-1-10 00:00:00'"
+            contains("partitions=2/5 (p10,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')>'2024-1-11 00:00:00'"
+            contains("partitions=2/5 (p10,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')>='2024-1-10 00:00:00'"
+            contains("partitions=3/5 (p0,p10,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')='2024-1-2 00:00:00'"
+            contains("partitions=1/5 (p0)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')='2020-10-01 00:00:00'"
+            contains("VEMPTYSET")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')<=>'2020-10-01 00:00:00'"
+            contains("VEMPTYSET")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')!='2020-10-01 00:00:00'"
+            contains("partitions=3/5 (p0,p10,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')<'2024-1-19 00:00:00' and a>10"
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')<='2024-1-10 00:00:00' and a=10"
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')>'2024-1-10 00:00:00' and a<2"
+            contains("partitions=2/5 (p10,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')='2024-1-10 00:00:00' and a >100"
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE date_trunc(dt,'day')='2024-1-1 00:00:00' and a <0"
+            contains("partitions=1/5 (p0)")
+        }
+
+
+        explain {
+            sql "SELECT * FROM t_multi_column_partition_datetime_first WHERE coalesce(date_trunc(dt,'day'), null, '2020-01-01') <'2021-1-07 00:00:00'"
+            contains("partitions=3/5 (p0,p10,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE coalesce(date_trunc(dt,'day') <'2021-1-07 00:00:00' , true, false) and a=9"
+            contains("partitions=1/5 (p0)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE if(date_trunc(dt,'day') <'2021-1-07 00:00:00', true, false) and a<20"
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE if(a>1, dt<'2001-1-01 00:00:00', dt<'2001-1-01 00:00:00')"
+            contains("partitions=4/5 (p0,p10,p20,p30)")
+        }
+        explain {
+            sql "SELECT * FROM t_multi_column_range_partition WHERE if(dt<'2021-1-01 00:00:00', dt<'2001-1-01 00:00:00', dt>'2031-1-01 00:00:00')"
+            contains("partitions=4/5 (p0,p10,p20,p30)")
+        }
+
+        explain {
+            sql """SELECT * FROM t_multi_column_range_partition WHERE case when dt<'2024-1-10 00:00:00' then false when dt<'2024-1-20 00:00:00' then false
+    else true end and a=10;"""
+            contains("partitions=2/5 (p0,p10)")
+        }
+
+        explain {
+            sql """SELECT * FROM t_multi_column_range_partition WHERE case when date_trunc(dt,'day')<'2024-1-10 00:00:00' then false when date_trunc(dt,'day')<'2024-1-20 00:00:00' then true
+    else true end and a<20;"""
+            contains("partitions=2/5 (p0,p10)")
+        }
+        explain {
+            sql """SELECT * FROM t_multi_column_range_partition WHERE case when date_trunc(dt,'day')<'2024-1-10 00:00:00' then dt
+    else '2023-01-01 00:00:00' end <'2021-01-06 00:00:00' and a<20;"""
+            contains("partitions=2/5 (p0,p10)")
+        }
+    }
+}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/one_col_list_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/one_col_list_partition.groovy
new file mode 100644
index 00000000000000..4de512376d1378
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/one_col_list_partition.groovy
@@ -0,0 +1,249 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("one_col_list_partition") {
+    sql "drop table if exists one_col_list_partition_date"
+    sql """create table one_col_list_partition_date(a int, dt datetime, d date, c varchar(100)) duplicate key(a)
+    partition by list(dt)
+    (
+            partition p1 values in ('2021-01-01 00:00:00', '2021-01-02 00:00:00'),
+    partition p2 values in ('2021-01-03 00:00:00', '2021-01-04 00:00:00'),
+    partition p3 values in ('2021-01-05 00:00:00', '2021-01-06 00:00:00'),
+    partition p4 values in ('2021-01-07 00:00:00', '2021-01-08 00:00:00'),
+    partition p5 values in ('2021-01-09 00:00:00', '2021-01-10 00:00:00'),
+    partition p6 values in ('2021-01-11 00:00:00', '2021-01-12 00:00:00'),
+    partition p7 values in ('2021-01-13 00:00:00', '2021-01-14 00:00:00'),
+    partition p8 values in ('2021-01-15 00:00:00', '2021-01-16 00:00:00'),
+    partition p9 values in ('2021-01-17 00:00:00', '2021-01-18 00:00:00')
+    )
+    distributed by hash(a)
+    properties("replication_num"="1");"""
+
+    sql """
+    insert into one_col_list_partition_date values(1,'2021-01-01 00:00:00',null, 'abc'),(1,'2021-01-02 00:00:00',null, 'abc')
+    ,(1,'2021-01-03 00:00:00',null, 'abc'),(1,'2021-01-04 00:00:00',null, 'abc')
+    ,(1,'2021-01-05 00:00:00',null, 'abc'),(1,'2021-01-06 00:00:00',null, 'abc'),(1,'2021-01-07 00:00:00',null, 'abc'),(1,'2021-01-08 00:00:00',null, 'abc')
+    ,(1,'2021-01-09 00:00:00',null, 'abc'),(1,'2021-01-11 00:00:00',null, 'abc'),(1,'2021-01-13 00:00:00',null, 'abc'),(1,'2021-01-15 00:00:00',null, 'abc')
+    ,(1,'2021-01-10 00:00:00',null, 'abc'),(1,'2021-01-12 00:00:00',null, 'abc'),(1,'2021-01-14 00:00:00',null, 'abc'),(1,'2021-01-16 00:00:00',null, 'abc')
+    """
+
+    sql "drop table if exists one_col_list_partition_date_has_null"
+    sql"""
+    CREATE TABLE `one_col_list_partition_date_has_null` (
+      `a` int NULL,
+      `dt` datetime NULL,
+      `d` date NULL,
+      `c` varchar(100) NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`a`)
+    PARTITION BY LIST (`dt`)
+    (PARTITION p10 VALUES IN (NULL),
+    PARTITION p1 VALUES IN ('2021-01-01 00:00:00','2021-01-02 00:00:00'),
+    PARTITION p2 VALUES IN ('2021-01-03 00:00:00','2021-01-04 00:00:00'),
+    PARTITION p3 VALUES IN ('2021-01-05 00:00:00','2021-01-06 00:00:00'),
+    PARTITION p4 VALUES IN ('2021-01-07 00:00:00','2021-01-08 00:00:00'),
+    PARTITION p5 VALUES IN ('2021-01-09 00:00:00','2021-01-10 00:00:00'),
+    PARTITION p6 VALUES IN ('2021-01-11 00:00:00','2021-01-12 00:00:00'),
+    PARTITION p7 VALUES IN ('2021-01-13 00:00:00','2021-01-14 00:00:00'),
+    PARTITION p8 VALUES IN ('2021-01-15 00:00:00','2021-01-16 00:00:00'),
+    PARTITION p9 VALUES IN ('2021-01-17 00:00:00','2021-01-18 00:00:00'))
+    DISTRIBUTED BY HASH(`a`) BUCKETS 10
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1")
+    """
+
+    sql """
+    insert into one_col_list_partition_date_has_null values(1,'2021-01-01 00:00:00',null, 'abc'),(1,'2021-01-02 00:00:00',null, 'abc')
+    ,(1,'2021-01-03 00:00:00',null, 'abc'),(1,'2021-01-04 00:00:00',null, 'abc')
+    ,(1,'2021-01-05 00:00:00',null, 'abc'),(1,'2021-01-06 00:00:00',null, 'abc'),(1,'2021-01-07 00:00:00',null, 'abc'),(1,'2021-01-08 00:00:00',null, 'abc')
+    ,(1,'2021-01-09 00:00:00',null, 'abc'),(1,'2021-01-11 00:00:00',null, 'abc'),(1,'2021-01-13 00:00:00',null, 'abc'),(1,'2021-01-15 00:00:00',null, 'abc')
+    ,(1,'2021-01-10 00:00:00',null, 'abc'),(1,'2021-01-12 00:00:00',null, 'abc'),(1,'2021-01-14 00:00:00',null, 'abc'),(1,'2021-01-16 00:00:00',null, 'abc')
+    ,(1,null,null,'bdb')
+    """
+
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt<'2021-01-10 00:00:00' and dt>'2021-01-7 00:00:00'"
+        contains("partitions=2/9 (p4,p5)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt<'2021-01-10 00:00:00' or dt>'2021-01-14 00:00:00'"
+        contains("partitions=6/9 (p1,p2,p3,p4,p5,p8)")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE (dt<'2021-01-10 00:00:00' or dt>'2021-01-10 00:00:00') and dt<=>'2021-01-01 00:00:00'"
+        contains("partitions=1/9 (p1)")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE (dt<'2021-01-10 00:00:00' xor dt>'2021-01-10 00:00:00')"
+        contains("partitions=8/9 (p1,p2,p3,p4,p5,p6,p7,p8)")
+
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt regexp '2020-10-01 00:00:00'"
+        contains("partitions=8/9 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt like '%2020-10-01 00:00:00'"
+        contains("partitions=8/9 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt like '2020-10-01 00:00:00'"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt in ('2021-01-10 00:00:00', '2021-01-13 00:00:00')"
+        contains("partitions=2/9 (p5,p7)")
+
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt not in ('2021-01-01 00:00:00', '2021-01-02 00:00:00')"
+        contains("partitions=7/9 (p2,p3,p4,p5,p6,p7,p8)")
+
+    }
+
+    explain {
+        sql "select * from one_col_list_partition_date where dt is null"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "select * from one_col_list_partition_date where dt is not null"
+        contains("partitions=8/9 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "select * from one_col_list_partition_date where not dt is null"
+        contains("partitions=8/9 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "select * from one_col_list_partition_date where !(dt is null)"
+        contains("partitions=8/9 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+
+    explain {
+        sql "select * from one_col_list_partition_date where dt <=> null"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "select * from one_col_list_partition_date where !(dt <=> null)"
+        contains("partitions=8/9 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "select * from one_col_list_partition_date_has_null where dt is null"
+        contains("partitions=1/10 (p10)")
+    }
+    explain {
+        sql "select * from one_col_list_partition_date_has_null where dt is not null"
+        contains("partitions=8/10 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "select * from one_col_list_partition_date_has_null where not dt is null"
+        contains("partitions=8/10 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "select * from one_col_list_partition_date_has_null where !(dt is null)"
+        contains("partitions=8/10 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+
+    explain {
+        sql "select * from one_col_list_partition_date_has_null where dt <=> null"
+        contains("partitions=1/10 (p10)")
+    }
+    explain {
+        sql "select * from one_col_list_partition_date_has_null where !(dt <=> null)"
+        contains("partitions=8/10 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+
+
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt<'2021-01-01 00:00:00'"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt<='2021-01-02 00:00:00'"
+        contains("partitions=1/9 (p1)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt>'2021-1-03 00:00:00'"
+        contains("partitions=7/9 (p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt>='2021-1-04 00:00:00'"
+        contains("partitions=7/9 (p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt='2021-1-05 00:00:00'"
+        contains("partitions=1/9 (p3)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt<=>'2021-1-01 00:00:00'"
+        contains("partitions=1/9 (p1)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE dt!='2021-1-01 00:00:00' and dt!='2021-1-02 00:00:00'"
+        contains("partitions=7/9 (p2,p3,p4,p5,p6,p7,p8)")
+    }
+
+
+    // has condition function
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE coalesce(dt, null, '2020-01-01') <'2021-1-07 00:00:00'"
+        contains("partitions=3/9 (p1,p2,p3)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE coalesce(dt <'2021-1-07 00:00:00' , true, false)"
+        contains("partitions=3/9 (p1,p2,p3)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE coalesce(dt >'2021-1-07 00:00:00' , true, false)"
+        contains("partitions=5/9 (p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE coalesce(null ,dt< '2021-1-07 00:00:00', dt> '2020-10-01 00:00:00' )"
+        contains("partitions=3/9 (p1,p2,p3)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE if(dt<'2021-1-01 00:00:00', true, false)"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE if(a>1, dt<'2001-1-01 00:00:00', dt<'2001-1-01 00:00:00')"
+        contains("partitions=8/9 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_list_partition_date WHERE if(dt<'2021-1-01 00:00:00', dt<'2001-1-01 00:00:00', dt>'2031-1-01 00:00:00')"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql """SELECT * FROM one_col_list_partition_date WHERE case when dt<'2021-1-01 00:00:00' then false when dt<'2021-5-01' then false
+        else true end;"""
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql """SELECT * FROM one_col_list_partition_date WHERE case when dt<'2021-1-01 00:00:00' then false when dt<'2021-5-01' then true
+        else true end;"""
+        contains("partitions=8/9 (p1,p2,p3,p4,p5,p6,p7,p8)")
+    }
+    explain {
+        sql """SELECT * FROM one_col_list_partition_date WHERE case when dt<'2022-01-01 00:00:00' then dt
+        else '2023-01-01 00:00:00' end <'2021-01-06 00:00:00' ;"""
+        contains("partitions=3/9 (p1,p2,p3)")
+    }
+}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy
new file mode 100644
index 00000000000000..e8b7621e6d7ff7
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy
@@ -0,0 +1,326 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("one_col_range_partition") {
+    sql "drop table if exists one_col_range_partition_date"
+    sql """
+    create table one_col_range_partition_date(a int, dt datetime, d date, c varchar(100)) duplicate key(a)
+    partition by range(dt)
+    (
+            from ("2020-01-01") to ("2024-12-31") interval 1 day
+    )
+    distributed by hash(a)
+    properties("replication_num"="1");
+    """
+    sql """
+    INSERT INTO one_col_range_partition_date SELECT number, 
+    date_add('2020-01-01 00:00:00', interval number hour), 
+    cast(date_add('2020-01-01 00:00:00', interval number hour) as date), cast(number as varchar(65533)) FROM numbers('number'='10000');
+    """
+
+    sql "drop table if exists one_col_range_partition_date_has_null"
+    sql """create table one_col_range_partition_date_has_null (a int, dt datetime, d date, c varchar(100)) duplicate key(a)
+    partition by range(dt) (
+            partition p1 values less than ("2017-01-01"),
+            partition p2 values less than ("2018-01-01"),
+            partition p3 values less than ("2019-01-01"),
+            partition p4 values less than ("2020-01-01"),
+            partition p5 values less than ("2021-01-01")
+    )distributed by hash(a)
+    properties("replication_num"="1");"""
+    sql """INSERT INTO one_col_range_partition_date_has_null SELECT number,
+    date_add('2017-01-01 00:00:00', interval number day),
+    cast(date_add('2022-01-01 00:00:00', interval number day) as date), cast(number as varchar(65533)) FROM numbers('number'='1000');"""
+    sql """INSERT INTO one_col_range_partition_date_has_null  values(3,null,null,null);"""
+
+
+
+    // or and
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt<'2020-10-01 00:00:00' and dt>'2020-9-01 00:00:00'"
+        contains("partitions=30/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt<'2020-10-01 00:00:00' or dt>'2020-9-01 00:00:00'"
+        contains("partitions=417/1826")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE (dt<'2020-10-01 00:00:00' or dt>'2020-9-01 00:00:00') and dt<=>'2020-9-5 00:00:00'"
+        contains("partitions=1/1826 (p_20200905)")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE (dt<'2020-10-01 00:00:00' xor dt>'2020-9-01 00:00:00')"
+        contains("partitions=417/1826")
+    }
+
+    explain {
+        sql """SELECT count(*) FROM one_col_range_partition_date WHERE
+        !(dt<'2020-10-01 00:00:00' and dt>'2020-9-01 00:00:00' or date_trunc(dt,'month')<'2020-7-01' and date_trunc(dt,'month')>'2020-6-01' )"""
+        contains("partitions=388/1826")
+    }
+
+    explain {
+        sql """SELECT count(*) FROM one_col_range_partition_date WHERE
+        (dt>='2020-10-01 00:00:00' or dt<='2020-9-01 00:00:00' ) and (date_trunc(dt,'month')>='2020-9-01' or date_trunc(dt,'month')<='2020-6-01' )"""
+        contains("partitions=327/1826")
+    }
+
+    // !
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE !('2020-09-01' not like '%2020-10-01 00:00:00%')"
+        contains("partitions=417/1826 ")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE  !('2020-09-01' not in ('2020-10-01','2020-09-01'))"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE  !('2020-09-01' in ('2020-10-01','2020-09-01'))"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE !('2020-10-02'>'2020-10-01')"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE !(dt>='2020-10-01 00:00:00' and a<100)"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE !(a<100 and  c>10)"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE !(a<100 or  c>10)"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE !(dt>='2020-10-01 00:00:00' or a<100 )"
+        contains("partitions=274/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE !(dt<'2020-2-01 1:00:00' or (a<100 and dt>='2020-10-01 12:00:00'))"
+        contains("partitions=386/1826")
+    }
+
+    // like regexp
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt regexp '2020-10-01 00:00:00'"
+        contains("partitions=417/1826")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt like '2020-10-01 00:00:00'"
+        contains("partitions=417/1826")
+    }
+
+    // in
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt in ('2020-10-01 00:10:00', '2020-10-02 00:00:00')"
+        contains("partitions=2/1826")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt not in ('2020-10-01 00:10:00', '2020-10-02 00:00:00')"
+        contains("partitions=417/1826")
+    }
+
+    // is null
+    //partition key has null value
+    explain {
+        sql "select * from one_col_range_partition_date_has_null where dt is null"
+        contains("partitions=1/5 (p1)")
+    }
+    explain {
+        sql "select * from one_col_range_partition_date_has_null where dt is not null"
+        contains("partitions=4/5 (p1,p2,p3,p4)")
+    }
+    explain {
+        sql "select * from one_col_range_partition_date_has_null where not dt is null"
+        contains("partitions=4/5 (p1,p2,p3,p4)")
+    }
+    explain {
+        sql "select * from one_col_range_partition_date_has_null where !(dt is null)"
+        contains("partitions=4/5 (p1,p2,p3,p4)")
+    }
+
+    explain {
+        sql "select * from one_col_range_partition_date_has_null where dt <=> null"
+        contains("partitions=1/5 (p1)")
+    }
+    explain {
+        sql "select * from one_col_range_partition_date_has_null where !(dt <=> null)"
+        contains("partitions=4/5 (p1,p2,p3,p4)")
+    }
+    explain {
+        sql "select * from one_col_range_partition_date where dt is null"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "select * from one_col_range_partition_date where dt is not null"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "select * from one_col_range_partition_date where not dt is null"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "select * from one_col_range_partition_date where !(dt is null)"
+        contains("partitions=417/1826")
+    }
+
+    explain {
+        sql "select * from one_col_range_partition_date where dt <=> null"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "select * from one_col_range_partition_date where !(dt <=> null)"
+        contains("partitions=417/1826")
+    }
+
+    // op
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt<'2020-10-01 00:00:00'"
+        contains("partitions=274/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt<='2020-10-01 00:00:00'"
+        contains("partitions=275/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt>'2020-10-01 00:00:00'"
+        contains("partitions=143/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt>='2020-10-01 00:00:00'"
+        contains("partitions=143/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt='2020-10-01 00:00:00'"
+        contains("partitions=1/1826 (p_20201001)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt<=>'2020-10-01 00:00:00'"
+        contains("partitions=1/1826 (p_20201001)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt!='2020-10-01 00:00:00'"
+        contains("partitions=417/1826")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE coalesce(dt, null, '2020-01-01') <'2020-10-01 00:00:00'"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE coalesce(dt <'2020-2-01 00:00:00' , true, false)"
+        contains("partitions=31/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE coalesce(dt >'2020-2-01 00:00:00' , true, false)"
+        contains("partitions=386/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE coalesce(null ,dt< '2001-1-01 00:00:00', dt> '2020-10-01 00:00:00' )"
+        contains("partitions=143/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE if(dt<'2001-1-01 00:00:00', true, false)"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE if(a>1, dt<'2001-1-01 00:00:00', dt>'2001-1-01 00:00:00')"
+        contains("partitions=417/1826 ")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE if(dt<'2021-1-01 00:00:00', dt<'2001-1-01 00:00:00', dt>'2001-1-01 00:00:00')"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql """SELECT * FROM one_col_range_partition_date WHERE case when dt<'2021-1-01 00:00:00' then false when dt<'2021-5-01' then false
+        else true end;"""
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql """SELECT * FROM one_col_range_partition_date WHERE case when dt<'2022-1-01 00:00:00' then dt
+        else '2023-1-01 00:00:00' end <'2021-1-01 00:00:00' ;"""
+        contains("partitions=417/1826")
+    }
+
+    // predicates has non partition column
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE '2020-09-01' not like '%2020-10-01 00:00:00%'"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE  '2020-09-01' not in ('2020-10-01','2020-09-01')"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE  '2020-09-01' in ('2020-10-01','2020-09-01')"
+        contains("partitions=417/1826")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE '2020-10-02'>'2020-10-01'"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt>='2020-10-01 00:00:00' and a<100"
+        contains("partitions=143/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE a<100 and  c>10"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE a<100 or  c>10"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt>='2020-10-01 00:00:00' or a<100"
+        contains("partitions=417/1826")
+    }
+
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE not (dt>='2020-10-01 00:00:00' and a<100)"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt='2020-10-01 00:00:00' or a<100"
+        contains("partitions=417/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE not(dt!='2020-10-01 00:00:00' or a<100)"
+        contains("partitions=1/1826 (p_20201001)")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt<'2020-1-01 1:00:00' or a<100 and dt>='2020-10-01 12:00:00'"
+        contains("partitions=144/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt>='2020-10-01 00:00:00' and dt=d"
+        contains("partitions=143/1826")
+    }
+    explain {
+        sql "SELECT * FROM one_col_range_partition_date WHERE dt>='2020-10-01 00:00:00' or dt=d"
+        contains("partitions=417/1826")
+    }
+
+}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy
new file mode 100644
index 00000000000000..0cfe4963f27b2a
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy
@@ -0,0 +1,251 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("varchar_as_date_partition_col") {
+    sql"""drop table if exists partition_varchar;"""
+    sql """CREATE TABLE partition_varchar(a int, dt varchar(10), rdt datetime) PARTITION BY list(dt) (
+    partition p20240101 values in ("20240101","20240102"),
+    partition p20240103 values  in ("20240103","20240104"),
+    partition p20240105 values  in  ("20240105","20240106"),
+    partition p20240107 values  in ("20240107","20240108"),
+    partition p20240109 values  in ("20240109","20240110"),
+    partition p20240111 values  in ("20240111","20240112"),
+    partition p20240113 values  in ("20240113","20240114"),
+    partition p20240115 values  in ("20240115","20240116"),
+    partition p20240117 values  in ("20240117","20240118")
+    )
+    distributed BY hash(a)
+    properties(
+    "replication_num"="1"
+    );"""
+    sql """
+    insert into partition_varchar 
+    SELECT number, DATE_FORMAT(date_add('2024-01-01', interval number day),"yyyyMMdd"),date_add('2024-01-01', interval number day)
+    FROM numbers('number'='10');"""
+
+    sql "drop table if exists partition_varchar_has_null"
+    sql """CREATE TABLE partition_varchar_has_null(a int, dt varchar(10), rdt datetime) PARTITION BY list(dt) (
+    partition p20240101 values in ("20240101","20240102",null),
+    partition p20240103 values  in ("20240103","20240104"),
+    partition p20240105 values  in  ("20240105","20240106"),
+    partition p20240107 values  in ("20240107","20240108"),
+    partition p20240109 values  in ("20240109","20240110"),
+    partition p20240111 values  in ("20240111","20240112"),
+    partition p20240113 values  in ("20240113","20240114"),
+    partition p20240115 values  in ("20240115","20240116"),
+    partition p20240117 values  in ("20240117","20240118")
+    )
+    distributed BY hash(a)
+    properties(
+    "replication_num"="1"
+    );"""
+
+    sql """insert into partition_varchar_has_null 
+    SELECT number, DATE_FORMAT(date_add('2024-01-01', interval number day),"yyyyMMdd"),date_add('2024-01-01', interval number day)
+    FROM numbers('number'='10');"""
+    sql """insert into partition_varchar_has_null values(11,null,null)"""
+
+
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt<'20240110' and dt>'20240104'"
+        contains("partitions=3/9 (p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt<'20240104' or dt>'20240110'"
+        contains("partitions=2/9 (p20240101,p20240103)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt<date_format(now(),'yyyyMMdd') or dt > date_format(cast(now() as date) - interval 12 month,'yyyyMMdd')"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE date_trunc(dt,'day')< '20240105'"
+        contains("partitions=2/9 (p20240101,p20240103)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE date_trunc(dt,'day')< date_format(cast(now() as date) - interval 10 month,'yyyyMMdd')"
+        contains(" partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE (date_trunc(dt,'day')<'20240105' or dt>='20240108') or dt<=>'20240110';"
+        contains("partitions=4/9 (p20240101,p20240103,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE (dt<'20240107' and dt>='20240104') or dt<=>'20240109'"
+        contains("partitions=3/9 (p20240103,p20240105,p20240109)")
+    }
+    explain {
+        sql "SELECT count(*) FROM partition_varchar WHERE (date_trunc(dt,'day')<'2024-1-03' and date_trunc(dt,'day')>='2024-1-01' )"
+        contains("partitions=1/9 (p20240101)")
+    }
+
+    explain {
+        sql """
+        SELECT count(*) FROM partition_varchar WHERE
+        !(date_trunc(dt,'day')<'2024-1-03' and date_trunc(dt,'day')>='2024-1-01' )
+        """
+        contains("partitions=4/9 (p20240103,p20240105,p20240107,p20240109)")
+    }
+
+
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE !('2020-09-01' not like '%2020-10-01 00:00:00%')"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE  !('2020-09-01' not in ('2020-10-01','2020-09-01'))"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE  !('2020-09-01' in ('2020-10-01','2020-09-01'))"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE !('2020-10-02'>'2020-10-01')"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE !(dt>='20241001' and a<100)"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE !(a<100 )"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE !(dt>='20240102' or a<100 )"
+        contains("partitions=1/9 (p20240101)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE !(dt>='20240102' or (a<100 and dt<'20240801'))"
+        contains("partitions=1/9 (p20240101)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt regexp '2024-10-01'"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt like '%20240101'"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt like '20240101'"
+        contains("partitions=1/9 (p20240101)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt in ('20240101', '20241002')"
+        contains("partitions=1/9 (p20240101)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt not in ('20241001', '20241002')"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+
+    //partition key has null value
+    explain {
+        sql "select * from partition_varchar_has_null where dt is null"
+        contains("partitions=1/9 (p20240101)")
+    }
+
+    explain {
+        sql "select * from partition_varchar_has_null where dt is not null"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+
+    explain {
+        sql "select * from partition_varchar_has_null where not dt is null"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "select * from partition_varchar_has_null where !(dt is null)"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "select * from partition_varchar_has_null where dt <=> null"
+        contains("partitions=1/9 (p20240101)")
+    }
+    explain {
+        sql "select * from partition_varchar_has_null where !(dt <=> null)"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    //partition key has no null value
+    explain {
+        sql "select * from partition_varchar where dt is null"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "select * from partition_varchar where dt is not null"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "select * from partition_varchar where not dt is null"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "select * from partition_varchar where !(dt is null)"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "select * from partition_varchar where dt <=> null"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "select * from partition_varchar where !(dt <=> null)"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "select * from partition_varchar where  date_trunc(dt,'month') is null"
+        contains("VEMPTYSET")
+    }
+    explain {
+        sql "select * from partition_varchar where  date_trunc(dt,'month') is not null"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt<'20240103'"
+        contains("partitions=1/9 (p20240101)")
+    }
+
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt<='20240103'"
+        contains("partitions=2/9 (p20240101,p20240103)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt>'20240103'"
+        contains("partitions=4/9 (p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt>='20240103'"
+        contains("partitions=4/9 (p20240103,p20240105,p20240107,p20240109)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt='20240103'"
+        contains("partitions=1/9 (p20240103)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt<=>'20240103'"
+        contains("partitions=1/9 (p20240103)")
+    }
+    explain {
+        sql "SELECT * FROM partition_varchar WHERE dt!='20240103'"
+        contains("partitions=5/9 (p20240101,p20240103,p20240105,p20240107,p20240109)")
+    }
+}
\ No newline at end of file

From 4b84de4086a9cfcfe94f1a1ca2e2225a2aaa32f6 Mon Sep 17 00:00:00 2001
From: seawinde <wusi@selectdb.com>
Date: Tue, 3 Dec 2024 22:21:57 +0800
Subject: [PATCH 144/399] [fix](mtmv) Fix mv rewrite fail when base table add
 column (#44867)

### What problem does this PR solve?

Problem Summary:

Fix mv rewrite fail when base table add column
Such as mv def is as following:

            select
            o_orderstatus,
            l_suppkey,
            sum_merge(sum_state(o_shippriority)),
            group_concat_merge(group_concat_state(o_orderstatus)),
            avg_merge(avg_state(l_linenumber)),
            max_by_merge(max_by_state(O_COMMENT,o_totalprice)),
            count_merge(count_state(l_orderkey)),
            multi_distinct_count_merge(multi_distinct_count_state(l_shipmode))
            from lineitem
            left join orders
            on l_orderkey = o_orderkey and l_shipdate = o_orderdate
            group by
            o_orderstatus,
            l_suppkey
            order by o_orderstatus;

if add column to table orders, would cause rewrite fail

### Release note

Fix mv rewrite fail when base table add column
---
 .../exploration/mv/mapping/SlotMapping.java   |  13 +-
 .../mv/dml/rewrite/rewrite_when_dml.out       |  19 ++
 .../mv/dml/rewrite/rewrite_when_dml.groovy    | 291 ++++++++++++++++++
 3 files changed, 316 insertions(+), 7 deletions(-)
 create mode 100644 regression-test/data/nereids_rules_p0/mv/dml/rewrite/rewrite_when_dml.out
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/dml/rewrite/rewrite_when_dml.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
index 882dec19ea82dd..6e481a101420a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
-import org.apache.doris.nereids.types.VariantType;
 
 import com.google.common.collect.BiMap;
 import com.google.common.collect.HashBiMap;
@@ -78,19 +77,19 @@ public static SlotMapping generate(RelationMapping relationMapping) {
             for (List<String> sourceSlotName : sourceSlotNameToSlotMap.keySet()) {
                 Slot sourceSlot = sourceSlotNameToSlotMap.get(sourceSlotName);
                 Slot targetSlot = targetSlotNameSlotMap.get(sourceSlotName);
-                // source slot can not map from target, bail out
-                if (targetSlot == null && !(((SlotReference) sourceSlot).getDataType() instanceof VariantType)) {
-                    LOG.warn(String.format("SlotMapping generate is null, source relation is %s, "
-                            + "target relation is %s", sourceRelation, targetRelation));
-                    return null;
-                }
                 if (targetSlot == null) {
+                    // there are two scenes in which targetSlot maybe null
+                    // 1
                     // if variant, though can not map slot from query to view, but we maybe derive slot from query
                     // variant self, such as query slot to view slot mapping is payload#4 -> payload#10
                     // and query has a variant which is payload['issue']['number']#20, this can not get from view.
                     // in this scene, we can derive
                     // payload['issue']['number']#20 -> element_at(element_at(payload#10, 'issue'), 'number') mapping
                     // in expression rewrite
+                    // 2
+                    // Maybe table add column after last refresh
+                    LOG.warn(String.format("SlotMapping generate is null, source relation is %s, "
+                            + "target relation is %s", sourceRelation, targetRelation));
                     continue;
                 }
                 relationSlotMap.put(MappedSlot.of(sourceSlot,
diff --git a/regression-test/data/nereids_rules_p0/mv/dml/rewrite/rewrite_when_dml.out b/regression-test/data/nereids_rules_p0/mv/dml/rewrite/rewrite_when_dml.out
new file mode 100644
index 00000000000000..639b3b0b977a55
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/mv/dml/rewrite/rewrite_when_dml.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !query1_0_before --
+o	3	21	o,o,o,o,o,o,o,o,o,o,o,o,o,o	4.571428571428571	mi	14	2
+o	4	4	o,o,o,o	4.0	yy	4	1
+
+-- !query1_0_after --
+o	3	21	o,o,o,o,o,o,o,o,o,o,o,o,o,o	4.571428571428571	mi	14	2
+o	4	4	o,o,o,o	4.0	yy	4	1
+
+-- !query2_0_before --
+o	3	21	o,o,o,o,o,o,o,o,o,o,o,o,o,o	4.571428571428571	mi	14	2
+o	4	4	o,o,o,o	4.0	yy	4	1
+
+-- !query3_0_before --
+o	25	o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o	4.444444444444445	18	2
+
+-- !query3_0_after --
+o	25	o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o	4.444444444444445	18	2
+
diff --git a/regression-test/suites/nereids_rules_p0/mv/dml/rewrite/rewrite_when_dml.groovy b/regression-test/suites/nereids_rules_p0/mv/dml/rewrite/rewrite_when_dml.groovy
new file mode 100644
index 00000000000000..c1c00d448635ee
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/dml/rewrite/rewrite_when_dml.groovy
@@ -0,0 +1,291 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("rewrite_when_dml") {
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    sql "set runtime_filter_mode=OFF";
+    sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "set enable_agg_state=true"
+
+    sql """
+    drop table if exists orders
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS orders  (
+      o_orderkey       INTEGER NOT NULL,
+      o_custkey        INTEGER NOT NULL,
+      o_orderstatus    CHAR(1) NOT NULL,
+      o_totalprice     DECIMALV3(15,2) NOT NULL,
+      o_orderdate      DATE NOT NULL,
+      o_orderpriority  CHAR(15) NOT NULL,  
+      o_clerk          CHAR(15) NOT NULL, 
+      o_shippriority   INTEGER NOT NULL,
+      O_COMMENT        VARCHAR(79) NOT NULL
+    )
+    DUPLICATE KEY(o_orderkey, o_custkey)
+    DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    );
+    """
+
+    sql """
+    drop table if exists lineitem
+    """
+
+    sql"""
+    CREATE TABLE IF NOT EXISTS lineitem (
+      l_orderkey    INTEGER NOT NULL,
+      l_partkey     INTEGER NOT NULL,
+      l_suppkey     INTEGER NOT NULL,
+      l_linenumber  INTEGER NOT NULL,
+      l_quantity    DECIMALV3(15,2) NOT NULL,
+      l_extendedprice  DECIMALV3(15,2) NOT NULL,
+      l_discount    DECIMALV3(15,2) NOT NULL,
+      l_tax         DECIMALV3(15,2) NOT NULL,
+      l_returnflag  CHAR(1) NOT NULL,
+      l_linestatus  CHAR(1) NOT NULL,
+      l_shipdate    DATE NOT NULL,
+      l_commitdate  DATE NOT NULL,
+      l_receiptdate DATE NOT NULL,
+      l_shipinstruct CHAR(25) NOT NULL,
+      l_shipmode     CHAR(10) NOT NULL,
+      l_comment      VARCHAR(44) NOT NULL
+    )
+    DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber)
+    DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    )
+    """
+
+    sql """
+    drop table if exists partsupp
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS partsupp (
+      ps_partkey     INTEGER NOT NULL,
+      ps_suppkey     INTEGER NOT NULL,
+      ps_availqty    INTEGER NOT NULL,
+      ps_supplycost  DECIMALV3(15,2)  NOT NULL,
+      ps_comment     VARCHAR(199) NOT NULL 
+    )
+    DUPLICATE KEY(ps_partkey, ps_suppkey)
+    DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    )
+    """
+
+    sql """ insert into lineitem values
+    (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+    (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+    (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+    (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+    (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');
+    """
+
+    sql """
+    insert into orders values
+    (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi');  
+    """
+
+    sql """
+    insert into partsupp values
+    (2, 3, 9, 10.01, 'supply1'),
+    (2, 3, 10, 11.01, 'supply2');
+    """
+
+    sql """analyze table partsupp with sync"""
+    sql """analyze table lineitem with sync"""
+    sql """analyze table orders with sync"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
+    // rewrite should success when table add column
+    def mv1_0 = """
+            select
+            o_orderstatus,
+            l_partkey,
+            l_suppkey,
+            sum_union(sum_state(o_shippriority)),
+            group_concat_union(group_concat_state(o_orderstatus)),
+            avg_union(avg_state(l_linenumber)),
+            max_by_union(max_by_state(O_COMMENT,o_totalprice)),
+            count_union(count_state(l_orderkey)),
+            multi_distinct_count_union(multi_distinct_count_state(l_shipmode))
+            from lineitem
+            left join orders
+            on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate
+            group by
+            o_orderstatus,
+            l_partkey,
+            l_suppkey;
+    """
+    def query1_0 = """
+            select
+            o_orderstatus,
+            l_suppkey,
+            sum_merge(sum_state(o_shippriority)),
+            group_concat_merge(group_concat_state(o_orderstatus)),
+            avg_merge(avg_state(l_linenumber)),
+            max_by_merge(max_by_state(O_COMMENT,o_totalprice)),
+            count_merge(count_state(l_orderkey)),
+            multi_distinct_count_merge(multi_distinct_count_state(l_shipmode))
+            from lineitem
+            left join orders 
+            on l_orderkey = o_orderkey and l_shipdate = o_orderdate
+            group by
+            o_orderstatus,
+            l_suppkey
+            order by o_orderstatus;
+    """
+    order_qt_query1_0_before "${query1_0}"
+    sql """ALTER TABLE orders ADD COLUMN new_col INT KEY DEFAULT "0";"""
+    async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0")
+    order_qt_query1_0_after "${query1_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0"""
+
+
+    // rewrite should fail when table drop column which is used by query
+    def mv2_0 = """
+            select
+            o_orderstatus,
+            l_partkey,
+            l_suppkey,
+            sum_union(sum_state(o_shippriority)),
+            group_concat_union(group_concat_state(o_orderstatus)),
+            avg_union(avg_state(l_linenumber)),
+            max_by_union(max_by_state(O_COMMENT,o_totalprice)),
+            count_union(count_state(l_orderkey)),
+            multi_distinct_count_union(multi_distinct_count_state(l_shipmode))
+            from lineitem
+            left join orders
+            on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate
+            group by
+            o_orderstatus,
+            l_partkey,
+            l_suppkey;
+    """
+    def query2_0 = """
+            select
+            o_orderstatus,
+            l_suppkey,
+            sum_merge(sum_state(o_shippriority)),
+            group_concat_merge(group_concat_state(o_orderstatus)),
+            avg_merge(avg_state(l_linenumber)),
+            max_by_merge(max_by_state(O_COMMENT,o_totalprice)),
+            count_merge(count_state(l_orderkey)),
+            multi_distinct_count_merge(multi_distinct_count_state(l_shipmode))
+            from lineitem
+            left join orders 
+            on l_orderkey = o_orderkey and l_shipdate = o_orderdate
+            group by
+            o_orderstatus,
+            l_suppkey
+            order by o_orderstatus;
+    """
+    order_qt_query2_0_before "${query2_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0"""
+    sql """
+        CREATE MATERIALIZED VIEW mv2_0
+        BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS ${mv2_0}
+    """
+    waitingMTMVTaskFinishedByMvName("mv2_0")
+    sql """ALTER TABLE lineitem DROP COLUMN l_suppkey;"""
+
+    try {
+        mv_not_part_in(query2_0, "mv2_0")
+    } catch (Exception e) {
+        // Unknown column because drop
+    }
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0"""
+
+
+    // rewrite should fail when table drop column which is not used by query
+    // can be optimized in the future
+    def mv3_0 = """
+            select
+            o_orderstatus,
+            l_partkey,
+            sum_union(sum_state(o_shippriority)),
+            group_concat_union(group_concat_state(o_orderstatus)),
+            avg_union(avg_state(l_linenumber)),
+            count_union(count_state(l_orderkey)),
+            multi_distinct_count_union(multi_distinct_count_state(l_shipmode))
+            from lineitem
+            left join orders
+            on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate
+            group by
+            o_orderstatus,
+            l_partkey;
+    """
+    def query3_0 = """
+            select
+            o_orderstatus,
+            sum_merge(sum_state(o_shippriority)),
+            group_concat_merge(group_concat_state(o_orderstatus)),
+            avg_merge(avg_state(l_linenumber)),
+            count_merge(count_state(l_orderkey)),
+            multi_distinct_count_merge(multi_distinct_count_state(l_shipmode))
+            from lineitem
+            left join orders 
+            on l_orderkey = o_orderkey and l_shipdate = o_orderdate
+            group by
+            o_orderstatus
+            order by o_orderstatus;
+    """
+    order_qt_query3_0_before "${query3_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_0"""
+    sql """
+        CREATE MATERIALIZED VIEW mv3_0
+        BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS ${mv3_0}
+    """
+    waitingMTMVTaskFinishedByMvName("mv3_0")
+    sql """ALTER TABLE orders DROP COLUMN O_COMMENT;"""
+    mv_not_part_in(query3_0, "mv3_0")
+
+    order_qt_query3_0_after "${query3_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_0"""
+}

From 80c2b8d7e7b5b08fcf4601c7853878d6a37adbad Mon Sep 17 00:00:00 2001
From: LiBinfeng <libinfeng@selectdb.com>
Date: Tue, 3 Dec 2024 22:48:54 +0800
Subject: [PATCH 145/399] [fix](Nereids) move tables from connect context to
 statement context (#44568)

Problem Summary:
When using tables in connect context, it would keep on memory in next
run in the same session, but it should not be in memory when running
next sql statement
---
 .../apache/doris/nereids/CascadesContext.java |  3 +-
 .../apache/doris/nereids/NereidsPlanner.java  |  4 +--
 .../doris/nereids/StatementContext.java       | 29 +++++++++++++++++++
 .../doris/nereids/minidump/Minidump.java      |  1 +
 .../doris/nereids/minidump/MinidumpUtils.java |  1 -
 .../nereids/rules/analysis/BindRelation.java  |  8 ++---
 .../trees/plans/commands/ReplayCommand.java   |  1 +
 .../org/apache/doris/qe/ConnectContext.java   | 29 -------------------
 .../doris/nereids/util/ReadLockTest.java      | 23 +++++----------
 9 files changed, 47 insertions(+), 52 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index 17ae5883063fb7..bb10996a11bf6a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -582,8 +582,9 @@ public static class Lock implements AutoCloseable {
         public Lock(LogicalPlan plan, CascadesContext cascadesContext) {
             this.cascadesContext = cascadesContext;
             // tables can also be load from dump file
-            if (cascadesContext.tables == null) {
+            if (cascadesContext.getTables() == null || cascadesContext.getTables().isEmpty()) {
                 cascadesContext.extractTables(plan);
+                cascadesContext.getStatementContext().setTables(cascadesContext.getTables());
             }
             for (TableIf table : cascadesContext.tables.values()) {
                 if (!table.needReadLockWhenPlan()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index c7478411a5de11..58af5cd3e92199 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -361,8 +361,8 @@ private void setRuntimeFilterWaitTimeByTableRowCountAndType() {
 
     private void initCascadesContext(LogicalPlan plan, PhysicalProperties requireProperties) {
         cascadesContext = CascadesContext.initContext(statementContext, plan, requireProperties);
-        if (statementContext.getConnectContext().getTables() != null) {
-            cascadesContext.setTables(statementContext.getConnectContext().getTables());
+        if (statementContext.getTables() != null) {
+            cascadesContext.setTables(statementContext.getTables());
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index b172f9dc591bd9..008a2c8ac70da2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -27,6 +27,7 @@
 import org.apache.doris.datasource.mvcc.MvccSnapshot;
 import org.apache.doris.datasource.mvcc.MvccTable;
 import org.apache.doris.datasource.mvcc.MvccTableInfo;
+import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.hint.Hint;
 import org.apache.doris.nereids.memo.Group;
 import org.apache.doris.nereids.rules.analysis.ColumnAliasGenerator;
@@ -53,6 +54,7 @@
 import org.apache.doris.system.Backend;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Stopwatch;
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
@@ -150,6 +152,9 @@ public class StatementContext implements Closeable {
     // placeholder params for prepared statement
     private List<Placeholder> placeholders;
 
+    // tables used for plan replayer
+    private Map<List<String>, TableIf> tables = null;
+
     // for create view support in nereids
     // key is the start and end position of the sql substring that needs to be replaced,
     // and value is the new string used for replacement.
@@ -213,6 +218,30 @@ public StatementContext(ConnectContext connectContext, OriginStatement originSta
         }
     }
 
+    public Map<List<String>, TableIf> getTables() {
+        if (tables == null) {
+            tables = Maps.newHashMap();
+        }
+        return tables;
+    }
+
+    public void setTables(Map<List<String>, TableIf> tables) {
+        this.tables = tables;
+    }
+
+    /** get table by table name, try to get from information from dumpfile first */
+    public TableIf getTableInMinidumpCache(List<String> tableQualifier) {
+        if (!getConnectContext().getSessionVariable().isPlayNereidsDump()) {
+            return null;
+        }
+        Preconditions.checkState(tables != null, "tables should not be null");
+        TableIf table = tables.getOrDefault(tableQualifier, null);
+        if (getConnectContext().getSessionVariable().isPlayNereidsDump() && table == null) {
+            throw new AnalysisException("Minidump cache can not find table:" + tableQualifier);
+        }
+        return table;
+    }
+
     public void setConnectContext(ConnectContext connectContext) {
         this.connectContext = connectContext;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java
index 5c324e1f364f15..37c7ff9a165b7c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java
@@ -123,6 +123,7 @@ public static void main(String[] args) {
 
         StatementContext statementContext = new StatementContext(ConnectContext.get(),
                 new OriginStatement(minidump.getSql(), 0));
+        statementContext.setTables(minidump.getTables());
         ConnectContext.get().setStatementContext(statementContext);
         JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql());
         JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
index fad7befc1626d6..c0f88b25341cde 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
@@ -227,7 +227,6 @@ public static void setConnectContext(Minidump minidump) {
         connectContext.setThreadLocalInfo();
         Env.getCurrentEnv().setColocateTableIndex(minidump.getColocateTableIndex());
         connectContext.setSessionVariable(minidump.getSessionVariable());
-        connectContext.setTables(minidump.getTables());
         connectContext.setDatabase(minidump.getDbName());
         connectContext.getSessionVariable().setPlanNereidsDump(true);
         connectContext.getSessionVariable().enableNereidsTimeout = false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index c62dda5a539df7..c7d4e9f975e50a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -171,7 +171,7 @@ private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRe
         List<String> tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(),
                 unboundRelation.getNameParts());
         TableIf table = null;
-        table = ConnectContext.get().getTableInMinidumpCache(tableQualifier);
+        table = ConnectContext.get().getStatementContext().getTableInMinidumpCache(tableQualifier);
         if (table == null) {
             if (customTableResolver.isPresent()) {
                 table = customTableResolver.get().apply(tableQualifier);
@@ -182,7 +182,7 @@ private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRe
         if (table == null) {
             table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv());
         }
-        ConnectContext.get().getTables().put(tableQualifier, table);
+        ConnectContext.get().getStatementContext().getTables().put(tableQualifier, table);
 
         // TODO: should generate different Scan sub class according to table's type
         LogicalPlan scan = getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
@@ -201,13 +201,13 @@ private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboun
         if (customTableResolver.isPresent()) {
             table = customTableResolver.get().apply(tableQualifier);
         }
-        table = ConnectContext.get().getTableInMinidumpCache(tableQualifier);
+        table = ConnectContext.get().getStatementContext().getTableInMinidumpCache(tableQualifier);
         // In some cases even if we have already called the "cascadesContext.getTableByName",
         // it also gets the null. So, we just check it in the catalog again for safety.
         if (table == null) {
             table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv());
         }
-        ConnectContext.get().getTables().put(tableQualifier, table);
+        ConnectContext.get().getStatementContext().getTables().put(tableQualifier, table);
         return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java
index 15cc2b3696a41e..4eabbb9e9595d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java
@@ -99,6 +99,7 @@ private void handleLoad() throws Exception {
         // 3. run nereids planner with sql in minidump file
         StatementContext statementContext = new StatementContext(ConnectContext.get(),
                 new OriginStatement(minidump.getSql(), 0));
+        statementContext.setTables(minidump.getTables());
         ConnectContext.get().setStatementContext(statementContext);
         JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql());
         JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index c21c9ee3f86db9..c81cf4920e1f7d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -31,7 +31,6 @@
 import org.apache.doris.catalog.DatabaseIf;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.FunctionRegistry;
-import org.apache.doris.catalog.TableIf;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.cloud.qe.ComputeGroupException;
 import org.apache.doris.cloud.system.CloudSystemInfoService;
@@ -54,7 +53,6 @@
 import org.apache.doris.mysql.ProxyMysqlChannel;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.nereids.StatementContext;
-import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.stats.StatsErrorEstimator;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.plsql.Exec;
@@ -73,7 +71,6 @@
 import org.apache.doris.transaction.TransactionEntry;
 import org.apache.doris.transaction.TransactionStatus;
 
-import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -267,8 +264,6 @@ public void setUserInsertTimeout(int insertTimeout) {
     // new planner
     private Map<String, PreparedStatementContext> preparedStatementContextMap = Maps.newHashMap();
 
-    private Map<List<String>, TableIf> tables = null;
-
     private Map<String, ColumnStatistic> totalColumnStatisticMap = new HashMap<>();
 
     public Map<String, ColumnStatistic> getTotalColumnStatisticMap() {
@@ -433,30 +428,6 @@ public PreparedStatementContext getPreparedStementContext(String stmtName) {
         return this.preparedStatementContextMap.get(stmtName);
     }
 
-    public Map<List<String>, TableIf> getTables() {
-        if (tables == null) {
-            tables = Maps.newHashMap();
-        }
-        return tables;
-    }
-
-    public void setTables(Map<List<String>, TableIf> tables) {
-        this.tables = tables;
-    }
-
-    /** get table by table name, try to get from information from dumpfile first */
-    public TableIf getTableInMinidumpCache(List<String> tableQualifier) {
-        if (!getSessionVariable().isPlayNereidsDump()) {
-            return null;
-        }
-        Preconditions.checkState(tables != null, "tables should not be null");
-        TableIf table = tables.getOrDefault(tableQualifier, null);
-        if (getSessionVariable().isPlayNereidsDump() && table == null) {
-            throw new AnalysisException("Minidump cache can not find table:" + tableQualifier);
-        }
-        return table;
-    }
-
     public void closeTxn() {
         if (isTxnModel()) {
             try {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java
index 3e1752e41bcdce..1e1535a573610b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java
@@ -18,7 +18,6 @@
 package org.apache.doris.nereids.util;
 
 import org.apache.doris.catalog.TableIf;
-import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.NereidsPlanner;
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.datasets.ssb.SSBTestBase;
@@ -48,10 +47,8 @@ public void testSimple() {
                 parser.parseSingle(sql),
                 PhysicalProperties.ANY
         );
-        CascadesContext cascadesContext = planner.getCascadesContext();
-
-        Map<List<String>, TableIf> f = cascadesContext.getTables();
-        Assertions.assertEquals(2, f.size());
+        Map<List<String>, TableIf> f = statementContext.getTables();
+        Assertions.assertEquals(1, f.size());
         Set<String> tableNames = new HashSet<>();
         for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
             TableIf table = entry.getValue();
@@ -75,8 +72,7 @@ public void testCTE() {
                 parser.parseSingle(sql),
                 PhysicalProperties.ANY
         );
-        CascadesContext cascadesContext = planner.getCascadesContext();
-        Map<List<String>, TableIf> f = cascadesContext.getTables();
+        Map<List<String>, TableIf> f = statementContext.getTables();
         Assertions.assertEquals(1, f.size());
         for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
             TableIf table = entry.getValue();
@@ -93,8 +89,7 @@ public void testSubQuery() {
                 parser.parseSingle(sql),
                 PhysicalProperties.ANY
         );
-        CascadesContext cascadesContext = planner.getCascadesContext();
-        Map<List<String>, TableIf> f = cascadesContext.getTables();
+        Map<List<String>, TableIf> f = statementContext.getTables();
         Assertions.assertEquals(1, f.size());
         for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
             TableIf table = entry.getValue();
@@ -111,8 +106,7 @@ public void testScalarSubQuery() {
                 parser.parseSingle(sql),
                 PhysicalProperties.ANY
         );
-        CascadesContext cascadesContext = planner.getCascadesContext();
-        Map<List<String>, TableIf> f = cascadesContext.getTables();
+        Map<List<String>, TableIf> f = statementContext.getTables();
         Assertions.assertEquals(2, f.size());
         Set<String> tableNames = new HashSet<>();
         for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
@@ -134,15 +128,14 @@ public void testInserInto() {
                 (LogicalPlan) insertIntoTableCommand.getExplainPlan(connectContext),
                 PhysicalProperties.ANY
         );
-        CascadesContext cascadesContext = planner.getCascadesContext();
-        Map<List<String>, TableIf> f = cascadesContext.getTables();
-        Assertions.assertEquals(2, f.size());
+        Map<List<String>, TableIf> f = statementContext.getTables();
+        // when table in insert would not be added to statement context, but be lock when insert
+        Assertions.assertEquals(1, f.size());
         Set<String> tableNames = new HashSet<>();
         for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
             TableIf table = entry.getValue();
             tableNames.add(table.getName());
         }
-        Assertions.assertTrue(tableNames.contains("supplier"));
         Assertions.assertTrue(tableNames.contains("lineorder"));
     }
 }

From 142c693784fd32ad3a4271bfee498e505ae8c86e Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Wed, 4 Dec 2024 09:06:05 +0800
Subject: [PATCH 146/399] [fix](hudi) upgrade hudi to 0.15.0 (#44267)

### What problem does this PR solve?

1. upgrade hudi to 0.15.0.
2. impl new hudi jni reader based on hudi-hadoop-mr
3. add session variable `hudi_jni_scanner` to choose which hudi jni
reader to use, "hadoop" means HadoopHudiJniReader, "spark" means old
HudiJniReader, default value is "hadoop"
4. support session variable `force_jni_scanner` for hudi
5. add more cases for hudi p2

### Release note
[opt](hudi) upgrade hudi to 0.15 and support hadoop jni reader
---
 .../vec/exec/format/table/hudi_jni_reader.cpp |  14 +-
 .../vec/exec/format/table/hudi_jni_reader.h   |   4 +-
 be/src/vec/exec/scan/vfile_scanner.cpp        |  17 +-
 build.sh                                      |   2 +
 conf/be.conf                                  |   2 +-
 .../hadoop-hudi-scanner/pom.xml               | 227 +++++++++++++++
 .../doris/hudi/HadoopHudiColumnValue.java     | 219 ++++++++++++++
 .../doris/hudi/HadoopHudiJniScanner.java      | 271 ++++++++++++++++++
 .../src/main/resources/package.xml            |  41 +++
 .../java/org/apache/doris/hudi/Utils.java     |   4 +-
 .../apache/doris/hudi/BaseSplitReader.scala   |  15 +-
 fe/be-java-extensions/pom.xml                 |   1 +
 .../doris/datasource/FileQueryScanNode.java   |   2 +
 .../hive/HiveMetaStoreClientHelper.java       |   5 +-
 .../doris/datasource/hudi/HudiUtils.java      |   2 +-
 .../hudi/source/COWIncrementalRelation.java   |  11 +-
 .../hudi/source/HudiLocalEngineContext.java   |  67 ++---
 .../hudi/source/HudiPartitionProcessor.java   |  14 +-
 .../datasource/hudi/source/HudiScanNode.java  |  61 ++--
 .../datasource/hudi/source/HudiSplit.java     |   3 +-
 .../hudi/source/MORIncrementalRelation.java   |  14 +-
 .../paimon/source/PaimonScanNode.java         |  39 ++-
 .../translator/PhysicalPlanTranslator.java    |   2 +-
 .../doris/planner/SingleNodePlanner.java      |   2 +-
 .../org/apache/doris/qe/SessionVariable.java  |  14 +
 fe/pom.xml                                    |   6 +-
 gensrc/thrift/PlanNodes.thrift                |   4 +-
 .../hudi/test_hudi_incremental.out            | 174 +++++++++++
 .../hudi/test_hudi_schema_evolution.out       |  32 +++
 .../hudi/test_hudi_snapshot.out               | Bin 348526 -> 696105 bytes
 .../hudi/test_hudi_timetravel.out             | 120 ++++++++
 .../hudi/test_hudi_catalog.groovy             |   2 +-
 .../hudi/test_hudi_incremental.groovy         |  16 +-
 .../hudi/test_hudi_schema_evolution.groovy    |  14 +-
 .../hudi/test_hudi_snapshot.groovy            |  13 +-
 .../hudi/test_hudi_timestamp.groovy           |   2 +-
 .../hudi/test_hudi_timetravel.groovy          |  15 +-
 37 files changed, 1312 insertions(+), 139 deletions(-)
 create mode 100644 fe/be-java-extensions/hadoop-hudi-scanner/pom.xml
 create mode 100644 fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java
 create mode 100644 fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
 create mode 100644 fe/be-java-extensions/hadoop-hudi-scanner/src/main/resources/package.xml

diff --git a/be/src/vec/exec/format/table/hudi_jni_reader.cpp b/be/src/vec/exec/format/table/hudi_jni_reader.cpp
index 33ba92b540a497..cb109bf05a2393 100644
--- a/be/src/vec/exec/format/table/hudi_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/hudi_jni_reader.cpp
@@ -18,7 +18,6 @@
 #include "hudi_jni_reader.h"
 
 #include <map>
-#include <ostream>
 
 #include "runtime/descriptors.h"
 #include "runtime/runtime_state.h"
@@ -65,7 +64,7 @@ HudiJniReader::HudiJniReader(const TFileScanRangeParams& scan_params,
             {"input_format", _hudi_params.input_format}};
 
     // Use compatible hadoop client to read data
-    for (auto& kv : _scan_params.properties) {
+    for (const auto& kv : _scan_params.properties) {
         if (kv.first.starts_with(HOODIE_CONF_PREFIX)) {
             params[kv.first] = kv.second;
         } else {
@@ -73,8 +72,15 @@ HudiJniReader::HudiJniReader(const TFileScanRangeParams& scan_params,
         }
     }
 
-    _jni_connector = std::make_unique<JniConnector>("org/apache/doris/hudi/HudiJniScanner", params,
-                                                    required_fields);
+    if (_hudi_params.hudi_jni_scanner == "hadoop") {
+        _jni_connector = std::make_unique<JniConnector>(
+                "org/apache/doris/hudi/HadoopHudiJniScanner", params, required_fields);
+    } else if (_hudi_params.hudi_jni_scanner == "spark") {
+        _jni_connector = std::make_unique<JniConnector>("org/apache/doris/hudi/HudiJniScanner",
+                                                        params, required_fields);
+    } else {
+        DCHECK(false) << "Unsupported hudi jni scanner: " << _hudi_params.hudi_jni_scanner;
+    }
 }
 
 Status HudiJniReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
diff --git a/be/src/vec/exec/format/table/hudi_jni_reader.h b/be/src/vec/exec/format/table/hudi_jni_reader.h
index e9bb55a69a77e7..bfa0291a61035c 100644
--- a/be/src/vec/exec/format/table/hudi_jni_reader.h
+++ b/be/src/vec/exec/format/table/hudi_jni_reader.h
@@ -17,9 +17,7 @@
 
 #pragma once
 
-#include <stddef.h>
-
-#include <memory>
+#include <cstddef>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp
index 3053adebbb521e..246a2edb082c92 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -23,18 +23,15 @@
 #include <gen_cpp/PaloInternalService_types.h>
 #include <gen_cpp/PlanNodes_types.h>
 
-#include <algorithm>
 #include <boost/iterator/iterator_facade.hpp>
 #include <iterator>
 #include <map>
-#include <ostream>
 #include <tuple>
 #include <utility>
 
 #include "common/compiler_util.h" // IWYU pragma: keep
 #include "common/config.h"
 #include "common/logging.h"
-#include "common/object_pool.h"
 #include "io/cache/block_file_cache_profile.h"
 #include "runtime/descriptors.h"
 #include "runtime/runtime_state.h"
@@ -47,7 +44,6 @@
 #include "vec/common/string_ref.h"
 #include "vec/core/column_with_type_and_name.h"
 #include "vec/core/columns_with_type_and_name.h"
-#include "vec/core/field.h"
 #include "vec/data_types/data_type.h"
 #include "vec/data_types/data_type_factory.hpp"
 #include "vec/data_types/data_type_nullable.h"
@@ -754,17 +750,16 @@ Status VFileScanner::_get_next_reader() {
 
         // create reader for specific format
         Status init_status;
-        TFileFormatType::type format_type = _params->format_type;
+        // for compatibility, if format_type is not set in range, use the format type of params
+        TFileFormatType::type format_type =
+                range.__isset.format_type ? range.format_type : _params->format_type;
         // JNI reader can only push down column value range
         bool push_down_predicates =
                 !_is_load && _params->format_type != TFileFormatType::FORMAT_JNI;
+        // for compatibility, this logic is deprecated in 3.1
         if (format_type == TFileFormatType::FORMAT_JNI && range.__isset.table_format_params) {
-            if (range.table_format_params.table_format_type == "hudi" &&
-                range.table_format_params.hudi_params.delta_logs.empty()) {
-                // fall back to native reader if there is no log file
-                format_type = TFileFormatType::FORMAT_PARQUET;
-            } else if (range.table_format_params.table_format_type == "paimon" &&
-                       !range.table_format_params.paimon_params.__isset.paimon_split) {
+            if (range.table_format_params.table_format_type == "paimon" &&
+                !range.table_format_params.paimon_params.__isset.paimon_split) {
                 // use native reader
                 auto format = range.table_format_params.paimon_params.file_format;
                 if (format == "orc") {
diff --git a/build.sh b/build.sh
index c90f6b14144600..da4e701f42146a 100755
--- a/build.sh
+++ b/build.sh
@@ -538,6 +538,7 @@ fi
 if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then
     modules+=("fe-common")
     modules+=("be-java-extensions/hudi-scanner")
+    modules+=("be-java-extensions/hadoop-hudi-scanner")
     modules+=("be-java-extensions/java-common")
     modules+=("be-java-extensions/java-udf")
     modules+=("be-java-extensions/jdbc-scanner")
@@ -825,6 +826,7 @@ EOF
     extensions_modules=("java-udf")
     extensions_modules+=("jdbc-scanner")
     extensions_modules+=("hudi-scanner")
+    extensions_modules+=("hadoop-hudi-scanner")
     extensions_modules+=("paimon-scanner")
     extensions_modules+=("trino-connector-scanner")
     extensions_modules+=("max-compute-scanner")
diff --git a/conf/be.conf b/conf/be.conf
index 5ad5e07176d545..ae9524313d860a 100644
--- a/conf/be.conf
+++ b/conf/be.conf
@@ -24,7 +24,7 @@ LOG_DIR="${DORIS_HOME}/log/"
 JAVA_OPTS="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xloggc:$LOG_DIR/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
 
 # For jdk 17, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED"
+JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djol.skipHotspotSAAttach=true -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED"
 
 # Set your own JAVA_HOME
 # JAVA_HOME=/path/to/jdk/
diff --git a/fe/be-java-extensions/hadoop-hudi-scanner/pom.xml b/fe/be-java-extensions/hadoop-hudi-scanner/pom.xml
new file mode 100644
index 00000000000000..4b80d49de17527
--- /dev/null
+++ b/fe/be-java-extensions/hadoop-hudi-scanner/pom.xml
@@ -0,0 +1,227 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <parent>
+        <artifactId>be-java-extensions</artifactId>
+        <groupId>org.apache.doris</groupId>
+        <version>${revision}</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>hadoop-hudi-scanner</artifactId>
+
+    <properties>
+        <doris.home>${basedir}/../../</doris.home>
+        <fe_ut_parallel>1</fe_ut_parallel>
+        <hudi.version>0.15.0</hudi.version>
+        <avro.version>1.11.3</avro.version>
+        <luben.zstd.jni.version>1.5.4-2</luben.zstd.jni.version>
+        <hive-apache.version>3.1.2-22</hive-apache.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.doris</groupId>
+            <artifactId>java-common</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.thrift</groupId>
+                    <artifactId>libthrift</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-hdfs-client</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-aws</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce-client-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.hudi/hudi-common -->
+        <dependency>
+            <groupId>org.apache.hudi</groupId>
+            <artifactId>hudi-common</artifactId>
+            <version>${hudi.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>hbase-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>hbase-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.thrift</groupId>
+                    <artifactId>libthrift</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.hudi/hudi-io -->
+        <dependency>
+            <groupId>org.apache.hudi</groupId>
+            <artifactId>hudi-io</artifactId>
+            <version>${hudi.version}</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.hudi/hudi-hadoop-mr -->
+        <dependency>
+            <groupId>org.apache.hudi</groupId>
+            <artifactId>hudi-hadoop-mr</artifactId>
+            <version>${hudi.version}</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop-bundle</artifactId>
+            <version>${parquet.version}</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro -->
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-avro</artifactId>
+            <version>${parquet.version}</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
+        <dependency>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro</artifactId>
+            <version>${avro.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-compress</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>concurrent</artifactId>
+            <version>202</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/io.airlift/aircompressor -->
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>aircompressor</artifactId>
+            <version>${aircompressor.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <version>${luben.zstd.jni.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.esotericsoftware</groupId>
+            <artifactId>kryo-shaded</artifactId>
+            <version>4.0.2</version>
+        </dependency>
+
+        <!-- hive -->
+        <dependency>
+            <groupId>io.trino.hive</groupId>
+            <artifactId>hive-apache</artifactId>
+            <version>${hive-apache.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.thrift</groupId>
+                    <artifactId>libthrift</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.parquet</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.avro</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>io.airlift</groupId>
+                    <artifactId>aircompressor</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <finalName>hadoop-hudi-scanner</finalName>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptors>
+                        <descriptor>src/main/resources/package.xml</descriptor>
+                    </descriptors>
+                    <archive>
+                        <manifest>
+                            <mainClass></mainClass>
+                        </manifest>
+                    </archive>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java
new file mode 100644
index 00000000000000..ae0199d07d27c5
--- /dev/null
+++ b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java
@@ -0,0 +1,219 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.hudi;
+
+import org.apache.doris.common.jni.vec.ColumnType;
+import org.apache.doris.common.jni.vec.ColumnValue;
+
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+import org.apache.hadoop.io.LongWritable;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.List;
+import java.util.Map;
+
+public class HadoopHudiColumnValue implements ColumnValue {
+    private ColumnType dorisType;
+    private ObjectInspector fieldInspector;
+    private Object fieldData;
+    private final ZoneId zoneId;
+
+    public HadoopHudiColumnValue(ZoneId zoneId) {
+        this.zoneId = zoneId;
+    }
+
+    public void setRow(Object record) {
+        this.fieldData = record;
+    }
+
+    public void setField(ColumnType dorisType, ObjectInspector fieldInspector) {
+        this.dorisType = dorisType;
+        this.fieldInspector = fieldInspector;
+    }
+
+    private Object inspectObject() {
+        return ((PrimitiveObjectInspector) fieldInspector).getPrimitiveJavaObject(fieldData);
+    }
+
+    @Override
+    public boolean getBoolean() {
+        return (boolean) inspectObject();
+    }
+
+    @Override
+    public short getShort() {
+        return (short) inspectObject();
+    }
+
+    @Override
+    public int getInt() {
+        return (int) inspectObject();
+    }
+
+    @Override
+    public float getFloat() {
+        return (float) inspectObject();
+    }
+
+    @Override
+    public long getLong() {
+        return (long) inspectObject();
+    }
+
+    @Override
+    public double getDouble() {
+        return (double) inspectObject();
+    }
+
+    @Override
+    public String getString() {
+        return inspectObject().toString();
+    }
+
+    @Override
+    public byte[] getBytes() {
+        return (byte[]) inspectObject();
+    }
+
+
+    @Override
+    public byte getByte() {
+        throw new UnsupportedOperationException("Hoodie type does not support tinyint");
+    }
+
+    @Override
+    public BigDecimal getDecimal() {
+        return ((HiveDecimal) inspectObject()).bigDecimalValue();
+    }
+
+    @Override
+    public LocalDate getDate() {
+        return LocalDate.ofEpochDay((((DateObjectInspector) fieldInspector).getPrimitiveJavaObject(fieldData))
+                .toEpochDay());
+    }
+
+    @Override
+    public LocalDateTime getDateTime() {
+        if (fieldData instanceof Timestamp) {
+            return ((Timestamp) fieldData).toLocalDateTime();
+        } else if (fieldData instanceof TimestampWritableV2) {
+            return LocalDateTime.ofInstant(Instant.ofEpochSecond((((TimestampObjectInspector) fieldInspector)
+                    .getPrimitiveJavaObject(fieldData)).toEpochSecond()), zoneId);
+        } else {
+            long datetime = ((LongWritable) fieldData).get();
+            long seconds;
+            long nanoseconds;
+            if (dorisType.getPrecision() == 3) {
+                seconds = datetime / 1000;
+                nanoseconds = (datetime % 1000) * 1000000;
+            } else if (dorisType.getPrecision() == 6) {
+                seconds = datetime / 1000000;
+                nanoseconds = (datetime % 1000000) * 1000;
+            } else {
+                throw new RuntimeException("Hoodie timestamp only support milliseconds and microseconds, "
+                        + "wrong precision = " + dorisType.getPrecision());
+            }
+            return LocalDateTime.ofInstant(Instant.ofEpochSecond(seconds, nanoseconds), zoneId);
+        }
+    }
+
+    @Override
+    public boolean canGetStringAsBytes() {
+        return false;
+    }
+
+    @Override
+    public boolean isNull() {
+        return fieldData == null;
+    }
+
+    @Override
+    public BigInteger getBigInteger() {
+        throw new UnsupportedOperationException("Hoodie type does not support largeint");
+    }
+
+    @Override
+    public byte[] getStringAsBytes() {
+        throw new UnsupportedOperationException("Hoodie type does not support getStringAsBytes");
+    }
+
+    @Override
+    public void unpackArray(List<ColumnValue> values) {
+        ListObjectInspector inspector = (ListObjectInspector) fieldInspector;
+        List<?> items = inspector.getList(fieldData);
+        ObjectInspector itemInspector = inspector.getListElementObjectInspector();
+        for (int i = 0; i < items.size(); i++) {
+            Object item = items.get(i);
+            HadoopHudiColumnValue childValue = new HadoopHudiColumnValue(zoneId);
+            childValue.setRow(item);
+            childValue.setField(dorisType.getChildTypes().get(0), itemInspector);
+            values.add(childValue);
+        }
+    }
+
+    @Override
+    public void unpackMap(List<ColumnValue> keys, List<ColumnValue> values) {
+        MapObjectInspector inspector = (MapObjectInspector) fieldInspector;
+        ObjectInspector keyObjectInspector = inspector.getMapKeyObjectInspector();
+        ObjectInspector valueObjectInspector = inspector.getMapValueObjectInspector();
+        for (Map.Entry kv : inspector.getMap(fieldData).entrySet()) {
+            HadoopHudiColumnValue key = new HadoopHudiColumnValue(zoneId);
+            key.setRow(kv.getKey());
+            key.setField(dorisType.getChildTypes().get(0), keyObjectInspector);
+            keys.add(key);
+
+            HadoopHudiColumnValue value = new HadoopHudiColumnValue(zoneId);
+            value.setRow(kv.getValue());
+            value.setField(dorisType.getChildTypes().get(1), valueObjectInspector);
+            values.add(value);
+        }
+    }
+
+    @Override
+    public void unpackStruct(List<Integer> structFieldIndex, List<ColumnValue> values) {
+        StructObjectInspector inspector = (StructObjectInspector) fieldInspector;
+        List<? extends StructField> fields = inspector.getAllStructFieldRefs();
+        for (int i = 0; i < structFieldIndex.size(); i++) {
+            Integer idx = structFieldIndex.get(i);
+            HadoopHudiColumnValue value = new HadoopHudiColumnValue(zoneId);
+            Object obj = null;
+            if (idx != null) {
+                StructField sf = fields.get(idx);
+                obj = inspector.getStructFieldData(fieldData, sf);
+            }
+            value.setRow(obj);
+            value.setField(dorisType.getChildTypes().get(i), fields.get(i).getFieldObjectInspector());
+            values.add(value);
+        }
+    }
+}
diff --git a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
new file mode 100644
index 00000000000000..f2b38815a366fe
--- /dev/null
+++ b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
@@ -0,0 +1,271 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.hudi;
+
+import org.apache.doris.common.classloader.ThreadClassLoaderContext;
+import org.apache.doris.common.jni.JniScanner;
+import org.apache.doris.common.jni.vec.ColumnType;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hudi.common.model.HoodieLogFile;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.hadoop.realtime.HoodieRealtimeFileSplit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * HadoopHudiJniScanner is a JniScanner implementation that reads Hudi data using hudi-hadoop-mr.
+ */
+public class HadoopHudiJniScanner extends JniScanner {
+    private static final Logger LOG = LoggerFactory.getLogger(HadoopHudiJniScanner.class);
+
+    private static final String HADOOP_CONF_PREFIX = "hadoop_conf.";
+
+    // Hudi data info
+    private final String basePath;
+    private final String dataFilePath;
+    private final long dataFileLength;
+    private final String[] deltaFilePaths;
+    private final String instantTime;
+    private final String serde;
+    private final String inputFormat;
+
+    // schema info
+    private final String hudiColumnNames;
+    private final String[] hudiColumnTypes;
+    private final String[] requiredFields;
+    private List<Integer> requiredColumnIds;
+    private ColumnType[] requiredTypes;
+
+    // Hadoop info
+    private RecordReader<NullWritable, ArrayWritable> reader;
+    private StructObjectInspector rowInspector;
+    private final ObjectInspector[] fieldInspectors;
+    private final StructField[] structFields;
+    private Deserializer deserializer;
+    private final Map<String, String> fsOptionsProps;
+
+    // scanner info
+    private final HadoopHudiColumnValue columnValue;
+    private final int fetchSize;
+    private final ClassLoader classLoader;
+
+    public HadoopHudiJniScanner(int fetchSize, Map<String, String> params) {
+        this.basePath = params.get("base_path");
+        this.dataFilePath = params.get("data_file_path");
+        this.dataFileLength = Long.parseLong(params.get("data_file_length"));
+        if (Strings.isNullOrEmpty(params.get("delta_file_paths"))) {
+            this.deltaFilePaths = new String[0];
+        } else {
+            this.deltaFilePaths = params.get("delta_file_paths").split(",");
+        }
+        this.instantTime = params.get("instant_time");
+        this.serde = params.get("serde");
+        this.inputFormat = params.get("input_format");
+
+        this.hudiColumnNames = params.get("hudi_column_names");
+        this.hudiColumnTypes = params.get("hudi_column_types").split("#");
+        this.requiredFields = params.get("required_fields").split(",");
+
+        this.fieldInspectors = new ObjectInspector[requiredFields.length];
+        this.structFields = new StructField[requiredFields.length];
+        this.fsOptionsProps = Maps.newHashMap();
+        for (Map.Entry<String, String> entry : params.entrySet()) {
+            if (entry.getKey().startsWith(HADOOP_CONF_PREFIX)) {
+                fsOptionsProps.put(entry.getKey().substring(HADOOP_CONF_PREFIX.length()), entry.getValue());
+            }
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("get hudi params {}: {}", entry.getKey(), entry.getValue());
+            }
+        }
+
+        ZoneId zoneId;
+        if (Strings.isNullOrEmpty(params.get("time_zone"))) {
+            zoneId = ZoneId.systemDefault();
+        } else {
+            zoneId = ZoneId.of(params.get("time_zone"));
+        }
+        this.columnValue = new HadoopHudiColumnValue(zoneId);
+        this.fetchSize = fetchSize;
+        this.classLoader = this.getClass().getClassLoader();
+    }
+
+    @Override
+    public void open() throws IOException {
+        try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) {
+            initRequiredColumnsAndTypes();
+            initTableInfo(requiredTypes, requiredFields, fetchSize);
+            Properties properties = getReaderProperties();
+            initReader(properties);
+        } catch (Exception e) {
+            close();
+            LOG.warn("failed to open hadoop hudi jni scanner", e);
+            throw new IOException("failed to open hadoop hudi jni scanner: " + e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public int getNext() throws IOException {
+        try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) {
+            NullWritable key = reader.createKey();
+            ArrayWritable value = reader.createValue();
+            int numRows = 0;
+            for (; numRows < fetchSize; numRows++) {
+                if (!reader.next(key, value)) {
+                    break;
+                }
+                Object rowData = deserializer.deserialize(value);
+                for (int i = 0; i < fields.length; i++) {
+                    Object fieldData = rowInspector.getStructFieldData(rowData, structFields[i]);
+                    columnValue.setRow(fieldData);
+                    // LOG.info("rows: {}, column: {}, col name: {}, col type: {}, inspector: {}",
+                    //        numRows, i, types[i].getName(), types[i].getType().name(),
+                    //        fieldInspectors[i].getTypeName());
+                    columnValue.setField(types[i], fieldInspectors[i]);
+                    appendData(i, columnValue);
+                }
+            }
+            return numRows;
+        } catch (Exception e) {
+            close();
+            LOG.warn("failed to get next in hadoop hudi jni scanner", e);
+            throw new IOException("failed to get next in hadoop hudi jni scanner: " + e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) {
+            if (reader != null) {
+                reader.close();
+            }
+        } catch (IOException e) {
+            LOG.warn("failed to close hadoop hudi jni scanner", e);
+            throw new IOException("failed to close hadoop hudi jni scanner: " + e.getMessage(), e);
+        }
+    }
+
+    private void initRequiredColumnsAndTypes() {
+        String[] splitHudiColumnNames = hudiColumnNames.split(",");
+
+        Map<String, Integer> hudiColNameToIdx =
+                IntStream.range(0, splitHudiColumnNames.length)
+                        .boxed()
+                        .collect(Collectors.toMap(i -> splitHudiColumnNames[i], i -> i));
+
+        Map<String, String> hudiColNameToType =
+                IntStream.range(0, splitHudiColumnNames.length)
+                        .boxed()
+                        .collect(Collectors.toMap(i -> splitHudiColumnNames[i], i -> hudiColumnTypes[i]));
+
+        requiredTypes = Arrays.stream(requiredFields)
+                .map(field -> ColumnType.parseType(field, hudiColNameToType.get(field)))
+                .toArray(ColumnType[]::new);
+
+        requiredColumnIds = Arrays.stream(requiredFields)
+                .mapToInt(hudiColNameToIdx::get)
+                .boxed().collect(Collectors.toList());
+    }
+
+    private Properties getReaderProperties() {
+        Properties properties = new Properties();
+        properties.setProperty("hive.io.file.readcolumn.ids", Joiner.on(",").join(requiredColumnIds));
+        properties.setProperty("hive.io.file.readcolumn.names", Joiner.on(",").join(this.requiredFields));
+        properties.setProperty("columns", this.hudiColumnNames);
+        properties.setProperty("columns.types", Joiner.on(",").join(hudiColumnTypes));
+        properties.setProperty("serialization.lib", this.serde);
+        properties.setProperty("hive.io.file.read.all.columns", "false");
+        fsOptionsProps.forEach(properties::setProperty);
+        return properties;
+    }
+
+    private void initReader(Properties properties) throws Exception {
+        String realtimePath = dataFileLength != -1 ? dataFilePath : deltaFilePaths[0];
+        long realtimeLength = dataFileLength != -1 ? dataFileLength : 0;
+        Path path = new Path(realtimePath);
+        FileSplit fileSplit = new FileSplit(path, 0, realtimeLength, (String[]) null);
+        List<HoodieLogFile> logFiles = Arrays.stream(deltaFilePaths).map(HoodieLogFile::new)
+                .collect(Collectors.toList());
+        FileSplit hudiSplit =
+                new HoodieRealtimeFileSplit(fileSplit, basePath, logFiles, instantTime, false, Option.empty());
+
+        JobConf jobConf = new JobConf(new Configuration());
+        properties.stringPropertyNames().forEach(name -> jobConf.set(name, properties.getProperty(name)));
+        InputFormat<?, ?> inputFormatClass = createInputFormat(jobConf, inputFormat);
+        reader = (RecordReader<NullWritable, ArrayWritable>) inputFormatClass
+                .getRecordReader(hudiSplit, jobConf, Reporter.NULL);
+
+        deserializer = getDeserializer(jobConf, properties, serde);
+        rowInspector = getTableObjectInspector(deserializer);
+        for (int i = 0; i < requiredFields.length; i++) {
+            StructField field = rowInspector.getStructFieldRef(requiredFields[i]);
+            structFields[i] = field;
+            fieldInspectors[i] = field.getFieldObjectInspector();
+        }
+    }
+
+    private InputFormat<?, ?> createInputFormat(Configuration conf, String inputFormat) throws Exception {
+        Class<?> clazz = conf.getClassByName(inputFormat);
+        Class<? extends InputFormat<?, ?>> cls =
+                (Class<? extends InputFormat<?, ?>>) clazz.asSubclass(InputFormat.class);
+        return ReflectionUtils.newInstance(cls, conf);
+    }
+
+    private Deserializer getDeserializer(Configuration configuration, Properties properties, String name)
+            throws Exception {
+        Class<? extends Deserializer> deserializerClass = Class.forName(name, true, JavaUtils.getClassLoader())
+                .asSubclass(Deserializer.class);
+        Deserializer deserializer = deserializerClass.getConstructor().newInstance();
+        deserializer.initialize(configuration, properties);
+        return deserializer;
+    }
+
+    private StructObjectInspector getTableObjectInspector(Deserializer deserializer) throws Exception {
+        ObjectInspector inspector = deserializer.getObjectInspector();
+        Preconditions.checkArgument(inspector.getCategory() == ObjectInspector.Category.STRUCT,
+                "expected STRUCT: %s", inspector.getCategory());
+        return (StructObjectInspector) inspector;
+    }
+}
diff --git a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/resources/package.xml b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/resources/package.xml
new file mode 100644
index 00000000000000..4bbb2610603363
--- /dev/null
+++ b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/resources/package.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
+    <id>jar-with-dependencies</id>
+    <formats>
+        <format>jar</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <unpack>true</unpack>
+            <scope>runtime</scope>
+            <unpackOptions>
+                <excludes>
+                    <exclude>**/Log4j2Plugins.dat</exclude>
+                </excludes>
+            </unpackOptions>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
index 5614f8bcc96eb1..3e07c8917905a3 100644
--- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
+++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
@@ -23,6 +23,7 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -75,7 +76,8 @@ public static void killProcess(long pid) {
     }
 
     public static HoodieTableMetaClient getMetaClient(Configuration conf, String basePath) {
+        HadoopStorageConfiguration hadoopStorageConfiguration = new HadoopStorageConfiguration(conf);
         return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf), () -> HoodieTableMetaClient.builder()
-                .setConf(conf).setBasePath(basePath).build());
+                .setConf(hadoopStorageConfiguration).setBasePath(basePath).build());
     }
 }
diff --git a/fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/BaseSplitReader.scala b/fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/BaseSplitReader.scala
index dcc068ad7006d8..fc8d74f9713c26 100644
--- a/fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/BaseSplitReader.scala
+++ b/fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/BaseSplitReader.scala
@@ -36,13 +36,15 @@ import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, T
 import org.apache.hudi.common.util.ValidationUtils.checkState
 import org.apache.hudi.common.util.{ConfigUtils, StringUtils}
 import org.apache.hudi.config.HoodieWriteConfig
-import org.apache.hudi.hadoop.CachingPath
+import org.apache.hudi.hadoop.fs.CachingPath
 import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter
 import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
 import org.apache.hudi.internal.schema.{HoodieSchemaException, InternalSchema}
-import org.apache.hudi.io.storage.HoodieAvroHFileReader
+import org.apache.hudi.io.hadoop.HoodieHBaseAvroHFileReader
 import org.apache.hudi.metadata.HoodieTableMetadataUtil
 import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, DataSourceWriteOptions, HoodieSparkConfUtils, HoodieTableSchema, HoodieTableState}
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
 import org.apache.log4j.Logger
 import org.apache.spark.sql.adapter.Spark3_4Adapter
 import org.apache.spark.sql.avro.{HoodieAvroSchemaConverters, HoodieSparkAvroSchemaConverters}
@@ -430,7 +432,7 @@ abstract class BaseSplitReader(val split: HoodieSplit) {
     try {
       if (shouldExtractPartitionValuesFromPartitionPath) {
         val filePath = new Path(split.dataFilePath)
-        val tablePathWithoutScheme = CachingPath.getPathWithoutSchemeAndAuthority(tableInformation.metaClient.getBasePathV2)
+        val tablePathWithoutScheme = CachingPath.getPathWithoutSchemeAndAuthority(new Path(tableInformation.metaClient.getBasePathV2.toUri))
         val partitionPathWithoutScheme = CachingPath.getPathWithoutSchemeAndAuthority(filePath.getParent)
         val relativePath = new URI(tablePathWithoutScheme.toString).relativize(new URI(partitionPathWithoutScheme.toString)).toString
         val hiveStylePartitioningEnabled = tableConfig.getHiveStylePartitioningEnable.toBoolean
@@ -497,8 +499,11 @@ abstract class BaseSplitReader(val split: HoodieSplit) {
                                 options: Map[String, String],
                                 hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = {
     partitionedFile => {
-      val reader = new HoodieAvroHFileReader(
-        hadoopConf, partitionedFile.filePath.toPath, new CacheConfig(hadoopConf))
+      var hadoopStorageConfiguration = new HadoopStorageConfiguration(hadoopConf);
+      var storagePath = new StoragePath(partitionedFile.toPath.toUri.getPath);
+      var emptySchema = org.apache.hudi.common.util.Option.empty[org.apache.avro.Schema]()
+      val reader = new HoodieHBaseAvroHFileReader(
+        hadoopStorageConfiguration, storagePath, emptySchema)
 
       val requiredRowSchema = requiredDataSchema.structTypeSchema
       // NOTE: Schema has to be parsed at this point, since Avro's [[Schema]] aren't serializable
diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml
index bbe056739d51ec..5d56ef76e7c3ef 100644
--- a/fe/be-java-extensions/pom.xml
+++ b/fe/be-java-extensions/pom.xml
@@ -22,6 +22,7 @@ under the License.
     <modelVersion>4.0.0</modelVersion>
     <modules>
         <module>hudi-scanner</module>
+        <module>hadoop-hudi-scanner</module>
         <module>java-common</module>
         <module>java-udf</module>
         <module>jdbc-scanner</module>
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index 3107b4ed5d7b85..2672cc016eb528 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -433,6 +433,8 @@ private TScanRangeLocations splitToScanRange(
             }
         }
 
+        // set file format type, and the type might fall back to native format in setScanParams
+        rangeDesc.setFormatType(getFileFormatType());
         setScanParams(rangeDesc, fileSplit);
         curLocations.getScanRange().getExtScanRange().getFileScanRange().addToRanges(rangeDesc);
         TScanRangeLocation location = new TScanRangeLocation();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index 0f839d238b2b1e..97f86612a495da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -64,6 +64,7 @@
 import org.apache.hudi.avro.HoodieAvroUtils;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.TableSchemaResolver;
+import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -831,8 +832,10 @@ public static <T> T ugiDoAs(Configuration conf, PrivilegedExceptionAction<T> act
     public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
         String hudiBasePath = table.getRemoteTable().getSd().getLocation();
         Configuration conf = getConfiguration(table);
+        HadoopStorageConfiguration hadoopStorageConfiguration = new HadoopStorageConfiguration(conf);
         return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf),
-                () -> HoodieTableMetaClient.builder().setConf(conf).setBasePath(hudiBasePath).build());
+                () -> HoodieTableMetaClient.builder().setConf(hadoopStorageConfiguration).setBasePath(hudiBasePath)
+                        .build());
     }
 
     public static Configuration getConfiguration(HMSExternalTable table) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
index d7803b1a516f9e..c98d994a28a08f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
@@ -86,7 +86,7 @@ public static String convertAvroToHiveType(Schema schema) {
             case LONG:
                 if (logicalType instanceof LogicalTypes.TimestampMillis
                         || logicalType instanceof LogicalTypes.TimestampMicros) {
-                    return logicalType.getName();
+                    return "timestamp";
                 }
                 if (logicalType instanceof LogicalTypes.TimeMicros) {
                     return handleUnsupportedType(schema);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/COWIncrementalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/COWIncrementalRelation.java
index 7981a0b4f261ff..843dded27969ad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/COWIncrementalRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/COWIncrementalRelation.java
@@ -37,6 +37,7 @@
 import org.apache.hudi.common.table.timeline.TimelineUtils.HollowCommitHandling;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.storage.StoragePath;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -105,7 +106,7 @@ public COWIncrementalRelation(Map<String, String> optParams, Configuration confi
         List<HoodieInstant> commitsToReturn = commitsTimelineToReturn.getInstants();
 
         // todo: support configuration hoodie.datasource.read.incr.filters
-        Path basePath = metaClient.getBasePathV2();
+        StoragePath basePath = metaClient.getBasePathV2();
         Map<String, String> regularFileIdToFullPath = new HashMap<>();
         Map<String, String> metaBootstrapFileIdToFullPath = new HashMap<>();
         HoodieTimeline replacedTimeline = commitsTimelineToReturn.getCompletedReplaceTimeline();
@@ -113,8 +114,8 @@ public COWIncrementalRelation(Map<String, String> optParams, Configuration confi
         for (HoodieInstant instant : replacedTimeline.getInstants()) {
             HoodieReplaceCommitMetadata.fromBytes(metaClient.getActiveTimeline().getInstantDetails(instant).get(),
                     HoodieReplaceCommitMetadata.class).getPartitionToReplaceFileIds().forEach(
-                        (key, value) -> value.forEach(
-                            e -> replacedFile.put(e, FSUtils.getPartitionPath(basePath, key).toString())));
+                            (key, value) -> value.forEach(
+                                    e -> replacedFile.put(e, FSUtils.constructAbsolutePath(basePath, key).toString())));
         }
 
         fileToWriteStat = new HashMap<>();
@@ -123,7 +124,7 @@ public COWIncrementalRelation(Map<String, String> optParams, Configuration confi
                     commitTimeline.getInstantDetails(commit).get(), HoodieCommitMetadata.class);
             metadata.getPartitionToWriteStats().forEach((partition, stats) -> {
                 for (HoodieWriteStat stat : stats) {
-                    fileToWriteStat.put(FSUtils.getPartitionPath(basePath, stat.getPath()).toString(), stat);
+                    fileToWriteStat.put(FSUtils.constructAbsolutePath(basePath, stat.getPath()).toString(), stat);
                 }
             });
             if (HoodieTimeline.METADATA_BOOTSTRAP_INSTANT_TS.equals(commit.getTimestamp())) {
@@ -158,7 +159,7 @@ public COWIncrementalRelation(Map<String, String> optParams, Configuration confi
 
         }
 
-        fs = basePath.getFileSystem(configuration);
+        fs = new Path(basePath.toUri().getPath()).getFileSystem(configuration);
         fullTableScan = shouldFullTableScan();
         includeStartTime = !fullTableScan;
         if (fullTableScan || commitsToReturn.isEmpty()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiLocalEngineContext.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiLocalEngineContext.java
index 26ef6fdfef7086..fecc026cf8d046 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiLocalEngineContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiLocalEngineContext.java
@@ -17,10 +17,6 @@
 
 package org.apache.doris.datasource.hudi.source;
 
-import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hudi.common.config.SerializableConfiguration;
 import org.apache.hudi.common.data.HoodieAccumulator;
 import org.apache.hudi.common.data.HoodieAtomicLongAccumulator;
 import org.apache.hudi.common.data.HoodieData;
@@ -39,7 +35,7 @@
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.collection.ImmutablePair;
 import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.storage.StorageConfiguration;
 
 import java.util.Collections;
 import java.util.Iterator;
@@ -50,18 +46,20 @@
 import java.util.stream.Stream;
 
 /**
- * This file is copied from org.apache.hudi.common.engine.HoodieLocalEngineContext.
+ * This file is copied from
+ * org.apache.hudi.common.engine.HudiLocalEngineContext.
  * Because we need set ugi in thread pool
- * A java based engine context, use this implementation on the query engine integrations if needed.
+ * A java based engine context, use this implementation on the query engine
+ * integrations if needed.
  */
 public final class HudiLocalEngineContext extends HoodieEngineContext {
 
-    public HudiLocalEngineContext(Configuration conf) {
+    public HudiLocalEngineContext(StorageConfiguration<?> conf) {
         this(conf, new LocalTaskContextSupplier());
     }
 
-    public HudiLocalEngineContext(Configuration conf, TaskContextSupplier taskContextSupplier) {
-        super(new SerializableConfiguration(conf), taskContextSupplier);
+    public HudiLocalEngineContext(StorageConfiguration<?> conf, TaskContextSupplier taskContextSupplier) {
+        super(conf, taskContextSupplier);
     }
 
     @Override
@@ -81,27 +79,18 @@ public <T> HoodieData<T> parallelize(List<T> data, int parallelism) {
 
     @Override
     public <I, O> List<O> map(List<I> data, SerializableFunction<I, O> func, int parallelism) {
-        return data.stream().parallel().map(v1 -> {
-            try {
-                return HiveMetaStoreClientHelper.ugiDoAs(getHadoopConf().get(), () -> func.apply(v1));
-            } catch (Exception e) {
-                throw new HoodieException("Error occurs when executing map", e);
-            }
-        }).collect(Collectors.toList());
+        return data.stream().parallel().map(FunctionWrapper.throwingMapWrapper(func)).collect(Collectors.toList());
     }
 
     @Override
     public <I, K, V> List<V> mapToPairAndReduceByKey(
-            List<I> data,
-            SerializablePairFunction<I, K, V> mapToPairFunc,
-            SerializableBiFunction<V, V, V> reduceFunc, int parallelism) {
+            List<I> data, SerializablePairFunction<I, K, V> mapToPairFunc, SerializableBiFunction<V, V, V> reduceFunc,
+            int parallelism) {
         return data.stream().parallel().map(FunctionWrapper.throwingMapToPairWrapper(mapToPairFunc))
-            .collect(Collectors.groupingBy(p -> p.getKey())).values().stream()
-            .map(list ->
-                    list.stream()
-                        .map(e -> e.getValue())
+                .collect(Collectors.groupingBy(p -> p.getKey())).values().stream()
+                .map(list -> list.stream().map(e -> e.getValue())
                         .reduce(FunctionWrapper.throwingReduceWrapper(reduceFunc)).get())
-            .collect(Collectors.toList());
+                .collect(Collectors.toList());
     }
 
     @Override
@@ -109,29 +98,28 @@ public <I, K, V> Stream<ImmutablePair<K, V>> mapPartitionsToPairAndReduceByKey(
             Stream<I> data, SerializablePairFlatMapFunction<Iterator<I>, K, V> flatMapToPairFunc,
             SerializableBiFunction<V, V, V> reduceFunc, int parallelism) {
         return FunctionWrapper.throwingFlatMapToPairWrapper(flatMapToPairFunc).apply(data.parallel().iterator())
-            .collect(Collectors.groupingBy(Pair::getKey)).entrySet().stream()
-            .map(entry -> new ImmutablePair<>(entry.getKey(), entry.getValue().stream().map(
-                Pair::getValue).reduce(FunctionWrapper.throwingReduceWrapper(reduceFunc)).orElse(null)))
-            .filter(Objects::nonNull);
+                .collect(Collectors.groupingBy(Pair::getKey)).entrySet().stream()
+                .map(entry -> new ImmutablePair<>(entry.getKey(), entry.getValue().stream().map(
+                        Pair::getValue).reduce(FunctionWrapper.throwingReduceWrapper(reduceFunc)).orElse(null)))
+                .filter(Objects::nonNull);
     }
 
     @Override
     public <I, K, V> List<V> reduceByKey(
             List<Pair<K, V>> data, SerializableBiFunction<V, V, V> reduceFunc, int parallelism) {
         return data.stream().parallel()
-            .collect(Collectors.groupingBy(p -> p.getKey())).values().stream()
-            .map(list ->
-                list.stream()
-                    .map(e -> e.getValue())
-                    .reduce(FunctionWrapper.throwingReduceWrapper(reduceFunc)).orElse(null))
-            .filter(Objects::nonNull)
-            .collect(Collectors.toList());
+                .collect(Collectors.groupingBy(p -> p.getKey())).values().stream()
+                .map(list -> list.stream().map(e -> e.getValue())
+                        .reduce(FunctionWrapper.throwingReduceWrapper(reduceFunc))
+                        .orElse(null))
+                .filter(Objects::nonNull)
+                .collect(Collectors.toList());
     }
 
     @Override
     public <I, O> List<O> flatMap(List<I> data, SerializableFunction<I, Stream<O>> func, int parallelism) {
-        return
-            data.stream().parallel().flatMap(FunctionWrapper.throwingFlatMapWrapper(func)).collect(Collectors.toList());
+        return data.stream().parallel().flatMap(FunctionWrapper.throwingFlatMapWrapper(func))
+                .collect(Collectors.toList());
     }
 
     @Override
@@ -142,8 +130,7 @@ public <I> void foreach(List<I> data, SerializableConsumer<I> consumer, int para
     @Override
     public <I, K, V> Map<K, V> mapToPair(List<I> data, SerializablePairFunction<I, K, V> func, Integer parallelism) {
         return data.stream().map(FunctionWrapper.throwingMapToPairWrapper(func)).collect(
-            Collectors.toMap(Pair::getLeft, Pair::getRight, (oldVal, newVal) -> newVal)
-        );
+                Collectors.toMap(Pair::getLeft, Pair::getRight, (oldVal, newVal) -> newVal));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiPartitionProcessor.java
index 738b2638588e03..0ab9fef951a378 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiPartitionProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiPartitionProcessor.java
@@ -21,7 +21,6 @@
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
 import org.apache.hudi.common.table.timeline.TimelineUtils;
-import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils;
 import org.apache.hudi.metadata.HoodieTableMetadata;
 import org.apache.hudi.metadata.HoodieTableMetadataUtil;
 
@@ -49,14 +48,15 @@ public List<String> getAllPartitionNames(HoodieTableMetaClient tableMetaClient)
                 .build();
 
         HoodieTableMetadata newTableMetadata = HoodieTableMetadata.create(
-                new HudiLocalEngineContext(tableMetaClient.getHadoopConf()), metadataConfig,
+                new HudiLocalEngineContext(tableMetaClient.getStorageConf()), tableMetaClient.getStorage(),
+                metadataConfig,
                 tableMetaClient.getBasePathV2().toString(), true);
 
         return newTableMetadata.getAllPartitionPaths();
     }
 
     public List<String> getPartitionNamesBeforeOrEquals(HoodieTimeline timeline, String timestamp) {
-        return new ArrayList<>(HoodieInputFormatUtils.getWritePartitionPaths(
+        return new ArrayList<>(HoodieTableMetadataUtil.getWritePartitionPaths(
                 timeline.findInstantsBeforeOrEquals(timestamp).getInstants().stream().map(instant -> {
                     try {
                         return TimelineUtils.getCommitMetadata(instant, timeline);
@@ -67,7 +67,7 @@ public List<String> getPartitionNamesBeforeOrEquals(HoodieTimeline timeline, Str
     }
 
     public List<String> getPartitionNamesInRange(HoodieTimeline timeline, String startTimestamp, String endTimestamp) {
-        return new ArrayList<>(HoodieInputFormatUtils.getWritePartitionPaths(
+        return new ArrayList<>(HoodieTableMetadataUtil.getWritePartitionPaths(
                 timeline.findInstantsInRange(startTimestamp, endTimestamp).getInstants().stream().map(instant -> {
                     try {
                         return TimelineUtils.getCommitMetadata(instant, timeline);
@@ -101,8 +101,10 @@ public static List<String> parsePartitionValues(List<String> partitionColumns, S
             } else {
                 // If the partition column size is not equal to the partition fragments size
                 // and the partition column size > 1, we do not know how to map the partition
-                // fragments to the partition columns and therefore return an empty tuple. We don't
-                // fail outright so that in some cases we can fallback to reading the table as non-partitioned
+                // fragments to the partition columns and therefore return an empty tuple. We
+                // don't
+                // fail outright so that in some cases we can fallback to reading the table as
+                // non-partitioned
                 // one
                 throw new RuntimeException("Failed to parse partition values of path: " + partitionPath);
             }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
index a8f2a362bfde8d..28805aae63c1e3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
@@ -37,6 +37,7 @@
 import org.apache.doris.planner.ListPartitionPrunerV2;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.thrift.TExplainLevel;
@@ -48,8 +49,6 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.avro.Schema;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
 import org.apache.hudi.avro.HoodieAvroUtils;
 import org.apache.hudi.common.fs.FSUtils;
 import org.apache.hudi.common.model.BaseFile;
@@ -62,6 +61,8 @@
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
 import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
 import org.apache.hudi.common.util.Option;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -87,7 +88,7 @@ public class HudiScanNode extends HiveScanNode {
 
     private static final Logger LOG = LogManager.getLogger(HudiScanNode.class);
 
-    private boolean isCowOrRoTable;
+    private boolean isCowTable;
 
     private final AtomicLong noLogsSplitNum = new AtomicLong(0);
 
@@ -113,19 +114,23 @@ public class HudiScanNode extends HiveScanNode {
     private boolean incrementalRead = false;
     private TableScanParams scanParams;
     private IncrementalRelation incrementalRelation;
+    private SessionVariable sessionVariable;
 
     /**
      * External file scan node for Query Hudi table
-     * needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column priv
+     * needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column
+     * priv
      * eg: s3 tvf
-     * These scan nodes do not have corresponding catalog/database/table info, so no need to do priv check
+     * These scan nodes do not have corresponding catalog/database/table info, so no
+     * need to do priv check
      */
     public HudiScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv,
-            Optional<TableScanParams> scanParams, Optional<IncrementalRelation> incrementalRelation) {
+            Optional<TableScanParams> scanParams, Optional<IncrementalRelation> incrementalRelation,
+            SessionVariable sessionVariable) {
         super(id, desc, "HUDI_SCAN_NODE", StatisticalType.HUDI_SCAN_NODE, needCheckColumnPriv);
-        isCowOrRoTable = hmsTable.isHoodieCowTable();
+        isCowTable = hmsTable.isHoodieCowTable();
         if (LOG.isDebugEnabled()) {
-            if (isCowOrRoTable) {
+            if (isCowTable) {
                 LOG.debug("Hudi table {} can read as cow/read optimize table", hmsTable.getFullQualifiers());
             } else {
                 LOG.debug("Hudi table {} is a mor table, and will use JNI to read data in BE",
@@ -136,11 +141,12 @@ public HudiScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumn
         this.scanParams = scanParams.orElse(null);
         this.incrementalRelation = incrementalRelation.orElse(null);
         this.incrementalRead = (this.scanParams != null && this.scanParams.incrementalRead());
+        this.sessionVariable = sessionVariable;
     }
 
     @Override
     public TFileFormatType getFileFormatType() throws UserException {
-        if (isCowOrRoTable) {
+        if (canUseNativeReader()) {
             return super.getFileFormatType();
         } else {
             // Use jni to read hudi table in BE
@@ -185,13 +191,13 @@ protected void doInitialize() throws UserException {
             throw new UserException("Not support function '" + scanParams.getParamType() + "' in hudi table");
         }
         if (incrementalRead) {
-            if (isCowOrRoTable) {
+            if (isCowTable) {
                 try {
                     Map<String, String> serd = hmsTable.getRemoteTable().getSd().getSerdeInfo().getParameters();
                     if ("true".equals(serd.get("hoodie.query.as.ro.table"))
                             && hmsTable.getRemoteTable().getTableName().endsWith("_ro")) {
                         // Incremental read RO table as RT table, I don't know why?
-                        isCowOrRoTable = false;
+                        isCowTable = false;
                         LOG.warn("Execute incremental read on RO table: {}", hmsTable.getFullQualifiers());
                     }
                 } catch (Exception e) {
@@ -236,7 +242,15 @@ protected Map<String, String> getLocationProperties() throws UserException {
     @Override
     protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
         if (split instanceof HudiSplit) {
-            setHudiParams(rangeDesc, (HudiSplit) split);
+            HudiSplit hudiSplit = (HudiSplit) split;
+            if (rangeDesc.getFormatType() == TFileFormatType.FORMAT_JNI
+                    && !sessionVariable.isForceJniScanner()
+                    && hudiSplit.getHudiDeltaLogs().isEmpty()) {
+                // no logs, is read optimize table, fallback to use native reader
+                // TODO: support read orc hudi table in native reader
+                rangeDesc.setFormatType(TFileFormatType.FORMAT_PARQUET);
+            }
+            setHudiParams(rangeDesc, hudiSplit);
         }
     }
 
@@ -255,10 +269,15 @@ private void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) {
         fileDesc.setColumnTypes(hudiSplit.getHudiColumnTypes());
         // TODO(gaoxin): support complex types
         // fileDesc.setNestedFields(hudiSplit.getNestedFields());
+        fileDesc.setHudiJniScanner(hudiSplit.getHudiJniScanner());
         tableFormatFileDesc.setHudiParams(fileDesc);
         rangeDesc.setTableFormatParams(tableFormatFileDesc);
     }
 
+    private boolean canUseNativeReader() {
+        return !sessionVariable.isForceJniScanner() && isCowTable;
+    }
+
     private List<HivePartition> getPrunedPartitions(
             HoodieTableMetaClient metaClient, Option<String> snapshotTimestamp) throws AnalysisException {
         List<Type> partitionColumnTypes = hmsTable.getPartitionColumnTypes();
@@ -304,7 +323,8 @@ private List<HivePartition> getPrunedPartitions(
                 }
             }
         }
-        // unpartitioned table, create a dummy partition to save location and inputformat,
+        // unpartitioned table, create a dummy partition to save location and
+        // inputformat,
         // so that we can unify the interface.
         HivePartition dummyPartition = new HivePartition(hmsTable.getDbName(), hmsTable.getName(), true,
                 hmsTable.getRemoteTable().getSd().getInputFormat(),
@@ -315,7 +335,7 @@ private List<HivePartition> getPrunedPartitions(
     }
 
     private List<Split> getIncrementalSplits() {
-        if (isCowOrRoTable) {
+        if (canUseNativeReader()) {
             List<Split> splits = incrementalRelation.collectSplits();
             noLogsSplitNum.addAndGet(splits.size());
             return splits;
@@ -336,15 +356,15 @@ private void getPartitionSplits(HivePartition partition, List<Split> splits) thr
             globPath = hudiClient.getBasePathV2().toString() + "/*";
         } else {
             partitionName = FSUtils.getRelativePartitionPath(hudiClient.getBasePathV2(),
-                    new Path(partition.getPath()));
+                    new StoragePath(partition.getPath()));
             globPath = String.format("%s/%s/*", hudiClient.getBasePathV2().toString(), partitionName);
         }
-        List<FileStatus> statuses = FSUtils.getGlobStatusExcludingMetaFolder(
-                hudiClient.getRawFs(), new Path(globPath));
+        List<StoragePathInfo> statuses = FSUtils.getGlobStatusExcludingMetaFolder(
+                hudiClient.getRawHoodieStorage(), new StoragePath(globPath));
         HoodieTableFileSystemView fileSystemView = new HoodieTableFileSystemView(hudiClient,
-                timeline, statuses.toArray(new FileStatus[0]));
+                timeline, statuses);
 
-        if (isCowOrRoTable) {
+        if (canUseNativeReader()) {
             fileSystemView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant).forEach(baseFile -> {
                 noLogsSplitNum.incrementAndGet();
                 String filePath = baseFile.getPath();
@@ -473,7 +493,7 @@ private HudiSplit generateHudiSplit(FileSlice fileSlice, List<String> partitionV
         fileSlice.getPartitionPath();
 
         List<String> logs = fileSlice.getLogFiles().map(HoodieLogFile::getPath)
-                .map(Path::toString)
+                .map(StoragePath::toString)
                 .collect(Collectors.toList());
         if (logs.isEmpty()) {
             noLogsSplitNum.incrementAndGet();
@@ -492,6 +512,7 @@ private HudiSplit generateHudiSplit(FileSlice fileSlice, List<String> partitionV
         split.setHudiColumnNames(columnNames);
         split.setHudiColumnTypes(columnTypes);
         split.setInstantTime(queryInstant);
+        split.setHudiJniScanner(sessionVariable.getHudiJniScanner());
         return split;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiSplit.java
index c72f7621feaa55..2270d2017937da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiSplit.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiSplit.java
@@ -40,6 +40,5 @@ public HudiSplit(LocationPath file, long start, long length, long fileLength, St
     private List<String> hudiColumnNames;
     private List<String> hudiColumnTypes;
     private List<String> nestedFields;
+    private String hudiJniScanner;
 }
-
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/MORIncrementalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/MORIncrementalRelation.java
index c06fcc2a578d43..7df013599229fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/MORIncrementalRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/MORIncrementalRelation.java
@@ -20,9 +20,7 @@
 import org.apache.doris.spi.Split;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.GlobPattern;
-import org.apache.hadoop.fs.Path;
 import org.apache.hudi.common.model.BaseFile;
 import org.apache.hudi.common.model.FileSlice;
 import org.apache.hudi.common.model.HoodieCommitMetadata;
@@ -34,6 +32,8 @@
 import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
 import org.apache.hudi.exception.HoodieException;
 import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils;
+import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.storage.StoragePathInfo;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -54,7 +54,7 @@ public class MORIncrementalRelation implements IncrementalRelation {
     private final boolean endInstantArchived;
     private final List<HoodieInstant> includedCommits;
     private final List<HoodieCommitMetadata> commitsMetadata;
-    private final FileStatus[] affectedFilesInCommits;
+    private final List<StoragePathInfo> affectedFilesInCommits;
     private final boolean fullTableScan;
     private final String globPattern;
     private final boolean includeStartTime;
@@ -96,7 +96,7 @@ public MORIncrementalRelation(Map<String, String> optParams, Configuration confi
         includedCommits = getIncludedCommits();
         commitsMetadata = getCommitsMetadata();
         affectedFilesInCommits = HoodieInputFormatUtils.listAffectedFilesForCommits(configuration,
-                new Path(metaClient.getBasePath()), commitsMetadata);
+                metaClient.getBasePathV2(), commitsMetadata);
         fullTableScan = shouldFullTableScan();
         if (hollowCommitHandling == HollowCommitHandling.USE_TRANSITION_TIME && fullTableScan) {
             throw new HoodieException("Cannot use stateTransitionTime while enables full table scan");
@@ -152,8 +152,8 @@ private boolean shouldFullTableScan() throws IOException {
         if (should) {
             return true;
         }
-        for (FileStatus fileStatus : affectedFilesInCommits) {
-            if (!metaClient.getFs().exists(fileStatus.getPath())) {
+        for (StoragePathInfo fileStatus : affectedFilesInCommits) {
+            if (!metaClient.getRawHoodieStorage().exists(fileStatus.getPath())) {
                 return true;
             }
         }
@@ -199,7 +199,7 @@ public List<FileSlice> collectFileSlices() throws HoodieException {
         String latestCommit = includedCommits.get(includedCommits.size() - 1).getTimestamp();
         HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient, scanTimeline,
                 affectedFilesInCommits);
-        Stream<FileSlice> fileSlices = HoodieInputFormatUtils.getWritePartitionPaths(commitsMetadata)
+        Stream<FileSlice> fileSlices = HoodieTableMetadataUtil.getWritePartitionPaths(commitsMetadata)
                 .stream().flatMap(relativePartitionPath ->
                         fsView.getLatestMergedFileSlicesBeforeOrOn(relativePartitionPath, latestCommit));
         if ("".equals(globPattern)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index 59f51c8425c7f2..bf917804fb7b6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -105,9 +105,9 @@ public String toString() {
     private String serializedTable;
 
     public PaimonScanNode(PlanNodeId id,
-                          TupleDescriptor desc,
-                          boolean needCheckColumnPriv,
-                          SessionVariable sessionVariable) {
+            TupleDescriptor desc,
+            boolean needCheckColumnPriv,
+            SessionVariable sessionVariable) {
         super(id, desc, "PAIMON_SCAN_NODE", StatisticalType.PAIMON_SCAN_NODE, needCheckColumnPriv);
         this.sessionVariable = sessionVariable;
     }
@@ -127,8 +127,7 @@ protected void convertPredicate() {
         predicates = paimonPredicateConverter.convertToPaimonExpr(conjuncts);
     }
 
-    private static final Base64.Encoder BASE64_ENCODER =
-            java.util.Base64.getUrlEncoder().withoutPadding();
+    private static final Base64.Encoder BASE64_ENCODER = java.util.Base64.getUrlEncoder().withoutPadding();
 
     public static <T> String encodeObjectToString(T t) {
         try {
@@ -156,11 +155,24 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit)
         tableFormatFileDesc.setTableFormatType(paimonSplit.getTableFormatType().value());
         TPaimonFileDesc fileDesc = new TPaimonFileDesc();
         org.apache.paimon.table.source.Split split = paimonSplit.getSplit();
+
+        String fileFormat = getFileFormat(paimonSplit.getPathString());
         if (split != null) {
             // use jni reader
+            rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI);
             fileDesc.setPaimonSplit(encodeObjectToString(split));
+        } else {
+            // use native reader
+            if (fileFormat.equals("orc")) {
+                rangeDesc.setFormatType(TFileFormatType.FORMAT_ORC);
+            } else if (fileFormat.equals("parquet")) {
+                rangeDesc.setFormatType(TFileFormatType.FORMAT_PARQUET);
+            } else {
+                throw new RuntimeException("Unsupported file format: " + fileFormat);
+            }
         }
-        fileDesc.setFileFormat(getFileFormat(paimonSplit.getPathString()));
+
+        fileDesc.setFileFormat(fileFormat);
         fileDesc.setPaimonPredicate(encodeObjectToString(predicates));
         fileDesc.setPaimonColumnNames(source.getDesc().getSlots().stream().map(slot -> slot.getColumn().getName())
                 .collect(Collectors.joining(",")));
@@ -172,7 +184,8 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit)
         fileDesc.setTblId(source.getTargetTable().getId());
         fileDesc.setLastUpdateTime(source.getTargetTable().getUpdateTime());
         fileDesc.setPaimonTable(encodeObjectToString(source.getPaimonTable()));
-        // The hadoop conf should be same with PaimonExternalCatalog.createCatalog()#getConfiguration()
+        // The hadoop conf should be same with
+        // PaimonExternalCatalog.createCatalog()#getConfiguration()
         fileDesc.setHadoopConf(source.getCatalog().getCatalogProperty().getHadoopProperties());
         Optional<DeletionFile> optDeletionFile = paimonSplit.getDeletionFile();
         if (optDeletionFile.isPresent()) {
@@ -190,8 +203,8 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit)
     @Override
     public List<Split> getSplits() throws UserException {
         boolean forceJniScanner = sessionVariable.isForceJniScanner();
-        SessionVariable.IgnoreSplitType ignoreSplitType =
-                SessionVariable.IgnoreSplitType.valueOf(sessionVariable.getIgnoreSplitType());
+        SessionVariable.IgnoreSplitType ignoreSplitType = SessionVariable.IgnoreSplitType
+                .valueOf(sessionVariable.getIgnoreSplitType());
         List<Split> splits = new ArrayList<>();
         int[] projected = desc.getSlots().stream().mapToInt(
                 slot -> (source.getPaimonTable().rowType().getFieldNames().indexOf(slot.getColumn().getName())))
@@ -288,7 +301,8 @@ public List<Split> getSplits() throws UserException {
         }
         this.selectedPartitionNum = selectedPartitionValues.size();
         // TODO: get total partition number
-        // We should set fileSplitSize at the end because fileSplitSize may be modified in splitFile.
+        // We should set fileSplitSize at the end because fileSplitSize may be modified
+        // in splitFile.
         splits.forEach(s -> s.setTargetSplitSize(fileSplitSize));
         return splits;
     }
@@ -318,8 +332,9 @@ public TFileFormatType getFileFormatType() throws DdlException, MetaNotFoundExce
 
     @Override
     public List<String> getPathPartitionKeys() throws DdlException, MetaNotFoundException {
-        //                return new ArrayList<>(source.getPaimonTable().partitionKeys());
-        //Paymon is not aware of partitions and bypasses some existing logic by returning an empty list
+        // return new ArrayList<>(source.getPaimonTable().partitionKeys());
+        // Paymon is not aware of partitions and bypasses some existing logic by
+        // returning an empty list
         return new ArrayList<>();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index c474e6bd56e1d7..468a58bd6f6815 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -649,7 +649,7 @@ public PlanFragment visitPhysicalHudiScan(PhysicalHudiScan fileScan, PlanTransla
                         + " for Hudi table");
         PhysicalHudiScan hudiScan = (PhysicalHudiScan) fileScan;
         ScanNode scanNode = new HudiScanNode(context.nextPlanNodeId(), tupleDescriptor, false,
-                hudiScan.getScanParams(), hudiScan.getIncrementalRelation());
+                hudiScan.getScanParams(), hudiScan.getIncrementalRelation(), ConnectContext.get().getSessionVariable());
         if (fileScan.getTableSnapshot().isPresent()) {
             ((FileQueryScanNode) scanNode).setQueryTableSnapshot(fileScan.getTableSnapshot().get());
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index 4091640066c1d7..ae5d562bd5bd30 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -1968,7 +1968,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s
                                     + "please set enable_nereids_planner = true to enable new optimizer");
                         }
                         scanNode = new HudiScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true,
-                                Optional.empty(), Optional.empty());
+                                Optional.empty(), Optional.empty(), ConnectContext.get().getSessionVariable());
                         break;
                     case ICEBERG:
                         scanNode = new IcebergScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 3c180be9d42802..374100598c8b75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -635,6 +635,8 @@ public class SessionVariable implements Serializable, Writable {
 
     public static final String FORCE_JNI_SCANNER = "force_jni_scanner";
 
+    public static final String HUDI_JNI_SCANNER = "hudi_jni_scanner";
+
     public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE = "enable_count_push_down_for_external_table";
 
     public static final String SHOW_ALL_FE_CONNECTION = "show_all_fe_connection";
@@ -2077,6 +2079,10 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
             description = {"强制使用jni方式读取外表", "Force the use of jni mode to read external table"})
     private boolean forceJniScanner = false;
 
+    @VariableMgr.VarAttr(name = HUDI_JNI_SCANNER, description = { "使用那种hudi jni scanner, 'hadoop' 或 'spark'",
+            "Which hudi jni scanner to use, 'hadoop' or 'spark'" })
+    private String hudiJniScanner = "hadoop";
+
     @VariableMgr.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE,
             description = {"对外表启用 count(*) 下推优化", "enable count(*) pushdown optimization for external table"})
     private boolean enableCountPushDownForExternalTable = true;
@@ -4510,6 +4516,10 @@ public boolean isForceJniScanner() {
         return forceJniScanner;
     }
 
+    public String getHudiJniScanner() {
+        return hudiJniScanner;
+    }
+
     public String getIgnoreSplitType() {
         return ignoreSplitType;
     }
@@ -4530,6 +4540,10 @@ public void setForceJniScanner(boolean force) {
         forceJniScanner = force;
     }
 
+    public void setHudiJniScanner(String hudiJniScanner) {
+        this.hudiJniScanner = hudiJniScanner;
+    }
+
     public boolean isEnableCountPushDownForExternalTable() {
         return enableCountPushDownForExternalTable;
     }
diff --git a/fe/pom.xml b/fe/pom.xml
index d78cfd50b819b4..0d2e3f70aa6e6a 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -317,7 +317,7 @@ under the License.
         <avro.version>1.11.4</avro.version>
         <arrow.version>17.0.0</arrow.version>
         <!-- hudi -->
-        <hudi.version>0.14.1</hudi.version>
+        <hudi.version>0.15.0</hudi.version>
         <presto.hadoop.version>2.7.4-11</presto.hadoop.version>
         <presto.hive.version>3.0.0-8</presto.hive.version>
         <!-- lakesoul -->
@@ -371,7 +371,7 @@ under the License.
         <trino.version>435</trino.version>
         <jakarta.annotation-api.version>2.1.1</jakarta.annotation-api.version>
         <asm.version>9.4</asm.version>
-        <airlift.version>202</airlift.version>
+        <airlift.concurrent.version>202</airlift.concurrent.version>
         <azure.sdk.version>1.2.27</azure.sdk.version>
         <azure.sdk.batch.version>12.22.0</azure.sdk.batch.version>
         <semver4j.version>5.3.0</semver4j.version>
@@ -1649,7 +1649,7 @@ under the License.
             <dependency>
                 <groupId>io.airlift</groupId>
                 <artifactId>concurrent</artifactId>
-                <version>${airlift.version}</version>
+                <version>${airlift.concurrent.version}</version>
             </dependency>
             <dependency>
                 <groupId>com.azure</groupId>
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 961433ab28d7e4..77d9c0ef672feb 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -353,7 +353,6 @@ struct TMaxComputeFileDesc {
     1: optional string partition_spec // deprecated 
     2: optional string session_id 
     3: optional string table_batch_read_session
-
 }
 
 struct THudiFileDesc {
@@ -367,6 +366,7 @@ struct THudiFileDesc {
     8: optional list<string> column_names;
     9: optional list<string> column_types;
     10: optional list<string> nested_fields;
+    11: optional string hudi_jni_scanner;
 }
 
 struct TLakeSoulFileDesc {
@@ -406,6 +406,7 @@ enum TTextSerdeType {
 struct TFileScanRangeParams {
     // deprecated, move to TFileScanRange
     1: optional Types.TFileType file_type;
+    // deprecated, move to TFileScanRange
     2: optional TFileFormatType format_type;
     // deprecated, move to TFileScanRange
     3: optional TFileCompressType compress_type;
@@ -480,6 +481,7 @@ struct TFileRangeDesc {
     // for hive table, different files may have different fs,
     // so fs_name should be with TFileRangeDesc
     12: optional string fs_name
+    13: optional TFileFormatType format_type;
 }
 
 struct TSplitSource {
diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out b/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
index b1bdad85013bfc..50644f34961942 100644
--- a/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
+++ b/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
@@ -347,3 +347,177 @@
 -- !incremental_9_10 --
 1000
 
+-- !incremental_1_end --
+9000
+
+-- !incremental_earliest_1 --
+1000
+
+-- !incremental_2_end --
+8000
+
+-- !incremental_earliest_2 --
+2000
+
+-- !incremental_1_2 --
+1000
+
+-- !incremental_3_end --
+7000
+
+-- !incremental_earliest_3 --
+3000
+
+-- !incremental_2_3 --
+1000
+
+-- !incremental_4_end --
+6000
+
+-- !incremental_earliest_4 --
+4000
+
+-- !incremental_3_4 --
+1000
+
+-- !incremental_5_end --
+5000
+
+-- !incremental_earliest_5 --
+5000
+
+-- !incremental_4_5 --
+1000
+
+-- !incremental_6_end --
+4000
+
+-- !incremental_earliest_6 --
+6000
+
+-- !incremental_5_6 --
+1000
+
+-- !incremental_7_end --
+3000
+
+-- !incremental_earliest_7 --
+7000
+
+-- !incremental_6_7 --
+1000
+
+-- !incremental_8_end --
+2000
+
+-- !incremental_earliest_8 --
+8000
+
+-- !incremental_7_8 --
+1000
+
+-- !incremental_9_end --
+1000
+
+-- !incremental_earliest_9 --
+9000
+
+-- !incremental_8_9 --
+1000
+
+-- !incremental_10_end --
+0
+
+-- !incremental_earliest_10 --
+10000
+
+-- !incremental_9_10 --
+1000
+
+-- !incremental_1_end --
+9000
+
+-- !incremental_earliest_1 --
+1000
+
+-- !incremental_2_end --
+8000
+
+-- !incremental_earliest_2 --
+2000
+
+-- !incremental_1_2 --
+1000
+
+-- !incremental_3_end --
+7000
+
+-- !incremental_earliest_3 --
+3000
+
+-- !incremental_2_3 --
+1000
+
+-- !incremental_4_end --
+6000
+
+-- !incremental_earliest_4 --
+4000
+
+-- !incremental_3_4 --
+1000
+
+-- !incremental_5_end --
+5000
+
+-- !incremental_earliest_5 --
+5000
+
+-- !incremental_4_5 --
+1000
+
+-- !incremental_6_end --
+4000
+
+-- !incremental_earliest_6 --
+6000
+
+-- !incremental_5_6 --
+1000
+
+-- !incremental_7_end --
+3000
+
+-- !incremental_earliest_7 --
+7000
+
+-- !incremental_6_7 --
+1000
+
+-- !incremental_8_end --
+2000
+
+-- !incremental_earliest_8 --
+8000
+
+-- !incremental_7_8 --
+1000
+
+-- !incremental_9_end --
+1000
+
+-- !incremental_earliest_9 --
+9000
+
+-- !incremental_8_9 --
+1000
+
+-- !incremental_10_end --
+0
+
+-- !incremental_earliest_10 --
+10000
+
+-- !incremental_9_10 --
+1000
+
diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
index 12dd0cf086d3f0..da7273d4c14ef9 100644
--- a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
+++ b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
@@ -31,3 +31,35 @@
 20241118012149007	20241118012149007_0_4	5		185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet	5	Eva	{"age":31.5, "address":"Chengdu"}
 20241118012149007	20241118012149007_0_5	6		185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet	6	Frank	{"age":29.2, "address":"Wuhan"}
 
+-- !adding_simple_columns_table --
+20241118012126237	20241118012126237_0_1	1		5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet	1	Alice	\N
+20241118012126237	20241118012126237_0_0	2		5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet	2	Bob	\N
+20241118012126237	20241118012126237_0_2	3		5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet	3	Cathy	\N
+20241118012132306	20241118012132306_0_3	4		5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet	4	David	25
+20241118012132306	20241118012132306_0_4	5		5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet	5	Eva	30
+20241118012132306	20241118012132306_0_5	6		5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet	6	Frank	28
+
+-- !altering_simple_columns_table --
+20241118012136512	20241118012136512_0_0	1		203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet	1	Alice	25.0
+20241118012136512	20241118012136512_0_2	2		203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet	2	Bob	30.0
+20241118012136512	20241118012136512_0_1	3		203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet	3	Cathy	28.0
+20241118012138287	20241118012138287_0_3	4		203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet	4	David	26.0
+20241118012138287	20241118012138287_0_4	5		203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet	5	Eva	31.5
+20241118012138287	20241118012138287_0_5	6		203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet	6	Frank	29.2
+
+-- !adding_complex_columns_table --
+20241118012144831	20241118012144831_0_1	1		3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet	1	Alice	{"age":25, "address":"Guangzhou", "email":null}
+20241118012144831	20241118012144831_0_0	2		3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet	2	Bob	{"age":30, "address":"Shanghai", "email":null}
+20241118012144831	20241118012144831_0_2	3		3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet	3	Cathy	{"age":28, "address":"Beijing", "email":null}
+20241118012146150	20241118012146150_0_3	4		3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet	4	David	{"age":25, "address":"Shenzhen", "email":"david@example.com"}
+20241118012146150	20241118012146150_0_4	5		3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet	5	Eva	{"age":30, "address":"Chengdu", "email":"eva@example.com"}
+20241118012146150	20241118012146150_0_5	6		3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet	6	Frank	{"age":28, "address":"Wuhan", "email":"frank@example.com"}
+
+-- !altering_complex_columns_table --
+20241118012147879	20241118012147879_0_0	1		185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet	1	Alice	{"age":25, "address":"Guangzhou"}
+20241118012147879	20241118012147879_0_2	2		185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet	2	Bob	{"age":30, "address":"Shanghai"}
+20241118012147879	20241118012147879_0_1	3		185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet	3	Cathy	{"age":28, "address":"Beijing"}
+20241118012149007	20241118012149007_0_3	4		185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet	4	David	{"age":26, "address":"Shenzhen"}
+20241118012149007	20241118012149007_0_4	5		185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet	5	Eva	{"age":31.5, "address":"Chengdu"}
+20241118012149007	20241118012149007_0_5	6		185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet	6	Frank	{"age":29.2, "address":"Wuhan"}
+
diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out b/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out
index efad67ffbfa8c407fa42d0dd28f569f492bd1d12..1e151c2a86fa200668caaaf943a01f9e4c11f5a3 100644
GIT binary patch
delta 40026
zcmeHQ36vDoxjwb@46PA`L5A62W>{6KseKtlLQoS?mO<PxV^#I7nZ}v!p{EA~jUzFM
zFL8O2Hr$g9LY|2xLBVhy?UE?QB*wTR>NCbAzHHAZZs#RAFHz6?ZdDI9karkHeG|0K
zsj9AfYrA#pzyJO3*Wdr&SHFGi_wGLa`c+q5doy3U_GaPvqvor+C7VfAR&_;Dbj4tM
z$mx(fOxx0vt}Hv6@1-S6@m)!`EJt!Q&5>-^Ry4y=JjImh`m2S714B78scWX~I7wX&
zpC~Dcr0C0JxlOa$WG7{7nxU%6LcX6Q^^`3q+vXat-SgvUnRDifXeU+9+VsU}XTd7=
zI}n~XEph6J#uIlTV>;HcSmkDH+OW=Q#|F#MSfi)Qrlgw53tGt~z1?mm+j?R0(pHaj
zWwPnkw${~MnQqcLv$?ge&>sZI=<Cmu*3aM+KT}wP^r^YN0?GI0@`dm$de%0vMYL^Q
zp>DuUoGj^jO1BhSw{ef|T(&b)=ueZ@Hc2t9lxfJQ2zu{##X8H;RYO;N$#nEU!Uymr
z+wh4Lc!BNPfvl^F%^r{`ke-gL+k;!Q_U1_+<zD4$Cc8bjGdo=9B5T-huD{TFA%1)F
z86T$_icOnu;u>l40YRaQ|0-73o}KCLc00pYE5FHtT$c0|*VT>}Eko02OO~HA#gPqJ
zO$WLpn=*>!S+?ZaS|C}n;n;!i+C;YL$`xGAG}0SZfbSMaXD*MLxu7+l=_8e;fIJnC
z>1?Er-7Kg_lI}jTrg<p}O6Zn$VLI<v#l@4R&_~w_4O3)awM<jiB+am7N%v7Dp61As
zFDoihWrOIZN9S!9kDKiJ`CMNgd*wXoAw3?+S3VJ<nZlbK$<=WRZJ8)2Q^J>QF28(u
zUHR?LA$^7BrNkq?Pn9#p1_7;ruKSJHO1DiD8XYI?yT0p6t||MdhcrPw=&qzGULeb|
zY7^Jl|MU2{tWWZE-VDB`xaqtEEo>K$Nhq?M(&@e1_(r<EMVLjib$nGLS=)SSF2B#m
zSOS4+UXbg`(l^fF$McR}T)C=+-a~|m#7nEPWqA?_R8P_k+m>9*RV2;vG?$=hs%rRE
zD<!3exS`ylnkqj~AAyV`FUaM4Dyz*@6}s!s;zWAoiQFuD(rH4nc7C3ai`lv5>RLor
zH!pPa^tQXX34-n@#d-HnrdRw~sG-a6;~S=%s*y~~p6xqoTGBLvItwgIa)_-+rlvZs
zqXvrS;BsNnzz|M!lZx)hwvsd|RED-pRhi0A<dkLFnxPG<3`0qxb+0}zYUw9NKj*Yq
z(M~sh>@nC>FU6)u9*DA;TA6}%A=c%|Q?O+xVwum>MxT0oOO))GISM<+VI^26HF;6?
z#AXRQmtjqaa+HOwdwH~5)-;-gdbyxAn+w~Dn+@Bl+ykvs7}Pw4-0Cb-Gh`LXR_dAa
zUBAoCuL=6|1={dO;doKAY?HoN6edoT>=Xn-F%^8C3KgShDO1)By7?)w{?ncn1g>X6
z)v2~2>FKlrQMP@_cMZ=`Y&+2W0|Yqe&gJsqH?L^a{n^Yuss-IcN~J4LOEl4E{=lW0
zmXbB3uX#xhuT&u}pU7v@ouoi-yP2yKZA-PNJDP7TZhO0t5X*xApewO8Re};sOLpLx
zl5P2cVmY!#oKQw9#o6tPM6se+g|2*VHGAOdT)sP9Da@9tq!hY&fmqLJbfv<}8-6N^
zEpGFfT{-B_iZY(!LIa#xAiZ5~HbaW;OSK}NuT$j|p_#S3rYRmFrVLFe`DssqTCzRK
zF?AEV+$ElZhH3b6QND5dafN)Qm)$u}20I3~y9XUbWw}&SNtsMH)fOMSsES^BGM`E=
zaQ&72q&t(PW0K-H@l$ShH(mL<SS!eyN{`X`8MIg8#uGWM2D;_jP>%$4W}zeY97B-;
z=wvx?bXWB}_V~&=ThWilBd@akqt}SK3zXOGt}L8k8uW^rggUz9BB6@5zAmQd>ko>_
z@$SX@R9UE{=I7|^lb{7;T;tYrlj+XW_!>HAq0lhhvFs$8Eg@;og057Fq`PTD^7K&r
zSix0dyDGhHE7ZUcPIQu*DWh!rSPoDFN}H~<8CuHFWk<IM)qrg!3uu0<`L<#`eR(qR
z>8Lrf27218+4q&AWY~lw%OOBcJBQ7RQ5kGmo36#$h;_A<#dalDEX$WCuw90=>tbw}
znGr3_*jT>&z76<!{#vF>K8Cd(D>F|fwX$2E%oNl8o3U{xO~v+?SFXYKtkYQrQxuw{
zsF6w1m+8!Af{gD*jVaUPc4o5ZQ(=De`2>4ytv!SkD)j_Y%nL{m<jESZKi^5Sv$^9$
zm=<!VD_S%&rJK6pI0j^)Qd#I)N;T2P(v}mrx&hyW@sTzOiV--HZkkMYqmk-{jz-}7
zY4^ZZ3F|4;XO$AxNBlx2mkosxg%3NXJn?h+eVV9(CK_t5-u^taTpyv&trJGm>*fne
z(-~xShUt@hp|Y*c%NDwF*)?>`Lt?Fn4n?EwKM-|#!F_z=blp^wx&vLO64Yd%!GIu8
ze6H^Ll4p3DE(d5oS|oHsIMGSMA~sbksjKgrtJ*TfY%`2D#Z1YL&5oAMRm)_3PqdUf
zAS6y?LV~5{*z*>Qj&`waVCL$qC^=zyR>R&jtiFv66U1Xp9or-OdnaCiUnYI~E%VN0
z;`pnpS6sFlzhZPWz4s*6`OGYLe`A@D_ny{1cXGKeQY2YRTCmdaIUr?gBe4oH-93=1
zAlKaueOh7Xo&$R^<M#CBvJk5^^twDhL$oYYp~_rtVvDAxOkK83Z9l0p%#>!UrUC0~
zwoo^ahGA)?J;~OfI(1zmlC2QJECS#2122tkXn(y09i4+zM22dG!uQ<J;4RBkfuMWb
z*Y-1I2$hQR3<_08nhyM(JCavqdfPL+T-@~8>f%^^oNzi>Rh0Dx-mvH!b7Al8vBhz8
zTS{n{ff6QN*EC(3q$M+uQDzS|mt#TK>VfWPw(Aoe-EcJW8N!)K6}`5pCsiyMVTB_f
zh!W%oi;R(vZL3N$aA6!{=wpR;48Gg3mMWjPcJFr`w|B25+FPE_E+1IVG>sZ%jDBfd
zS+|v~n+Z|=QYMEF|L|OFnQFtrG$a-+_2{?o(}A@!%GtYZKO4KF95>2fk~od*Ra1%s
zduDZ4u65=>b8CM#jc<S|FE?@Q!3-ezRT&cY&4nCnfbcUxfpxMDyA8Hb*ehX}#fDq3
zNHJ!p5WA&rHXRy0?5w^51OR8v&t<brvG^F}un&x$)$OJa{fSEomMl|uu`sDwL9Z`k
z9BCh*=rANuEtmpxxw<ogAPuWovn4|%CbLm>$uaPq(+Xq}-@?~alRcucCrvX84SAJ%
zuF&NcC}#`f1$67ocCIUKd%B9gxe_M+w+6WC39#=g&F9e8Smb7X`eFw+PQXY1N=u@d
z_TIoX*}50FuIdo9PZ-a-W2Pn7&@9Qc4NJ3h#|=z<|1yzGp^NVN7qMo1Zne9way6my
zO}cBHP|rEEeYv2txT`PDWFws#zDG2WJrhH*u4Y&uIj^|#$2EejTgCQC&BfRLt*MQe
zK|qM=N<_oJLl10JlZHm1LYKM>8GtRhe~~ghy{=!Nn>LE|O~eg9R9|5Y2GbSeQs{=1
zR&05wp8j$IryQ}+_xquD`a8QQcez-Tn8(I*bp1`-c+mmOfGV0yA8X<3PxQ33>g%o}
z1wJc}Y6U1yT9ce0z(7qk3?oqXFAwSI?ar+sgg%-V#!v3=cFU$M1~%cIm+7o5nxoUZ
z@8p{3;?1H$pZ`5qoj4m(+*jQ7AIA%-p;7Ml!sIW{tq(NEFg!((Q50t0XtE+XDlA!5
zg9xS5mS<T9*snb>qr#f6&}Y4FZ&Z!~kcW#aOJeFN81c-0HR;MJocys*y7{$k^ZZQS
z?<REfq{KM-x%xzn2;ndXrKq~J^C{t^sq|-~`Kg>v+qJ|rSiiY@#A%#Lch&J>stG^J
zRXg7jN;}^Y|7Q$6Z)&3M1SsncsGy<1ICS%QTn(*SD=v#spBVLtQJ)Z$`$(cbW9gos
zi*?5<2H@hM;684sI-C#=@f8L%rva_mu+sHFff{!VSF%;h3lzij4AYk64n?;C5g35E
zY|EizmWho62gjE_NZ6<hHIEo<)H5rFjbhm7!v!0SrK!J(bteG#>rm98AV)xu1s<a#
zGil2{uuR{POx1x&==v}TUBv)OZn&yxIgUf{FGh!At7SFfU>w+z5*uRN2_w1~cY05_
zQ`q}yirt|P1$lzdo}4(=0{r4CdfHHQz#v9|zE8s+Z8nf({7Yc#x~t1+T~{Hrnu2@=
z+sh2k4}~_ytzz8jK-}tBy5~z`-EpebArB3>g7zG5I!PG=d>12XNt5v}tFD;r|5nw&
znAXz*TS==xyO<oR_+JKpe_)`gTlBK~p*)8IkHw%@40<uDL=1X`WQ-W}+RG4mzd^6Z
zk54>36wr$T^qNq-<*cUdkFM$v>Be6Eh~nrco35OYC{0LkPVy902EoOMK>9&KRG4!p
zgs397c+tQRh*KXR9qOI|2JVj>9jZXj-X=E3e5jZY)z<ovM8?Jx=QlJBl@3*Ge==r6
z#cZgU4fSDTLlv*v&@f|&WGJUN?--a32PI8?kX)$9Y#2E%)ai^^6=PEG0h7{LzcXWq
zR4Av&tq1?^pp>Z(k_iQo8&M|I1!l~I`Ugy?8jOn?XE^H6@T73WGqN<O<pU#2gK8ca
zSsGLaefUq@sF(*8^PplL)D2uUWjrWqU&+g*@w2$crVN1*6@o)o?Oeu{b}r+8JBF^h
zjITX(TGXq%xR@3dJHzdTI%8+JkD@c&Qxo|K!=y#&hGIH!meJCnGQeTRlRQ}g2P$o)
z;s0Z)9^4}%T9gTD6ets>N!<l})4)N!?mkdfR5WxPIaZXLik<0VXSxrSGu>GF={?-=
zSW(22EhX?74GPpK-A}_0in()JnkU1tfaq{xDYK%=TEwyG8#i(>D=PNJ+v_+rTyMN5
zC-Qa24}%p2$66h(H(*7<djf7^Hj!M{u;J$cf|Tm`L^I*M7P6v3p+yJ^GZNaE6&16h
z4rE1*qf4IRAc==ei*hwhLF5MlT9gIf9vfahwnt!yrG0olf-wb8z%nh$1T{*5FStc-
z>xK4=X;HB+VC)OHH&RXP3pk>_fS^Ss_|gkk^1CMw75}2wzbG`(@7>Jr*uI$CBG9cV
zp^>I;;Y+0@=X2w#H~c%lyOcXZY~$(9&*6xCJC3}1J3mDz{pJ|)VSyg;0wUb-JCVEj
zSn=yT^UUA9ePZLUtLQNg@{gBZyo8gg>7uKJ2CD4lOFPzamx|GyzVI}z>G=Zp^8~xc
z?$TUC{E4vPYk0;_ev2bl{EnYKs&wls{K;Zz<Ei4W6ZGf@@ub)Pf@AV1?#L>7(GR#I
z>D;^cCi>#r{EpJbclf)+Qu}%0(rT95NFO;uD3v~Q2X{{uUZRn9jzv<wmwR4}iZgM7
zP}*@nw<qy#4U}GZn7dPm4qq}^C~co~*-y(gGJb}zWBZS;cs9W*X!rJ&SKTK@Rn?{A
zwl7>QZWW>ezF`YHN(0x6e-*>0-g^=b56l(rs4C4pRd}p=`#axhuc9Bni*KNxU4}!h
z$cub*<wfTpy>hE~S#22JfbPg4{hRy6O<@H!(pjsJ?tMgTs|uag=_fve7r0y~R+rkJ
z6SpVAs>@uBQXl=hI};QUzZ&RU>v737pWr`P{sfI*L;ATl#dh{x8|f#%jb!uR#hc1C
z_KWW$ofWv31orV7_EvUF6?k(0M1jKwLogg##IVr)eOh8*q#}luecI{rZDM^)l#7XS
zA5o&*$OH}>R2hdBF%0AsEGt-OjS-0$R;I`u7sV=!BZkEh!{Ufx|3_%CgRtZV+y9}3
z3JaTmz#PCwCRA7%n_Lu!3Ol?)g$+vEp~VO*hX{kTjZBQNa)>ZFjuH0H5F>1O!NJP0
z!J-(4BM=;{92=~MMLqg|3l2u_{S#tq#Sy{ch+uI<FfNP;25ikeC2_l0nrm`x0t15@
z=`Ak;nc8Y59^@nZXvyz^_Kg06Fd&qUIw|2+g#glvPa_kT0^a)apM~9}=5O;~P7DD{
zL}W2R;35d|<u8d1bUi0N7D0$%s1N#;q!<E;Qw0Vew)6Crqs2W8L%dUD$l>P&fQHXh
z(Iwvy>u7Vc7$S!=t7-P+L_M85Lwr0!4ljHbhqp*L93h9dj6*uDi%&54@K%wbhRdqy
zz{ALO=iw~ihj}3?z|T%W8c5<Vsw(kAl%kG)<4ktw9sa+B==rP8#SMWZ&KVWqhtUms
zx{wP1@i~!|s*awN6?aDf;<lG?lZpMf2>`@91iEdF_)3HwqAKd?y357KBkV9tF8RE;
zD*_Oa+{3WLuZj#Y{9S?pg?~t}GVEb!;oDWwgU`QFd;+-Pk}6hbb#&toP>u*U3~Ot{
ztw;km?3%#P!Y@`ajBr{tkinf*6!o=-!GoKsfC}D`peX4c1_+*7u37O-6dHiwiK7?}
zc*7`${tec`kE#=R-%(sk6+L%qqW;jCk?&K$xItqoM*HGv0^(@`4zp<j4%2{fgF0te
za3RABMiMT}&IkjC$8cc`7asm_;b9snZcu^5c;iQHYJ$TtMBJdX#&~7rOc{sioCb$u
zY`7Tge0QR`gUo~wgPrm82l4a=<w-QEOQ1ZyQ2};3K;K>h=JuqK*v-JslE5I&Ej&ZQ
z>Zvvdxd1yCRx^xqTVlxY=H57R#Ya~$qU3-WVVlKkH#8O#N0lOqWby5zcQbTzDPLT?
zrJ;Co4fuP<hz|+HM{1v7Am`V31nYPLK<6#hj2L-Bb@95{O~v{XB3dM_ZY<{H(soJt
zD%gUCc&~utPcTgL%xc`Csrak;C5CABA%8<daq6c)dSuMVV($FL;;$A}@*<0iwlo$W
zIIF}^&6k8?@A;2M%*Zgm<du1mVd<Bw2x*beieX`@dm~aLi<!sXl_8q93v`RjH5J`U
z86$Fo2rTo~Dj=8}5)tt+dj9og+9Phdn*o`BOyIE%#mqNL49k3{3T19A&il>|Aeoa<
z{fze*Uihi4A?xwdS{BHpxbgmw^SCh)zHsd$A>+}hLU4-RKs4{ID!y}cL-D82mKdn{
z!$er*_TMtf<26+b*E}VvQt{0)%`tq#S$`|D9NF8`Z9Er}96=Th!OefB5P*Yr*s!3^
zQwK(9D%3KuaUs2Mn}|vO;we$%DN*APfCrlq^?eEfIH)m(1ysho#{K7xif2NNXF?tB
zOsFFe`){!I4+~{P0EZD}HOdR)2~dYQ0qRHu{Tr0EVPS-jwGrhsW@p9oo>tC#Iug<T
z-V??bMVlEpLSt)pOlXX!IE|+`9f>JU_YLj0q5|#vn8+}`U-B$=DYfSS*a6sg3(r6z
zdiTeX1lD&Ekf+&A^Z^;kh}gJxAvbhlV~G4w1POdOA~!C6imRuWoX73D3I5Te%C6DZ
ztm5nF_tV_tWp3lv^O4@u$vshal9s=Ui?(*-v5}KB3z$OZ^m4l*(&KBZxtAFrIAIi6
zjzNNP8@GoD;7<g4)D>JE<=5lVh}(!;y<Da>B3Mv8z4e>iuCjwPZc|S$y%DGGb&&o`
znbsH{>ez-uk%M$tz#DHzf#D#%rJB(iA4q`A_;eKm{JtYb>_xomD-m%qd~5qT6gc#J
z4wHNS6Ss>zpK<#-`mcXPNkY$O7RG2#namg-YX5IkMZ{<f3qAiWZg<3J3=8rFfe#st
z0CFS$XVlLtW!GoCZymj2D!;4j`iux4^>oWj{_%+5h?;Mp7wPac51pXtQ9sAws@X`t
zFq{8whzY8rBA4edxDuVU6vrZ;XFPjPneezz)q(KXcNsqn!sGiC3UbiW8Wsu|yEm_4
zgL=$yT{*R~q1d$fCa#et9}pC}_^)Dh?b(>;zC5YCA5DzCAeSY5#dWpgMGMo{)0Qkh
zXNn^mvdRK!$)=3xe4b@Xo~;Fvg-K%Uz;|sT+jQj$u4WqP4X3`YoGd&J1sR8eJQxnR
z&kVUE77B7u<qQi(d=19=i1Hm<r(cM_&ttx0%y;}S@Eu_x9eyz&2NlS$kjU7-`UI|S
zz&8z_q)j5JW`Gc$rfEqI!Nk|Pp?ie*e%f^o7=872yP>&LX?64wzmUmg!@o9q$`(%9
z74t$^M!orPazaH_Ql`Sv{rUGW{fJHv$Ka2H(mE_GGxpE+JwNc$cA)M5_gS9w<seRk
z<yVTN=XSEJvM?5i>vxfSrG0M7kVdsa8avW-;P2d#Jm&7c?HOJ!Zu)F>ajZU0_y;qW
z9F|cZ-xC-bdv|MwY8w!jO1iG@q!9U2qbuifb!P-Y+S6RkmJF3(!tKD)CC89GDXl<m
zZ3Lag6vz8ZZ#uU+)S8u&Tj+8Nl(U66#$z1gQF2sGl~fHPlujdJv2{Rq+~Zz+fbRPM
zroK(^6fquOdDq1otVbY}b@bV50Hj4O-oVM~=-01DV8w|42%xN<&b|TZ(Ak^eiuLsR
XuLBH=K+&0-0fr6D+gnhMIO2Z+z=&te

delta 18840
zcmeHvdypK*eV+Dld%H)I0(eK@-naL_lbl8Oyyqm41dkBG2Ot~(3S<!!&@<C>x5VDg
zVIKfb5_KdM7p2&d0IsaI64N0Zr5L#=5wxrzPKhEBmn)HMhd7aP%#>IpVpbA4RU{E5
zWhG_N`Fds#oI9jS_fPVN4Ae~T-0onfx99i#e&27t`Tgzh{@zWePm1dOnSrOyeKURj
z+&43?TrUP+PY-RO)Glz-&~@K+6;1a%MJF~@9E);==v+4m)wrXIGyfyAZ*dJL@`6Jx
zT{8;URHEt%r3z6G5p9w>la!PU$I*yUXw;g#pqCt{F!0{XFVVZq+>gGC18n`+k9T0m
z#DCvJe{dE@bJ!O*XRDvBH*j=cBmQ!tE2>p^XpB`C8nfl<%-Ai(F>f{u>s%h)Er(`&
z%qnwXy)pK1;j?32I5)>@UYS+KCdcZ{nHi1`aHh|iGqVj|!@D)kDwPHK=6r=!qc=QW
z?Q4uZCjWoAv4FRx!e*^e<qP$)&&9?2I>ype7wh#xv1pV`)wE1%;Qv>`>P)%O^!eDN
zqM0Sz#G<t5KkXaWbk{I7)mL<nVF+ALVa)UukLujAENc3yxcp9GTQBqH%GGkcQDcpA
zSe0|Cm*?gy3*v0p(VekyZ=QItYcS=A6HX@?IW!y2v3hYY<}Xie&vk!TYZUjh+HtY#
z@pNxm*IT2T$He^e`QajSnMRqac$OyT!Y#$s17D$xaM#n!K<xT%sz+^l9<SG<Z)6QV
z6V|W-Sa$jAqE!mR+StQ#xoTw(Kh-3rI8(|FrgX9Eiyh?pyO>)m^GflaX4PwmCvMGl
zXYXNiTr7Px)1A>xU0irIJJDKqU|>9OY#JDb;!wj;bj@}Z=39nBOka0ZRd*N-S}$zb
zRIJN2tw#6Fn`PA*p1izfIbwP$Gaz31bXFbS%c|AleXQy)3rKwb4mS8HR%^5#AKjQW
zG|Lue|6^)gkl)W{#p%}z8)gF6<r*<H#WVw^=mzG)3|CbsH$ByIJ;&#U{O#oipF3K`
zuYpW7!fF{Zw0fNa7E9O+t0$*v5a@$oKg{-v{a^2(BYVp;&E-XE-|F`LI~X<_v*Pl1
zQ@t5SrDFZfsfpIJ*Y!_$x@i#$`l@Oe^dRsQn{ckUu1$5@Cs+tyq@GWA-x5#2OWeFu
zVePn_s9Om`KRGQbPGs^!xOGeU3>mqnS#Ol9#l4*Q;i>3{#ml#!HE^$ww=TTipR!xi
zBjc@Sf0`5j`L8mYw`hi42y~4(%q0rdJjfMyeZ{pMOEGoVVzx_&VT#e`^7|Ioa3U*E
zhZ3`(5vlc>LU9wwA)-%e+9a_{n&YTCEd;DmZ`+jEa$p?1a)g5ivlu>l<(&ub9>;O-
z@%W9lJ;9gjuqSoi5TmuO?mHkMzWmVv#J3t9)#ss0ksX<3l>qmgvl>iA!Z7T{-7ely
zTktq+hc8XTc!m;3tRGGf7j&oOsHS0P;=+N>?wsj3hNg*Kx1{?FH()Tfp5j|pWEBjB
z>AI@GG83CvP$}lgsU+oXz9!C0=X!36gf5|fmSvV4F}g87DAsN3*fG4@mp`OloC;6!
zSc}^_z~&%v;!-QypSGZ@uck-EtG8x{OwaQ)PF*OT=0Ny$j@!p=qUf#*YoU3j?~v7#
zsn3fP%XDv@ZLb;ZqdbLsVKtm9dr1*9HM=eODJa0VQe@*{Ebl_`Q)MRqOIzyqmX8bi
z{akNG)gdhP^td?oiNc_-8PqmyUr|-f#ng06&Eq(NJ%z9GJw4D@Ppv%1X3EvbDI`^3
zMoPZCUWLK9CD%_(|M!j|@%}<OEB@+%{Ep%B4E}$;xIZjc8_5Nnnr+s+S#gG?`_j59
zF4Dr7c=lWQp@#^(pw0CFnhD3MGj1xbPjFuy7C`(d*WBpR$-BA0*=#hOgqC_?WrcFu
zswI|ww{xKN>^)iGeJfS!xQErls-V4{-C}NUzBAjLsW7qr(d@=Ftl)Y#wMmTpdPjc{
zI8Z#7dWxrcHl{&bD5P#EoVXUTRXd=*yL#EG;niza;U{_ITA=;2$u-k7wWPO}F7%7{
z{S@s#T$4gsJUAaqLHm|U-4JJ=OmD~{3LxUdV&}M6_i{cb9{<BampJ!sW@xKv>IK!c
zZBN%M#nOohH{ke+jhn96e!x}3w&6HEK|c=jv4#^1wyo<_E2!!f^P#GTs6MHhlf)@u
z5j3JE%?H0?BkaTXEoyh*&re=?3kQ#4--&(OdoM4(bl1W4pTqH+U&e4hc5ET`M@<Z7
zZj(I3-g~6*DZIqnFkS)&!)LIs|NKUbO(uuWW9YmU1GXQYi(jvkC&t?&3%e{Fb*pn*
zZ<nvwALzsI81^^*;7UY9qy?*SkC#uvD<WW-<8y8l*0|hH8mcHNxR=#<)osF|LlNo?
zh#qX(UMAk3?dr|yrh}MfI6X3I%Fs=xuv|DT5!6siEm7ECRj}(beTx=xTzG9maRUeb
zo7<LRBfwBR*JZlzdBkB?*^S1+{8gHHY8DeYiAm#Xv4O<_qOujbU{V-UiVq-T2m<Q-
zs3>hsT`&H6U5XCus+6G@#lw6>AS9!Xx;XPrc0-mpRBt_DjJM9c)sG*5aL3_9u}Gjn
zJ0NS!vl!wI7q*5Y`Y^9j*EvkF);jxQ_r!4?tzHn;5d6p)mm9N~d(xmdE&9Po%l-2m
z0|ypZwfHH-X9ykT$w|$4@rL{WJiIEFUQ2ClJ@cIn<0=fUsl#xnT!SxgWoYCwQ!#z&
z83FYi*B48_)zu@0zM5Y*!Vm}HEB$iaYp%3t@R21`TzV$eFC)d#Jk2$02z=|};@kO~
zvY%pA=#w<Hnr6!I;#SA^5uvIwG_@_<NoLuK!{ClgjcT^d0z>1`wUP!5KPo&WZnq2l
z>k_}+P^GaS-I^L~75D8B=l)f?vq#3Fr&zRV?ccwLi4!jrdNY=3wpcJKZtv^rjtzq0
zKsy~1K1hW*^$7wU=9miZmZfkT$%*fCle(T@uc!AeuI0>v=Gca17EIc<2ne9eL)4li
zbW+ty1TL9ci5%rf0UCC%b)fOaZ{px1*mp>6jx~-FBskExFlg}VSnKlHo*nWYASID6
zjb^PHDPaY|8!2Ifd$ZNB63#@QODidM{Uo=s^}=AT^H90U5JQT0ztlMa&v+u88y!`2
zv!oG9=~b$SbjorFtY{(CH}1hMYOvvo=TkTw8sL6$<VTvvb&ndFM*?x-WWIY-eA|+4
z5L&;G6jn#KTl6)VQk?ltmW=GKK$VKSADyp3URK_;gAL}13w<4ZS)$qS(Mo1atn1GY
z*qUJ(oKZ)CvxD?NixkI0@a!2>cQq$Kj-krsNlMT0a>Em6-p_Y$nU^*xilQ6rcqC=j
zuptwRgj6|16}!}IpO}9-n-f1<Oz#+EHAsD1wd9w#C2Jo)RlHL?eh|W?TCh`plOGr7
zUMOV6H{Q)`+e#d*;2FM0bVFB&Z%0aI!#5&gbuhRN^N|NrDP8*(*KlIN)OCY81;c7v
zWHU;D$s{}u)k_q9&rlP>1wl$<mvQK5Us1sew!2>Y^@%!OKN5~k#qX4*#PR8!7~9`#
zBMO$5p2bE7i-t8S&$D>t=EHiqvFriDs?5s9PR*7pJm~@UvW2<~Qsru`w*3ygrkMUO
zso^dKHXa(GIz&v5b#-?TI8?;1U+?OhFkJ_+z3wWu=E;1GVC4)QIiwX>7N>-pnk&|A
z&-RGSxx)JKTIe^u<uumil`?Gn%A1NyJd~Ds<K^BI8J-Hmd0s2-Uyj}4sC6GA@Y&*>
ztTny2Co6q>J9T>@)9-SEB#ro7;XZtiP7&eQ#C8<ZRy|L591fSXTJUOoj+LX#I;j<2
zvjXK^F+GDq6%pd}r700-3pp}!A8Xdw)ydI)5ZmHCA#$%=8VRlunY7j`zc;k#ncUVK
zB>O&07qAxJR&1LGimy^++sNCvrLCUdLOY;N&Tfv?YUQ@Mzmv~}9^irK|5f3XB%qN{
z70o6k+r}-PZ?=BQ*0(Mm>ktQikloQg6+*QU*4`WYueQD*vulV;3!Qx#3vT_yaAs7z
z^@VKV{MqrsU!>1J@=W^W?D=zAhm#TeFXac%&%T{G(7%X`#x`^YJO&O^8fZjp0nrp@
zdW^ZgX#m?O7+uIyb;~y`1>ylsGaMv2jvHVN5P^EC$q8|Bnr*uV@l2-p2$L1v!VPpS
z9|#HVrN<qg5M9S<F5$S<I#v8BGeB};B3H3J4S(AP(qM!iUO?q}K8GGlYzrcZ!ACIe
z!Zt?9vtb0>^UQ)FF8#c-SClRk{@YWFf15eK__x`guM_`ObZy+G5$C8=P&JCY#qktm
z)wp%4?ZH5JM6q?>#`R2F<KoP1=^hJ+msy|)DIA%cc-(-;a$%2MAO4f*rsm^_xQ?cf
zqZ-!8)S=2Y6)!%P9a>z&Re-S}1;P)D+!w4C@?q<cs!bZsB+*JxT%udafEG66<2S3f
zeg4K{IC>z3LBFxGsVehdY%-3-=40RV-r~zI9jSh9UHlft2X4IhrH|uvgG`>W9Vtx4
z2RGnl8v9WE=MKEM{cXz>Ex<!#m2wan?>S_hb8UkTD-#8EE}vy5%Mvw0fa)^uILy;5
z0EXT~A{}Yb9T4-X^oh4sQe#;Yt19TF-0+xzfRw`Ptcpir6Rnb|Ihv}qPF(Ew2)6>{
z5g1{fqzJh1J&M6~XdzdDve~P(t%kH+nyC70d0u{&)H>#tE7Arfl*@3$CpUKuiEn={
zPdn~_-}1$Uucmv%S8mOAh+}&S{TbYkR_Zq=VxQyR)UkQHN}Z#CU4757WLyQH4N~b7
zOwK@%r>Z8=Jj=6yJBepM$lYW~;DZLP0lx`rhUibXVL_lG&@bZIp6cq%wF8I~evtz3
zQGukIL|p3Y8dzM*WeNsNH;@lKN|7K@hN2pWh%>1pCelil3Oq6y5aDJ&G?2sSAK!|x
z_;0_CvHLSpvarp63`6cGe{iG<dDtg0V4~nBJOqb)vRsMoxEG<Sr@|((Tdc9q16~DC
zJ2_Ua`g~=U0-Ww7J`Ah)zm*ti&DfeMF8pa`WK<_5ojRIrtr7!jSS4V3z{+HB<QVX|
zHez_iuz+dk78V_0HH-@P8Fv6`0AE}!2R?7GvPk`}RQL81%`zMT=Gj0B(pVPdFbGx#
z`P4GSG=d=*SiS$juvwic-hn7P8Rg&C^q5GU@7$PGiRnn#OptG8`WGDv3m5?Ybzo8?
zTtH#S06RK?2a`hx^Np*bZOCGb@h0E`Aj36io1AN%#ji7R&Pj=d4wWKo`byg5iyhCW
z3ayL(Qy;mzb_!^p%napsv^_$kh*P*V3nEodZOH1-J8@zry>p!Lz+gVa(&Tbs9k^R&
z8~9VzJQvOvY9}uJAlI$RNJM^NVy-h`w504&`BDj2<|{UEq1V!bX#(edFhjD3kz0t%
zJ5$}6z2Sm5@q=`4Rs$R?F2B*arS<j)8^!@YFk(_$2?7)GuYt8>o@*&8phjd=9y49B
z^yRMZTdsyQ!g-15HLI&M8<tq?8bvVTL`Z0EKcc7dyeMAIZzwFACV&~;xs7m?22#l}
zQ9RsnqqvO~`nFMG!(|wTp_<5;kOWIOn@Jbu17dgJK@f48s*M=?wsbctBMNZ}8r0V_
zF>}IfU^Fg)W2Eq;)P&yyE<zDfLCFz9#y$h!zb2+fvIB6xzw|=DMu~w4reJ&sqY_J2
zW>f^GYDv|RK$uClaUGd~B3l&<vP$#N>tDpEA31^%+eh|d_yqP%x#`7rvF7z}AC5oV
z@-V=Wm-2^4Fy44C1_^br-THweRUE{dgh8-XoYC#^(YmR2tcyQi{TK4%`#u)LE+t+c
z0J9QjmoB3&(SfLfBT<2!%lS7f2&yGbfEbXeIrd0ci$lf`E+iTF?rGMLf4gBVM9t@G
zV;x(ws$&sxVKg&5j%Zi4u~MsTih?YvWLUrkq-*KB7iJ@10Mwko(GhzD9JiIjr*n$}
z<KvoRUZt{Ouv!U#&7hbfmi{ueF=N}7IQILnAE!Gz#Cv~~-67t7Cfn6_pk1F@4$mN~
zMVWXl2=?2VjTu5L@#5D~qvG@%si8#|VXtrLIy_zk^J%bjmWOZ{rp`g6V+X-i@g*!B
zuVLA)($HljmjhXs6Nl4=5lGq+#61ukeAKoj;`xi6A5Cek)DjW%-!EkQ>UDNCYI-MJ
ze{22sdb29k46*bjWQ~vSOAk!Ap5<sJ44|cBU3CwU6w{zMnr=HffK-$cT(Lff>d{P%
z&58HFm+$%LirK*D?SC#aD2iT=wn6XW!Pzi2-pTd5GZdFQayMmkl+Wm6x$WZ4Te1^3
zOTVnz$lM(6Qx%{9lSn-DPDYhP3AkagfT+|$Qgi0jR5$KBKt)Uw{uq%S@?Bu%o=hA8
zULx~mX5bMMv6_tZRF$+NJ!w5-tzE-~;F^hzV0Ss2w4!WMpVZY!qL+wj+B&K&AJ#sk
ze%*~-zRHlU6Z?dR;fMeFSpD^9ad6iS64AuAczkDk5RU^V?-|3<#AXRtp14VB?5D9y
z)!r@B(naj?CVf6O`85;L=P$^bEVc*Zmv}rGzx?|rr25C3OkuZvwjCtOwTp3+0BI{!
zdD#m@uTiOKfyb2=S9&mLHfQ3(pQO7xc1b)_-2PN%a~2Uf0P(|xks_rf-B2|o-_aLb
zDZ2ueMz-J>;>=R2Zw9#`B6k#}9T#?3w`4?xqLPg1HRf8b3Eh=5NdglUOapHveV4>1
z<$E){3NsXCw27F|*2HLUdMHK3g`cI#4Q(jAjd_VDewf>maU4@z{+$jW5j!CG79*N#
zM&%Y2sf7a!()Ju%fx;s2a#X|OmiYde{MPa1_<1&L#4x-EtYoDQ2zD~lFN#a)oS0(i
zQtR~g?1t3^;R7L7uo2ZQvW~5)trLH<^Qglu9_YYp7%qrvBE%at0}m;k$^wl6DEFfB
zOmc0O?eMDgk=ZFceqv1wt2O=Pka%%_jt<<#Y750Z2w#$+(NtKimU(Q`5IM+V6be3n
znd%Uq-q&@5IC*_X?{<eC1z&+W1hoN;K|ghPRhbw9Z%`FOWrk@wC=mHH(oY+qCzK-~
zuIGC$YFUW;ka(evKnYW@w7!l^06r1k_oxI3WFiJg&5G4?E!Qd71bBz>p?+#$Bpg!d
zqz=Xdh2#M}B}x6vZ@F*acnylMtkBUM;n~VFF$96kqX~Y(*e!b_Os49$v;R{7l~D$c
zydTj05QWJ}1kN7l2T~Ff?1fl15{c9&EA_CIx_C3<W>}tgI!20$rIl<VLlgYh1e^~a
zXCOF)vgA7kCQx^DH7x+vYH%zxXAF5Wvtf@_R2Uu4F@d^X9L{gb0DBT2yqK5eX;r-a
z4?73p15VsZig&PzF9XVz2t!oPhDd^sF#Tq_FJq(jxc(4uhTC@+M#g;Ig2F1?(BT7+
z#{(fSkk>OPfel{<I9a5o(wll@TCgIpH2^_Q_oW8~d9#Cd-pS$K*{on6q`LD5LLW$L
zlebcDY|K(%eS(f>$6Lo(|G286$V8DI0#Zld7yv>=tfgXwxMLx)fSVVobiUhY&I3GK
zCFfU)n|L1#4OtpX$UFtBAcC_nujvfw2MK`}KLK~tuExi)i43+|m;OV4+R$3>ePffj
z*qhFt|4&b*JJRAM3jpW-r@LPJ5O&!*yy&Rh@_mY0H8Mvej)28o6Y_82J<SB};YL`C
z!vh#Jssjyy^U^K&C7Y=z0NRck*qTcbC<7dEP+VrjLXHgWK|R$(93x>+5H5VeJfcym
z;xyey;NyWPVod=$hmAr>*meR`dUSYk)pabCNaT4Q08SrOJ7765WmE!OG6IG*K@e*@
z27s;@Yv~g^tkj@*{NHu_OMoqz?`FR6i|qMdWK)L=;{8AC7~H%@X42Wk+%;y>HD=OE
z|9p*^1cQ|H(|_NYNv)0_Z`lIObS-qF)~<D^BmQy%(=}ewHD1y+UQ*2X{1xUUUDx``
z#|GC-OKPR=yv9np#!7<wy#lXaV<p8I>aPYXsV6cAYb7PcvBVnKOV>C_D*!g&Tr0|~
ze_uFB|Gb#`=+ig%ubGw<$=6!=OaBjPNx@6kXi320uF;YnLdos_H!Vqkmh^G4bD}`R
z^>R=)6^@CwzmPsAej)#Sa~#8*Ji7P~(i_El0G7@#mQq9Gi$J>qwDI_00io$a2a^_f
z7HY4k6%Y@QE(j3-s=74WH*IuZ;6usdF=TTbno*R%2eXk%fZpXe%>)mFXkeuJc7#de
zds)E1e*|L?C0`HRrU0j@mg!r>r;0b_cu~g=zL<dC%Val$CV45AJkQk_A*v7JG{~)(
z9JpePdEkavCYtn^X#|o06*HFh-IneX$NpLFM-oN*!;E<Ax!hkr^>>F;=l||-`hi~)
zL*wcGt*VY{v~Hta2n;L*B|?y|0vG%Ol{i2JP4s<Xj+!r~dpdhSt_7pj0>PLW8W^Oi
z1^7^v4FjeRBC^I*&cG@GVH&WxrmA3Xq4tkDI~DX#Qv-`@xQ<oOQN;%T7F5WnO%Nq*
zaF{5HT6EGdO9n`kD=4_tiYyX;SYR4`7=Oz49!XVde?>IyhsL&}XKz6Q=zwHV*Ngyj
z{wR2avWCP^@s0V8kD!;M^=*5@@L|BQ=uDGsPOG_)cZ1Dy`78MipfI8nMDmV`DlMs?
z7lGf3?@u7-4roUzMWYJJO#NdFttc)7BL~cK&?0ffP-;+A$}QDIMbXj3D=&6#61V@8
zT*olJ=v2F34{i^&>jZ=h1o`$)b`CzhM6>rUfc|pqYv6j{`$wI>CenUp1NcyA@`#{p
zw1YV|XAaO;NHZ8~xWTC1*gA-Dz_Gz(1*e%`UED%9<a|>+@j|A1sJ>7~FPmJqQ<6lQ
zm<3f$iR#Cd=^LR^*VVw+C{6)@M<MRTpMl?*ekOHeRzvGo>+(0oBMe*o=(*hHZPYZ6
zq5$&C(G+p}_c}MAlv+Tc3F-p!y=Miou#SE<RKQSY*9^lnP$x4{Tm=il1ho@_2QD|7
z6e7V}!&R<fHrFtlYnaWiK4v3ssCV{lr}j}86pQHrr3~i6je7AcP1dzxAW0CQnT!z-
zL&X`Ra{~yqF8EiDDmhoKB;q+BT!DUPgP*Mhu4$VLR6=Qd;58%LV@apyqz4z*aE$`$
zaE=c46?)NRYrO1Ur-!ILDK`gvMNM~B_py!!@Nd|DB2tg+T8{$*`^`Ha!<dtOW)U0f
zt$&o4<P1ss_}FzA7O-F0ZkMEvc<YPK-V*;Id+f12fPLq~7;fHHIe-`0h$hR^H)8)c
zr`sfs^x|BbGa}g_KmH92Bln-&{nk@B*n|DZ5qbTCVf%L>PNATC`rMV?4F1HEz4FmP
z)GsNIKqLYi)NjVb4+fTE%n_F4uCO-4s^!nbyb9^FW2Xd;{^Zy$)W*w_KANa+gPJ(x
zFSEJim&a}tAvsz<_OQA6h*glaD7D1;J(-a)tTvnoc%!SFI6^zL)FAfLT4%r5H|{f&
znW&7K93DsK;59OEag+eA2c&fkR07=mGhJJ5L5r(=3_uifeArF!e_;!wc}kq!jd*R{
zJLz2O-3z6OUG-To(a@hBH@dV*N^veR#bh5@xhj_K2agziSK`7_X0$awhdu%|z&{Y#
zEeEm)(vgLD6D=5^KcLmYKzpurRd+gwq0tPJL>3NOZ7r-yPHTK8>fs!3l+h*EzHikA
zS_2cI0Q&RM3?h0cD$~O!a<o&n2ZiVunBI`@7QgdkI-i;mm-nW6vX%onR@4%7`pxvf
zI9Q@+x(W~pp?w%`U53=I2PUK86HcLrzTH~d-Mu|#0!yJua7E*XH{i|o0|p%QQG?ri
zBHiB_exHu)0qq)6alBr<tAZvh`Jdz3`T^$6@=9xZX=6%lk>Smv_=VK^$Tma_O*GJG
z2xmcTAs~i>!lg&e2uKFd>yL^z291f)|DEbV%r6<5J__tE*qMqSFi>Q;2EHpru^jYk
z5+F)Kj0L@p7@F<aL=!K*l^b0A<(IL+%)~!T$%+V?_#SXcj6;-6>U2^gB?`JIHIiHs
zN$1Cw#(w`}Kl#>eQjJf>I=D=jgaFOedF9Y@-}mxEFzTUK=FxL3!ucx=IhCfQV1quA
zRIYoPET&#373*%F&4#UMw;NJvM*OQW3gQSj)sQ<Ze<CnOTu)1;18oyaKTh|9s0*eu
z=r=kV!63LM1j2+EpvgmH%mN46qOFS?whYabYY``GrWT%RtmrgG1TD_~d2T?w|5HTe
z>V?jdc<+@|r}$B4rbnFqL>>$eGz*IpzndC=`nG<XQP2^5$?-AeqX^JQ3=%HxnIwIf
z%=N$@ze-h3miZ|$|5~P}KVp{FLp*sSsRX74@tf`i)v)#(c`|}lXxU$nepAV6O32?`
zlBDVjdTZy#lr7FZm)#^@IhxO=9u~j&PUog=rVL{X=oJqb)1+n~M*yD#d?r6YDk)o2
zK%PLCfVlMKRF6p&je;Z$#bRJ5f)<0kjG%jlA@jg`+5!5a4J;;p+fgvM+8#>m|2&AH
zYq*G7&~y;z(SD6Ba!_W}24J8ma?;RCIw*{I+)9FM*b9N%jQvP;>rT8#--OMKUogHQ
zc46F%)NN6IAW#3?Aq<j~INH8O^rO%l5cS|gbn5f%hPFBG!_Y^o$>!vPMk3eXd1xqy
zPpG1&Kl%)9xJW|qRvn#<38%0Jw3>F)0B(qwdMY!KhO<spJ4Z%TG{f4)YU_^{S)}l2
z!GQ7aoiOo)j{xn93YapWje%B!hDtop0{J)is2q<B(JV3A$aM?tXW8}Jj#tA|74F9k
zxC@f}|DnDlBu57~b|0QBfEM}d4mRs(1wK?`Wl-DyIJ-G*QE~cD3KK%wO9Ndzgffr-
z(G>>u&|q(DNyAcoG<Ucv_YAJeORT8!NBxuH+`akSx)nTKN=a+<=Y!&{KhD$E{?obM
zDfrvy@gHqN870BHSWS%1=X$c#L0b#GF1w|5@y|Bk=?}=r&}gfGC?w?!a0y}}^sE^m
zTpD-=iECfQDHdm*&vkFWlWNKpH2fqMz;M8cO5?#4^G+v4gCCyya7Qf+lXHp4^#`F*
z(PT29V2P!F+%?)F1BLVNJemHBO^ddVtWU$TaMOauZ~!If>p`~bIzDqOmE*ZBKvB?b
zYBLLXjH7{_t>bnA>p(X%(LE>`1wRVC^5`~0|A)uHxYsoNDjkF-^sdP+HqT>d7({D0
zv2dD6EIbqgy^eG@aws4o=+&_;%|t7ofu}3L0YD~jo(6tafF5Z)X+gzlt_xKKT}v@2
PdgD~x1=mwfuIv8*SMN=H

diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out b/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out
index a9b5d23595a8e3..00d15805baf04e 100644
--- a/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out
+++ b/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out
@@ -119,3 +119,123 @@
 -- !timetravel10 --
 10000
 
+-- !timetravel1 --
+1000
+
+-- !timetravel2 --
+2000
+
+-- !timetravel3 --
+3000
+
+-- !timetravel4 --
+4000
+
+-- !timetravel5 --
+5000
+
+-- !timetravel6 --
+6000
+
+-- !timetravel7 --
+7000
+
+-- !timetravel8 --
+8000
+
+-- !timetravel9 --
+9000
+
+-- !timetravel10 --
+10000
+
+-- !timetravel1 --
+1000
+
+-- !timetravel2 --
+2000
+
+-- !timetravel3 --
+3000
+
+-- !timetravel4 --
+4000
+
+-- !timetravel5 --
+5000
+
+-- !timetravel6 --
+6000
+
+-- !timetravel7 --
+7000
+
+-- !timetravel8 --
+8000
+
+-- !timetravel9 --
+9000
+
+-- !timetravel10 --
+10000
+
+-- !timetravel1 --
+1000
+
+-- !timetravel2 --
+2000
+
+-- !timetravel3 --
+3000
+
+-- !timetravel4 --
+4000
+
+-- !timetravel5 --
+5000
+
+-- !timetravel6 --
+6000
+
+-- !timetravel7 --
+7000
+
+-- !timetravel8 --
+8000
+
+-- !timetravel9 --
+9000
+
+-- !timetravel10 --
+10000
+
+-- !timetravel1 --
+1000
+
+-- !timetravel2 --
+2000
+
+-- !timetravel3 --
+3000
+
+-- !timetravel4 --
+4000
+
+-- !timetravel5 --
+5000
+
+-- !timetravel6 --
+6000
+
+-- !timetravel7 --
+7000
+
+-- !timetravel8 --
+8000
+
+-- !timetravel9 --
+9000
+
+-- !timetravel10 --
+10000
+
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_catalog.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_catalog.groovy
index f2082ef89c7a50..149eecf5817bd4 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_catalog.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_catalog.groovy
@@ -36,4 +36,4 @@ suite("test_hudi_catalog", "p2,external,hudi,external_remote,external_remote_hud
     def tables = sql """ show tables; """
     assertTrue(tables.size() > 0)
     sql """drop catalog if exists ${catalog_name};"""
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
index 8cc1d2a852b8c4..885903646cc5b5 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
@@ -60,7 +60,6 @@ suite("test_hudi_incremental", "p2,external,hudi,external_remote,external_remote
         "20241114152009764",
         "20241114152011901",
     ]
-    test_hudi_incremental_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
 
     // spark-sql "select distinct _hoodie_commit_time from user_activity_log_cow_partition order by _hoodie_commit_time;"
     def timestamps_cow_partition = [
@@ -75,7 +74,6 @@ suite("test_hudi_incremental", "p2,external,hudi,external_remote,external_remote
         "20241114152147114",
         "20241114152156417",
     ]
-    test_hudi_incremental_querys("user_activity_log_cow_partition", timestamps_cow_partition)
 
     // spark-sql "select distinct _hoodie_commit_time from user_activity_log_mor_non_partition order by _hoodie_commit_time;"
     def timestamps_mor_non_partition = [
@@ -90,7 +88,6 @@ suite("test_hudi_incremental", "p2,external,hudi,external_remote,external_remote
         "20241114152028770",
         "20241114152030746",
     ]
-    test_hudi_incremental_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition)
 
     // spark-sql "select distinct _hoodie_commit_time from user_activity_log_mor_partition order by _hoodie_commit_time;"
     def timestamps_mor_partition = [
@@ -105,7 +102,18 @@ suite("test_hudi_incremental", "p2,external,hudi,external_remote,external_remote
         "20241114152323587",
         "20241114152334111",
     ]
+
+    test_hudi_incremental_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
+    test_hudi_incremental_querys("user_activity_log_cow_partition", timestamps_cow_partition)
+    test_hudi_incremental_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition)
+    test_hudi_incremental_querys("user_activity_log_mor_partition", timestamps_mor_partition)
+    sql """set force_jni_scanner=true;"""
+    // don't support incremental query for cow table by jni reader
+    // test_hudi_incremental_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
+    // test_hudi_incremental_querys("user_activity_log_cow_partition", timestamps_cow_partition)
+    test_hudi_incremental_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition)
     test_hudi_incremental_querys("user_activity_log_mor_partition", timestamps_mor_partition)
+    // sql """set force_jni_scanner=false;"""
 
     sql """drop catalog if exists ${catalog_name};"""
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
index b247aaf492400d..0da88447cdef15 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
@@ -33,7 +33,18 @@ suite("test_hudi_schema_evolution", "p2,external,hudi,external_remote,external_r
     sql """ switch ${catalog_name};"""
     sql """ use regression_hudi;""" 
     sql """ set enable_fallback_to_original_planner=false """
+    
+    qt_adding_simple_columns_table """ select * from adding_simple_columns_table order by id """
+    qt_altering_simple_columns_table """ select * from altering_simple_columns_table order by id """
+    // qt_deleting_simple_columns_table """ select * from deleting_simple_columns_table order by id """
+    // qt_renaming_simple_columns_table """ select * from renaming_simple_columns_table order by id """
 
+    qt_adding_complex_columns_table """ select * from adding_complex_columns_table order by id """
+    qt_altering_complex_columns_table """ select * from altering_complex_columns_table order by id """
+    // qt_deleting_complex_columns_table """ select * from deleting_complex_columns_table order by id """
+    // qt_renaming_complex_columns_table """ select * from renaming_complex_columns_table order by id """
+    
+    sql """set force_jni_scanner = true;"""
     qt_adding_simple_columns_table """ select * from adding_simple_columns_table order by id """
     qt_altering_simple_columns_table """ select * from altering_simple_columns_table order by id """
     // qt_deleting_simple_columns_table """ select * from deleting_simple_columns_table order by id """
@@ -43,6 +54,7 @@ suite("test_hudi_schema_evolution", "p2,external,hudi,external_remote,external_r
     qt_altering_complex_columns_table """ select * from altering_complex_columns_table order by id """
     // qt_deleting_complex_columns_table """ select * from deleting_complex_columns_table order by id """
     // qt_renaming_complex_columns_table """ select * from renaming_complex_columns_table order by id """
+    sql """set force_jni_scanner = false;"""
 
     sql """drop catalog if exists ${catalog_name};"""
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
index 53c09e6d5a9031..89d89709b3c822 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
@@ -64,7 +64,7 @@ suite("test_hudi_snapshot", "p2,external,hudi,external_remote,external_remote_hu
         qt_q09 """SELECT * FROM ${table_name} WHERE struct_element(struct_element(address, 'coordinates'), 'latitude') BETWEEN 0 AND 100 AND struct_element(struct_element(address, 'coordinates'), 'longitude') BETWEEN 0 AND 100 ORDER BY event_time LIMIT 5;"""
 
         // Query records with ratings above a specific value and limit output
-        qt_q10 """SELECT * FROM ${table_name} WHERE rating > 4.5 ORDER BY rating DESC LIMIT 5;"""
+        qt_q10 """SELECT * FROM ${table_name} WHERE rating > 4.5 ORDER BY event_time DESC LIMIT 5;"""
 
         // Query all users' signup dates and limit output
         qt_q11 """SELECT user_id, signup_date FROM ${table_name} ORDER BY signup_date DESC LIMIT 10;"""
@@ -79,13 +79,20 @@ suite("test_hudi_snapshot", "p2,external,hudi,external_remote,external_remote_hu
         qt_q14 """SELECT * FROM ${table_name} WHERE signup_date = '2024-01-15' ORDER BY user_id LIMIT 5;"""
 
         // Query the total count of purchases for each user and limit output
-        qt_q15 """SELECT user_id, array_size(purchases) AS purchase_count FROM ${table_name} ORDER BY purchase_count DESC LIMIT 5;"""
+        qt_q15 """SELECT user_id, array_size(purchases) AS purchase_count FROM ${table_name} ORDER BY user_id LIMIT 5;"""
     }
 
+    test_hudi_snapshot_querys("user_activity_log_mor_non_partition")
+    test_hudi_snapshot_querys("user_activity_log_mor_partition")
     test_hudi_snapshot_querys("user_activity_log_cow_non_partition")
     test_hudi_snapshot_querys("user_activity_log_cow_partition")
+
+    sql """set force_jni_scanner=true;"""
     test_hudi_snapshot_querys("user_activity_log_mor_non_partition")
     test_hudi_snapshot_querys("user_activity_log_mor_partition")
+    test_hudi_snapshot_querys("user_activity_log_cow_non_partition")
+    test_hudi_snapshot_querys("user_activity_log_cow_partition")
+    sql """set force_jni_scanner=false;"""
 
     sql """drop catalog if exists ${catalog_name};"""
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy
index c1ba630e4a7d01..36309322558f52 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy
@@ -59,4 +59,4 @@ suite("test_hudi_timestamp", "p2,external,hudi,external_remote,external_remote_h
 // INSERT OVERWRITE hudi_table_with_timestamp VALUES
 // ('1', 'Alice', timestamp('2024-10-25 08:00:00')),
 // ('2', 'Bob', timestamp('2024-10-25 09:30:00')),
-// ('3', 'Charlie', timestamp('2024-10-25 11:00:00'));
\ No newline at end of file
+// ('3', 'Charlie', timestamp('2024-10-25 11:00:00'));
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
index 4d458dc4381dcf..cceeaa412202c6 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
@@ -54,7 +54,6 @@ suite("test_hudi_timetravel", "p2,external,hudi,external_remote,external_remote_
         "20241114152009764",
         "20241114152011901",
     ]
-    test_hudi_timetravel_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
 
     // spark-sql "select distinct _hoodie_commit_time from user_activity_log_cow_partition order by _hoodie_commit_time;"
     def timestamps_cow_partition = [
@@ -69,7 +68,6 @@ suite("test_hudi_timetravel", "p2,external,hudi,external_remote,external_remote_
         "20241114152147114",
         "20241114152156417",
     ]
-    test_hudi_timetravel_querys("user_activity_log_cow_partition", timestamps_cow_partition)
 
     // spark-sql "select distinct _hoodie_commit_time from user_activity_log_mor_non_partition order by _hoodie_commit_time;"
     def timestamps_mor_non_partition = [
@@ -84,7 +82,6 @@ suite("test_hudi_timetravel", "p2,external,hudi,external_remote,external_remote_
         "20241114152028770",
         "20241114152030746",
     ]
-    test_hudi_timetravel_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition)
 
     // spark-sql "select distinct _hoodie_commit_time from user_activity_log_mor_partition order by _hoodie_commit_time;"
     def timestamps_mor_partition = [
@@ -99,7 +96,17 @@ suite("test_hudi_timetravel", "p2,external,hudi,external_remote,external_remote_
         "20241114152323587",
         "20241114152334111",
     ]
+
+    test_hudi_timetravel_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
+    test_hudi_timetravel_querys("user_activity_log_cow_partition", timestamps_cow_partition)
+    test_hudi_timetravel_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition)
+    test_hudi_timetravel_querys("user_activity_log_mor_partition", timestamps_mor_partition)
+    sql """set force_jni_scanner=true;"""
+    test_hudi_timetravel_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
+    test_hudi_timetravel_querys("user_activity_log_cow_partition", timestamps_cow_partition)
+    test_hudi_timetravel_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition)
     test_hudi_timetravel_querys("user_activity_log_mor_partition", timestamps_mor_partition)
+    sql """set force_jni_scanner=false;"""
 
     sql """drop catalog if exists ${catalog_name};"""
-}
\ No newline at end of file
+}

From 6f596586163b5dedaf1b8e062c8e3ecd654cee22 Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Wed, 4 Dec 2024 10:01:27 +0800
Subject: [PATCH 147/399] [fix](cluster key) fix cluster key duplicated key
 (#44776)

the duplicated keys is happend when:
1. do schema change and not finished
2. write data and the publish phase skip calculate delete bitmap
3. compact the newly rowsets without delete bitmap: the mow with cluster
key need to do compaction with delete bitmap
---
 be/src/cloud/cloud_cumulative_compaction.cpp  |   2 -
 .../cloud_engine_calc_delete_bitmap_task.cpp  |   2 +-
 be/src/olap/compaction.cpp                    |  57 +++++++
 be/src/olap/compaction.h                      |   6 +
 be/src/olap/cumulative_compaction.cpp         |   2 +-
 .../org/apache/doris/master/MasterImpl.java   |   6 +-
 .../test_schema_change_and_compaction.out     |   8 +
 .../test_schema_change_and_compaction.groovy  | 145 ++++++++++++++++++
 8 files changed, 221 insertions(+), 7 deletions(-)
 create mode 100644 regression-test/data/unique_with_mow_c_p0/test_schema_change_and_compaction.out
 create mode 100644 regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy

diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp
index c466c35e2a2ab6..4e25cd74209e6e 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -375,11 +375,9 @@ Status CloudCumulativeCompaction::modify_rowsets() {
 Status CloudCumulativeCompaction::process_old_version_delete_bitmap() {
     // agg previously rowset old version delete bitmap
     std::vector<RowsetSharedPtr> pre_rowsets {};
-    std::vector<std::string> pre_rowset_ids {};
     for (const auto& it : cloud_tablet()->rowset_map()) {
         if (it.first.second < _input_rowsets.front()->start_version()) {
             pre_rowsets.emplace_back(it.second);
-            pre_rowset_ids.emplace_back(it.second->rowset_id().to_string());
         }
     }
     std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator);
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
index 91611d20c6270b..336117d1012d4d 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
@@ -227,7 +227,7 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
         }
     }
     auto total_update_delete_bitmap_time_us = MonotonicMicros() - t3;
-    LOG(INFO) << "calculate delete bitmap successfully on tablet"
+    LOG(INFO) << "finish calculate delete bitmap on tablet"
               << ", table_id=" << tablet->table_id() << ", transaction_id=" << _transaction_id
               << ", tablet_id=" << tablet->tablet_id()
               << ", get_tablet_time_us=" << get_tablet_time_us
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index e71e1862dc8dbb..c85ce36a1033f5 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -191,6 +191,9 @@ Status Compaction::merge_input_rowsets() {
         SCOPED_TIMER(_merge_rowsets_latency_timer);
         // 1. Merge segment files and write bkd inverted index
         if (_is_vertical) {
+            if (!_tablet->tablet_schema()->cluster_key_idxes().empty()) {
+                RETURN_IF_ERROR(update_delete_bitmap());
+            }
             res = Merger::vertical_merge_rowsets(_tablet, compaction_type(), *_cur_tablet_schema,
                                                  input_rs_readers, _output_rs_writer.get(),
                                                  get_avg_segment_rows(), way_num, &_stats);
@@ -872,6 +875,60 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
     }
 }
 
+Status CompactionMixin::update_delete_bitmap() {
+    // for mow with cluster keys, compaction read data with delete bitmap
+    // if tablet is not ready(such as schema change), we need to update delete bitmap
+    {
+        std::shared_lock meta_rlock(_tablet->get_header_lock());
+        if (_tablet->tablet_state() != TABLET_NOTREADY) {
+            return Status::OK();
+        }
+    }
+    OlapStopWatch watch;
+    std::vector<RowsetSharedPtr> rowsets;
+    for (const auto& rowset : _input_rowsets) {
+        std::lock_guard rwlock(tablet()->get_rowset_update_lock());
+        std::shared_lock rlock(_tablet->get_header_lock());
+        Status st = _tablet->update_delete_bitmap_without_lock(_tablet, rowset, &rowsets);
+        if (!st.ok()) {
+            LOG(INFO) << "failed update_delete_bitmap_without_lock for tablet_id="
+                      << _tablet->tablet_id() << ", st=" << st.to_string();
+            return st;
+        }
+        rowsets.push_back(rowset);
+    }
+    LOG(INFO) << "finish update delete bitmap for tablet: " << _tablet->tablet_id()
+              << ", rowsets: " << _input_rowsets.size() << ", cost: " << watch.get_elapse_time_us()
+              << "(us)";
+    return Status::OK();
+}
+
+Status CloudCompactionMixin::update_delete_bitmap() {
+    // for mow with cluster keys, compaction read data with delete bitmap
+    // if tablet is not ready(such as schema change), we need to update delete bitmap
+    {
+        std::shared_lock meta_rlock(_tablet->get_header_lock());
+        if (_tablet->tablet_state() != TABLET_NOTREADY) {
+            return Status::OK();
+        }
+    }
+    OlapStopWatch watch;
+    std::vector<RowsetSharedPtr> rowsets;
+    for (const auto& rowset : _input_rowsets) {
+        Status st = _tablet->update_delete_bitmap_without_lock(_tablet, rowset, &rowsets);
+        if (!st.ok()) {
+            LOG(INFO) << "failed update_delete_bitmap_without_lock for tablet_id="
+                      << _tablet->tablet_id() << ", st=" << st.to_string();
+            return st;
+        }
+        rowsets.push_back(rowset);
+    }
+    LOG(INFO) << "finish update delete bitmap for tablet: " << _tablet->tablet_id()
+              << ", rowsets: " << _input_rowsets.size() << ", cost: " << watch.get_elapse_time_us()
+              << "(us)";
+    return Status::OK();
+}
+
 Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx) {
     // only do index compaction for dup_keys and unique_keys with mow enabled
     if (config::inverted_index_compaction_enable &&
diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h
index 06ef4268529247..7f92a6c5f4d7dc 100644
--- a/be/src/olap/compaction.h
+++ b/be/src/olap/compaction.h
@@ -84,6 +84,8 @@ class Compaction {
 
     int64_t merge_way_num();
 
+    virtual Status update_delete_bitmap() = 0;
+
     // the root tracker for this compaction
     std::shared_ptr<MemTrackerLimiter> _mem_tracker;
 
@@ -146,6 +148,8 @@ class CompactionMixin : public Compaction {
 
     virtual Status modify_rowsets();
 
+    Status update_delete_bitmap() override;
+
     StorageEngine& _engine;
 
 private:
@@ -175,6 +179,8 @@ class CloudCompactionMixin : public Compaction {
 protected:
     CloudTablet* cloud_tablet() { return static_cast<CloudTablet*>(_tablet.get()); }
 
+    Status update_delete_bitmap() override;
+
     virtual void garbage_collection();
 
     CloudStorageEngine& _engine;
diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp
index b961c694ede4d0..2dfd30fb86ed9a 100644
--- a/be/src/olap/cumulative_compaction.cpp
+++ b/be/src/olap/cumulative_compaction.cpp
@@ -145,7 +145,7 @@ Status CumulativeCompaction::pick_rowsets_to_compact() {
         DCHECK(missing_versions.size() % 2 == 0);
         LOG(WARNING) << "There are missed versions among rowsets. "
                      << "total missed version size: " << missing_versions.size() / 2
-                     << " first missed version prev rowset verison=" << missing_versions[0]
+                     << ", first missed version prev rowset verison=" << missing_versions[0]
                      << ", first missed version next rowset version=" << missing_versions[1]
                      << ", tablet=" << _tablet->tablet_id();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
index 4010a9b564d0a0..09318af34bd381 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
@@ -681,9 +681,9 @@ private void finishCalcDeleteBitmap(AgentTask task, TFinishTaskRequest request)
             CalcDeleteBitmapTask calcDeleteBitmapTask = (CalcDeleteBitmapTask) task;
             if (request.getTaskStatus().getStatusCode() != TStatusCode.OK) {
                 calcDeleteBitmapTask.countDownToZero(request.getTaskStatus().getStatusCode(),
-                        "backend: " + task.getBackendId() + ", error_tablet_size: "
-                                + request.getErrorTabletIdsSize() + ", err_msg: "
-                                + request.getTaskStatus().getErrorMsgs().toString());
+                        "backend: " + task.getBackendId() + ", error_tablet_size: " + request.getErrorTabletIdsSize()
+                                + ", error_tablets: " + request.getErrorTabletIds()
+                                + ", err_msg: " + request.getTaskStatus().getErrorMsgs().toString());
             } else if (request.isSetRespPartitions()
                     && calcDeleteBitmapTask.isFinishRequestStale(request.getRespPartitions())) {
                 LOG.warn("get staled response from backend: {}, report version: {}. calcDeleteBitmapTask's"
diff --git a/regression-test/data/unique_with_mow_c_p0/test_schema_change_and_compaction.out b/regression-test/data/unique_with_mow_c_p0/test_schema_change_and_compaction.out
new file mode 100644
index 00000000000000..35f26a488beff3
--- /dev/null
+++ b/regression-test/data/unique_with_mow_c_p0/test_schema_change_and_compaction.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select1 --
+10	20	35	40
+
+-- !select2 --
+10	20	40	37
+11	20	40	37
+
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy
new file mode 100644
index 00000000000000..dfb7facf5ee4cb
--- /dev/null
+++ b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy
@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// The cases is copied from https://github.com/trinodb/trino/tree/master
+// /testing/trino-product-tests/src/main/resources/sql-tests/testcases
+// and modified by Doris.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_schema_change_and_compaction", "nonConcurrent") {
+    def tableName = "test_schema_change_and_compaction"
+
+    def getAlterTableState = { job_state ->
+        def retry = 0
+        def last_state = ""
+        while (true) {
+            sleep(2000)
+            def state = sql " show alter table column where tablename = '${tableName}' order by CreateTime desc limit 1"
+            logger.info("alter table state: ${state}")
+            last_state = state[0][9]
+            if (state.size() > 0 && state[0][9] == job_state) {
+                return
+            }
+            retry++
+            if (retry >= 10) {
+                break
+            }
+        }
+        assertTrue(false, "alter table job state is ${last_state}, not ${job_state} after retry ${retry} times")
+    }
+
+    def block_convert_historical_rowsets = {
+        if (isCloudMode()) {
+            GetDebugPoint().enableDebugPointForAllBEs("CloudSchemaChangeJob::_convert_historical_rowsets.block")
+        } else {
+            GetDebugPoint().enableDebugPointForAllBEs("SchemaChangeJob::_convert_historical_rowsets.block")
+        }
+    }
+
+    def unblock = {
+        if (isCloudMode()) {
+            GetDebugPoint().disableDebugPointForAllBEs("CloudSchemaChangeJob::_convert_historical_rowsets.block")
+        } else {
+            GetDebugPoint().disableDebugPointForAllBEs("SchemaChangeJob::_convert_historical_rowsets.block")
+        }
+    }
+
+    onFinish {
+        unblock()
+    }
+
+    sql """ DROP TABLE IF EXISTS ${tableName} force """
+    sql """
+        CREATE TABLE ${tableName} ( `k1` int(11), `k2` int(11), `v1` int(11), `v2` int(11) ) ENGINE=OLAP
+        unique KEY(`k1`, `k2`) cluster by(v1) DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+        PROPERTIES ( "replication_num" = "1" );
+    """
+    sql """ insert into ${tableName} values(10, 20, 30, 40); """
+
+    // alter table
+    block_convert_historical_rowsets()
+    sql """ alter table ${tableName} order by(k1, k2, v2, v1); """
+    getAlterTableState("RUNNING")
+
+    def tablets = sql_return_maparray """ show tablets from ${tableName}; """
+    logger.info("tablets: ${tablets}")
+    assertEquals(2, tablets.size())
+    String alterTabletId = ""
+    String alterTabletBackendId = ""
+    String alterTabletCompactionUrl = ""
+    for (Map<String, String> tablet : tablets) {
+        if (tablet["State"] == "ALTER") {
+            alterTabletId = tablet["TabletId"].toLong()
+            alterTabletBackendId = tablet["BackendId"]
+            alterTabletCompactionUrl = tablet["CompactionStatus"]
+        }
+    }
+    logger.info("alterTabletId: ${alterTabletId}, alterTabletBackendId: ${alterTabletBackendId}, alterTabletCompactionUrl: ${alterTabletCompactionUrl}")
+    assertTrue(!alterTabletId.isEmpty())
+
+    // write some data
+    sql """ insert into ${tableName} values(10, 20, 31, 40); """
+    sql """ insert into ${tableName} values(10, 20, 32, 40); """
+    sql """ insert into ${tableName} values(10, 20, 33, 40); """
+    sql """ insert into ${tableName} values(10, 20, 34, 40); """
+    sql """ insert into ${tableName} values(10, 20, 35, 40); """
+    order_qt_select1 """ select * from ${tableName}; """
+
+    // trigger compaction
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort)
+    logger.info("ip: " + backendId_to_backendIP.get(alterTabletBackendId) + ", port: " + backendId_to_backendHttpPort.get(alterTabletBackendId))
+    def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(alterTabletBackendId), backendId_to_backendHttpPort.get(alterTabletBackendId), alterTabletId+"")
+    logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+
+    // wait for compaction done
+    def enable_new_tablet_do_compaction = get_be_param.call("enable_new_tablet_do_compaction")
+    logger.info("enable_new_tablet_do_compaction: " + enable_new_tablet_do_compaction)
+    boolean enable = enable_new_tablet_do_compaction.get(alterTabletBackendId).toBoolean()
+    logger.info("enable: " + enable)
+    for (int i = 0; i < 10; i++) {
+        (code, out, err) = curl("GET", alterTabletCompactionUrl)
+        logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+        assertEquals(code, 0)
+        def tabletJson = parseJson(out.trim())
+        assert tabletJson.rowsets instanceof List
+        if (isCloudMode()) {
+            if (enable) {
+                if(tabletJson.rowsets.size() < 5) {
+                    break
+                }
+            } else {
+                // "msg": "invalid tablet state. tablet_id="
+                break
+            }
+        } else {
+            if(tabletJson.rowsets.size() < 5) {
+                break
+            }
+        }
+        sleep(2000)
+    }
+
+    // unblock
+    unblock()
+    sql """ insert into ${tableName}(k1, k2, v1, v2) values(10, 20, 36, 40), (11, 20, 36, 40); """
+    sql """ insert into ${tableName}(k1, k2, v1, v2) values(10, 20, 37, 40), (11, 20, 37, 40); """
+    getAlterTableState("FINISHED")
+    order_qt_select2 """ select * from ${tableName}; """
+}

From b8dcac05bae57f2a2c87f5f2f2d1acdcf5912850 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Wed, 4 Dec 2024 10:03:54 +0800
Subject: [PATCH 148/399] [fix](scanner) Delete meaningless finish dependency
 in schema scanner (#44915)

Finish dependency is used to block `close` phase in order to release
some necessary resources safely. In schema scanner, an async thread is
used to do scanning and finish dependency is used to wait it done. But
in fact, we could close schema scanner directly now so this PR delete
this finish dependency.
---
 be/src/exec/schema_scanner.cpp                | 4 ----
 be/src/exec/schema_scanner.h                  | 7 +------
 be/src/pipeline/exec/schema_scan_operator.cpp | 5 +----
 be/src/pipeline/exec/schema_scan_operator.h   | 5 -----
 4 files changed, 2 insertions(+), 19 deletions(-)

diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp
index 39dd45163322ac..4b430f04289d04 100644
--- a/be/src/exec/schema_scanner.cpp
+++ b/be/src/exec/schema_scanner.cpp
@@ -124,7 +124,6 @@ Status SchemaScanner::get_next_block_async(RuntimeState* state) {
                 }
                 SCOPED_ATTACH_TASK(state);
                 _async_thread_running = true;
-                _finish_dependency->block();
                 if (!_opened) {
                     _data_block = vectorized::Block::create_unique();
                     _init_block(_data_block.get());
@@ -140,9 +139,6 @@ Status SchemaScanner::get_next_block_async(RuntimeState* state) {
                 _eos = eos;
                 _async_thread_running = false;
                 _dependency->set_ready();
-                if (eos) {
-                    _finish_dependency->set_ready();
-                }
             }));
     return Status::OK();
 }
diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h
index 440912bff1d729..6e7a229b7fd7b9 100644
--- a/be/src/exec/schema_scanner.h
+++ b/be/src/exec/schema_scanner.h
@@ -106,11 +106,7 @@ class SchemaScanner {
     // factory function
     static std::unique_ptr<SchemaScanner> create(TSchemaTableType::type type);
     TSchemaTableType::type type() const { return _schema_table_type; }
-    void set_dependency(std::shared_ptr<pipeline::Dependency> dep,
-                        std::shared_ptr<pipeline::Dependency> fin_dep) {
-        _dependency = dep;
-        _finish_dependency = fin_dep;
-    }
+    void set_dependency(std::shared_ptr<pipeline::Dependency> dep) { _dependency = dep; }
     Status get_next_block_async(RuntimeState* state);
 
 protected:
@@ -139,7 +135,6 @@ class SchemaScanner {
     RuntimeProfile::Counter* _fill_block_timer = nullptr;
 
     std::shared_ptr<pipeline::Dependency> _dependency = nullptr;
-    std::shared_ptr<pipeline::Dependency> _finish_dependency = nullptr;
 
     std::unique_ptr<vectorized::Block> _data_block;
     AtomicStatus _scanner_status;
diff --git a/be/src/pipeline/exec/schema_scan_operator.cpp b/be/src/pipeline/exec/schema_scan_operator.cpp
index 006ecf8ad82e84..ddc2821cac14a1 100644
--- a/be/src/pipeline/exec/schema_scan_operator.cpp
+++ b/be/src/pipeline/exec/schema_scan_operator.cpp
@@ -48,7 +48,7 @@ Status SchemaScanLocalState::init(RuntimeState* state, LocalStateInfo& info) {
     // new one scanner
     _schema_scanner = SchemaScanner::create(schema_table->schema_table_type());
 
-    _schema_scanner->set_dependency(_data_dependency, _finish_dependency);
+    _schema_scanner->set_dependency(_data_dependency);
     if (nullptr == _schema_scanner) {
         return Status::InternalError("schema scanner get nullptr pointer.");
     }
@@ -266,9 +266,6 @@ Status SchemaScanOperatorX::get_block(RuntimeState* state, vectorized::Block* bl
     } while (block->rows() == 0 && !*eos);
 
     local_state.reached_limit(block, eos);
-    if (*eos) {
-        local_state._finish_dependency->set_always_ready();
-    }
     return Status::OK();
 }
 
diff --git a/be/src/pipeline/exec/schema_scan_operator.h b/be/src/pipeline/exec/schema_scan_operator.h
index 03cf422fbc52e6..c8ddf885e98a0f 100644
--- a/be/src/pipeline/exec/schema_scan_operator.h
+++ b/be/src/pipeline/exec/schema_scan_operator.h
@@ -36,9 +36,6 @@ class SchemaScanLocalState final : public PipelineXLocalState<> {
 
     SchemaScanLocalState(RuntimeState* state, OperatorXBase* parent)
             : PipelineXLocalState<>(state, parent) {
-        _finish_dependency =
-                std::make_shared<Dependency>(parent->operator_id(), parent->node_id(),
-                                             parent->get_name() + "_FINISH_DEPENDENCY", true);
         _data_dependency = std::make_shared<Dependency>(parent->operator_id(), parent->node_id(),
                                                         parent->get_name() + "_DEPENDENCY", true);
     }
@@ -48,7 +45,6 @@ class SchemaScanLocalState final : public PipelineXLocalState<> {
 
     Status open(RuntimeState* state) override;
 
-    Dependency* finishdependency() override { return _finish_dependency.get(); }
     std::vector<Dependency*> dependencies() const override { return {_data_dependency.get()}; }
 
 private:
@@ -57,7 +53,6 @@ class SchemaScanLocalState final : public PipelineXLocalState<> {
     SchemaScannerParam _scanner_param;
     std::unique_ptr<SchemaScanner> _schema_scanner;
 
-    std::shared_ptr<Dependency> _finish_dependency;
     std::shared_ptr<Dependency> _data_dependency;
 };
 

From 1aa60f5186d19c3cb01a6d9732306ecf6df94606 Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Wed, 4 Dec 2024 10:15:54 +0800
Subject: [PATCH 149/399] [opt](exec) Use PASSTHROUGH to improve the
 concurrency of the ADAPTIVE_PASSTHROUGH SINK. (#44925)

before
```
op -> local sink(1) -> local source (n)
```
now
```
op -> local passthrough(1) -> local passthrough(n) ->  local sink(n) -> local source (n)
```

profile
```
                  Pipeline  :  1(instance_num=3):
                      AGGREGATION_SINK_OPERATOR  (id=4  ,  nereids_id=255):
                          CROSS_JOIN_OPERATOR  (id=3  ,  nereids_id=245):
                              LOCAL_EXCHANGE_OPERATOR  (ADAPTIVE_PASSTHROUGH)  (id=-5):
                  Pipeline  :  2(instance_num=3):
                      LOCAL_EXCHANGE_SINK_OPERATOR  (ADAPTIVE_PASSTHROUGH)  (id=-5):
                          LOCAL_EXCHANGE_OPERATOR  (PASSTHROUGH)  (id=-6):
                  Pipeline  :  3(instance_num=1):
                      LOCAL_EXCHANGE_SINK_OPERATOR  (PASSTHROUGH)  (id=-6):
                          OLAP_SCAN_OPERATOR  (id=2.  nereids_id=234.  table  name  =  nums1(nums1)):
```
---
 be/src/pipeline/pipeline.h                    |  8 ++++++++
 be/src/pipeline/pipeline_fragment_context.cpp | 10 +++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/be/src/pipeline/pipeline.h b/be/src/pipeline/pipeline.h
index b969186b178bf7..afbe6c77596432 100644
--- a/be/src/pipeline/pipeline.h
+++ b/be/src/pipeline/pipeline.h
@@ -73,6 +73,14 @@ class Pipeline : public std::enable_shared_from_this<Pipeline> {
         return idx == ExchangeType::HASH_SHUFFLE || idx == ExchangeType::BUCKET_HASH_SHUFFLE;
     }
 
+    // For HASH_SHUFFLE, BUCKET_HASH_SHUFFLE, and ADAPTIVE_PASSTHROUGH,
+    // data is processed and shuffled on the sink.
+    // Compared to PASSTHROUGH, this is a relatively heavy operation.
+    static bool heavy_operations_on_the_sink(ExchangeType idx) {
+        return idx == ExchangeType::HASH_SHUFFLE || idx == ExchangeType::BUCKET_HASH_SHUFFLE ||
+               idx == ExchangeType::ADAPTIVE_PASSTHROUGH;
+    }
+
     bool need_to_local_exchange(const DataDistribution target_data_distribution,
                                 const int idx) const;
     void init_data_distribution() {
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index 0775ef3fb19826..e6f257f9da792a 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -814,7 +814,7 @@ Status PipelineFragmentContext::_add_local_exchange_impl(
     }
     case ExchangeType::ADAPTIVE_PASSTHROUGH:
         shared_state->exchanger = AdaptivePassthroughExchanger::create_unique(
-                cur_pipe->num_tasks(), _num_instances,
+                std::max(cur_pipe->num_tasks(), _num_instances), _num_instances,
                 _runtime_state->query_options().__isset.local_exchange_free_blocks_limit
                         ? cast_set<int>(
                                   _runtime_state->query_options().local_exchange_free_blocks_limit)
@@ -915,9 +915,13 @@ Status PipelineFragmentContext::_add_local_exchange(
             << " cur_pipe->operators().size(): " << cur_pipe->operators().size()
             << " new_pip->operators().size(): " << new_pip->operators().size();
 
-    // Add passthrough local exchanger if necessary
+    // There are some local shuffles with relatively heavy operations on the sink.
+    // If the local sink concurrency is 1 and the local source concurrency is n, the sink becomes a bottleneck.
+    // Therefore, local passthrough is used to increase the concurrency of the sink.
+    // op -> local sink(1) -> local source (n)
+    // op -> local passthrough(1) -> local passthrough(n) ->  local sink(n) -> local source (n)
     if (cur_pipe->num_tasks() > 1 && new_pip->num_tasks() == 1 &&
-        Pipeline::is_hash_exchange(data_distribution.distribution_type)) {
+        Pipeline::heavy_operations_on_the_sink(data_distribution.distribution_type)) {
         RETURN_IF_ERROR(_add_local_exchange_impl(
                 cast_set<int>(new_pip->operators().size()), pool, new_pip,
                 add_pipeline(new_pip, pip_idx + 2), DataDistribution(ExchangeType::PASSTHROUGH),

From 170b5d33de53d900873934ba9c9525c85a3d7e43 Mon Sep 17 00:00:00 2001
From: airborne12 <jiangkai@selectdb.com>
Date: Wed, 4 Dec 2024 10:35:24 +0800
Subject: [PATCH 150/399] [fix](recycler) fix idx file leak in recycler
 (#44908)

Related PR: #38306

Problem Summary:
Non inverted index storage format for old version will cause leaking
recycling idx files.
---
 cloud/src/recycler/recycler.cpp |  28 +++++----
 cloud/test/recycler_test.cpp    | 105 ++++++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+), 11 deletions(-)

diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 1b21ec68916be6..f7000ea379225f 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -417,8 +417,12 @@ class InstanceRecycler::InvertedIndexIdCache {
             LOG(WARNING) << "malformed schema value, key=" << hex(schema_key);
             return -1;
         }
-        if (schema.index_size() > 0 && schema.has_inverted_index_storage_format()) {
-            res.first = schema.inverted_index_storage_format();
+        if (schema.index_size() > 0) {
+            InvertedIndexStorageFormatPB index_format = InvertedIndexStorageFormatPB::V1;
+            if (schema.has_inverted_index_storage_format()) {
+                index_format = schema.inverted_index_storage_format();
+            }
+            res.first = index_format;
             res.second.reserve(schema.index_size());
             for (auto& i : schema.index()) {
                 if (i.has_index_type() && i.index_type() == IndexType::INVERTED) {
@@ -1382,17 +1386,19 @@ int InstanceRecycler::delete_rowset_data(const doris::RowsetMetaCloudPB& rs_meta
     }
     std::vector<std::string> file_paths;
     auto tablet_schema = rs_meta_pb.tablet_schema();
+    auto index_storage_format = InvertedIndexStorageFormatPB::V1;
     for (int64_t i = 0; i < num_segments; ++i) {
         file_paths.push_back(segment_path(tablet_id, rowset_id, i));
         if (tablet_schema.has_inverted_index_storage_format()) {
-            if (tablet_schema.inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
-                for (const auto& index_id : index_ids) {
-                    file_paths.push_back(inverted_index_path_v1(tablet_id, rowset_id, i,
-                                                                index_id.first, index_id.second));
-                }
-            } else if (!index_ids.empty()) {
-                file_paths.push_back(inverted_index_path_v2(tablet_id, rowset_id, i));
+            index_storage_format = tablet_schema.inverted_index_storage_format();
+        }
+        if (index_storage_format == InvertedIndexStorageFormatPB::V1) {
+            for (const auto& index_id : index_ids) {
+                file_paths.push_back(inverted_index_path_v1(tablet_id, rowset_id, i, index_id.first,
+                                                            index_id.second));
             }
+        } else if (!index_ids.empty()) {
+            file_paths.push_back(inverted_index_path_v2(tablet_id, rowset_id, i));
         }
     }
     // TODO(AlexYue): seems could do do batch
@@ -1429,8 +1435,8 @@ int InstanceRecycler::delete_rowset_data(const std::vector<doris::RowsetMetaClou
 
         // Process inverted indexes
         std::vector<std::pair<int64_t, std::string>> index_ids;
-        // default format as v2.
-        InvertedIndexStorageFormatPB index_format = InvertedIndexStorageFormatPB::V2;
+        // default format as v1.
+        InvertedIndexStorageFormatPB index_format = InvertedIndexStorageFormatPB::V1;
 
         if (rs.has_tablet_schema()) {
             for (const auto& index : rs.tablet_schema().index()) {
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index feecf9552f933b..0bc16644a82041 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -3077,4 +3077,109 @@ TEST(RecyclerTest, delete_rowset_data) {
     }
 }
 
+TEST(RecyclerTest, delete_rowset_data_without_inverted_index_storage_format) {
+    auto txn_kv = std::make_shared<MemTxnKv>();
+    ASSERT_EQ(txn_kv->init(), 0);
+
+    InstanceInfoPB instance;
+    instance.set_instance_id(instance_id);
+    auto obj_info = instance.add_obj_info();
+    obj_info->set_id("recycle_tmp_rowsets");
+    obj_info->set_ak(config::test_s3_ak);
+    obj_info->set_sk(config::test_s3_sk);
+    obj_info->set_endpoint(config::test_s3_endpoint);
+    obj_info->set_region(config::test_s3_region);
+    obj_info->set_bucket(config::test_s3_bucket);
+    obj_info->set_prefix("recycle_tmp_rowsets");
+
+    std::vector<doris::TabletSchemaCloudPB> schemas;
+    for (int i = 0; i < 5; ++i) {
+        auto& schema = schemas.emplace_back();
+        schema.set_schema_version(i);
+        //schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
+        for (int j = 0; j < i; ++j) {
+            auto index = schema.add_index();
+            index->set_index_id(j);
+            index->set_index_type(IndexType::INVERTED);
+        }
+    }
+
+    {
+        InstanceRecycler recycler(txn_kv, instance, thread_group,
+                                  std::make_shared<TxnLazyCommitter>(txn_kv));
+        ASSERT_EQ(recycler.init(), 0);
+        auto accessor = recycler.accessor_map_.begin()->second;
+        int64_t txn_id_base = 114115;
+        int64_t tablet_id_base = 10015;
+        int64_t index_id_base = 1000;
+        // Delete each rowset directly using one RowsetPB
+        for (int i = 0; i < 100; ++i) {
+            int64_t txn_id = txn_id_base + i;
+            for (int j = 0; j < 20; ++j) {
+                auto rowset = create_rowset("recycle_tmp_rowsets", tablet_id_base + j,
+                                            index_id_base + j % 4, 5, schemas[i % 5], txn_id);
+                create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, i & 1);
+                ASSERT_EQ(0, recycler.delete_rowset_data(rowset));
+            }
+        }
+
+        std::unique_ptr<ListIterator> list_iter;
+        ASSERT_EQ(0, accessor->list_all(&list_iter));
+        ASSERT_FALSE(list_iter->has_next());
+    }
+    {
+        InstanceInfoPB tmp_instance;
+        std::string resource_id = "recycle_tmp_rowsets";
+        tmp_instance.set_instance_id(instance_id);
+        auto tmp_obj_info = tmp_instance.add_obj_info();
+        tmp_obj_info->set_id(resource_id);
+        tmp_obj_info->set_ak(config::test_s3_ak);
+        tmp_obj_info->set_sk(config::test_s3_sk);
+        tmp_obj_info->set_endpoint(config::test_s3_endpoint);
+        tmp_obj_info->set_region(config::test_s3_region);
+        tmp_obj_info->set_bucket(config::test_s3_bucket);
+        tmp_obj_info->set_prefix(resource_id);
+
+        InstanceRecycler recycler(txn_kv, tmp_instance, thread_group,
+                                  std::make_shared<TxnLazyCommitter>(txn_kv));
+        ASSERT_EQ(recycler.init(), 0);
+        auto accessor = recycler.accessor_map_.begin()->second;
+        // Delete multiple rowset files using one series of RowsetPB
+        constexpr int index_id = 10001, tablet_id = 10002;
+        std::vector<doris::RowsetMetaCloudPB> rowset_pbs;
+        for (int i = 0; i < 10; ++i) {
+            auto rowset = create_rowset(resource_id, tablet_id, index_id, 5, schemas[i % 5]);
+            create_recycle_rowset(
+                    txn_kv.get(), accessor.get(), rowset,
+                    static_cast<RecycleRowsetPB::Type>(i % (RecycleRowsetPB::Type_MAX + 1)), true);
+
+            rowset_pbs.emplace_back(std::move(rowset));
+        }
+        ASSERT_EQ(0, recycler.delete_rowset_data(rowset_pbs));
+        std::unique_ptr<ListIterator> list_iter;
+        ASSERT_EQ(0, accessor->list_all(&list_iter));
+        ASSERT_FALSE(list_iter->has_next());
+    }
+    {
+        InstanceRecycler recycler(txn_kv, instance, thread_group,
+                                  std::make_shared<TxnLazyCommitter>(txn_kv));
+        ASSERT_EQ(recycler.init(), 0);
+        auto accessor = recycler.accessor_map_.begin()->second;
+        // Delete multiple rowset files using one series of RowsetPB
+        constexpr int index_id = 20001, tablet_id = 20002;
+        // Delete each rowset file directly using it's id to construct one path
+        for (int i = 0; i < 1000; ++i) {
+            auto rowset =
+                    create_rowset("recycle_tmp_rowsets", tablet_id, index_id, 5, schemas[i % 5]);
+            create_recycle_rowset(txn_kv.get(), accessor.get(), rowset, RecycleRowsetPB::COMPACT,
+                                  true);
+            ASSERT_EQ(0, recycler.delete_rowset_data(rowset.resource_id(), rowset.tablet_id(),
+                                                     rowset.rowset_id_v2()));
+        }
+        std::unique_ptr<ListIterator> list_iter;
+        ASSERT_EQ(0, accessor->list_all(&list_iter));
+        ASSERT_FALSE(list_iter->has_next());
+    }
+}
+
 } // namespace doris::cloud

From bb96c24dc771704cec2960db348261a2a26d8550 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Wed, 4 Dec 2024 10:43:56 +0800
Subject: [PATCH 151/399] [fix](memory) Fix metadata memory tracking and
 profile (#44739)

### What problem does this PR solve?

Problem Summary:

1. `MemTrackerLimiter` and `MemoryProfile` add metadata and cache type.
2. metadata memory consists of rowsets, segments, and tablets.
3. `all_segments_mem_tracker` excludes the segments in SegmentCache, and
all segments use the same memory calculation method.
4. segments memory is estimated, which is not accurate and can only be
used as a reference.
5. rowsets and tablets only count part of the memory, mainly PB.

TODO, Metadata also has some memory that can be allocated using Doris
Allocator.

```
MemoryProfile:
  MemoryOverviewSnapshot:
     - PhysicalMemory(VmRSS) Current: 5.85 GB (Peak: 7.13 GB)
     - VirtualMemory(VmSize) Current: 61.35 GB (Peak: 61.67 GB)
    UntrackedMemory:
       - Memory Current: 3.61 GB (Peak: 3.77 GB)
    TrackedMemory:
       - Memory Current: 2.24 GB (Peak: 3.48 GB)
      TasksMemory:
         - Memory Current: 0 (Peak: 749.17 MB)
           - ReservedMemory Current: 0 (Peak: 0)
        Details:
           - Compaction Current: 0 (Peak: 13.60 MB)
           - Load Current: 0 (Peak: 749.17 MB)
             - AllMemTablesMemory Current: 0 (Peak: 853.20 MB)
           - Other Current: 0 (Peak: 0)
           - Query Current: 0 (Peak: 65.20 MB)
           - SchemaChange Current: 0 (Peak: 0)
      GlobalMemory:
         - Memory Current: 55.03 MB (Peak: 70.39 MB)
      MetadataMemory:
         - Memory Current: 972.13 MB (Peak: 972.13 MB)
      CacheMemory:
         - Memory Current: 360.21 MB (Peak: 675.32 MB)
      JemallocMemory:
         - Memory Current: 905.55 MB (Peak: 1.80 GB)
        Details:
           - Cache Current: 646.47 MB (Peak: 1.59 GB)
           - Metadata Current: 259.08 MB (Peak: 259.72 MB)
  GlobalMemorySnapshot:
    Orphan@global@id=3144f940fa84f930-d82d459289789494:
       - Memory Current: 1.73 MB (Peak: 1.73 MB)
    IOBufBlockMemory@global@id=1f4316bc5fb4c3d4-73f14d3652ed269b:
       - Memory Current: 2.65 GB (Peak: 2.65 GB)
    PointQueryExecutor@global@id=794878deaa4d8f56-769fbc87ccb7ec8d:
       - Memory Current: 36.90 MB (Peak: 36.90 MB)
    BlockCompression@global@id=0a4d429b1915dd8b-c2e429b1186d69a8:
       - Memory Current: 563.01 MB (Peak: 563.01 MB)
    RowIdStorageReader@global@id=a546ca5878c9204e-c4c72ea9088ca9a0:
       - Memory Current: 101.25 KB (Peak: 101.25 KB)
    SubcolumnsTree@global@id=294bde93c01d4ba6-4ce7b3290489a1b8:
       - Memory Current: 95.61 MB (Peak: 95.61 MB)
    S3FileBuffer@global@id=bc437d624d9a234d-03eb4bbc3d030a99:
       - Memory Current: 0 (Peak: 0)
  MetadataMemorySnapshot:
    Tablets(not in SchemaCache, TabletSchemaCache)@metadata@id=7646d676ad9f8de9-98beb5f1a8aa739b:
       - Memory Current: 26.84 GB (Peak: 26.84 GB)
    Segments(not in SegmentCache)@metadata@id=864b4887722b327a-516c912d4602b59b:
       - Memory Current: 4.98 MB (Peak: 4.98 MB)
    Rowsets@metadata@id=f0440aa247730497-752a527c71e3c393:
       - Memory Current: 5.57 GB (Peak: 5.57 GB)
    SegmentCache[size]@metadata@id=b34e798db2c25109-04c7ff3c722e4cbb:
       - Memory Current: 4.86 GB (Peak: 4.86 GB)
    SchemaCache[number]@metadata@id=9f41f61531da3fd9-163834a6a6f83eba:
       - Memory Current: 103.38 MB (Peak: 103.38 MB)
    TabletSchemaCache[number]@metadata@id=f9475c16ba7ea1ab-1d28b38568ef0b8d:
       - Memory Current: 170.65 MB (Peak: 170.65 MB)
  CacheMemorySnapshot:
    QueryCache@cache@id=504af426a68103cf-a60f36a48bc1998a:
       - Memory Current: 0 (Peak: 0)
    DataPageCache[size]@cache@id=f744bb4ef53c307b-4c021a8f9b8a77a5:
       - Memory Current: 14.33 GB (Peak: 14.33 GB)
    IndexPageCache[size]@cache@id=cb4ed0e456f081d4-d635edda9aa2778b:
       - Memory Current: 5.25 GB (Peak: 5.25 GB)
    PKIndexPageCache[size]@cache@id=8e4bf12b4d39133a-e0d878a18d73619b:
       - Memory Current: 2.72 GB (Peak: 2.72 GB)
    PointQueryRowCache[size]@cache@id=024dc624c80205f1-ca3221a154e4b68a:
       - Memory Current: 94.84 KB (Peak: 94.84 KB)
    CommonObjLRUCache[number]@cache@id=af40d4a03b391540-43338381c8e826a4:
       - Memory Current: 0 (Peak: 0)
    PointQueryLookupConnectionCache[number]@cache@id=284fa848dd97ccc8-56c61386a4950784:
       - Memory Current: 337.97 KB (Peak: 337.97 KB)
    InvertedIndexSearcherCache[size]@cache@id=f5404416ad0a9465-3cf12f5224276d8a:
       - Memory Current: 111.52 MB (Peak: 111.52 MB)
    InvertedIndexQueryCache[size]@cache@id=6f4d3c55508e3531-3fafe328c6d3e1ab:
       - Memory Current: 51.97 MB (Peak: 51.97 MB)
    QueryCache[size]@cache@id=274d47670e0796ad-542ad958334ff988:
       - Memory Current: 0 (Peak: 0)
    LastSuccessChannelCache[size]@cache@id=08449badfd6abb81-5b30599bfc60f79f:
       - Memory Current: 0 (Peak: 0)
    TabletColumnObjectPool[number]@cache@id=7b42da8391945265-06697f83a6bf01a7:
       - Memory Current: 151.67 MB (Peak: 151.67 MB)
    MowTabletVersionCache[number]@cache@id=f345a1adbd2a87dd-74b9caa3b30cf18b:
       - Memory Current: 150.94 MB (Peak: 150.94 MB)
    CreateTabletRRIdxCache[number]@cache@id=f043eee91e9aef2d-ef3a2245d50545ae:
       - Memory Current: 15.20 MB (Peak: 15.20 MB)
    MowDeleteBitmapAggCache[size]@cache@id=df47a8ed5bf3f11f-ac944d7d7e79d4b3:
       - Memory Current: 181.11 MB (Peak: 181.11 MB)
  TopMemoryTasksSnapshot:
    SnapshotManager@other_task@id=11480ad6017368b9-8aef2150427d229f:
       - Memory Current: 0 (Peak: 0)
    StreamLoadPipe@load@id=024958077445f4f7-d739fd5231dca890:
       - Memory Current: 1.39 GB (Peak: 1.39 GB)
    SegCompaction@compaction@id=2d4d5c796a9f48ee-fdf5dd9f8faf83a8:
       - Memory Current: 0 (Peak: 0)
    Load#Id=544468c7f5a9bac6-2c78fc9962f3908d@load@id=52497774c12c5ad1-1fda35667a3d0e8b:
       - Limit: 2.00 GB
       - Memory Current: 6.06 MB (Peak: 6.06 MB)
    Load#Id=624e7947099ef183-577a22cca9d34d89@load@id=ed4691ce5a748ca8-e49ef7cc9be10b95:
       - Limit: 2.00 GB
       - Memory Current: 6.06 MB (Peak: 6.06 MB)
 ```
---
 be/src/common/daemon.cpp                      |  11 +-
 be/src/olap/delta_writer_v2.h                 |   1 -
 be/src/olap/memtable_writer.h                 |   1 -
 be/src/olap/metadata_adder.h                  |  73 +++--
 .../segment_v2/indexed_column_reader.cpp      |   5 +-
 .../rowset/segment_v2/indexed_column_reader.h |  15 +-
 be/src/olap/rowset/segment_v2/page_handle.h   |  12 +-
 be/src/olap/rowset/segment_v2/segment.cpp     |  17 +-
 be/src/olap/rowset/segment_v2/segment.h       |   7 +-
 be/src/olap/rowset_builder.h                  |   1 -
 be/src/olap/segment_loader.cpp                |   3 +-
 be/src/olap/segment_loader.h                  |  13 +-
 be/src/olap/tablet_manager.cpp                |  19 +-
 be/src/olap/tablet_manager.h                  |   3 -
 be/src/olap/tablet_meta.cpp                   |   6 -
 be/src/olap/tablet_meta.h                     |   4 -
 be/src/olap/tablet_schema.cpp                 |   2 -
 be/src/olap/tablet_schema.h                   |   7 +-
 be/src/olap/tablet_schema_cache.cpp           |   2 +-
 be/src/runtime/exec_env.h                     |  16 +-
 be/src/runtime/exec_env_init.cpp              |  13 +-
 be/src/runtime/memory/cache_policy.h          |   6 +
 be/src/runtime/memory/lru_cache_policy.h      |  38 ++-
 be/src/runtime/memory/lru_cache_value_base.h  |  11 +-
 be/src/runtime/memory/mem_tracker_limiter.cpp |  58 +++-
 be/src/runtime/memory/mem_tracker_limiter.h   |  14 +-
 be/src/runtime/memory/memory_profile.cpp      | 305 +++++++++---------
 be/src/runtime/memory/memory_profile.h        |  69 +++-
 be/src/util/runtime_profile.cpp               |   4 +-
 be/src/util/runtime_profile.h                 |  45 ++-
 be/src/vec/runtime/partitioner.h              |   1 -
 be/src/vec/sink/vdata_stream_sender.h         |   1 -
 32 files changed, 464 insertions(+), 319 deletions(-)

diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index ce2a6878dba034..d3d55f10dde5fb 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -500,15 +500,18 @@ void Daemon::cache_adjust_capacity_thread() {
 void Daemon::cache_prune_stale_thread() {
     int32_t interval = config::cache_periodic_prune_stale_sweep_sec;
     while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))) {
-        if (interval <= 0) {
-            LOG(WARNING) << "config of cache clean interval is illegal: [" << interval
-                         << "], force set to 3600 ";
-            interval = 3600;
+        if (config::cache_periodic_prune_stale_sweep_sec <= 0) {
+            LOG(WARNING) << "config of cache clean interval is: [" << interval
+                         << "], it means the cache prune stale thread is disabled, will wait 3s "
+                            "and check again.";
+            interval = 3;
+            continue;
         }
         if (config::disable_memory_gc) {
             continue;
         }
         CacheManager::instance()->for_each_cache_prune_stale();
+        interval = config::cache_periodic_prune_stale_sweep_sec;
     }
 }
 
diff --git a/be/src/olap/delta_writer_v2.h b/be/src/olap/delta_writer_v2.h
index beeb3d3ecd3ec5..f9c2800a68f499 100644
--- a/be/src/olap/delta_writer_v2.h
+++ b/be/src/olap/delta_writer_v2.h
@@ -46,7 +46,6 @@ namespace doris {
 
 class FlushToken;
 class MemTable;
-class MemTracker;
 class Schema;
 class StorageEngine;
 class TupleDescriptor;
diff --git a/be/src/olap/memtable_writer.h b/be/src/olap/memtable_writer.h
index fb07e740fa3cf6..713400793a1754 100644
--- a/be/src/olap/memtable_writer.h
+++ b/be/src/olap/memtable_writer.h
@@ -45,7 +45,6 @@ namespace doris {
 
 class FlushToken;
 class MemTable;
-class MemTracker;
 class StorageEngine;
 class TupleDescriptor;
 class SlotDescriptor;
diff --git a/be/src/olap/metadata_adder.h b/be/src/olap/metadata_adder.h
index 559c5db873b2f6..5b5ba16322490e 100644
--- a/be/src/olap/metadata_adder.h
+++ b/be/src/olap/metadata_adder.h
@@ -20,6 +20,8 @@
 #include <bvar/bvar.h>
 #include <stdint.h>
 
+#include "runtime/exec_env.h"
+#include "runtime/memory/mem_tracker_limiter.h"
 #include "util/runtime_profile.h"
 
 namespace doris {
@@ -27,8 +29,8 @@ namespace doris {
 inline bvar::Adder<int64_t> g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");
 
-inline bvar::Adder<int64_t> g_all_rowsets_mem_bytes("doris_all_rowsets_mem_bytes");
-inline bvar::Adder<int64_t> g_all_rowsets_num("doris_all_rowsets_num");
+inline bvar::Adder<int64_t> g_rowset_mem_bytes("doris_rowset_mem_bytes");
+inline bvar::Adder<int64_t> g_rowset_num("doris_rowset_num");
 
 inline bvar::Adder<int64_t> g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");
@@ -42,8 +44,9 @@ inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
 inline bvar::Adder<int64_t> g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
 inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");
 
-inline bvar::Adder<int64_t> g_all_segments_mem_bytes("doris_all_segments_mem_bytes");
-inline bvar::Adder<int64_t> g_all_segments_num("doris_all_segments_num");
+inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
+inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
+inline bvar::Adder<int64_t> g_segment_estimate_mem_bytes("doris_segment_estimate_mem_bytes");
 
 inline bvar::Adder<int64_t> g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
 inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
@@ -96,6 +99,10 @@ class ZoneMapIndexReader;
     When a derived Class extends MetadataAdder, then the Class's number and fixed length field's memory can be counted automatically.
     But if the Class has variable length field, then you should overwrite get_metadata_size and call update_metadata_size when the Class's memory changes.
 
+    get_metadata_size is only the memory of the metadata object itself, not include child objects,
+    for example, TabletMeta::get_metadata_size does not include the memory of TabletSchema.
+    Note, the memory allocated by Doris Allocator is not included.
+
     There are some special situations that need to be noted:
     1. when the derived Class override copy constructor, you'd better update memory size(call update_metadata_size) if derived class's 
     memory changed in its copy constructor or you not call MetadataAdder's copy constructor.
@@ -111,6 +118,31 @@ class MetadataAdder {
 
     static void dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot);
 
+    static int64_t get_all_tablets_size() {
+        return g_tablet_meta_mem_bytes.get_value() + g_tablet_column_mem_bytes.get_value() +
+               g_tablet_index_mem_bytes.get_value() + g_tablet_schema_mem_bytes.get_value();
+    }
+
+    static int64_t get_all_rowsets_size() {
+        return g_rowset_meta_mem_bytes.get_value() + g_rowset_mem_bytes.get_value();
+    }
+
+    static int64_t get_all_segments_size() {
+        return g_segment_mem_bytes.get_value() + g_column_reader_mem_bytes.get_value() +
+               g_bitmap_index_reader_mem_bytes.get_value() +
+               g_bloom_filter_index_reader_mem_bytes.get_value() +
+               g_index_page_reader_mem_bytes.get_value() +
+               g_indexed_column_reader_mem_bytes.get_value() +
+               g_inverted_index_reader_mem_bytes.get_value() +
+               g_ordinal_index_reader_mem_bytes.get_value() +
+               g_zone_map_index_reader_mem_bytes.get_value();
+    }
+
+    // Doris currently uses the estimated segments memory as the basis, maybe it is more realistic.
+    static int64_t get_all_segments_estimate_size() {
+        return g_segment_estimate_mem_bytes.get_value();
+    }
+
 protected:
     MetadataAdder(const MetadataAdder& other);
 
@@ -122,7 +154,6 @@ class MetadataAdder {
 
     MetadataAdder<T>& operator=(const MetadataAdder<T>& other) = default;
 
-private:
     int64_t _current_meta_size {0};
 
     void add_mem_size(int64_t val);
@@ -167,7 +198,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, Rowset>) {
-        g_all_rowsets_mem_bytes << val;
+        g_rowset_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -177,7 +208,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_all_segments_mem_bytes << val;
+        g_segment_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_mem_bytes << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -208,7 +239,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
     if constexpr (std::is_same_v<T, RowsetMeta>) {
         g_rowset_meta_num << val;
     } else if constexpr (std::is_same_v<T, Rowset>) {
-        g_all_rowsets_num << val;
+        g_rowset_num << val;
     } else if constexpr (std::is_same_v<T, TabletMeta>) {
         g_tablet_meta_num << val;
     } else if constexpr (std::is_same_v<T, TabletColumn>) {
@@ -218,7 +249,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
     } else if constexpr (std::is_same_v<T, TabletSchema>) {
         g_tablet_schema_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
-        g_all_segments_num << val;
+        g_segment_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
         g_column_reader_num << val;
     } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
@@ -250,12 +281,12 @@ void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_sna
     COUNTER_SET(rowset_meta_mem_bytes_counter, g_rowset_meta_mem_bytes.get_value());
     COUNTER_SET(rowset_meta_num_counter, g_rowset_meta_num.get_value());
 
-    RuntimeProfile::Counter* all_rowsets_mem_bytes_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsMemBytes", TUnit::BYTES);
-    RuntimeProfile::Counter* all_rowsets_num_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsNum", TUnit::UNIT);
-    COUNTER_SET(all_rowsets_mem_bytes_counter, g_all_rowsets_mem_bytes.get_value());
-    COUNTER_SET(all_rowsets_num_counter, g_all_rowsets_num.get_value());
+    RuntimeProfile::Counter* rowset_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* rowset_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "RowsetNum", TUnit::UNIT);
+    COUNTER_SET(rowset_mem_bytes_counter, g_rowset_mem_bytes.get_value());
+    COUNTER_SET(rowset_num_counter, g_rowset_num.get_value());
 
     RuntimeProfile::Counter* tablet_meta_mem_bytes_counter =
             ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaMemBytes", TUnit::BYTES);
@@ -285,12 +316,12 @@ void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_sna
     COUNTER_SET(tablet_schema_mem_bytes_counter, g_tablet_schema_mem_bytes.get_value());
     COUNTER_SET(tablet_schema_num_counter, g_tablet_schema_num.get_value());
 
-    RuntimeProfile::Counter* all_segments_mem_bytes_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsMemBytes", TUnit::BYTES);
-    RuntimeProfile::Counter* all_segments_num_counter =
-            ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsNum", TUnit::UNIT);
-    COUNTER_SET(all_segments_mem_bytes_counter, g_all_segments_mem_bytes.get_value());
-    COUNTER_SET(all_segments_num_counter, g_all_segments_num.get_value());
+    RuntimeProfile::Counter* segment_mem_bytes_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "SegmentMemBytes", TUnit::BYTES);
+    RuntimeProfile::Counter* segment_num_counter =
+            ADD_COUNTER(object_heap_dump_snapshot, "SegmentNum", TUnit::UNIT);
+    COUNTER_SET(segment_mem_bytes_counter, g_segment_mem_bytes.get_value());
+    COUNTER_SET(segment_num_counter, g_segment_num.get_value());
 
     RuntimeProfile::Counter* column_reader_mem_bytes_counter =
             ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderMemBytes", TUnit::BYTES);
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index 3028211f266157..da6beff5d8d6a2 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -81,7 +81,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
             _sole_data_page = PagePointer(_meta.ordinal_index_meta().root_page());
         } else {
             RETURN_IF_ERROR(load_index_page(_meta.ordinal_index_meta().root_page(),
-                                            &_ordinal_index_page_handle, &_ordinal_index_reader));
+                                            &_ordinal_index_page_handle,
+                                            _ordinal_index_reader.get()));
             _has_index_page = true;
         }
     }
@@ -92,7 +93,7 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory,
             _sole_data_page = PagePointer(_meta.value_index_meta().root_page());
         } else {
             RETURN_IF_ERROR(load_index_page(_meta.value_index_meta().root_page(),
-                                            &_value_index_page_handle, &_value_index_reader));
+                                            &_value_index_page_handle, _value_index_reader.get()));
             _has_index_page = true;
         }
     }
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index c3469f9f6bed0d..c9640c0007c153 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -50,9 +50,12 @@ class EncodingInfo;
 class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
 public:
     explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const IndexedColumnMetaPB& meta)
-            : _file_reader(std::move(file_reader)), _meta(meta) {}
+            : _file_reader(std::move(file_reader)), _meta(meta) {
+        _ordinal_index_reader = std::make_unique<IndexPageReader>();
+        _value_index_reader = std::make_unique<IndexPageReader>();
+    }
 
-    ~IndexedColumnReader();
+    ~IndexedColumnReader() override;
 
     Status load(bool use_page_cache, bool kept_in_memory,
                 OlapReaderStatistics* index_load_stats = nullptr);
@@ -90,8 +93,8 @@ class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
     bool _has_index_page = false;
     // valid only when the column contains only one data page
     PagePointer _sole_data_page;
-    IndexPageReader _ordinal_index_reader;
-    IndexPageReader _value_index_reader;
+    std::unique_ptr<IndexPageReader> _ordinal_index_reader;
+    std::unique_ptr<IndexPageReader> _value_index_reader;
     PageHandle _ordinal_index_page_handle;
     PageHandle _value_index_page_handle;
 
@@ -108,8 +111,8 @@ class IndexedColumnIterator {
     explicit IndexedColumnIterator(const IndexedColumnReader* reader,
                                    OlapReaderStatistics* stats = nullptr)
             : _reader(reader),
-              _ordinal_iter(&reader->_ordinal_index_reader),
-              _value_iter(&reader->_value_index_reader),
+              _ordinal_iter(reader->_ordinal_index_reader.get()),
+              _value_iter(reader->_value_index_reader.get()),
               _stats(stats) {}
 
     // Seek to the given ordinal entry. Entry 0 is the first entry.
diff --git a/be/src/olap/rowset/segment_v2/page_handle.h b/be/src/olap/rowset/segment_v2/page_handle.h
index b1e53ee808697e..d4dfdfb2ff3c55 100644
--- a/be/src/olap/rowset/segment_v2/page_handle.h
+++ b/be/src/olap/rowset/segment_v2/page_handle.h
@@ -23,6 +23,10 @@
 #include "util/slice.h" // for Slice
 
 namespace doris {
+
+// After disable page cache, sometimes we need to know the percentage of data pages in query memory.
+inline bvar::Adder<int64_t> g_page_no_cache_mem_bytes("doris_page_no_cache_mem_bytes");
+
 namespace segment_v2 {
 
 // When a column page is read into memory, we use this to store it.
@@ -37,8 +41,7 @@ class PageHandle {
     // This class will take the ownership of input data's memory. It will
     // free it when deconstructs.
     PageHandle(DataPage* data) : _is_data_owner(true), _data(data) {
-        _page_tracker = ExecEnv::GetInstance()->page_no_cache_mem_tracker();
-        _page_tracker->consume(_data->capacity());
+        g_page_no_cache_mem_bytes << _data->capacity();
     }
 
     // This class will take the content of cache data, and will make input
@@ -51,20 +54,18 @@ class PageHandle {
         // we can use std::exchange if we switch c++14 on
         std::swap(_is_data_owner, other._is_data_owner);
         std::swap(_data, other._data);
-        _page_tracker = ExecEnv::GetInstance()->page_no_cache_mem_tracker();
     }
 
     PageHandle& operator=(PageHandle&& other) noexcept {
         std::swap(_is_data_owner, other._is_data_owner);
         std::swap(_data, other._data);
         _cache_data = std::move(other._cache_data);
-        _page_tracker = ExecEnv::GetInstance()->page_no_cache_mem_tracker();
         return *this;
     }
 
     ~PageHandle() {
         if (_is_data_owner) {
-            _page_tracker->release(_data->capacity());
+            g_page_no_cache_mem_bytes << -_data->capacity();
             delete _data;
         } else {
             DCHECK(_data == nullptr);
@@ -85,7 +86,6 @@ class PageHandle {
     // otherwise _cache_data is valid, and data is belong to cache.
     bool _is_data_owner = false;
     DataPage* _data = nullptr;
-    std::shared_ptr<MemTracker> _page_tracker;
     PageCacheHandle _cache_data;
 
     // Don't allow copy and assign
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 0ad799683fc458..d2c2920d4e6f74 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -163,7 +163,11 @@ Segment::Segment(uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr table
           _tablet_schema(std::move(tablet_schema)),
           _idx_file_info(idx_file_info) {}
 
-Segment::~Segment() = default;
+Segment::~Segment() {
+    g_segment_estimate_mem_bytes << -_tracked_meta_mem_usage;
+    // if failed, fix `_tracked_meta_mem_usage` accuracy
+    DCHECK(_tracked_meta_mem_usage == meta_mem_usage());
+}
 
 io::UInt128Wrapper Segment::file_cache_key(std::string_view rowset_id, uint32_t seg_id) {
     return io::BlockFileCache::hash(fmt::format("{}_{}.dat", rowset_id, seg_id));
@@ -174,6 +178,12 @@ int64_t Segment::get_metadata_size() const {
            (_pk_index_meta ? _pk_index_meta->ByteSizeLong() : 0);
 }
 
+void Segment::update_metadata_size() {
+    MetadataAdder::update_metadata_size();
+    g_segment_estimate_mem_bytes << _meta_mem_usage - _tracked_meta_mem_usage;
+    _tracked_meta_mem_usage = _meta_mem_usage;
+}
+
 Status Segment::_open() {
     _footer_pb = std::make_unique<SegmentFooterPB>();
     RETURN_IF_ERROR(_parse_footer(_footer_pb.get()));
@@ -191,8 +201,6 @@ Status Segment::_open() {
         _meta_mem_usage += _pk_index_meta->ByteSizeLong();
     }
 
-    update_metadata_size();
-
     _meta_mem_usage += sizeof(*this);
     _meta_mem_usage += _tablet_schema->num_columns() * config::estimated_mem_per_column_reader;
 
@@ -201,6 +209,8 @@ Status Segment::_open() {
     // 0.01 comes from PrimaryKeyIndexBuilder::init
     _meta_mem_usage += BloomFilter::optimal_bit_num(_num_rows, 0.01) / 8;
 
+    update_metadata_size();
+
     return Status::OK();
 }
 
@@ -467,6 +477,7 @@ Status Segment::_load_pk_bloom_filter() {
         // for BE UT "segment_cache_test"
         return _load_pk_bf_once.call([this] {
             _meta_mem_usage += 100;
+            update_metadata_size();
             return Status::OK();
         });
     }
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index bc5ab1e1fdc80a..1b20c1f066bdf9 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -57,7 +57,6 @@ class IDataType;
 class ShortKeyIndexDecoder;
 class Schema;
 class StorageReadOptions;
-class MemTracker;
 class PrimaryKeyIndexReader;
 class RowwiseIterator;
 struct RowLocation;
@@ -93,6 +92,7 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
     ~Segment();
 
     int64_t get_metadata_size() const override;
+    void update_metadata_size();
 
     Status new_iterator(SchemaSPtr schema, const StorageReadOptions& read_options,
                         std::unique_ptr<RowwiseIterator>* iter);
@@ -163,6 +163,8 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
 
     io::FileReaderSPtr file_reader() { return _file_reader; }
 
+    // Including the column reader memory.
+    // another method `get_metadata_size` not include the column reader, only the segment object itself.
     int64_t meta_mem_usage() const { return _meta_mem_usage; }
 
     // Identify the column by unique id or path info
@@ -249,9 +251,8 @@ class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdd
     // 1. Tracking memory use by segment meta data such as footer or index page.
     // 2. Tracking memory use by segment column reader
     // The memory consumed by querying is tracked in segment iterator.
-    // TODO: Segment::_meta_mem_usage Unknown value overflow, causes the value of SegmentMeta mem tracker
-    // is similar to `-2912341218700198079`. So, temporarily put it in experimental type tracker.
     int64_t _meta_mem_usage;
+    int64_t _tracked_meta_mem_usage = 0;
 
     RowsetId _rowset_id;
     TabletSchemaSPtr _tablet_schema;
diff --git a/be/src/olap/rowset_builder.h b/be/src/olap/rowset_builder.h
index 7fd578037363a0..fb2294d1770cc4 100644
--- a/be/src/olap/rowset_builder.h
+++ b/be/src/olap/rowset_builder.h
@@ -38,7 +38,6 @@ namespace doris {
 class CalcDeleteBitmapToken;
 class FlushToken;
 class MemTable;
-class MemTracker;
 class StorageEngine;
 class TupleDescriptor;
 class SlotDescriptor;
diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
index 26ac54c699b81a..4240f7e250a06b 100644
--- a/be/src/olap/segment_loader.cpp
+++ b/be/src/olap/segment_loader.cpp
@@ -77,9 +77,8 @@ Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
         }
         if (use_cache && !config::disable_segment_cache) {
             // memory of SegmentCache::CacheValue will be handled by SegmentCache
-            auto* cache_value = new SegmentCache::CacheValue();
+            auto* cache_value = new SegmentCache::CacheValue(segment);
             _cache_mem_usage += segment->meta_mem_usage();
-            cache_value->segment = std::move(segment);
             _segment_cache->insert(cache_key, *cache_value, cache_handle);
         } else {
             cache_handle->push_segment(std::move(segment));
diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h
index 834906da93bf74..2c5b1ed200dde7 100644
--- a/be/src/olap/segment_loader.h
+++ b/be/src/olap/segment_loader.h
@@ -75,9 +75,9 @@ class SegmentCache : public LRUCachePolicy {
     // Holding all opened segments of a rowset.
     class CacheValue : public LRUCacheValueBase {
     public:
-        ~CacheValue() override { segment.reset(); }
+        CacheValue(segment_v2::SegmentSharedPtr segment_) : segment(std::move(segment_)) {}
 
-        segment_v2::SegmentSharedPtr segment;
+        const segment_v2::SegmentSharedPtr segment;
     };
 
     SegmentCache(size_t memory_bytes_limit, size_t segment_num_limit)
@@ -124,8 +124,13 @@ class SegmentLoader {
 
     void erase_segments(const RowsetId& rowset_id, int64_t num_segments);
 
-    // Just used for BE UT
-    int64_t cache_mem_usage() const { return _cache_mem_usage; }
+    int64_t cache_mem_usage() const {
+#ifdef BE_TEST
+        return _cache_mem_usage;
+#else
+        return _segment_cache->value_mem_consumption();
+#endif
+    }
 
 private:
     SegmentLoader();
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index a18dc0b4a161b6..59f0e1893d84d2 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -57,8 +57,6 @@
 #include "olap/tablet_schema.h"
 #include "olap/txn_manager.h"
 #include "runtime/exec_env.h"
-#include "runtime/memory/mem_tracker.h"
-#include "runtime/thread_context.h"
 #include "service/backend_options.h"
 #include "util/defer_op.h"
 #include "util/doris_metrics.h"
@@ -83,28 +81,18 @@ using std::vector;
 namespace doris {
 using namespace ErrorCode;
 
-DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(tablet_meta_mem_consumption, MetricUnit::BYTES, "",
-                                   mem_consumption, Labels({{"type", "tablet_meta"}}));
-
 bvar::Adder<int64_t> g_tablet_meta_schema_columns_count("tablet_meta_schema_columns_count");
 
 TabletManager::TabletManager(StorageEngine& engine, int32_t tablet_map_lock_shard_size)
         : _engine(engine),
-          _tablet_meta_mem_tracker(std::make_shared<MemTracker>("TabletMeta(experimental)")),
           _tablets_shards_size(tablet_map_lock_shard_size),
           _tablets_shards_mask(tablet_map_lock_shard_size - 1) {
     CHECK_GT(_tablets_shards_size, 0);
     CHECK_EQ(_tablets_shards_size & _tablets_shards_mask, 0);
     _tablets_shards.resize(_tablets_shards_size);
-    REGISTER_HOOK_METRIC(tablet_meta_mem_consumption,
-                         [this]() { return _tablet_meta_mem_tracker->consumption(); });
 }
 
-TabletManager::~TabletManager() {
-#ifndef BE_TEST
-    DEREGISTER_HOOK_METRIC(tablet_meta_mem_consumption);
-#endif
-}
+TabletManager::~TabletManager() = default;
 
 Status TabletManager::_add_tablet_unlocked(TTabletId tablet_id, const TabletSharedPtr& tablet,
                                            bool update_meta, bool force, RuntimeProfile* profile) {
@@ -242,10 +230,6 @@ Status TabletManager::_add_tablet_to_map_unlocked(TTabletId tablet_id,
     tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
     tablet_map[tablet_id] = tablet;
     _add_tablet_to_partition(tablet);
-    // TODO: remove multiply 2 of tablet meta mem size
-    // Because table schema will copy in tablet, there will be double mem cost
-    // so here multiply 2
-    _tablet_meta_mem_tracker->consume(tablet->tablet_meta()->mem_size() * 2);
     g_tablet_meta_schema_columns_count << tablet->tablet_meta()->tablet_columns_num();
     COUNTER_UPDATE(ADD_CHILD_TIMER(profile, "RegisterTabletInfo", "AddTablet"),
                    static_cast<int64_t>(watch.reset()));
@@ -599,7 +583,6 @@ Status TabletManager::_drop_tablet(TTabletId tablet_id, TReplicaId replica_id, b
     }
 
     to_drop_tablet->deregister_tablet_from_dir();
-    _tablet_meta_mem_tracker->release(to_drop_tablet->tablet_meta()->mem_size() * 2);
     g_tablet_meta_schema_columns_count << -to_drop_tablet->tablet_meta()->tablet_columns_num();
     return Status::OK();
 }
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 42623cf05f2aea..6b6e7998f9cee1 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -251,9 +251,6 @@ class TabletManager {
 
     StorageEngine& _engine;
 
-    // TODO: memory size of TabletSchema cannot be accurately tracked.
-    std::shared_ptr<MemTracker> _tablet_meta_mem_tracker;
-
     const int32_t _tablets_shards_size;
     const int32_t _tablets_shards_mask;
     std::vector<tablets_shard> _tablets_shards;
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 0570aff349c583..fc9fc034b0bcd7 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -781,12 +781,6 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
             time_series_compaction_level_threshold());
 }
 
-int64_t TabletMeta::mem_size() const {
-    auto size = sizeof(TabletMeta);
-    size += _schema->mem_size();
-    return size;
-}
-
 void TabletMeta::to_json(string* json_string, json2pb::Pb2JsonOptions& options) {
     TabletMetaPB tablet_meta_pb;
     to_meta_pb(&tablet_meta_pb);
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 34794ef6d0a760..a8c82a4abd2921 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -140,10 +140,6 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
 
     void to_meta_pb(TabletMetaPB* tablet_meta_pb);
     void to_json(std::string* json_string, json2pb::Pb2JsonOptions& options);
-    // Don't use.
-    // TODO: memory size of TabletSchema cannot be accurately tracked.
-    // In some places, temporarily use num_columns() as TabletSchema size.
-    int64_t mem_size() const;
     size_t tablet_columns_num() const { return _schema->num_columns(); }
 
     TabletTypePB tablet_type() const { return _tablet_type; }
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index c4f96e2214853d..47179ce19b23e7 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -41,8 +41,6 @@
 #include "olap/tablet_column_object_pool.h"
 #include "olap/types.h"
 #include "olap/utils.h"
-#include "runtime/memory/lru_cache_policy.h"
-#include "runtime/thread_context.h"
 #include "tablet_meta.h"
 #include "vec/aggregate_functions/aggregate_function_simple_factory.h"
 #include "vec/aggregate_functions/aggregate_function_state_union.h"
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 5fb3deafd77319..9a0cd53f7b1786 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -332,10 +332,8 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
     void copy_from(const TabletSchema& tablet_schema);
     void update_index_info_from(const TabletSchema& tablet_schema);
     std::string to_key() const;
-    // Don't use.
-    // TODO: memory size of TabletSchema cannot be accurately tracked.
-    // In some places, temporarily use num_columns() as TabletSchema size.
-    int64_t mem_size() const { return _mem_size; }
+    // get_metadata_size is only the memory of the TabletSchema itself, not include child objects.
+    int64_t mem_size() const { return get_metadata_size(); }
     size_t row_size() const;
     int32_t field_index(const std::string& field_name) const;
     int32_t field_index(const vectorized::PathInData& path) const;
@@ -573,7 +571,6 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
     int64_t _db_id = -1;
     bool _disable_auto_compaction = false;
     bool _enable_single_replica_compaction = false;
-    int64_t _mem_size = 0;
     bool _store_row_column = false;
     bool _skip_write_index_on_load = false;
     InvertedIndexStorageFormatPB _inverted_index_storage_format = InvertedIndexStorageFormatPB::V1;
diff --git a/be/src/olap/tablet_schema_cache.cpp b/be/src/olap/tablet_schema_cache.cpp
index fd238fa5affb3f..e044ef9c0426f4 100644
--- a/be/src/olap/tablet_schema_cache.cpp
+++ b/be/src/olap/tablet_schema_cache.cpp
@@ -56,7 +56,7 @@ std::pair<Cache::Handle*, TabletSchemaSPtr> TabletSchemaCache::insert(const std:
         tablet_schema_ptr->init_from_pb(pb, false, true);
         value->tablet_schema = tablet_schema_ptr;
         lru_handle = LRUCachePolicy::insert(key_signature, value, tablet_schema_ptr->num_columns(),
-                                            0, CachePriority::NORMAL);
+                                            tablet_schema_ptr->mem_size(), CachePriority::NORMAL);
         g_tablet_schema_cache_count << 1;
         g_tablet_schema_cache_columns_count << tablet_schema_ptr->num_columns();
     }
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index a27936f5f0d88b..636ce2bf288b58 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -178,7 +178,6 @@ class ExecEnv {
     std::vector<TrackerLimiterGroup> mem_tracker_limiter_pool;
     void init_mem_tracker();
     std::shared_ptr<MemTrackerLimiter> orphan_mem_tracker() { return _orphan_mem_tracker; }
-    std::shared_ptr<MemTracker> page_no_cache_mem_tracker() { return _page_no_cache_mem_tracker; }
     std::shared_ptr<MemTrackerLimiter> brpc_iobuf_block_memory_tracker() {
         return _brpc_iobuf_block_memory_tracker;
     }
@@ -188,6 +187,15 @@ class ExecEnv {
     std::shared_ptr<MemTrackerLimiter> stream_load_pipe_tracker() {
         return _stream_load_pipe_tracker;
     }
+    std::shared_ptr<MemTrackerLimiter> tablets_no_cache_mem_tracker() {
+        return _tablets_no_cache_mem_tracker;
+    }
+    std::shared_ptr<MemTrackerLimiter> rowsets_no_cache_mem_tracker() {
+        return _rowsets_no_cache_mem_tracker;
+    }
+    std::shared_ptr<MemTrackerLimiter> segments_no_cache_mem_tracker() {
+        return _segments_no_cache_mem_tracker;
+    }
     std::shared_ptr<MemTrackerLimiter> point_query_executor_mem_tracker() {
         return _point_query_executor_mem_tracker;
     }
@@ -377,13 +385,15 @@ class ExecEnv {
     // Ideally, all threads are expected to attach to the specified tracker, so that "all memory has its own ownership",
     // and the consumption of the orphan mem tracker is close to 0, but greater than 0.
     std::shared_ptr<MemTrackerLimiter> _orphan_mem_tracker;
-    // page size not in cache, data page/index page/etc.
-    std::shared_ptr<MemTracker> _page_no_cache_mem_tracker;
     std::shared_ptr<MemTrackerLimiter> _brpc_iobuf_block_memory_tracker;
     // Count the memory consumption of segment compaction tasks.
     std::shared_ptr<MemTrackerLimiter> _segcompaction_mem_tracker;
     std::shared_ptr<MemTrackerLimiter> _stream_load_pipe_tracker;
 
+    std::shared_ptr<MemTrackerLimiter> _tablets_no_cache_mem_tracker;
+    std::shared_ptr<MemTrackerLimiter> _rowsets_no_cache_mem_tracker;
+    std::shared_ptr<MemTrackerLimiter> _segments_no_cache_mem_tracker;
+
     // Tracking memory may be shared between multiple queries.
     std::shared_ptr<MemTrackerLimiter> _point_query_executor_mem_tracker;
     std::shared_ptr<MemTrackerLimiter> _block_compression_mem_tracker;
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index b7f926cc3b4512..a371cdb947ff56 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -599,15 +599,20 @@ void ExecEnv::init_mem_tracker() {
     _s_tracking_memory = true;
     _orphan_mem_tracker =
             MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "Orphan");
-    _page_no_cache_mem_tracker = std::make_shared<MemTracker>("PageNoCache");
     _brpc_iobuf_block_memory_tracker =
             MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "IOBufBlockMemory");
     _segcompaction_mem_tracker =
-            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "SegCompaction");
+            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::COMPACTION, "SegCompaction");
+    _tablets_no_cache_mem_tracker = MemTrackerLimiter::create_shared(
+            MemTrackerLimiter::Type::METADATA, "Tablets(not in SchemaCache, TabletSchemaCache)");
+    _segments_no_cache_mem_tracker = MemTrackerLimiter::create_shared(
+            MemTrackerLimiter::Type::METADATA, "Segments(not in SegmentCache)");
+    _rowsets_no_cache_mem_tracker =
+            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::METADATA, "Rowsets");
     _point_query_executor_mem_tracker =
             MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "PointQueryExecutor");
     _query_cache_mem_tracker =
-            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "QueryCache");
+            MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::CACHE, "QueryCache");
     _block_compression_mem_tracker =
             MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "BlockCompression");
     _rowid_storage_reader_tracker =
@@ -716,7 +721,7 @@ void ExecEnv::destroy() {
     _file_cache_open_fd_cache.reset();
     SAFE_STOP(_write_cooldown_meta_executors);
 
-    // StorageEngine must be destoried before _page_no_cache_mem_tracker.reset and _cache_manager destory
+    // StorageEngine must be destoried before _cache_manager destory
     SAFE_STOP(_storage_engine);
     _storage_engine.reset();
 
diff --git a/be/src/runtime/memory/cache_policy.h b/be/src/runtime/memory/cache_policy.h
index e7e1c73e7cbb41..8f077a4eb45bb1 100644
--- a/be/src/runtime/memory/cache_policy.h
+++ b/be/src/runtime/memory/cache_policy.h
@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include <vector>
+
 #include "util/runtime_profile.h"
 
 namespace doris {
@@ -123,6 +125,7 @@ class CachePolicy {
             {"CloudTabletCache", CacheType::CLOUD_TABLET_CACHE},
             {"CloudTxnDeleteBitmapCache", CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE},
             {"ForUTCacheNumber", CacheType::FOR_UT_CACHE_NUMBER},
+            {"QueryCache", CacheType::QUERY_CACHE},
             {"TabletColumnObjectPool", CacheType::TABLET_COLUMN_OBJECT_POOL}};
 
     static CacheType string_to_type(std::string type) {
@@ -133,6 +136,9 @@ class CachePolicy {
         }
     }
 
+    inline static std::vector<CacheType> MetadataCache {
+            CacheType::SEGMENT_CACHE, CacheType::SCHEMA_CACHE, CacheType::TABLET_SCHEMA_CACHE};
+
     CachePolicy(CacheType type, size_t capacity, uint32_t stale_sweep_time_s, bool enable_prune);
     virtual ~CachePolicy();
 
diff --git a/be/src/runtime/memory/lru_cache_policy.h b/be/src/runtime/memory/lru_cache_policy.h
index ea34e2837f1313..3fdb43facd7715 100644
--- a/be/src/runtime/memory/lru_cache_policy.h
+++ b/be/src/runtime/memory/lru_cache_policy.h
@@ -104,20 +104,26 @@ class LRUCachePolicy : public CachePolicy {
         return _mem_tracker->consumption();
     }
 
+    int64_t value_mem_consumption() {
+        DCHECK(_value_mem_tracker != nullptr);
+        return _value_mem_tracker->consumption();
+    }
+
     // Insert will consume tracking_bytes to _mem_tracker and cache value destroy will release tracking_bytes.
-    // If LRUCacheType::SIZE, tracking_bytes usually equal to charge.
-    // If LRUCacheType::NUMBER, tracking_bytes usually not equal to charge, at this time charge is an weight.
-    // If LRUCacheType::SIZE and tracking_bytes equals 0, memory must be tracked in Doris Allocator,
+    // If LRUCacheType::SIZE, value_tracking_bytes usually equal to charge.
+    // If LRUCacheType::NUMBER, value_tracking_bytes usually not equal to charge, at this time charge is an weight.
+    // If LRUCacheType::SIZE and value_tracking_bytes equals 0, memory must be tracked in Doris Allocator,
     //    cache value is allocated using Alloctor.
-    // If LRUCacheType::NUMBER and tracking_bytes equals 0, usually currently cannot accurately tracking memory size,
+    // If LRUCacheType::NUMBER and value_tracking_bytes equals 0, usually currently cannot accurately tracking memory size,
     //    only tracking handle_size(106).
-    Cache::Handle* insert(const CacheKey& key, void* value, size_t charge, size_t tracking_bytes,
+    Cache::Handle* insert(const CacheKey& key, void* value, size_t charge,
+                          size_t value_tracking_bytes,
                           CachePriority priority = CachePriority::NORMAL) {
-        size_t tracking_bytes_with_handle = sizeof(LRUHandle) - 1 + key.size() + tracking_bytes;
+        size_t tracking_bytes = sizeof(LRUHandle) - 1 + key.size() + value_tracking_bytes;
         if (value != nullptr) {
-            mem_tracker()->consume(tracking_bytes_with_handle);
             ((LRUCacheValueBase*)value)
-                    ->set_tracking_bytes(tracking_bytes_with_handle, _mem_tracker);
+                    ->set_tracking_bytes(tracking_bytes, _mem_tracker, value_tracking_bytes,
+                                         _value_mem_tracker);
         }
         return _cache->insert(key, value, charge, priority);
     }
@@ -265,9 +271,18 @@ class LRUCachePolicy : public CachePolicy {
 
 protected:
     void _init_mem_tracker(const std::string& type_name) {
-        _mem_tracker = MemTrackerLimiter::create_shared(
-                MemTrackerLimiter::Type::GLOBAL,
-                fmt::format("{}[{}]", type_string(_type), type_name));
+        if (std::find(CachePolicy::MetadataCache.begin(), CachePolicy::MetadataCache.end(),
+                      _type) == CachePolicy::MetadataCache.end()) {
+            _mem_tracker = MemTrackerLimiter::create_shared(
+                    MemTrackerLimiter::Type::CACHE,
+                    fmt::format("{}[{}]", type_string(_type), type_name));
+        } else {
+            _mem_tracker = MemTrackerLimiter::create_shared(
+                    MemTrackerLimiter::Type::METADATA,
+                    fmt::format("{}[{}]", type_string(_type), type_name));
+        }
+        _value_mem_tracker = std::make_shared<MemTracker>(
+                fmt::format("{}::Value[{}]", type_string(_type), type_name));
     }
 
     // if check_capacity failed, will return dummy lru cache,
@@ -277,6 +292,7 @@ class LRUCachePolicy : public CachePolicy {
     LRUCacheType _lru_cache_type;
 
     std::shared_ptr<MemTrackerLimiter> _mem_tracker;
+    std::shared_ptr<MemTracker> _value_mem_tracker;
 };
 
 } // namespace doris
diff --git a/be/src/runtime/memory/lru_cache_value_base.h b/be/src/runtime/memory/lru_cache_value_base.h
index f9e534e6600df8..a9a3ae5ddab632 100644
--- a/be/src/runtime/memory/lru_cache_value_base.h
+++ b/be/src/runtime/memory/lru_cache_value_base.h
@@ -28,18 +28,27 @@ class LRUCacheValueBase {
     virtual ~LRUCacheValueBase() {
         if (_tracking_bytes > 0) {
             _mem_tracker->release(_tracking_bytes);
+            _value_mem_tracker->release(_value_tracking_bytes);
         }
     }
 
     void set_tracking_bytes(size_t tracking_bytes,
-                            const std::shared_ptr<MemTrackerLimiter>& mem_tracker) {
+                            const std::shared_ptr<MemTrackerLimiter>& mem_tracker,
+                            size_t value_tracking_bytes,
+                            const std::shared_ptr<MemTracker>& value_mem_tracker) {
         this->_tracking_bytes = tracking_bytes;
         this->_mem_tracker = mem_tracker;
+        this->_value_tracking_bytes = value_tracking_bytes;
+        this->_value_mem_tracker = value_mem_tracker;
+        _mem_tracker->consume(_tracking_bytes);
+        _value_mem_tracker->consume(_value_tracking_bytes);
     }
 
 protected:
     size_t _tracking_bytes = 0;
+    size_t _value_tracking_bytes = 0;
     std::shared_ptr<MemTrackerLimiter> _mem_tracker;
+    std::shared_ptr<MemTracker> _value_mem_tracker;
 };
 
 } // namespace doris
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp
index 05ff13f0e7c646..ac4684835a670c 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -66,9 +66,13 @@ MemTrackerLimiter::MemTrackerLimiter(Type type, const std::string& label, int64_
     _uid = UniqueId::gen_uid();
     if (_type == Type::GLOBAL) {
         _group_num = 0;
+    } else if (_type == Type::METADATA) {
+        _group_num = 1;
+    } else if (_type == Type::CACHE) {
+        _group_num = 2;
     } else {
         _group_num =
-                mem_tracker_limiter_group_counter.fetch_add(1) % (MEM_TRACKER_GROUP_NUM - 1) + 1;
+                mem_tracker_limiter_group_counter.fetch_add(1) % (MEM_TRACKER_GROUP_NUM - 3) + 3;
     }
 
     // currently only select/load need runtime query statistics
@@ -208,24 +212,20 @@ std::string MemTrackerLimiter::print_address_sanitizers() {
 RuntimeProfile* MemTrackerLimiter::make_profile(RuntimeProfile* profile) const {
     RuntimeProfile* profile_snapshot = profile->create_child(
             fmt::format("{}@{}@id={}", _label, type_string(_type), _uid.to_string()), true, false);
-    RuntimeProfile::Counter* current_usage_counter =
-            ADD_COUNTER(profile_snapshot, "CurrentUsage", TUnit::BYTES);
-    RuntimeProfile::Counter* peak_usage_counter =
-            ADD_COUNTER(profile_snapshot, "PeakUsage", TUnit::BYTES);
-    COUNTER_SET(current_usage_counter, consumption());
-    COUNTER_SET(peak_usage_counter, peak_consumption());
+    RuntimeProfile::HighWaterMarkCounter* usage_counter =
+            profile_snapshot->AddHighWaterMarkCounter("Memory", TUnit::BYTES);
+    COUNTER_SET(usage_counter, peak_consumption());
+    COUNTER_SET(usage_counter, consumption());
     if (has_limit()) {
         RuntimeProfile::Counter* limit_counter =
                 ADD_COUNTER(profile_snapshot, "Limit", TUnit::BYTES);
         COUNTER_SET(limit_counter, _limit);
     }
     if (reserved_peak_consumption() != 0) {
-        RuntimeProfile::Counter* reserved_counter =
-                ADD_COUNTER(profile_snapshot, "ReservedMemory", TUnit::BYTES);
-        RuntimeProfile::Counter* reserved_peak_counter =
-                ADD_COUNTER(profile_snapshot, "ReservedPeakMemory", TUnit::BYTES);
+        RuntimeProfile::HighWaterMarkCounter* reserved_counter =
+                profile_snapshot->AddHighWaterMarkCounter("ReservedMemory", TUnit::BYTES);
+        COUNTER_SET(reserved_counter, reserved_peak_consumption());
         COUNTER_SET(reserved_counter, reserved_consumption());
-        COUNTER_SET(reserved_peak_counter, reserved_peak_consumption());
     }
     return profile_snapshot;
 }
@@ -268,8 +268,26 @@ void MemTrackerLimiter::make_type_trackers_profile(RuntimeProfile* profile,
                 tracker->make_profile(profile);
             }
         }
+    } else if (type == Type::METADATA) {
+        std::lock_guard<std::mutex> l(
+                ExecEnv::GetInstance()->mem_tracker_limiter_pool[1].group_lock);
+        for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[1].trackers) {
+            auto tracker = trackerWptr.lock();
+            if (tracker != nullptr) {
+                tracker->make_profile(profile);
+            }
+        }
+    } else if (type == Type::CACHE) {
+        std::lock_guard<std::mutex> l(
+                ExecEnv::GetInstance()->mem_tracker_limiter_pool[2].group_lock);
+        for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[2].trackers) {
+            auto tracker = trackerWptr.lock();
+            if (tracker != nullptr) {
+                tracker->make_profile(profile);
+            }
+        }
     } else {
-        for (unsigned i = 1; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
+        for (unsigned i = 3; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
             std::lock_guard<std::mutex> l(
                     ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].group_lock);
             for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].trackers) {
@@ -296,8 +314,8 @@ void MemTrackerLimiter::make_top_consumption_tasks_tracker_profile(RuntimeProfil
     std::unique_ptr<RuntimeProfile> tmp_profile_snapshot =
             std::make_unique<RuntimeProfile>("tmpSnapshot");
     std::priority_queue<std::pair<int64_t, RuntimeProfile*>> max_pq;
-    // start from 2, not include global type.
-    for (unsigned i = 1; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
+    // start from 3, not include global/metadata/cache type.
+    for (unsigned i = 3; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
         std::lock_guard<std::mutex> l(
                 ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].group_lock);
         for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].trackers) {
@@ -326,13 +344,19 @@ void MemTrackerLimiter::make_all_tasks_tracker_profile(RuntimeProfile* profile)
     types_profile[Type::SCHEMA_CHANGE] = profile->create_child("SchemaChangeTasks", true, false);
     types_profile[Type::OTHER] = profile->create_child("OtherTasks", true, false);
 
-    // start from 2, not include global type.
-    for (unsigned i = 1; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
+    // start from 3, not include global/metadata/cache type.
+    for (unsigned i = 3; i < ExecEnv::GetInstance()->mem_tracker_limiter_pool.size(); ++i) {
         std::lock_guard<std::mutex> l(
                 ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].group_lock);
         for (auto trackerWptr : ExecEnv::GetInstance()->mem_tracker_limiter_pool[i].trackers) {
             auto tracker = trackerWptr.lock();
             if (tracker != nullptr) {
+                // BufferControlBlock will continue to exist for 5 minutes after the query ends, even if the
+                // result buffer is empty, and will not be shown in the profile. of course, this code is tricky.
+                if (tracker->consumption() == 0 &&
+                    tracker->label().starts_with("BufferControlBlock")) {
+                    continue;
+                }
                 tracker->make_profile(types_profile[tracker->type()]);
             }
         }
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h b/be/src/runtime/memory/mem_tracker_limiter.h
index 445856b1f6af83..43b20a410ff27c 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -77,12 +77,14 @@ class MemTrackerLimiter final {
     enum class GCType { PROCESS = 0, WORK_LOAD_GROUP = 1 };
 
     enum class Type {
-        GLOBAL = 0,        // Life cycle is the same as the process, e.g. Cache and default Orphan
+        GLOBAL = 0,        // Life cycle is the same as the process, except cache and metadata.
         QUERY = 1,         // Count the memory consumption of all Query tasks.
         LOAD = 2,          // Count the memory consumption of all Load tasks.
         COMPACTION = 3,    // Count the memory consumption of all Base and Cumulative tasks.
         SCHEMA_CHANGE = 4, // Count the memory consumption of all SchemaChange tasks.
-        OTHER = 5,
+        METADATA = 5,      // Count the memory consumption of all Metadata.
+        CACHE = 6,         // Count the memory consumption of all Cache.
+        OTHER = 7, // Count the memory consumption of all other tasks, such as Clone, Snapshot, etc..
     };
 
     static std::string type_string(Type type) {
@@ -97,8 +99,12 @@ class MemTrackerLimiter final {
             return "compaction";
         case Type::SCHEMA_CHANGE:
             return "schema_change";
+        case Type::METADATA:
+            return "metadata";
+        case Type::CACHE:
+            return "cache";
         case Type::OTHER:
-            return "other";
+            return "other_task";
         default:
             LOG(FATAL) << "not match type of mem tracker limiter :" << static_cast<int>(type);
         }
@@ -158,6 +164,8 @@ class MemTrackerLimiter final {
     int64_t consumption() const { return _mem_counter.current_value(); }
     int64_t peak_consumption() const { return _mem_counter.peak_value(); }
 
+    // Use carefully! only memory that cannot be allocated using Doris Allocator needs to be consumed manually.
+    // Ideally, all memory should use Doris Allocator.
     void consume(int64_t bytes) {
         _mem_counter.add(bytes);
         if (_query_statistics) {
diff --git a/be/src/runtime/memory/memory_profile.cpp b/be/src/runtime/memory/memory_profile.cpp
index 8dbdcbdd3af769..5d649c526014af 100644
--- a/be/src/runtime/memory/memory_profile.cpp
+++ b/be/src/runtime/memory/memory_profile.cpp
@@ -18,6 +18,9 @@
 #include "runtime/memory/memory_profile.h"
 
 #include "bvar/reducer.h"
+#include "olap/metadata_adder.h"
+#include "olap/schema_cache.h"
+#include "olap/tablet_schema_cache.h"
 #include "runtime/exec_env.h"
 #include "runtime/memory/global_memory_arbitrator.h"
 #include "runtime/memory/mem_tracker_limiter.h"
@@ -28,6 +31,9 @@ namespace doris {
 
 static bvar::Adder<int64_t> memory_all_tracked_sum_bytes("memory_all_tracked_sum_bytes");
 static bvar::Adder<int64_t> memory_global_trackers_sum_bytes("memory_global_trackers_sum_bytes");
+static bvar::Adder<int64_t> memory_metadata_trackers_sum_bytes(
+        "memory_metadata_trackers_sum_bytes");
+static bvar::Adder<int64_t> memory_cache_trackers_sum_bytes("memory_cache_trackers_sum_bytes");
 static bvar::Adder<int64_t> memory_query_trackers_sum_bytes("memory_query_trackers_sum_bytes");
 static bvar::Adder<int64_t> memory_load_trackers_sum_bytes("memory_load_trackers_sum_bytes");
 static bvar::Adder<int64_t> memory_compaction_trackers_sum_bytes(
@@ -40,140 +46,122 @@ static bvar::Adder<int64_t> memory_all_tasks_memory_bytes("memory_all_tasks_memo
 static bvar::Adder<int64_t> memory_untracked_memory_bytes("memory_untracked_memory_bytes");
 
 MemoryProfile::MemoryProfile() {
-    _memory_overview_profile.set(std::make_unique<RuntimeProfile>("MemoryOverviewSnapshot"));
+#ifdef ADDRESS_SANITIZER
+    _memory_overview_profile = std::make_unique<RuntimeProfile>("[ASAN]MemoryOverviewSnapshot");
+#else
+    _memory_overview_profile = std::make_unique<RuntimeProfile>("MemoryOverviewSnapshot");
+#endif
     _global_memory_profile.set(std::make_unique<RuntimeProfile>("GlobalMemorySnapshot"));
+    _metadata_memory_profile.set(std::make_unique<RuntimeProfile>("MetadataMemorySnapshot"));
+    _cache_memory_profile.set(std::make_unique<RuntimeProfile>("CacheMemorySnapshot"));
     _top_memory_tasks_profile.set(std::make_unique<RuntimeProfile>("TopMemoryTasksSnapshot"));
     _tasks_memory_profile.set(std::make_unique<RuntimeProfile>("TasksMemorySnapshot"));
+    init_memory_overview_counter();
 }
 
-void MemoryProfile::refresh_memory_overview_profile() {
-#ifdef ADDRESS_SANITIZER
-    std::unique_ptr<RuntimeProfile> memory_overview_profile =
-            std::make_unique<RuntimeProfile>("[ASAN]MemoryOverviewSnapshot");
-#else
-    std::unique_ptr<RuntimeProfile> memory_overview_profile =
-            std::make_unique<RuntimeProfile>("MemoryOverviewSnapshot");
-#endif
-    std::unique_ptr<RuntimeProfile> global_memory_profile =
-            std::make_unique<RuntimeProfile>("GlobalMemorySnapshot");
-    std::unique_ptr<RuntimeProfile> top_memory_tasks_profile =
-            std::make_unique<RuntimeProfile>("TopMemoryTasksSnapshot");
-
-    // 1. create profile
+void MemoryProfile::init_memory_overview_counter() {
     RuntimeProfile* untracked_memory_profile =
-            memory_overview_profile->create_child("UntrackedMemory", true, false);
+            _memory_overview_profile->create_child("UntrackedMemory", true, false);
     RuntimeProfile* tracked_memory_profile =
-            memory_overview_profile->create_child("TrackedMemory", true, false);
+            _memory_overview_profile->create_child("TrackedMemory", true, false);
     RuntimeProfile* tasks_memory_overview_profile =
             tracked_memory_profile->create_child("TasksMemory", true, false);
     RuntimeProfile* tasks_memory_overview_details_profile =
             tasks_memory_overview_profile->create_child("Details", true, false);
     RuntimeProfile* global_memory_overview_profile =
             tracked_memory_profile->create_child("GlobalMemory", true, false);
+    RuntimeProfile* metadata_memory_overview_profile =
+            tracked_memory_profile->create_child("MetadataMemory", true, false);
+    RuntimeProfile* cache_memory_overview_profile =
+            tracked_memory_profile->create_child("CacheMemory", true, false);
     RuntimeProfile* jemalloc_memory_profile =
             tracked_memory_profile->create_child("JemallocMemory", true, false);
     RuntimeProfile* jemalloc_memory_details_profile =
             jemalloc_memory_profile->create_child("Details", true, false);
 
-    // 2. add counter
-    // 2.1 add process memory counter
-    RuntimeProfile::Counter* process_physical_memory_current_usage_counter =
-            ADD_COUNTER(memory_overview_profile, "PhysicalMemory(VmRSS)", TUnit::BYTES);
-    RuntimeProfile::Counter* process_physical_memory_peak_usage_counter =
-            memory_overview_profile->AddHighWaterMarkCounter("PhysicalMemoryPeak", TUnit::BYTES);
-    RuntimeProfile::Counter* process_virtual_memory_current_usage_counter =
-            ADD_COUNTER(memory_overview_profile, "VirtualMemory(VmSize)", TUnit::BYTES);
-    RuntimeProfile::Counter* process_virtual_memory_peak_usage_counter =
-            memory_overview_profile->AddHighWaterMarkCounter("VirtualMemoryPeak", TUnit::BYTES);
-
-    // 2.2 add untracked memory counter
-    RuntimeProfile::Counter* untracked_memory_current_usage_counter =
-            ADD_COUNTER(untracked_memory_profile, "CurrentUsage", TUnit::BYTES);
-    RuntimeProfile::Counter* untracked_memory_peak_usage_counter =
-            untracked_memory_profile->AddHighWaterMarkCounter("PeakUsage", TUnit::BYTES);
-
-    // 2.3 add tracked memory counter
-    RuntimeProfile::Counter* tracked_memory_current_usage_counter =
-            ADD_COUNTER(tracked_memory_profile, "CurrentUsage", TUnit::BYTES);
-    RuntimeProfile::Counter* tracked_memory_peak_usage_counter =
-            tracked_memory_profile->AddHighWaterMarkCounter("PeakUsage", TUnit::BYTES);
-
-    // 2.4 add jemalloc memory counter
-    RuntimeProfile::Counter* jemalloc_memory_current_usage_counter =
-            ADD_COUNTER(jemalloc_memory_profile, "CurrentUsage", TUnit::BYTES);
-    RuntimeProfile::Counter* jemalloc_memory_peak_usage_counter =
-            jemalloc_memory_profile->AddHighWaterMarkCounter("PeakUsage", TUnit::BYTES);
-    RuntimeProfile::Counter* jemalloc_cache_current_usage_counter =
-            ADD_COUNTER(jemalloc_memory_details_profile, "Cache", TUnit::BYTES);
-    RuntimeProfile::Counter* jemalloc_cache_peak_usage_counter =
-            jemalloc_memory_details_profile->AddHighWaterMarkCounter("CachePeak", TUnit::BYTES);
-    RuntimeProfile::Counter* jemalloc_metadata_current_usage_counter =
-            ADD_COUNTER(jemalloc_memory_details_profile, "Metadata", TUnit::BYTES);
-    RuntimeProfile::Counter* jemalloc_metadata_peak_usage_counter =
-            jemalloc_memory_details_profile->AddHighWaterMarkCounter("MetadataPeak", TUnit::BYTES);
-
-    // 2.5 add global memory counter
-    RuntimeProfile::Counter* global_current_usage_counter =
-            ADD_COUNTER(global_memory_overview_profile, "CurrentUsage", TUnit::BYTES);
-    RuntimeProfile::Counter* global_peak_usage_counter =
-            global_memory_overview_profile->AddHighWaterMarkCounter("PeakUsage", TUnit::BYTES);
-
-    // 2.6 add tasks memory counter
-    RuntimeProfile::Counter* tasks_memory_current_usage_counter =
-            ADD_COUNTER_WITH_LEVEL(tasks_memory_overview_profile, "CurrentUsage", TUnit::BYTES, 1);
+    // 1 add process memory counter
+    _process_physical_memory_usage_counter = _memory_overview_profile->AddHighWaterMarkCounter(
+            "PhysicalMemory(VmRSS)", TUnit::BYTES);
+    _process_virtual_memory_usage_counter = _memory_overview_profile->AddHighWaterMarkCounter(
+            "VirtualMemory(VmSize)", TUnit::BYTES);
+
+    // 2 add untracked/tracked memory counter
+    _untracked_memory_usage_counter =
+            untracked_memory_profile->AddHighWaterMarkCounter("Memory", TUnit::BYTES);
+    _tracked_memory_usage_counter =
+            tracked_memory_profile->AddHighWaterMarkCounter("Memory", TUnit::BYTES);
+
+    // 3 add Jemalloc memory counter
+    _jemalloc_memory_usage_counter =
+            jemalloc_memory_profile->AddHighWaterMarkCounter("Memory", TUnit::BYTES);
+    _jemalloc_cache_usage_counter =
+            jemalloc_memory_details_profile->AddHighWaterMarkCounter("Cache", TUnit::BYTES);
+    _jemalloc_metadata_usage_counter =
+            jemalloc_memory_details_profile->AddHighWaterMarkCounter("Metadata", TUnit::BYTES);
+
+    // 4 add global/metadata/cache memory counter
+    _global_usage_counter =
+            global_memory_overview_profile->AddHighWaterMarkCounter("Memory", TUnit::BYTES);
+    _metadata_usage_counter =
+            metadata_memory_overview_profile->AddHighWaterMarkCounter("Memory", TUnit::BYTES);
+    _cache_usage_counter =
+            cache_memory_overview_profile->AddHighWaterMarkCounter("Memory", TUnit::BYTES);
+
+    // 5 add tasks memory counter
+    _tasks_memory_usage_counter =
+            tasks_memory_overview_profile->AddHighWaterMarkCounter("Memory", TUnit::BYTES);
     // Reserved memory is the sum of all task reserved memory, is duplicated with all task memory counter.
-    RuntimeProfile::Counter* reserved_memory_current_usage_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
-            tasks_memory_overview_profile, "ReservedMemory", TUnit::BYTES, "CurrentUsage", 1);
-    RuntimeProfile::Counter* reserved_memory_peak_usage_counter =
-            tasks_memory_overview_profile->AddHighWaterMarkCounter("ReservedMemoryPeak",
-                                                                   TUnit::BYTES, "CurrentUsage", 1);
-    RuntimeProfile::Counter* tasks_memory_peak_usage_counter =
-            tasks_memory_overview_profile->AddHighWaterMarkCounter("PeakUsage", TUnit::BYTES);
-    RuntimeProfile::Counter* query_current_usage_counter =
-            ADD_COUNTER_WITH_LEVEL(tasks_memory_overview_details_profile, "Query", TUnit::BYTES, 1);
-    RuntimeProfile::Counter* query_peak_usage_counter =
-            tasks_memory_overview_details_profile->AddHighWaterMarkCounter(
-                    "QueryPeak", TUnit::BYTES, "Query", 1);
-    RuntimeProfile::Counter* load_current_usage_counter =
-            ADD_COUNTER_WITH_LEVEL(tasks_memory_overview_details_profile, "Load", TUnit::BYTES, 1);
-    RuntimeProfile::Counter* load_peak_usage_counter =
-            tasks_memory_overview_details_profile->AddHighWaterMarkCounter("LoadPeak", TUnit::BYTES,
-                                                                           "Load", 1);
-    RuntimeProfile::Counter* load_all_memtables_current_usage_counter =
-            ADD_CHILD_COUNTER_WITH_LEVEL(tasks_memory_overview_details_profile,
-                                         "AllMemTablesMemory", TUnit::BYTES, "Load", 1);
-    RuntimeProfile::Counter* load_all_memtables_peak_usage_counter =
-            ADD_CHILD_COUNTER_WITH_LEVEL(tasks_memory_overview_details_profile,
-                                         "AllMemTablesMemoryPeak", TUnit::BYTES, "Load", 1);
-    RuntimeProfile::Counter* compaction_current_usage_counter = ADD_COUNTER_WITH_LEVEL(
-            tasks_memory_overview_details_profile, "Compaction", TUnit::BYTES, 1);
-    RuntimeProfile::Counter* compaction_peak_usage_counter =
-            tasks_memory_overview_details_profile->AddHighWaterMarkCounter(
-                    "CompactionPeak", TUnit::BYTES, "Compaction", 1);
-    RuntimeProfile::Counter* schema_change_current_usage_counter = ADD_COUNTER_WITH_LEVEL(
-            tasks_memory_overview_details_profile, "SchemaChange", TUnit::BYTES, 1);
-    RuntimeProfile::Counter* schema_change_peak_usage_counter =
-            tasks_memory_overview_details_profile->AddHighWaterMarkCounter(
-                    "SchemaChangePeak", TUnit::BYTES, "SchemaChange", 1);
-    RuntimeProfile::Counter* other_current_usage_counter =
-            ADD_COUNTER_WITH_LEVEL(tasks_memory_overview_details_profile, "Other", TUnit::BYTES, 1);
-    RuntimeProfile::Counter* other_peak_usage_counter =
-            tasks_memory_overview_details_profile->AddHighWaterMarkCounter(
-                    "OtherPeak", TUnit::BYTES, "Other", 1);
-    // 3. refresh counter
-    // 3.1 refresh process memory counter
-    COUNTER_SET(process_physical_memory_current_usage_counter,
+    _reserved_memory_usage_counter = tasks_memory_overview_profile->AddHighWaterMarkCounter(
+            "ReservedMemory", TUnit::BYTES, "Memory", 1);
+    _query_usage_counter =
+            tasks_memory_overview_details_profile->AddHighWaterMarkCounter("Query", TUnit::BYTES);
+    _load_usage_counter =
+            tasks_memory_overview_details_profile->AddHighWaterMarkCounter("Load", TUnit::BYTES);
+    _load_all_memtables_usage_counter =
+            tasks_memory_overview_details_profile->AddHighWaterMarkCounter("AllMemTablesMemory",
+                                                                           TUnit::BYTES, "Load", 1);
+    _compaction_usage_counter = tasks_memory_overview_details_profile->AddHighWaterMarkCounter(
+            "Compaction", TUnit::BYTES);
+    _schema_change_usage_counter = tasks_memory_overview_details_profile->AddHighWaterMarkCounter(
+            "SchemaChange", TUnit::BYTES);
+    _other_usage_counter =
+            tasks_memory_overview_details_profile->AddHighWaterMarkCounter("Other", TUnit::BYTES);
+}
+
+void MemoryProfile::refresh_memory_overview_profile() {
+    // 1 create profile
+    std::unique_ptr<RuntimeProfile> global_memory_profile =
+            std::make_unique<RuntimeProfile>("GlobalMemorySnapshot");
+    std::unique_ptr<RuntimeProfile> metadata_memory_profile =
+            std::make_unique<RuntimeProfile>("MetadataMemorySnapshot");
+    std::unique_ptr<RuntimeProfile> cache_memory_profile =
+            std::make_unique<RuntimeProfile>("CacheMemorySnapshot");
+    std::unique_ptr<RuntimeProfile> top_memory_tasks_profile =
+            std::make_unique<RuntimeProfile>("TopMemoryTasksSnapshot");
+
+    // 2 refresh process memory counter
+    COUNTER_SET(_process_physical_memory_usage_counter,
                 PerfCounters::get_vm_rss()); // from /proc VmRSS VmHWM
-    COUNTER_SET(process_physical_memory_peak_usage_counter, PerfCounters::get_vm_hwm());
-    COUNTER_SET(process_virtual_memory_current_usage_counter,
+    COUNTER_SET(_process_virtual_memory_usage_counter,
                 PerfCounters::get_vm_size()); // from /proc VmSize VmPeak
-    COUNTER_SET(process_virtual_memory_peak_usage_counter, PerfCounters::get_vm_peak());
 
-    // 3.2 refresh tracked memory counter
+    // 2 refresh metadata memory tracker
+    ExecEnv::GetInstance()->tablets_no_cache_mem_tracker()->set_consumption(
+            MetadataAdder<TabletMeta>::get_all_tablets_size() -
+            TabletSchemaCache::instance()->value_mem_consumption() -
+            SchemaCache::instance()->value_mem_consumption());
+    ExecEnv::GetInstance()->rowsets_no_cache_mem_tracker()->set_consumption(
+            MetadataAdder<RowsetMeta>::get_all_rowsets_size());
+    ExecEnv::GetInstance()->segments_no_cache_mem_tracker()->set_consumption(
+            MetadataAdder<segment_v2::Segment>::get_all_segments_estimate_size() -
+            SegmentLoader::instance()->cache_mem_usage());
+
+    // 4 refresh tracked memory counter
     std::unordered_map<MemTrackerLimiter::Type, int64_t> type_mem_sum = {
             {MemTrackerLimiter::Type::GLOBAL, 0},        {MemTrackerLimiter::Type::QUERY, 0},
             {MemTrackerLimiter::Type::LOAD, 0},          {MemTrackerLimiter::Type::COMPACTION, 0},
-            {MemTrackerLimiter::Type::SCHEMA_CHANGE, 0}, {MemTrackerLimiter::Type::OTHER, 0}};
+            {MemTrackerLimiter::Type::SCHEMA_CHANGE, 0}, {MemTrackerLimiter::Type::METADATA, 0},
+            {MemTrackerLimiter::Type::CACHE, 0},         {MemTrackerLimiter::Type::OTHER, 0}};
     // always ExecEnv::ready(), because Daemon::_stop_background_threads_latch
     for (auto& group : ExecEnv::GetInstance()->mem_tracker_limiter_pool) {
         std::lock_guard<std::mutex> l(group.group_lock);
@@ -191,42 +179,46 @@ void MemoryProfile::refresh_memory_overview_profile() {
         all_tracked_mem_sum += it.second;
         switch (it.first) {
         case MemTrackerLimiter::Type::GLOBAL:
-            COUNTER_SET(global_current_usage_counter, it.second);
-            COUNTER_SET(global_peak_usage_counter, it.second);
+            COUNTER_SET(_global_usage_counter, it.second);
             memory_global_trackers_sum_bytes
                     << it.second - memory_global_trackers_sum_bytes.get_value();
             break;
         case MemTrackerLimiter::Type::QUERY:
-            COUNTER_SET(query_current_usage_counter, it.second);
-            COUNTER_SET(query_peak_usage_counter, it.second);
+            COUNTER_SET(_query_usage_counter, it.second);
             tasks_trackers_mem_sum += it.second;
             memory_query_trackers_sum_bytes
                     << it.second - memory_query_trackers_sum_bytes.get_value();
             break;
         case MemTrackerLimiter::Type::LOAD:
-            COUNTER_SET(load_current_usage_counter, it.second);
-            COUNTER_SET(load_peak_usage_counter, it.second);
+            COUNTER_SET(_load_usage_counter, it.second);
             tasks_trackers_mem_sum += it.second;
             memory_load_trackers_sum_bytes
                     << it.second - memory_load_trackers_sum_bytes.get_value();
             break;
         case MemTrackerLimiter::Type::COMPACTION:
-            COUNTER_SET(compaction_current_usage_counter, it.second);
-            COUNTER_SET(compaction_peak_usage_counter, it.second);
+            COUNTER_SET(_compaction_usage_counter, it.second);
             tasks_trackers_mem_sum += it.second;
             memory_compaction_trackers_sum_bytes
                     << it.second - memory_compaction_trackers_sum_bytes.get_value();
             break;
         case MemTrackerLimiter::Type::SCHEMA_CHANGE:
-            COUNTER_SET(schema_change_current_usage_counter, it.second);
-            COUNTER_SET(schema_change_peak_usage_counter, it.second);
+            COUNTER_SET(_schema_change_usage_counter, it.second);
             tasks_trackers_mem_sum += it.second;
             memory_schema_change_trackers_sum_bytes
                     << it.second - memory_schema_change_trackers_sum_bytes.get_value();
             break;
+        case MemTrackerLimiter::Type::METADATA:
+            COUNTER_SET(_metadata_usage_counter, it.second);
+            memory_metadata_trackers_sum_bytes
+                    << it.second - memory_metadata_trackers_sum_bytes.get_value();
+            break;
+        case MemTrackerLimiter::Type::CACHE:
+            COUNTER_SET(_cache_usage_counter, it.second);
+            memory_cache_trackers_sum_bytes
+                    << it.second - memory_cache_trackers_sum_bytes.get_value();
+            break;
         case MemTrackerLimiter::Type::OTHER:
-            COUNTER_SET(other_current_usage_counter, it.second);
-            COUNTER_SET(other_peak_usage_counter, it.second);
+            COUNTER_SET(_other_usage_counter, it.second);
             tasks_trackers_mem_sum += it.second;
             memory_other_trackers_sum_bytes
                     << it.second - memory_other_trackers_sum_bytes.get_value();
@@ -235,60 +227,52 @@ void MemoryProfile::refresh_memory_overview_profile() {
 
     MemTrackerLimiter::make_type_trackers_profile(global_memory_profile.get(),
                                                   MemTrackerLimiter::Type::GLOBAL);
+    MemTrackerLimiter::make_type_trackers_profile(metadata_memory_profile.get(),
+                                                  MemTrackerLimiter::Type::METADATA);
+    MemTrackerLimiter::make_type_trackers_profile(cache_memory_profile.get(),
+                                                  MemTrackerLimiter::Type::CACHE);
 
     MemTrackerLimiter::make_top_consumption_tasks_tracker_profile(top_memory_tasks_profile.get(),
                                                                   15);
 
-    COUNTER_SET(tasks_memory_current_usage_counter, tasks_trackers_mem_sum);
-    COUNTER_SET(tasks_memory_peak_usage_counter, tasks_trackers_mem_sum);
+    COUNTER_SET(_tasks_memory_usage_counter, tasks_trackers_mem_sum);
     memory_all_tasks_memory_bytes << tasks_trackers_mem_sum -
                                              memory_all_tasks_memory_bytes.get_value();
 
-    COUNTER_SET(reserved_memory_current_usage_counter,
-                GlobalMemoryArbitrator::process_reserved_memory());
-    COUNTER_SET(reserved_memory_peak_usage_counter,
-                GlobalMemoryArbitrator::process_reserved_memory());
+    COUNTER_SET(_reserved_memory_usage_counter, GlobalMemoryArbitrator::process_reserved_memory());
     memory_reserved_memory_bytes << GlobalMemoryArbitrator::process_reserved_memory() -
                                             memory_reserved_memory_bytes.get_value();
 
     all_tracked_mem_sum += MemInfo::allocator_cache_mem();
-    COUNTER_SET(jemalloc_cache_current_usage_counter,
-                static_cast<int64_t>(MemInfo::allocator_cache_mem()));
-    COUNTER_SET(jemalloc_cache_peak_usage_counter,
+    COUNTER_SET(_jemalloc_cache_usage_counter,
                 static_cast<int64_t>(MemInfo::allocator_cache_mem()));
     all_tracked_mem_sum += MemInfo::allocator_metadata_mem();
-    COUNTER_SET(jemalloc_metadata_current_usage_counter,
-                static_cast<int64_t>(MemInfo::allocator_metadata_mem()));
-    COUNTER_SET(jemalloc_metadata_peak_usage_counter,
+    COUNTER_SET(_jemalloc_metadata_usage_counter,
                 static_cast<int64_t>(MemInfo::allocator_metadata_mem()));
-    COUNTER_SET(jemalloc_memory_current_usage_counter,
-                jemalloc_cache_current_usage_counter->value() +
-                        jemalloc_metadata_current_usage_counter->value());
-    COUNTER_SET(jemalloc_memory_peak_usage_counter,
-                jemalloc_cache_current_usage_counter->value() +
-                        jemalloc_metadata_current_usage_counter->value());
-
-    COUNTER_SET(tracked_memory_current_usage_counter, all_tracked_mem_sum);
-    COUNTER_SET(tracked_memory_peak_usage_counter, all_tracked_mem_sum);
+    COUNTER_SET(_jemalloc_memory_usage_counter,
+                _jemalloc_cache_usage_counter->current_value() +
+                        _jemalloc_metadata_usage_counter->current_value());
+
+    COUNTER_SET(_tracked_memory_usage_counter, all_tracked_mem_sum);
     memory_all_tracked_sum_bytes << all_tracked_mem_sum - memory_all_tracked_sum_bytes.get_value();
 
-    // 3.3 refresh untracked memory counter
+    // 5 refresh untracked memory counter
     int64_t untracked_memory =
-            process_physical_memory_current_usage_counter->value() - all_tracked_mem_sum;
-    COUNTER_SET(untracked_memory_current_usage_counter, untracked_memory);
-    COUNTER_SET(untracked_memory_peak_usage_counter, untracked_memory);
+            _process_physical_memory_usage_counter->current_value() - all_tracked_mem_sum;
+    COUNTER_SET(_untracked_memory_usage_counter, untracked_memory);
     memory_untracked_memory_bytes << untracked_memory - memory_untracked_memory_bytes.get_value();
 
-    // 3.4 refresh additional tracker printed when memory exceeds limit.
-    COUNTER_SET(load_all_memtables_current_usage_counter,
-                ExecEnv::GetInstance()->memtable_memory_limiter()->mem_tracker()->consumption());
+    // 6 refresh additional tracker printed when memory exceeds limit.
     COUNTER_SET(
-            load_all_memtables_peak_usage_counter,
+            _load_all_memtables_usage_counter,
             ExecEnv::GetInstance()->memtable_memory_limiter()->mem_tracker()->peak_consumption());
+    COUNTER_SET(_load_all_memtables_usage_counter,
+                ExecEnv::GetInstance()->memtable_memory_limiter()->mem_tracker()->consumption());
 
-    // 4. reset profile
-    _memory_overview_profile.set(std::move(memory_overview_profile));
+    // 7. reset profile
     _global_memory_profile.set(std::move(global_memory_profile));
+    _metadata_memory_profile.set(std::move(metadata_memory_profile));
+    _cache_memory_profile.set(std::move(cache_memory_profile));
     _top_memory_tasks_profile.set(std::move(top_memory_tasks_profile));
 }
 
@@ -302,16 +286,25 @@ void MemoryProfile::refresh_tasks_memory_profile() {
 void MemoryProfile::make_memory_profile(RuntimeProfile* profile) const {
     RuntimeProfile* memory_profile_snapshot = profile->create_child("MemoryProfile", true, false);
 
-    auto memory_overview_version_ptr = _memory_overview_profile.get();
     RuntimeProfile* memory_overview_profile =
-            memory_profile_snapshot->create_child(memory_overview_version_ptr->name(), true, false);
-    memory_overview_profile->merge(const_cast<RuntimeProfile*>(memory_overview_version_ptr.get()));
+            memory_profile_snapshot->create_child(_memory_overview_profile->name(), true, false);
+    memory_overview_profile->merge(const_cast<RuntimeProfile*>(_memory_overview_profile.get()));
 
     auto global_memory_version_ptr = _global_memory_profile.get();
     RuntimeProfile* global_memory_profile =
             memory_profile_snapshot->create_child(global_memory_version_ptr->name(), true, false);
     global_memory_profile->merge(const_cast<RuntimeProfile*>(global_memory_version_ptr.get()));
 
+    auto metadata_memory_version_ptr = _metadata_memory_profile.get();
+    RuntimeProfile* metadata_memory_profile =
+            memory_profile_snapshot->create_child(metadata_memory_version_ptr->name(), true, false);
+    metadata_memory_profile->merge(const_cast<RuntimeProfile*>(metadata_memory_version_ptr.get()));
+
+    auto cache_memory_version_ptr = _cache_memory_profile.get();
+    RuntimeProfile* cache_memory_profile =
+            memory_profile_snapshot->create_child(cache_memory_version_ptr->name(), true, false);
+    cache_memory_profile->merge(const_cast<RuntimeProfile*>(cache_memory_version_ptr.get()));
+
     auto top_memory_tasks_version_ptr = _top_memory_tasks_profile.get();
     RuntimeProfile* top_memory_tasks_profile = memory_profile_snapshot->create_child(
             top_memory_tasks_version_ptr->name(), true, false);
@@ -346,6 +339,8 @@ void MemoryProfile::print_log_process_usage() {
         LOG(WARNING) << "Process Memory Summary: " + GlobalMemoryArbitrator::process_mem_log_str();
         LOG(WARNING) << "\n" << print_memory_overview_profile();
         LOG(WARNING) << "\n" << print_global_memory_profile();
+        LOG(WARNING) << "\n" << print_metadata_memory_profile();
+        LOG(WARNING) << "\n" << print_cache_memory_profile();
         LOG(WARNING) << "\n" << print_top_memory_tasks_profile();
     }
 }
diff --git a/be/src/runtime/memory/memory_profile.h b/be/src/runtime/memory/memory_profile.h
index 9f1bab0c02a802..c6aefb72f22e1a 100644
--- a/be/src/runtime/memory/memory_profile.h
+++ b/be/src/runtime/memory/memory_profile.h
@@ -33,31 +33,27 @@ class MemoryProfile {
     void make_memory_profile(RuntimeProfile* profile) const;
 
     std::string print_memory_overview_profile() const {
-        std::stringstream ss;
-        auto version_ptr = _memory_overview_profile.get();
-        version_ptr->pretty_print(&ss);
-        return ss.str();
+        return return_memory_profile_str(_memory_overview_profile.get());
     }
 
     std::string print_global_memory_profile() const {
-        std::stringstream ss;
-        auto version_ptr = _global_memory_profile.get();
-        version_ptr->pretty_print(&ss);
-        return ss.str();
+        return return_memory_profile_str(_global_memory_profile.get().get());
+    }
+
+    std::string print_metadata_memory_profile() const {
+        return return_memory_profile_str(_metadata_memory_profile.get().get());
+    }
+
+    std::string print_cache_memory_profile() const {
+        return return_memory_profile_str(_cache_memory_profile.get().get());
     }
 
     std::string print_top_memory_tasks_profile() const {
-        std::stringstream ss;
-        auto version_ptr = _top_memory_tasks_profile.get();
-        version_ptr->pretty_print(&ss);
-        return ss.str();
+        return return_memory_profile_str(_top_memory_tasks_profile.get().get());
     }
 
     std::string print_tasks_memory_profile() const {
-        std::stringstream ss;
-        auto version_ptr = _tasks_memory_profile.get();
-        version_ptr->pretty_print(&ss);
-        return ss.str();
+        return return_memory_profile_str(_tasks_memory_profile.get().get());
     }
 
     static int64_t query_current_usage();
@@ -71,11 +67,50 @@ class MemoryProfile {
     void print_log_process_usage();
 
 private:
-    MultiVersion<RuntimeProfile> _memory_overview_profile;
+    std::string return_memory_profile_str(const RuntimeProfile* profile) const {
+        std::stringstream ss;
+        profile->pretty_print(&ss);
+        return ss.str();
+    }
+
+    void init_memory_overview_counter();
+
+    std::unique_ptr<RuntimeProfile> _memory_overview_profile;
     MultiVersion<RuntimeProfile> _global_memory_profile;
+    MultiVersion<RuntimeProfile> _metadata_memory_profile;
+    MultiVersion<RuntimeProfile> _cache_memory_profile;
     MultiVersion<RuntimeProfile> _top_memory_tasks_profile;
     MultiVersion<RuntimeProfile> _tasks_memory_profile;
 
+    // process memory counter
+    RuntimeProfile::HighWaterMarkCounter* _process_physical_memory_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _process_virtual_memory_usage_counter;
+
+    // untracked/tracked memory counter
+    RuntimeProfile::HighWaterMarkCounter* _untracked_memory_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _tracked_memory_usage_counter;
+
+    // Jemalloc memory counter
+    RuntimeProfile::HighWaterMarkCounter* _jemalloc_memory_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _jemalloc_cache_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _jemalloc_metadata_usage_counter;
+
+    // global/metadata/cache memory counter
+    RuntimeProfile::HighWaterMarkCounter* _global_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _metadata_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _cache_usage_counter;
+
+    // tasks memory counter
+    RuntimeProfile::HighWaterMarkCounter* _tasks_memory_usage_counter;
+    // reserved memory is the sum of all task reserved memory, is duplicated with all task memory counter.
+    RuntimeProfile::HighWaterMarkCounter* _reserved_memory_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _query_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _load_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _load_all_memtables_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _compaction_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _schema_change_usage_counter;
+    RuntimeProfile::HighWaterMarkCounter* _other_usage_counter;
+
     std::atomic<bool> _enable_print_log_process_usage {true};
 };
 
diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp
index 45db607a342743..8b9e97bfc9cb54 100644
--- a/be/src/util/runtime_profile.cpp
+++ b/be/src/util/runtime_profile.cpp
@@ -28,6 +28,7 @@
 #include <algorithm>
 #include <iomanip>
 #include <iostream>
+#include <type_traits>
 
 #include "common/object_pool.h"
 #include "util/container_util.hpp"
@@ -72,8 +73,7 @@ void RuntimeProfile::merge(RuntimeProfile* other) {
             dst_iter = _counter_map.find(src_iter->first);
 
             if (dst_iter == _counter_map.end()) {
-                _counter_map[src_iter->first] = _pool->add(
-                        new Counter(src_iter->second->type(), src_iter->second->value()));
+                _counter_map[src_iter->first] = _pool->add(src_iter->second->clone());
             } else {
                 DCHECK(dst_iter->second->type() == src_iter->second->type());
 
diff --git a/be/src/util/runtime_profile.h b/be/src/util/runtime_profile.h
index 6e393ac673a628..7130acbd2f9427 100644
--- a/be/src/util/runtime_profile.h
+++ b/be/src/util/runtime_profile.h
@@ -100,6 +100,8 @@ class RuntimeProfile {
                 : _value(value), _type(type), _level(level) {}
         virtual ~Counter() = default;
 
+        virtual Counter* clone() const { return new Counter(type(), value(), _level); }
+
         virtual void update(int64_t delta) { _value.fetch_add(delta, std::memory_order_relaxed); }
 
         void bit_or(int64_t delta) { _value.fetch_or(delta, std::memory_order_relaxed); }
@@ -137,7 +139,7 @@ class RuntimeProfile {
 
         TUnit::type type() const { return _type; }
 
-        virtual int64_t level() { return _level; }
+        virtual int64_t level() const { return _level; }
 
     private:
         friend class RuntimeProfile;
@@ -151,8 +153,16 @@ class RuntimeProfile {
     /// as value()) and the current value.
     class HighWaterMarkCounter : public Counter {
     public:
-        HighWaterMarkCounter(TUnit::type unit, int64_t level, const std::string& parent_name)
-                : Counter(unit, 0, level), current_value_(0), _parent_name(parent_name) {}
+        HighWaterMarkCounter(TUnit::type unit, int64_t level, const std::string& parent_name,
+                             int64_t value = 0, int64_t current_value = 0)
+                : Counter(unit, value, level),
+                  current_value_(current_value),
+                  _parent_name(parent_name) {}
+
+        virtual Counter* clone() const override {
+            return new HighWaterMarkCounter(type(), level(), parent_name(), value(),
+                                            current_value());
+        }
 
         void add(int64_t delta) {
             current_value_.fetch_add(delta, std::memory_order_relaxed);
@@ -188,10 +198,9 @@ class RuntimeProfile {
         virtual void pretty_print(std::ostream* s, const std::string& prefix,
                                   const std::string& name) const override {
             std::ostream& stream = *s;
-            stream << prefix << "   - " << name << ": "
-                   << PrettyPrinter::print(current_value(), type()) << std::endl;
-            stream << prefix << "      - " << name << "Peak: "
-                   << PrettyPrinter::print(_value.load(std::memory_order_relaxed), type())
+            stream << prefix << "   - " << name
+                   << " Current: " << PrettyPrinter::print(current_value(), type()) << " (Peak: "
+                   << PrettyPrinter::print(_value.load(std::memory_order_relaxed), type()) << ")"
                    << std::endl;
         }
 
@@ -217,6 +226,8 @@ class RuntimeProfile {
 
         int64_t current_value() const { return current_value_.load(std::memory_order_relaxed); }
 
+        std::string parent_name() const { return _parent_name; }
+
     private:
         /// Set '_value' to 'v' if 'v' is larger than '_value'. The entire operation is
         /// atomic.
@@ -247,8 +258,13 @@ class RuntimeProfile {
     // Do not call Set() and Update().
     class DerivedCounter : public Counter {
     public:
-        DerivedCounter(TUnit::type type, const DerivedCounterFunction& counter_fn)
-                : Counter(type, 0), _counter_fn(counter_fn) {}
+        DerivedCounter(TUnit::type type, const DerivedCounterFunction& counter_fn,
+                       int64_t value = 0, int64_t level = 1)
+                : Counter(type, value, level), _counter_fn(counter_fn) {}
+
+        virtual Counter* clone() const override {
+            return new DerivedCounter(type(), _counter_fn, value(), level());
+        }
 
         int64_t value() const override { return _counter_fn(); }
 
@@ -259,8 +275,13 @@ class RuntimeProfile {
     // NonZeroCounter will not be converted to Thrift if the value is 0.
     class NonZeroCounter : public Counter {
     public:
-        NonZeroCounter(TUnit::type type, int64_t level, const std::string& parent_name)
-                : Counter(type, 0, level), _parent_name(parent_name) {}
+        NonZeroCounter(TUnit::type type, int64_t level, const std::string& parent_name,
+                       int64_t value = 0)
+                : Counter(type, value, level), _parent_name(parent_name) {}
+
+        virtual Counter* clone() const override {
+            return new NonZeroCounter(type(), level(), parent_name(), value());
+        }
 
         void to_thrift(const std::string& name, std::vector<TCounter>& tcounters,
                        std::map<std::string, std::set<std::string>>& child_counters_map) override {
@@ -272,6 +293,8 @@ class RuntimeProfile {
             }
         }
 
+        std::string parent_name() const { return _parent_name; }
+
     private:
         const std::string _parent_name;
     };
diff --git a/be/src/vec/runtime/partitioner.h b/be/src/vec/runtime/partitioner.h
index 39ed0e899411b8..d5492c3be87683 100644
--- a/be/src/vec/runtime/partitioner.h
+++ b/be/src/vec/runtime/partitioner.h
@@ -23,7 +23,6 @@
 
 namespace doris {
 #include "common/compile_check_begin.h"
-class MemTracker;
 
 namespace vectorized {
 
diff --git a/be/src/vec/sink/vdata_stream_sender.h b/be/src/vec/sink/vdata_stream_sender.h
index 5fe35e4da119d0..4999602fdf49a7 100644
--- a/be/src/vec/sink/vdata_stream_sender.h
+++ b/be/src/vec/sink/vdata_stream_sender.h
@@ -56,7 +56,6 @@ namespace doris {
 #include "common/compile_check_begin.h"
 class ObjectPool;
 class RuntimeState;
-class MemTracker;
 class RowDescriptor;
 class TDataSink;
 class TDataStreamSink;

From fa278fe0843a24af1f6e9eb0c4a21afff8e5362d Mon Sep 17 00:00:00 2001
From: LiBinfeng <libinfeng@selectdb.com>
Date: Wed, 4 Dec 2024 10:45:24 +0800
Subject: [PATCH 152/399] [feat](Nereids) support admin commands (#44861)

Support admin commands
```ADMIN SHOW REPLICA STATUS FROM baseTableRef wildWhere?                        #adminShowReplicaStatus ShowReplicaStatusStmt*
    | ADMIN COMPACT TABLE baseTableRef wildWhere?                                   #adminCompactTable AdminCompactTableStmt*
```
note that adminCompactTable only supported in syntax, but this feature
is not supported when execute, which it would throw an exception when
running in backend
---
 .../org/apache/doris/nereids/DorisParser.g4   |   6 +-
 .../java/org/apache/doris/catalog/Env.java    |   6 +-
 .../apache/doris/catalog/MetadataViewer.java  | 123 ++++++++++++
 .../nereids/parser/LogicalPlanBuilder.java    |  32 ++++
 .../doris/nereids/trees/plans/PlanType.java   |   2 +
 .../commands/AdminCompactTableCommand.java    | 136 +++++++++++++
 .../AdminShowReplicaStatusCommand.java        | 178 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  11 ++
 .../control_p0/set_replica_status.groovy      |   3 +
 9 files changed, 493 insertions(+), 4 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCompactTableCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminShowReplicaStatusCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index b51b342e24b895..dcc87851196bfb 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -487,6 +487,8 @@ unsupportedCancelStatement
 supportedAdminStatement
     : ADMIN SHOW REPLICA DISTRIBUTION FROM baseTableRef                             #adminShowReplicaDistribution
     | ADMIN DIAGNOSE TABLET tabletId=INTEGER_VALUE                                  #adminDiagnoseTablet
+    | ADMIN SHOW REPLICA STATUS FROM baseTableRef (WHERE STATUS EQ|NEQ STRING_LITERAL)?   #adminShowReplicaStatus
+    | ADMIN COMPACT TABLE baseTableRef (WHERE TYPE EQ STRING_LITERAL)?              #adminCompactTable
     ;
 
 supportedRecoverStatement
@@ -498,12 +500,10 @@ supportedRecoverStatement
     ;
 
 unsupportedAdminStatement
-    : ADMIN SHOW REPLICA STATUS FROM baseTableRef wildWhere?                        #adminShowReplicaStatus
-    | ADMIN SET REPLICA STATUS PROPERTIES LEFT_PAREN propertyItemList RIGHT_PAREN   #adminSetReplicaStatus
+    : ADMIN SET REPLICA STATUS PROPERTIES LEFT_PAREN propertyItemList RIGHT_PAREN   #adminSetReplicaStatus
     | ADMIN SET REPLICA VERSION PROPERTIES LEFT_PAREN propertyItemList RIGHT_PAREN  #adminSetReplicaVersion
     | ADMIN REPAIR TABLE baseTableRef                                               #adminRepairTable
     | ADMIN CANCEL REPAIR TABLE baseTableRef                                        #adminCancelRepairTable
-    | ADMIN COMPACT TABLE baseTableRef wildWhere?                                   #adminCompactTable
     | ADMIN SET (FRONTEND | (ALL FRONTENDS)) CONFIG
         (LEFT_PAREN propertyItemList RIGHT_PAREN)? ALL?                             #adminSetFrontendConfig
     | ADMIN CHECK tabletList properties=propertyClause?                             #adminCheckTablets
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 7597a7d256b460..12eff06acbcbfa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -6555,14 +6555,18 @@ public void compactTable(AdminCompactTableStmt stmt) throws DdlException {
         String dbName = stmt.getDbName();
         String tableName = stmt.getTblName();
         String type = stmt.getCompactionType();
+        List<String> partitionNames = stmt.getPartitions();
+        compactTable(dbName, tableName, type, partitionNames);
+    }
 
+    public void compactTable(String dbName, String tableName, String type, List<String> partitionNames)
+            throws DdlException {
         Database db = getInternalCatalog().getDbOrDdlException(dbName);
         OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
 
         AgentBatchTask batchTask = new AgentBatchTask();
         olapTable.readLock();
         try {
-            List<String> partitionNames = stmt.getPartitions();
             LOG.info("Table compaction. database: {}, table: {}, partition: {}, type: {}", dbName, tableName,
                     Joiner.on(", ").join(partitionNames), type);
             for (String parName : partitionNames) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
index 4da77267857f67..501d49e4add8d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
@@ -28,6 +28,8 @@
 import org.apache.doris.common.Config;
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.FeConstants;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.resource.Tag;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
@@ -152,6 +154,113 @@ private static List<List<String>> getTabletStatus(String dbName, String tblName,
         return result;
     }
 
+    /**
+     * Used for Nereids
+     */
+    public static List<List<String>> getTabletStatus(String dbName, String tblName, List<String> partitions,
+                                                      ReplicaStatus statusFilter, Expression op) throws DdlException {
+        List<List<String>> result = Lists.newArrayList();
+
+        Env env = Env.getCurrentEnv();
+        SystemInfoService infoService = Env.getCurrentSystemInfo();
+
+        Database db = env.getInternalCatalog().getDbOrDdlException(dbName);
+        OlapTable olapTable = db.getOlapTableOrDdlException(tblName);
+
+        olapTable.readLock();
+        try {
+            if (partitions.isEmpty()) {
+                partitions.addAll(olapTable.getPartitionNames());
+            } else {
+                // check partition
+                for (String partName : partitions) {
+                    Partition partition = olapTable.getPartition(partName);
+                    if (partition == null) {
+                        throw new DdlException("Partition does not exist: " + partName);
+                    }
+                }
+            }
+
+            for (String partName : partitions) {
+                Partition partition = olapTable.getPartition(partName);
+                long visibleVersion = partition.getVisibleVersion();
+                short replicationNum = olapTable.getPartitionInfo()
+                        .getReplicaAllocation(partition.getId()).getTotalReplicaNum();
+
+                for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
+                    int schemaHash = olapTable.getSchemaHashByIndexId(index.getId());
+                    for (Tablet tablet : index.getTablets()) {
+                        long tabletId = tablet.getId();
+                        int count = replicationNum;
+                        for (Replica replica : tablet.getReplicas()) {
+                            --count;
+                            List<String> row = Lists.newArrayList();
+
+                            ReplicaStatus status = ReplicaStatus.OK;
+                            Backend be = infoService.getBackend(replica.getBackendIdWithoutException());
+                            if (be == null || !be.isAlive() || replica.isBad()) {
+                                status = ReplicaStatus.DEAD;
+                            } else if (replica.getVersion() < visibleVersion
+                                    || replica.getLastFailedVersion() > 0) {
+                                status = ReplicaStatus.VERSION_ERROR;
+
+                            } else if (replica.getSchemaHash() != -1 && replica.getSchemaHash() != schemaHash) {
+                                status = ReplicaStatus.SCHEMA_ERROR;
+                            } else if (replica.isUserDrop()) {
+                                status = ReplicaStatus.DROP;
+                            }
+
+                            if (filterReplica(status, statusFilter, op)) {
+                                continue;
+                            }
+
+                            row.add(String.valueOf(tabletId));
+                            row.add(String.valueOf(replica.getId()));
+                            row.add(String.valueOf(replica.getBackendIdWithoutException()));
+                            row.add(String.valueOf(replica.getVersion()));
+                            row.add(String.valueOf(replica.getLastFailedVersion()));
+                            row.add(String.valueOf(replica.getLastSuccessVersion()));
+                            row.add(String.valueOf(visibleVersion));
+                            row.add(String.valueOf(replica.getSchemaHash()));
+                            row.add(String.valueOf(replica.getTotalVersionCount()));
+                            row.add(String.valueOf(replica.isBad()));
+                            row.add(String.valueOf(replica.isUserDrop()));
+                            row.add(replica.getState().name());
+                            row.add(status.name());
+                            result.add(row);
+                        }
+
+                        if (filterReplica(ReplicaStatus.MISSING, statusFilter, op)) {
+                            continue;
+                        }
+
+                        // get missing replicas
+                        for (int i = 0; i < count; ++i) {
+                            List<String> row = Lists.newArrayList();
+                            row.add(String.valueOf(tabletId));
+                            row.add("-1");
+                            row.add("-1");
+                            row.add("-1");
+                            row.add("-1");
+                            row.add("-1");
+                            row.add("-1");
+                            row.add("-1");
+                            row.add(FeConstants.null_string);
+                            row.add(FeConstants.null_string);
+                            row.add(FeConstants.null_string);
+                            row.add(ReplicaStatus.MISSING.name());
+                            result.add(row);
+                        }
+                    }
+                }
+            }
+        } finally {
+            olapTable.readUnlock();
+        }
+
+        return result;
+    }
+
     private static boolean filterReplica(ReplicaStatus status, ReplicaStatus statusFilter, Operator op) {
         if (statusFilter == null) {
             return false;
@@ -163,6 +272,20 @@ private static boolean filterReplica(ReplicaStatus status, ReplicaStatus statusF
         }
     }
 
+    /**
+     * Used for Nereids
+     */
+    private static boolean filterReplica(ReplicaStatus status, ReplicaStatus statusFilter, Expression op) {
+        if (statusFilter == null) {
+            return false;
+        }
+        if (op instanceof EqualTo) {
+            return status != statusFilter;
+        } else {
+            return status == statusFilter;
+        }
+    }
+
     public static List<List<String>> getTabletDistribution(ShowReplicaDistributionStmt stmt) throws DdlException {
         return getTabletDistribution(stmt.getDbName(), stmt.getTblName(), stmt.getPartitionNames());
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 7804959aa79f9e..705acb4788fda1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -50,8 +50,10 @@
 import org.apache.doris.mtmv.MTMVRefreshTriggerInfo;
 import org.apache.doris.nereids.DorisParser;
 import org.apache.doris.nereids.DorisParser.AddConstraintContext;
+import org.apache.doris.nereids.DorisParser.AdminCompactTableContext;
 import org.apache.doris.nereids.DorisParser.AdminDiagnoseTabletContext;
 import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext;
+import org.apache.doris.nereids.DorisParser.AdminShowReplicaStatusContext;
 import org.apache.doris.nereids.DorisParser.AggClauseContext;
 import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext;
 import org.apache.doris.nereids.DorisParser.AliasQueryContext;
@@ -465,6 +467,8 @@
 import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
 import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
 import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterSqlBlockRuleCommand;
@@ -1159,6 +1163,18 @@ public CancelMTMVTaskCommand visitCancelMTMVTask(CancelMTMVTaskContext ctx) {
         return new CancelMTMVTaskCommand(new CancelMTMVTaskInfo(new TableNameInfo(nameParts), taskId));
     }
 
+    @Override
+    public AdminCompactTableCommand visitAdminCompactTable(AdminCompactTableContext ctx) {
+        TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef());
+        EqualTo equalTo = null;
+        if (ctx.WHERE() != null) {
+            StringLiteral left = new StringLiteral(stripQuotes(ctx.TYPE().getText()));
+            StringLiteral right = new StringLiteral(stripQuotes(ctx.STRING_LITERAL().getText()));
+            equalTo = new EqualTo(left, right);
+        }
+        return new AdminCompactTableCommand(tableRefInfo, equalTo);
+    }
+
     @Override
     public AlterMTMVCommand visitAlterMTMV(AlterMTMVContext ctx) {
         List<String> nameParts = visitMultipartIdentifier(ctx.mvName);
@@ -4541,6 +4557,22 @@ public LogicalPlan visitShowTrash(ShowTrashContext ctx) {
         return new ShowTrashCommand();
     }
 
+    @Override
+    public LogicalPlan visitAdminShowReplicaStatus(AdminShowReplicaStatusContext ctx) {
+        Expression where = null;
+        if (ctx.WHERE() != null) {
+            StringLiteral left = new StringLiteral(stripQuotes(ctx.STATUS().toString()));
+            StringLiteral right = new StringLiteral(stripQuotes(ctx.STRING_LITERAL().getText()));
+            if (ctx.NEQ() != null) {
+                where = new Not(new EqualTo(left, right));
+            } else {
+                where = new EqualTo(left, right);
+            }
+        }
+        TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef());
+        return new AdminShowReplicaStatusCommand(tableRefInfo, where);
+    }
+
     @Override
     public LogicalPlan visitShowRepositories(ShowRepositoriesContext ctx) {
         return new ShowRepositoriesCommand();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 48e64acaab6377..5d523ce3c1d425 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -152,6 +152,7 @@ public enum PlanType {
     RESUME_JOB_COMMAND,
     ALTER_MTMV_COMMAND,
     ADD_CONSTRAINT_COMMAND,
+    ADMIN_COMPACT_TABLE_COMMAND,
     DROP_CONSTRAINT_COMMAND,
     SHOW_CONSTRAINTS_COMMAND,
     REFRESH_MTMV_COMMAND,
@@ -222,6 +223,7 @@ public enum PlanType {
     SHOW_PLUGINS_COMMAND,
     SHOW_PRIVILEGES_COMMAND,
     SHOW_REPLICA_DISTRIBUTION_COMMAND,
+    SHOW_REPLICA_STATUS_COMMAND,
     SHOW_REPOSITORIES_COMMAND,
     SHOW_ROLE_COMMAND,
     SHOW_SMALL_FILES_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCompactTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCompactTableCommand.java
new file mode 100644
index 00000000000000..cb1b75af54d796
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCompactTableCommand.java
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+
+/**
+ * AdminCompactTableCommand
+ */
+public class AdminCompactTableCommand extends Command implements ForwardWithSync {
+    private static final Logger LOG = LogManager.getLogger(AdminCompactTableCommand.class);
+    private TableRefInfo tableRefInfo;
+    private EqualTo where;
+
+    /**
+     * compact type
+     */
+    public enum CompactionType {
+        CUMULATIVE,
+        BASE
+    }
+
+    private CompactionType typeFilter;
+
+    public AdminCompactTableCommand(TableRefInfo tableRefInfo, EqualTo where) {
+        super(PlanType.ADD_CONSTRAINT_COMMAND);
+        this.tableRefInfo = tableRefInfo;
+        this.where = where;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        String dbName = tableRefInfo.getTableNameInfo().getDb();
+        String tableName = tableRefInfo.getTableNameInfo().getTbl();
+        String type = getCompactionType();
+        List<String> partitionNames = tableRefInfo.getPartitionNamesInfo().getPartitionNames();
+        ctx.getEnv().compactTable(dbName, tableName, type, partitionNames);
+    }
+
+    private void validate(ConnectContext ctx) throws UserException {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+        tableRefInfo.analyze(ctx);
+        Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName());
+
+        List<String> partitionNames = tableRefInfo.getPartitionNamesInfo().getPartitionNames();
+        if (partitionNames != null) {
+            if (partitionNames.size() != 1) {
+                throw new AnalysisException("Only support single partition for compaction");
+            }
+        } else {
+            throw new AnalysisException("No partition selected for compaction");
+        }
+
+        // analyze where clause if not null
+        if (where == null) {
+            throw new AnalysisException("Compaction type must be specified in"
+                + " Where clause like: type = 'BASE/CUMULATIVE'");
+        }
+
+        if (!analyzeWhere()) {
+            throw new AnalysisException(
+                "Where clause should looks like: type = 'BASE/CUMULATIVE'");
+        }
+    }
+
+    private boolean analyzeWhere() {
+        try {
+            typeFilter = CompactionType.valueOf(((StringLiteral) where.right()).getStringValue().toUpperCase());
+        } catch (Exception e) {
+            return false;
+        }
+
+        if (typeFilter == null || (typeFilter != CompactionType.CUMULATIVE && typeFilter != CompactionType.BASE)) {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAdminCompactTableCommand(this, context);
+    }
+
+    private String getCompactionType() {
+        if (typeFilter == CompactionType.CUMULATIVE) {
+            return "cumulative";
+        } else {
+            return "base";
+        }
+    }
+
+    @Override
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
+        LOG.info("AdminCompactTableCommand not supported in cloud mode");
+        throw new DdlException("Unsupported operation");
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminShowReplicaStatusCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminShowReplicaStatusCommand.java
new file mode 100644
index 00000000000000..89ff5c8e2c7147
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminShowReplicaStatusCommand.java
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.RedirectStatus;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MetadataViewer;
+import org.apache.doris.catalog.Replica;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.mysql.privilege.Auth;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+
+/**
+ * ShowReplicaStatusCommand
+ */
+public class AdminShowReplicaStatusCommand extends ShowCommand {
+    public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
+            .add("TabletId").add("ReplicaId").add("BackendId").add("Version").add("LastFailedVersion")
+            .add("LastSuccessVersion").add("CommittedVersion").add("SchemaHash").add("VersionNum")
+            .add("IsBad").add("IsUserDrop").add("State").add("Status")
+            .build();
+    private static final Logger LOG = LogManager.getLogger(AdminShowReplicaStatusCommand.class);
+
+    private TableRefInfo tableRefInfo;
+    private Expression where;
+
+    private Replica.ReplicaStatus statusFilter;
+
+    public AdminShowReplicaStatusCommand(TableRefInfo tableRefInfo, Expression where) {
+        super(PlanType.SHOW_REPLICA_STATUS_COMMAND);
+        this.tableRefInfo = tableRefInfo;
+        this.where = where;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        List<List<String>> results;
+        try {
+            results = MetadataViewer.getTabletStatus(tableRefInfo.getTableNameInfo().getDb(),
+                tableRefInfo.getTableNameInfo().getTbl(),
+                tableRefInfo.getPartitionNamesInfo().getPartitionNames(), statusFilter, where);
+        } catch (DdlException e) {
+            throw new AnalysisException(e.getMessage());
+        }
+        return new ShowResultSet(getMetaData(), results);
+    }
+
+    public ShowResultSetMetaData getMetaData() {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        for (String title : TITLE_NAMES) {
+            builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
+        }
+        return builder.build();
+    }
+
+    /**
+     * validate for show replica
+     */
+    public void validate(ConnectContext ctx) throws AnalysisException, UserException {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        // bind relation
+        tableRefInfo.analyze(ctx);
+        Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName());
+
+        if (!validateWhere()) {
+            throw new AnalysisException(
+                "Where clause should looks like: status =/!= 'OK/DEAD/VERSION_ERROR/SCHEMA_ERROR/MISSING'");
+        }
+    }
+
+    private boolean validateWhere() throws AnalysisException {
+        // analyze where clause if not null
+        if (where == null) {
+            return true;
+        }
+
+        if (!(where instanceof EqualTo || (where instanceof Not && where.child(0) instanceof EqualTo))) {
+            return false;
+        }
+
+        EqualTo equalTo = where instanceof EqualTo ? (EqualTo) where : (EqualTo) ((Not) where).child();
+
+        Expression leftChild = equalTo.child(0);
+        if (!(leftChild instanceof UnboundSlot)) {
+            return false;
+        }
+
+        String leftKey = ((UnboundSlot) leftChild).getName();
+        if (!leftKey.equalsIgnoreCase("status")) {
+            return false;
+        }
+
+        Expression rightChild = equalTo.child(1);
+        if (!(rightChild instanceof StringLikeLiteral)) {
+            return false;
+        }
+
+        try {
+            statusFilter = Replica.ReplicaStatus
+                    .valueOf(((StringLikeLiteral) rightChild).getStringValue().toUpperCase());
+        } catch (Exception e) {
+            return false;
+        }
+
+        if (statusFilter == null) {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAdminShowReplicaStatusCommand(this, context);
+    }
+
+    @Override
+    public RedirectStatus toRedirectStatus() {
+        if (ConnectContext.get().getSessionVariable().getForwardToMaster()) {
+            return RedirectStatus.FORWARD_NO_SYNC;
+        } else {
+            return RedirectStatus.NO_FORWARD;
+        }
+    }
+
+    @Override
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
+        if (!ctx.getCurrentUserIdentity().getUser().equals(Auth.ROOT_USER)) {
+            LOG.info("ShowReplicaStatusCommand not supported in cloud mode");
+            throw new DdlException("Unsupported operation");
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 8561c934ec976f..23ceadbc3168f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -18,6 +18,8 @@
 package org.apache.doris.nereids.trees.plans.visitor;
 
 import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterJobStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterRoleCommand;
@@ -207,6 +209,10 @@ default R visitAddConstraintCommand(AddConstraintCommand addConstraintCommand, C
         return visitCommand(addConstraintCommand, context);
     }
 
+    default R visitAdminCompactTableCommand(AdminCompactTableCommand adminCompactTableCommand, C context) {
+        return visitCommand(adminCompactTableCommand, context);
+    }
+
     default R visitDropConstraintCommand(DropConstraintCommand dropConstraintCommand, C context) {
         return visitCommand(dropConstraintCommand, context);
     }
@@ -400,6 +406,11 @@ default R visitShowTriggersCommand(ShowTriggersCommand showTriggersCommand, C co
         return visitCommand(showTriggersCommand, context);
     }
 
+    default R visitAdminShowReplicaStatusCommand(AdminShowReplicaStatusCommand adminShowReplicaStatusCommand,
+                                                    C context) {
+        return visitCommand(adminShowReplicaStatusCommand, context);
+    }
+
     default R visitShowRepositoriesCommand(ShowRepositoriesCommand showRepositoriesCommand, C context) {
         return visitCommand(showRepositoriesCommand, context);
     }
diff --git a/regression-test/suites/control_p0/set_replica_status.groovy b/regression-test/suites/control_p0/set_replica_status.groovy
index 367893c8e9cdc4..627f48755ee6c0 100644
--- a/regression-test/suites/control_p0/set_replica_status.groovy
+++ b/regression-test/suites/control_p0/set_replica_status.groovy
@@ -50,6 +50,7 @@ suite('test_set_replica_status', 'nonConcurrent') {
             break
         }
         sql """ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "${tabletId}", "backend_id" = "${backendId}", "status" = "bad");"""
+        checkNereidsExecute("""SHOW REPLICA STATUS FROM ${tableName}""")
         result = sql_return_maparray """SHOW REPLICA STATUS FROM ${tableName}"""
         for (def res : result) {
             if (res.TabletId == tabletId && res.BackendId == backendId) {
@@ -58,6 +59,7 @@ suite('test_set_replica_status', 'nonConcurrent') {
             }
         }
         sql """ADMIN SET REPLICA STATUS PROPERTIES("tablet_id" = "${tabletId}", "backend_id" = "${backendId}", "status" = "ok");"""
+        checkNereidsExecute("""SHOW REPLICA STATUS FROM ${tableName}""")
         result = sql_return_maparray """SHOW REPLICA STATUS FROM ${tableName}"""
         for (def res : result) {
             if (res.TabletId == tabletId && res.BackendId == backendId) {
@@ -66,6 +68,7 @@ suite('test_set_replica_status', 'nonConcurrent') {
             }
         }
         sql """ADMIN SET REPLICA VERSION PROPERTIES("tablet_id" = "${tabletId}", "backend_id" = "${backendId}", "last_failed_version" = "10");"""
+        checkNereidsExecute("""SHOW REPLICA STATUS FROM ${tableName}""")
         result = sql_return_maparray """SHOW REPLICA STATUS FROM ${tableName}"""
         for (def res : result) {
             if (res.TabletId == tabletId && res.BackendId == backendId) {

From c36b0ccde25c90e9f83c974f367f252499fe3ecd Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Wed, 4 Dec 2024 10:46:56 +0800
Subject: [PATCH 153/399] [fix](arrow-flight-sql) Fix Arrow Flight SQL
 regression-test (#44894)

### What problem does this PR solve?

Problem Summary:

Fix Arrow Flight SQL profile
Fix regression-test frametwork
---
 .../arrowflight/FlightSqlConnectProcessor.java     |  2 +-
 regression-test/framework/pom.xml                  |  1 +
 .../org/apache/doris/regression/Config.groovy      | 14 +++++++++-----
 .../doris/regression/suite/SuiteContext.groovy     |  2 +-
 .../nereids_p0/show/test_show_create_view.groovy   |  2 +-
 5 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java
index 3fba602a1c1e2f..345d7d824a23ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java
@@ -196,11 +196,11 @@ public void fetchArrowFlightSchema(int timeoutMs) {
     public void close() throws Exception {
         ctx.setCommand(MysqlCommand.COM_SLEEP);
         ctx.clear();
-        // TODO support query profile
         for (StmtExecutor asynExecutor : returnResultFromRemoteExecutor) {
             asynExecutor.finalizeQuery();
         }
         returnResultFromRemoteExecutor.clear();
+        executor.finalizeQuery();
         ConnectContext.remove();
     }
 }
diff --git a/regression-test/framework/pom.xml b/regression-test/framework/pom.xml
index 6b749bf0fd1dae..813659989ae60c 100644
--- a/regression-test/framework/pom.xml
+++ b/regression-test/framework/pom.xml
@@ -75,6 +75,7 @@ under the License.
         <antlr.version>4.9.3</antlr.version>
         <hadoop.version>2.8.0</hadoop.version>
         <aws-java-sdk-s3.version>1.11.95</aws-java-sdk-s3.version>
+        <!-- Arrow 18 only supports jdk17 -->
         <arrow.version>17.0.0</arrow.version>
     </properties>
     <build>
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 2f632f5c9252a7..e08ec60457707f 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -1064,19 +1064,23 @@ class Config {
         return buildUrlWithDb(jdbcUrl, dbName)
     }
 
-    Connection getConnectionByArrowFlightSql(String dbName) {
+    Connection getConnectionByArrowFlightSqlDbName(String dbName) {
         Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver")
         String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost")
         String arrowFlightSqlPort = otherConfigs.get("extArrowFlightSqlPort")
         String arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" +
                 "/?useServerPrepStmts=false&useSSL=false&useEncryption=false"
-        // TODO jdbc:arrow-flight-sql not support connect db
-        String dbUrl = buildUrlWithDbImpl(arrowFlightSqlUrl, dbName)
+        // Arrow 17.0.0-rc03 support jdbc:arrow-flight-sql connect db
+        // https://github.com/apache/arrow/issues/41947
+        if (dbName?.trim()) {
+            arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" +
+                "/catalog=" + dbName + "?useServerPrepStmts=false&useSSL=false&useEncryption=false"
+        }
         tryCreateDbIfNotExist(dbName)
-        log.info("connect to ${dbUrl}".toString())
+        log.info("connect to ${arrowFlightSqlUrl}".toString())
         String arrowFlightSqlJdbcUser = otherConfigs.get("extArrowFlightSqlUser")
         String arrowFlightSqlJdbcPassword = otherConfigs.get("extArrowFlightSqlPassword")
-        return DriverManager.getConnection(dbUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword)
+        return DriverManager.getConnection(arrowFlightSqlUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword)
     }
 
     Connection getDownstreamConnection() {
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
index d2d49efa1a86de..88ad1fac355d17 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
@@ -156,7 +156,7 @@ class SuiteContext implements Closeable {
         def threadConnInfo = threadArrowFlightSqlConn.get()
         if (threadConnInfo == null) {
             threadConnInfo = new ConnectionInfo()
-            threadConnInfo.conn = config.getConnectionByArrowFlightSql(dbName)
+            threadConnInfo.conn = config.getConnectionByArrowFlightSqlDbName(dbName)
             threadConnInfo.username = config.jdbcUser
             threadConnInfo.password = config.jdbcPassword
             threadArrowFlightSqlConn.set(threadConnInfo)
diff --git a/regression-test/suites/nereids_p0/show/test_show_create_view.groovy b/regression-test/suites/nereids_p0/show/test_show_create_view.groovy
index 65ed4758a9ae66..be3e691f6dfc38 100644
--- a/regression-test/suites/nereids_p0/show/test_show_create_view.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_create_view.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_show_create_view", "query,arrow_flight_sql") {
+suite("test_show_create_view", "query") {
     String view_name = "view_show_create_view";
     String table_name = "table_for_view_test";
     try {

From e9d267309b7f06b12cb956d44bb66aeffbccdda3 Mon Sep 17 00:00:00 2001
From: zxealous <zhouchangyue@baidu.com>
Date: Wed, 4 Dec 2024 11:21:24 +0800
Subject: [PATCH 154/399] [fix](func) Fix BE core dump caused by buffer access
 out of bounds (#44919)

Fix BE core dump caused by buffer access out of bounds
---
 be/src/vec/functions/function_string.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/vec/functions/function_string.cpp b/be/src/vec/functions/function_string.cpp
index cee141db2a7820..89737d03453ae3 100644
--- a/be/src/vec/functions/function_string.cpp
+++ b/be/src/vec/functions/function_string.cpp
@@ -894,7 +894,7 @@ struct StringSpace {
             if (data[i] > 0) {
                 buffer.resize(data[i]);
                 for (size_t j = 0; j < data[i]; ++j) {
-                    buffer[i] = ' ';
+                    buffer[j] = ' ';
                 }
                 StringOP::push_value_string(std::string_view(buffer.data(), buffer.size()), i,
                                             res_data, res_offsets);

From 3dee2e51b9221a5529ce051c065084df46f6c91e Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Wed, 4 Dec 2024 11:36:24 +0800
Subject: [PATCH 155/399] [fix](cluster key) fix cluster key too many segment
 after compaction (#44927)

---
 be/src/common/config.cpp                      |  3 ++
 be/src/common/config.h                        |  3 ++
 .../olap/rowset/segment_v2/segment_writer.cpp | 31 ++++++++++---------
 .../olap/rowset/segment_v2/segment_writer.h   |  4 +++
 .../rowset/vertical_beta_rowset_writer.cpp    |  6 ++--
 ...ompaction_with_multi_append_columns.groovy | 16 +++++++---
 6 files changed, 40 insertions(+), 23 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 63989a76261bb6..b3e7d0bce5ee4d 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1166,6 +1166,9 @@ DEFINE_mBool(enable_missing_rows_correctness_check, "false");
 // When the number of missing versions is more than this value, do not directly
 // retry the publish and handle it through async publish.
 DEFINE_mInt32(mow_publish_max_discontinuous_version_num, "20");
+// When the size of primary keys in memory exceeds this value, finish current segment
+// and create a new segment, used in compaction. Default 50MB.
+DEFINE_mInt64(mow_primary_key_index_max_size_in_memory, "52428800");
 // When the version is not continuous for MOW table in publish phase and the gap between
 // current txn's publishing version and the max version of the tablet exceeds this value,
 // don't print warning log
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 29e55e6406390e..59fc61e8cb3e5a 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1236,6 +1236,9 @@ DECLARE_mBool(enable_missing_rows_correctness_check);
 // When the number of missing versions is more than this value, do not directly
 // retry the publish and handle it through async publish.
 DECLARE_mInt32(mow_publish_max_discontinuous_version_num);
+// When the size of primary keys in memory exceeds this value, finish current segment
+// and create a new segment, used in compaction.
+DECLARE_mInt64(mow_primary_key_index_max_size_in_memory);
 // When the version is not continuous for MOW table in publish phase and the gap between
 // current txn's publishing version and the max version of the tablet exceeds this value,
 // don't print warning log
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index fc22c3570e52a2..c6c9664be4b232 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -1016,6 +1016,18 @@ Status SegmentWriter::finalize_columns_index(uint64_t* index_size) {
     *index_size = _file_writer->bytes_appended() - index_start;
     if (_has_key) {
         if (_is_mow_with_cluster_key()) {
+            // 1. sort primary keys
+            std::sort(_primary_keys.begin(), _primary_keys.end());
+            // 2. write primary keys index
+            std::string last_key;
+            for (const auto& key : _primary_keys) {
+                DCHECK(key.compare(last_key) > 0)
+                        << "found duplicate key or key is not sorted! current key: " << key
+                        << ", last key: " << last_key;
+                RETURN_IF_ERROR(_primary_key_index_builder->add_item(key));
+                last_key = key;
+            }
+
             RETURN_IF_ERROR(_write_short_key_index());
             *index_size = _file_writer->bytes_appended() - index_start;
             RETURN_IF_ERROR(_write_primary_key_index());
@@ -1236,27 +1248,16 @@ Status SegmentWriter::_generate_primary_key_index(
             last_key = std::move(key);
         }
     } else { // mow table with cluster key
-        // 1. generate primary keys in memory
-        std::vector<std::string> primary_keys;
+        // generate primary keys in memory
         for (uint32_t pos = 0; pos < num_rows; pos++) {
             std::string key = _full_encode_keys(primary_key_coders, primary_key_columns, pos);
             _maybe_invalid_row_cache(key);
             if (_tablet_schema->has_sequence_col()) {
                 _encode_seq_column(seq_column, pos, &key);
             }
-            _encode_rowid(pos, &key);
-            primary_keys.emplace_back(std::move(key));
-        }
-        // 2. sort primary keys
-        std::sort(primary_keys.begin(), primary_keys.end());
-        // 3. write primary keys index
-        std::string last_key;
-        for (const auto& key : primary_keys) {
-            DCHECK(key.compare(last_key) > 0)
-                    << "found duplicate key or key is not sorted! current key: " << key
-                    << ", last key: " << last_key;
-            RETURN_IF_ERROR(_primary_key_index_builder->add_item(key));
-            last_key = key;
+            _encode_rowid(pos + _num_rows_written, &key);
+            _primary_keys_size += key.size();
+            _primary_keys.emplace_back(std::move(key));
         }
     }
     return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h
index 9a8af131087f92..a1b7491a669673 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.h
+++ b/be/src/olap/rowset/segment_v2/segment_writer.h
@@ -155,6 +155,8 @@ class SegmentWriter {
         return Status::OK();
     }
 
+    uint64_t primary_keys_size() const { return _primary_keys_size; }
+
 private:
     DISALLOW_COPY_AND_ASSIGN(SegmentWriter);
     Status _create_column_writer(uint32_t cid, const TabletColumn& column,
@@ -260,6 +262,8 @@ class SegmentWriter {
     std::map<RowsetId, RowsetSharedPtr> _rsid_to_rowset;
     // contains auto generated columns, should be nullptr if no variants's subcolumns
     TabletSchemaSPtr _flush_schema = nullptr;
+    std::vector<std::string> _primary_keys;
+    uint64_t _primary_keys_size = 0;
 };
 
 } // namespace segment_v2
diff --git a/be/src/olap/rowset/vertical_beta_rowset_writer.cpp b/be/src/olap/rowset/vertical_beta_rowset_writer.cpp
index ee9bfd97745c9b..f493f21ac97fb7 100644
--- a/be/src/olap/rowset/vertical_beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/vertical_beta_rowset_writer.cpp
@@ -72,10 +72,9 @@ Status VerticalBetaRowsetWriter<T>::add_columns(const vectorized::Block* block,
         _cur_writer_idx = 0;
         RETURN_IF_ERROR(_segment_writers[_cur_writer_idx]->append_block(block, 0, num_rows));
     } else if (is_key) {
-        // TODO for cluster key, always create new segment writer because the primary keys are
-        // sorted in SegmentWriter::_generate_primary_key_index, will cause too many segments
         if (_segment_writers[_cur_writer_idx]->num_rows_written() > max_rows_per_segment ||
-            has_cluster_key) {
+            (has_cluster_key && _segment_writers[_cur_writer_idx]->primary_keys_size() >
+                                        config::mow_primary_key_index_max_size_in_memory)) {
             // segment is full, need flush columns and create new segment writer
             RETURN_IF_ERROR(_flush_columns(_segment_writers[_cur_writer_idx].get(), true));
 
@@ -181,6 +180,7 @@ Status VerticalBetaRowsetWriter<T>::_create_segment_writer(
     writer_options.enable_unique_key_merge_on_write = context.enable_unique_key_merge_on_write;
     writer_options.rowset_ctx = &context;
     writer_options.max_rows_per_segment = context.max_rows_per_segment;
+    // TODO if support VerticalSegmentWriter, also need to handle cluster key primary key index
     *writer = std::make_unique<segment_v2::SegmentWriter>(
             segment_file_writer.get(), seg_id, context.tablet_schema, context.tablet,
             context.data_dir, writer_options, inverted_index_file_writer.get());
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_compaction_with_multi_append_columns.groovy b/regression-test/suites/unique_with_mow_c_p0/test_compaction_with_multi_append_columns.groovy
index 8403b17cce5b6d..acac719b8c5fd0 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_compaction_with_multi_append_columns.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_compaction_with_multi_append_columns.groovy
@@ -128,11 +128,17 @@ suite("test_compaction_with_multi_append_columns", "p0") {
             assertEquals("success", compactJson.status.toLowerCase())
         }
 
-        (code, out, err) = be_show_tablet_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
-        logger.info("Show tablet status: code=" + code + ", out=" + out + ", err=" + err)
-        assertEquals(code, 0)
-        def json = parseJson(out.trim())
-        logger.info("tablet rowset: " + json)
+        for (int i = 0; i < 10; i++) {
+            (code, out, err) = be_show_tablet_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+            logger.info("loop " + i + ", Show tablet status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def json = parseJson(out.trim())
+            logger.info("tablet rowsets: " + json)
+            if (json.rowsets.size() <= 5) {
+                break
+            }
+            sleep(2000)
+        }
     }
     checkNoDuplicatedKeys(tableName)
 

From a2ebbe2bffdbef6804dfcfd7c09ad109b4cef6fe Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Wed, 4 Dec 2024 11:36:44 +0800
Subject: [PATCH 156/399] [fix](cluster key) fix cluster key in topn (#44880)

---
 be/src/olap/tablet_reader.cpp                 | 33 +++++++--
 .../data/unique_with_mow_c_p0/test_select.out |  9 +++
 .../unique_with_mow_c_p0/test_select.groovy   | 70 +++++++++++++++++++
 3 files changed, 105 insertions(+), 7 deletions(-)
 create mode 100644 regression-test/data/unique_with_mow_c_p0/test_select.out
 create mode 100644 regression-test/suites/unique_with_mow_c_p0/test_select.groovy

diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 7410b70f4aa471..cfd5aa2068bf33 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -464,13 +464,32 @@ Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) {
     // UNIQUE_KEYS will compare all keys as before
     if (_tablet_schema->keys_type() == DUP_KEYS || (_tablet_schema->keys_type() == UNIQUE_KEYS &&
                                                     _tablet->enable_unique_key_merge_on_write())) {
-        // find index in vector _return_columns
-        //   for the read_orderby_key_num_prefix_columns orderby keys
-        for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) {
-            for (uint32_t idx = 0; idx < _return_columns.size(); idx++) {
-                if (_return_columns[idx] == i) {
-                    _orderby_key_columns.push_back(idx);
-                    break;
+        if (!_tablet_schema->cluster_key_idxes().empty()) {
+            for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) {
+                auto cid = _tablet_schema->cluster_key_idxes()[i];
+                auto index = _tablet_schema->field_index(cid);
+                if (index < 0) {
+                    return Status::Error<ErrorCode::INTERNAL_ERROR>(
+                            "could not find cluster key column with unique_id=" +
+                            std::to_string(cid) +
+                            " in tablet schema, tablet_id=" + std::to_string(_tablet->tablet_id()));
+                }
+                for (uint32_t idx = 0; idx < _return_columns.size(); idx++) {
+                    if (_return_columns[idx] == index) {
+                        _orderby_key_columns.push_back(idx);
+                        break;
+                    }
+                }
+            }
+        } else {
+            // find index in vector _return_columns
+            //   for the read_orderby_key_num_prefix_columns orderby keys
+            for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) {
+                for (uint32_t idx = 0; idx < _return_columns.size(); idx++) {
+                    if (_return_columns[idx] == i) {
+                        _orderby_key_columns.push_back(idx);
+                        break;
+                    }
                 }
             }
         }
diff --git a/regression-test/data/unique_with_mow_c_p0/test_select.out b/regression-test/data/unique_with_mow_c_p0/test_select.out
new file mode 100644
index 00000000000000..11c66ecf4dbc47
--- /dev/null
+++ b/regression-test/data/unique_with_mow_c_p0/test_select.out
@@ -0,0 +1,9 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql0 --
+0
+can
+
+-- !sql1 --
+0
+can
+
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_select.groovy b/regression-test/suites/unique_with_mow_c_p0/test_select.groovy
new file mode 100644
index 00000000000000..8cf169e6272dd0
--- /dev/null
+++ b/regression-test/suites/unique_with_mow_c_p0/test_select.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_select") {
+    sql """ DROP TABLE IF EXISTS test_select0 """
+    sql """ DROP TABLE IF EXISTS test_select1 """
+
+    sql """ 
+        create table test_select0 (
+            pk int,
+            v1 char(255) null,
+            v2 varchar(255) not null, 
+            v3 varchar(1000) not null 
+        ) engine=olap
+        UNIQUE KEY(pk)
+        distributed by hash(pk) buckets 10
+        properties("replication_num" = "1");
+    """
+
+    sql """ 
+        create table test_select1 (
+            pk int,
+            v1 char(255) null,
+            v2 varchar(255) not null, 
+            v3 varchar(1000) not null 
+        ) engine=olap
+        UNIQUE KEY(pk)
+        cluster by(v2, v3)
+        distributed by hash(pk) buckets 10
+        properties("replication_num" = "1");
+    """
+
+    sql """
+        insert into test_select0 values 
+        (0,null,'a','1'),
+        (1,'r','0','9999-12-31 23:59:59'),
+        (2,'n','she','i'),
+        (3,'but','can','2024-08-03 13:08:30');
+    """
+
+    sql """
+        insert into test_select1 values 
+        (0,null,'a','1'),
+        (1,'r','0','9999-12-31 23:59:59'),
+        (2,'n','she','i'),
+        (3,'but','can','2024-08-03 13:08:30');
+    """
+
+    order_qt_sql0 """ 
+        select v2  from test_select0 where v1 is not null  ORDER BY v2 LIMIT 2 ;
+    """
+
+    order_qt_sql1 """ 
+        select v2  from test_select1 where v1 is not null  ORDER BY v2 LIMIT 2 ;
+    """
+}

From 5d5f1e54cdd445ffb4f8c32376fe82c36496cb6d Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Wed, 4 Dec 2024 11:38:37 +0800
Subject: [PATCH 157/399] [fix](cluster key) fix bug if be enable
 enable_rowid_conversion_correctness_check (#44914)

fix:
```
terminate called after throwing an instance of 'std::system_error'
  what():  Resource deadlock avoided
*** Query id: 0-0 ***
*** is nereids: 0 ***
*** tablet id: 1980528 ***
*** Aborted at 1733280745 (unix time) try "date -d @1733280745" if you are using GNU date ***
*** Current BE git commitID: 009e5a82c4 ***
*** SIGABRT unknown detail explain (@0x4290022570e) received by PID 2250510 (TID 2251330 OR 0x7f21b45dd700) from PID 2250510; stack trace: ***
 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /mnt/disk2/meiyi/clion_workspace/doris/be/src/common/signal_handler.h:421
 1# 0x00007F24197AFB50 in /lib64/libc.so.6
 2# gsignal in /lib64/libc.so.6
 3# __GI_abort in /lib64/libc.so.6
 4# __gnu_cxx::__verbose_terminate_handler() [clone .cold] at ../../../../libstdc++-v3/libsupc++/vterminate.cc:75
 5# __cxxabiv1::__terminate(void (*)()) at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48
 6# 0x0000564E1F9917D1 in /mnt/disk2/meiyi/deployment/doris/output/be/lib/doris_be
 7# 0x0000564E1F991924 in /mnt/disk2/meiyi/deployment/doris/output/be/lib/doris_be
 8# std::__throw_system_error(int) at ../../../../../libstdc++-v3/src/c++11/system_error.cc:338
 9# std::__shared_mutex_pthread::lock_shared() at /mnt/disk2/meiyi/soft/ldb_toolchain_2/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/shared_mutex:232
10# std::shared_mutex::lock_shared() at /mnt/disk2/meiyi/soft/ldb_toolchain_2/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/shared_mutex:426
11# std::shared_lock<std::shared_mutex>::shared_lock(std::shared_mutex&) at /mnt/disk2/meiyi/soft/ldb_toolchain_2/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/shared_mutex:727
12# doris::BaseTablet::tablet_schema() const at /mnt/disk2/meiyi/clion_workspace/doris/be/src/olap/base_tablet.h:91
13# doris::BaseTablet::check_rowid_conversion(std::shared_ptr<doris::Rowset>, std::map<std::shared_ptr<doris::Rowset>, std::__cxx11::list<std::pair<doris::RowLocation, doris::RowLocation>, std::allocator<std::pair<doris::RowLocation, doris::RowLocation> > >, std::less<std::shared_ptr<doris::Rowset> >, std::allocator<std::pair<std::shared_ptr<doris::Rowset> const, std::__cxx11::list<std::pair<doris::RowLocation, doris::RowLocation>, std::allocator<std::pair<doris::RowLocation, doris::RowLocation> > > > > > const&) at /mnt/disk2/meiyi/clion_workspace/doris/be/src/olap/base_tablet.cpp:1607
14# doris::CompactionMixin::modify_rowsets() at /mnt/disk2/meiyi/clion_workspace/doris/be/src/olap/compaction.cpp:1087
15# doris::CompactionMixin::execute_compact_impl(long) at /mnt/disk2/meiyi/clion_workspace/doris/be/src/olap/compaction.cpp:472
16# doris::CompactionMixin::execute_compact() at /mnt/disk2/meiyi/clion_workspace/doris/be/src/olap/compaction.cpp:425
17# doris::CumulativeCompaction::execute_compact() at /mnt/disk2/meiyi/clion_workspace/doris/be/src/olap/cumulative_compaction.cpp:111
18# doris::Tablet::execute_compaction(doris::CompactionMixin&)::$_0::operator()() const at /mnt/disk2/meiyi/clion_workspace/doris/be/src/olap/tablet.cpp:1850
19# doris::Tablet::execute_compaction(doris::CompactionMixin&) at /mnt/disk2/meiyi/clion_workspace/doris/be/src/olap/tablet.cpp:1850
```
---
 be/src/cloud/cloud_tablet.cpp                                 | 3 ++-
 be/src/olap/base_tablet.cpp                                   | 4 ----
 be/src/olap/compaction.cpp                                    | 3 ++-
 .../src/main/groovy/org/apache/doris/regression/Config.groovy | 1 +
 4 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index ea86f3b40ff1dc..c27353be5235d0 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -775,7 +775,8 @@ Status CloudTablet::calc_delete_bitmap_for_compaction(
     }
 
     std::unique_ptr<std::map<RowsetSharedPtr, RowLocationPairList>> location_map;
-    if (config::enable_rowid_conversion_correctness_check) {
+    if (config::enable_rowid_conversion_correctness_check &&
+        tablet_schema()->cluster_key_idxes().empty()) {
         location_map = std::make_unique<std::map<RowsetSharedPtr, RowLocationPairList>>();
         LOG(INFO) << "Location Map inited succ for tablet:" << tablet_id();
     }
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index a499a27b07f6e2..104dc44ebec8ae 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -1604,10 +1604,6 @@ Status BaseTablet::check_rowid_conversion(
         VLOG_DEBUG << "check_rowid_conversion, location_map is empty";
         return Status::OK();
     }
-    if (!tablet_schema()->cluster_key_idxes().empty()) {
-        VLOG_DEBUG << "skip check_rowid_conversion for mow tables with cluster keys";
-        return Status::OK();
-    }
     std::vector<segment_v2::SegmentSharedPtr> dst_segments;
 
     RETURN_IF_ERROR(
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index c85ce36a1033f5..98d7787550d197 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -965,7 +965,8 @@ Status CompactionMixin::modify_rowsets() {
             LOG(INFO) << "RowLocation Set inited succ for tablet:" << _tablet->tablet_id();
         }
         std::unique_ptr<std::map<RowsetSharedPtr, RowLocationPairList>> location_map;
-        if (config::enable_rowid_conversion_correctness_check) {
+        if (config::enable_rowid_conversion_correctness_check &&
+            tablet()->tablet_schema()->cluster_key_idxes().empty()) {
             location_map = std::make_unique<std::map<RowsetSharedPtr, RowLocationPairList>>();
             LOG(INFO) << "Location Map inited succ for tablet:" << _tablet->tablet_id();
         }
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index e08ec60457707f..aca1596f863769 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -1031,6 +1031,7 @@ class Config {
             excludeDirectorySet.add("schema_change_p0/unique_ck")
             List<String> excludeCases = ["test_table_properties", "test_create_table"
                 , "test_default_hll", "test_default_pi", "test_default_bitmap_empty"
+                , "test_full_compaction", "test_full_compaction_by_table_id"
                 // partial update
                 , "txn_insert", "test_update_schema_change", "test_generated_column_update", "test_nested_type_with_rowstore", "test_partial_update_generated_column", "nereids_partial_update_native_insert_stmt"
                 , "partial_update", "nereids_update_on_current_timestamp", "update_on_current_timestamp", "nereids_delete_mow_partial_update", "delete_mow_partial_update", "test_unique_table_auto_inc"

From 864ebb82d196e8d831c955a33821144193d412da Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Wed, 4 Dec 2024 14:31:16 +0800
Subject: [PATCH 158/399] [fix](compile) Fix MacOS compile failed (#44901)

---
 be/src/vec/columns/column_decimal.h                   | 2 +-
 be/src/vec/columns/column_vector.cpp                  | 2 +-
 be/src/vec/functions/array/function_array_shuffle.cpp | 3 ++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h
index a8b1f9573ef185..dde7a1c6237481 100644
--- a/be/src/vec/columns/column_decimal.h
+++ b/be/src/vec/columns/column_decimal.h
@@ -259,7 +259,7 @@ class ColumnDecimal final : public COWHelper<IColumn, ColumnDecimal<T>> {
         for (U i = 0; i < s; ++i) res[i] = i;
 
         auto sort_end = res.end();
-        if (limit && limit < static_cast<double>(s) / 8.0L) {
+        if (limit && static_cast<double>(limit) < static_cast<double>(s) / 8.0) {
             sort_end = res.begin() + limit;
             if (reverse)
                 std::partial_sort(res.begin(), sort_end, res.end(),
diff --git a/be/src/vec/columns/column_vector.cpp b/be/src/vec/columns/column_vector.cpp
index f0f0bec8b99bb1..396edcc2ad4c3c 100644
--- a/be/src/vec/columns/column_vector.cpp
+++ b/be/src/vec/columns/column_vector.cpp
@@ -243,7 +243,7 @@ void ColumnVector<T>::get_permutation(bool reverse, size_t limit, int nan_direct
     if (s == 0) return;
 
     // std::partial_sort need limit << s can get performance benefit
-    if (limit > (s / 8.0L)) limit = 0;
+    if (static_cast<double>(limit) > (static_cast<double>(s) / 8.0)) limit = 0;
 
     if (limit) {
         for (size_t i = 0; i < s; ++i) res[i] = i;
diff --git a/be/src/vec/functions/array/function_array_shuffle.cpp b/be/src/vec/functions/array/function_array_shuffle.cpp
index fbf5bcfcdf1686..b9e3206ba0baa1 100644
--- a/be/src/vec/functions/array/function_array_shuffle.cpp
+++ b/be/src/vec/functions/array/function_array_shuffle.cpp
@@ -79,7 +79,8 @@ class FunctionArrayShuffle : public IFunction {
             seed = assert_cast<const ColumnInt64*>(seed_column.get())->get_element(0);
         }
 
-        std::mt19937 g(seed);
+        // time() and seed will not exceed the range of uint32.
+        std::mt19937 g(cast_set<uint32>(seed));
         auto dest_column_ptr = _execute(src_column_array, g);
         if (!dest_column_ptr) {
             return Status::RuntimeError(

From 06466c891bf3fe86bd02e99a4f5cdd4895d4c8f5 Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Wed, 4 Dec 2024 14:51:26 +0800
Subject: [PATCH 159/399] [fix](auth)fix be enable http auth, some request link
 never return. (#44959)

if you `enable_all_http_auth = true` in be.conf, then restart be, and
keep using `curl -u "xxxx:xxxx" http://127.0.0.1:8040/api/health` while
be is starting. You may encounter a situation where the link does not
return.
Reason:
When be is still starting, there is no information about fe master. When
you make an api request to be http port, be needs to request
authentication information from fe, which will cause it to request a
machine with empty ip and port 0. This rpc call will definitely fail
(this is not equivalent to a password error). After receiving this
failure, be does not `send_reply` to the api requester, so this api
request cannot be returned.
---
 be/src/http/http_handler_with_auth.cpp | 11 +++++++++++
 be/test/http/http_client_test.cpp      |  8 ++++----
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/be/src/http/http_handler_with_auth.cpp b/be/src/http/http_handler_with_auth.cpp
index 518b9868de191e..ae5c024e76d093 100644
--- a/be/src/http/http_handler_with_auth.cpp
+++ b/be/src/http/http_handler_with_auth.cpp
@@ -35,6 +35,7 @@ HttpHandlerWithAuth::HttpHandlerWithAuth(ExecEnv* exec_env, TPrivilegeHier::type
         : _exec_env(exec_env), _hier(hier), _type(type) {}
 
 int HttpHandlerWithAuth::on_header(HttpRequest* req) {
+    //if u return value isn't 0,u should `send_reply`,Avoid requesting links that never return.
     TCheckAuthRequest auth_request;
     TCheckAuthResult auth_result;
     AuthInfo auth_info;
@@ -83,6 +84,11 @@ int HttpHandlerWithAuth::on_header(HttpRequest* req) {
 
 #ifndef BE_TEST
     TNetworkAddress master_addr = _exec_env->cluster_info()->master_fe_addr;
+    if (master_addr.hostname.empty() || master_addr.port == 0) {
+        LOG(WARNING) << "Not found master fe, Can't auth API request: " << req->debug_string();
+        HttpChannel::send_error(req, HttpStatus::SERVICE_UNAVAILABLE);
+        return -1;
+    }
     {
         auto status = ThriftRpcHelper::rpc<FrontendServiceClient>(
                 master_addr.hostname, master_addr.port,
@@ -90,6 +96,10 @@ int HttpHandlerWithAuth::on_header(HttpRequest* req) {
                     client->checkAuth(auth_result, auth_request);
                 });
         if (!status) {
+            LOG(WARNING) << "CheckAuth Rpc Fail.Fe Ip:" << master_addr.hostname
+                         << ", Fe port:" << master_addr.port << ".Status:" << status.to_string()
+                         << ".Request: " << req->debug_string();
+            HttpChannel::send_error(req, HttpStatus::SERVICE_UNAVAILABLE);
             return -1;
         }
     }
@@ -98,6 +108,7 @@ int HttpHandlerWithAuth::on_header(HttpRequest* req) {
         auth_result.status.status_code = TStatusCode::type::OK;
         auth_result.status.error_msgs.clear();
     } else {
+        HttpChannel::send_reply(req, HttpStatus::FORBIDDEN);
         return -1;
     }
 #endif
diff --git a/be/test/http/http_client_test.cpp b/be/test/http/http_client_test.cpp
index d42e0a6775ed4b..c98328d7c8e37c 100644
--- a/be/test/http/http_client_test.cpp
+++ b/be/test/http/http_client_test.cpp
@@ -413,7 +413,7 @@ TEST_F(HttpClientTest, enable_http_auth) {
         EXPECT_TRUE(!st.ok());
         std::cout << "response = " << response << "\n";
         std::cout << "st.msg() = " << st.msg() << "\n";
-        EXPECT_TRUE(st.msg().find("Operation timed out after") != std::string::npos);
+        EXPECT_TRUE(st.msg().find("403") != std::string::npos);
     }
 
     {
@@ -474,7 +474,7 @@ TEST_F(HttpClientTest, enable_http_auth) {
         EXPECT_TRUE(!st.ok());
         std::cout << "response = " << response << "\n";
         std::cout << "st.msg() = " << st.msg() << "\n";
-        EXPECT_TRUE(st.msg().find("Operation timed out after") != std::string::npos);
+        EXPECT_TRUE(st.msg().find("403") != std::string::npos);
     }
 
     // valid token
@@ -521,7 +521,7 @@ TEST_F(HttpClientTest, enable_http_auth) {
         EXPECT_TRUE(!st.ok());
         std::cout << "response = " << response << "\n";
         std::cout << "st.msg() = " << st.msg() << "\n";
-        EXPECT_TRUE(st.msg().find("Operation timed out after") != std::string::npos);
+        EXPECT_TRUE(st.msg().find("403") != std::string::npos);
     }
 
     std::vector<std::string> check_get_list = {"/api/clear_cache/aa",
@@ -566,7 +566,7 @@ TEST_F(HttpClientTest, enable_http_auth) {
             EXPECT_TRUE(!st.ok());
             std::cout << "response = " << response << "\n";
             std::cout << "st.msg() = " << st.msg() << "\n";
-            EXPECT_TRUE(st.msg().find("Operation timed out after") != std::string::npos);
+            EXPECT_TRUE(st.msg().find("403") != std::string::npos);
         }
     }
 }

From 04e58bc531bd401b0000463b2e930b7f1a37b7d1 Mon Sep 17 00:00:00 2001
From: TsukiokaKogane <cby141994@gmail.com>
Date: Wed, 4 Dec 2024 15:00:24 +0800
Subject: [PATCH 160/399] [fix](ctas) fix wrong resultType of column typeDef
 (#43828)

wrong resultType could lead to definition change of `Type.VARCHAR`
---
 .../main/java/org/apache/doris/datasource/InternalCatalog.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 64a759b9747044..65f77fd039137d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1416,7 +1416,7 @@ public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlExceptio
                     // we change it to tinyint nullable.
                     typeDef = TypeDef.create(PrimitiveType.TINYINT);
                 } else {
-                    typeDef = new TypeDef(resultExpr.getType());
+                    typeDef = new TypeDef(resultType);
                 }
                 if (i == 0) {
                     // If this is the first column, because olap table does not support the first column to be

From 4c55b53be45940d3392d633c76d2b151e32fe8c7 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <chenkaijie@selectdb.com>
Date: Wed, 4 Dec 2024 15:36:06 +0800
Subject: [PATCH 161/399] [test](load) add p0 cases for insert docs (#44364)

Add cases in INSERT INTO VALUES and INSERT INTO docs demos.
---
 .../data/insert_p0/test_insert_docs_demo.out  |   6 +
 .../insert_p0/test_insert_docs_demo.groovy    | 111 ++++++++++++++++++
 2 files changed, 117 insertions(+)
 create mode 100644 regression-test/data/insert_p0/test_insert_docs_demo.out
 create mode 100644 regression-test/suites/insert_p0/test_insert_docs_demo.groovy

diff --git a/regression-test/data/insert_p0/test_insert_docs_demo.out b/regression-test/data/insert_p0/test_insert_docs_demo.out
new file mode 100644
index 00000000000000..f984fb09c6d433
--- /dev/null
+++ b/regression-test/data/insert_p0/test_insert_docs_demo.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !insert --
+5	Ava	17
+1	Emily	25
+3	Olivia	28
+
diff --git a/regression-test/suites/insert_p0/test_insert_docs_demo.groovy b/regression-test/suites/insert_p0/test_insert_docs_demo.groovy
new file mode 100644
index 00000000000000..b56df70e344ef3
--- /dev/null
+++ b/regression-test/suites/insert_p0/test_insert_docs_demo.groovy
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import com.mysql.cj.jdbc.StatementImpl
+
+suite("test_insert_docs_demo") {
+    def dbName = "regression_test_insert_p0"
+    def srcTableName = "test_insert_docs_demo_src"
+    def dstTableName = "test_insert_docs_demo_dst"
+    def srcName = dbName + "." + srcTableName
+    def dstName = dbName + "." + dstTableName
+
+    // create table
+    sql """ drop table if exists ${srcName}; """
+    sql """
+        CREATE TABLE ${srcName}(
+            user_id            BIGINT       NOT NULL COMMENT "user id",
+            name               VARCHAR(20)           COMMENT "name",
+            age                INT                   COMMENT "age"
+        ) DUPLICATE KEY(user_id)
+        DISTRIBUTED BY HASH(user_id) BUCKETS 10;
+    """
+
+    sql """ drop table if exists ${dstName}; """
+    sql """
+        CREATE TABLE ${dstName}(
+            user_id            BIGINT       NOT NULL COMMENT "user id",
+            name               VARCHAR(20)           COMMENT "name",
+            age                INT                   COMMENT "age"
+        ) DUPLICATE KEY(user_id)
+        DISTRIBUTED BY HASH(user_id) BUCKETS 10;
+    """
+
+    sql """ SET enable_insert_strict = true; """
+
+    // insert into values
+    sql """
+        INSERT INTO ${srcName} (user_id, name, age)
+        VALUES (1, "Emily", 25),
+            (2, "Benjamin", 35),
+            (3, "Olivia", 28),
+            (4, "Alexander", 60),
+            (5, "Ava", 17);
+    """
+
+    // insert into
+    sql """
+        INSERT INTO ${dstName}
+        SELECT * FROM ${srcName} WHERE age < 30;
+    """
+
+    qt_insert """
+        SELECT * FROM ${dstName} ORDER BY age;
+    """
+
+    // The error message may vary due to variations in fuzzy execution instance number or batch size.
+    // like this:
+    // Insert has filtered data in strict mode. url: http://172.16.0.10:8041/api/_load_error_log?
+    // file=__shard_303/error_log_insert_stmt_a1ccfb9c67ba40f5-900d0db1d06a19dd_a1ccfb9c67ba40f5_900d0db1d06a19dd
+    // or like this:
+    // [DATA_QUALITY_ERROR]Encountered unqualified data, stop processing. url: http://172.16.0.10:8041/api/_load_error_log?
+    // file=__shard_303/error_log_insert_stmt_a1ccfb9c67ba40f5-900d0db1d06a19dd_a1ccfb9c67ba40f5_900d0db1d06a19dd
+    expectExceptionLike({
+        sql """
+            INSERT INTO ${srcName} (user_id, name, age)
+            VALUES (1, "Emily", 25),
+                (2, "Benjamin", 35),
+                (3, "Olivia", 28),
+                (NULL, "Alexander", 60),
+                (5, "Ava", 17);
+        """
+    }, "error_log")
+
+    sql """ SET enable_insert_strict = false; """
+
+    sql """
+        INSERT INTO ${srcName} (user_id, name, age)
+        VALUES (1, "Emily", 25),
+            (2, "Benjamin", 35),
+            (3, "Olivia", 28),
+            (NULL, "Alexander", 60),
+            (5, "Ava", 17);
+    """
+
+    sql """ SET insert_max_filter_ratio = 0.1 """
+
+    expectExceptionLike({
+        sql """
+            INSERT INTO ${srcName} (user_id, name, age)
+            VALUES (1, "Emily", 25),
+                (2, "Benjamin", 35),
+                (3, "Olivia", 28),
+                (NULL, "Alexander", 60),
+                (5, "Ava", 17);
+        """
+    }, "Insert has too many filtered data 1/5 insert_max_filter_ratio is 0.1")
+}

From 78bd61e6d5e7fb1e869cbbd7c1f987a2dff01918 Mon Sep 17 00:00:00 2001
From: zzzxl <yangsiyu@selectdb.com>
Date: Wed, 4 Dec 2024 17:12:30 +0800
Subject: [PATCH 162/399] [feat](inverted index) Adding Storage Format V3 for
 Inverted Index (#44414)

Problem Summary:

1. "Mainly added the functionality for compressing inverted index
position information and dictionary information."

2. "Position information compression must be enabled by setting
inverted_index_storage_format to v3 when creating the table."

e.g.
```
    CREATE TABLE tbl (
          ...
    ) ENGINE=OLAP
    DUPLICATE KEY(`x`)
    COMMENT "OLAP"
    DISTRIBUTED BY RANDOM BUCKETS 1
    PROPERTIES (
    "inverted_index_storage_format" = "V3"
    );
```
4. "The dictionary compression feature requires setting
inverted_index_storage_format to v3 and configuring dict_compression to
true in the properties."

e.g.
```
INDEX x_idx (`x`) USING INVERTED PROPERTIES("dict_compression" = "true") COMMENT ''
```
---
 be/src/clucene                                |   2 +-
 be/src/olap/inverted_index_parser.cpp         |   9 ++
 be/src/olap/inverted_index_parser.h           |   5 +
 be/src/olap/rowset/beta_rowset.cpp            |  18 ++-
 .../char_filter/char_replace_char_filter.h    |   2 +-
 .../segment_v2/inverted_index_file_reader.cpp |   8 +-
 .../segment_v2/inverted_index_file_reader.h   |   2 +-
 .../segment_v2/inverted_index_file_writer.cpp |  22 ++--
 .../segment_v2/inverted_index_file_writer.h   |  10 +-
 .../segment_v2/inverted_index_writer.cpp      |  22 ++++
 be/src/olap/tablet_meta.cpp                   |   3 +
 .../query/phrase_query_test.cpp               |   1 -
 .../inverted_index_file_writer_test.cpp       |   8 +-
 .../doris/analysis/InvertedIndexUtil.java     |  12 +-
 .../datasource/CloudInternalCatalog.java      |   6 +-
 .../doris/common/util/PropertyAnalyzer.java   |   6 +
 .../doris/analysis/InvertedIndexUtilTest.java |  47 +++++++
 .../doris/common/PropertyAnalyzerTest.java    |  50 ++++++++
 gensrc/proto/olap_file.proto                  |   1 +
 gensrc/thrift/Types.thrift                    |   3 +-
 .../test_inverted_index_v3.out                |  25 ++++
 ...t_inverted_index_v3_fault_injection.groovy |  60 +++++++++
 .../test_inverted_index_v3.groovy             | 117 ++++++++++++++++++
 23 files changed, 406 insertions(+), 33 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexUtilTest.java
 create mode 100644 regression-test/data/inverted_index_p0/test_inverted_index_v3.out
 create mode 100644 regression-test/suites/fault_injection_p0/test_inverted_index_v3_fault_injection.groovy
 create mode 100644 regression-test/suites/inverted_index_p0/test_inverted_index_v3.groovy

diff --git a/be/src/clucene b/be/src/clucene
index 48fa9cc4ec32b4..a506dbb6c523aa 160000
--- a/be/src/clucene
+++ b/be/src/clucene
@@ -1 +1 @@
-Subproject commit 48fa9cc4ec32b40bf3b02338d0a1b2cdbc6408cf
+Subproject commit a506dbb6c523aa65044eb1c527a066d236172543
diff --git a/be/src/olap/inverted_index_parser.cpp b/be/src/olap/inverted_index_parser.cpp
index f7e511970d91f2..f1de5a5e0c10fc 100644
--- a/be/src/olap/inverted_index_parser.cpp
+++ b/be/src/olap/inverted_index_parser.cpp
@@ -136,4 +136,13 @@ std::string get_parser_stopwords_from_properties(
     }
 }
 
+std::string get_parser_dict_compression_from_properties(
+        const std::map<std::string, std::string>& properties) {
+    if (properties.find(INVERTED_INDEX_PARSER_DICT_COMPRESSION_KEY) != properties.end()) {
+        return properties.at(INVERTED_INDEX_PARSER_DICT_COMPRESSION_KEY);
+    } else {
+        return "";
+    }
+}
+
 } // namespace doris
diff --git a/be/src/olap/inverted_index_parser.h b/be/src/olap/inverted_index_parser.h
index 0b8426d74c7ab3..f1f85995a206a8 100644
--- a/be/src/olap/inverted_index_parser.h
+++ b/be/src/olap/inverted_index_parser.h
@@ -83,6 +83,8 @@ const std::string INVERTED_INDEX_PARSER_LOWERCASE_KEY = "lower_case";
 
 const std::string INVERTED_INDEX_PARSER_STOPWORDS_KEY = "stopwords";
 
+const std::string INVERTED_INDEX_PARSER_DICT_COMPRESSION_KEY = "dict_compression";
+
 std::string inverted_index_parser_type_to_string(InvertedIndexParserType parser_type);
 
 InvertedIndexParserType get_inverted_index_parser_type_from_string(const std::string& parser_str);
@@ -119,4 +121,7 @@ std::string get_parser_lowercase_from_properties(
 std::string get_parser_stopwords_from_properties(
         const std::map<std::string, std::string>& properties);
 
+std::string get_parser_dict_compression_from_properties(
+        const std::map<std::string, std::string>& properties);
+
 } // namespace doris
diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp
index bbb2ca72b4ae7f..cd52deed0c8a4d 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -703,10 +703,24 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value,
                                           rapidjson::Document::AllocatorType& allocator) {
     const auto& fs = _rowset_meta->fs();
     auto storage_format = _schema->get_inverted_index_storage_format();
-    auto format_str = storage_format == InvertedIndexStorageFormatPB::V1 ? "V1" : "V2";
+    std::string format_str;
+    switch (storage_format) {
+    case InvertedIndexStorageFormatPB::V1:
+        format_str = "V1";
+        break;
+    case InvertedIndexStorageFormatPB::V2:
+        format_str = "V2";
+        break;
+    case InvertedIndexStorageFormatPB::V3:
+        format_str = "V3";
+        break;
+    default:
+        return Status::InternalError("inverted index storage format error");
+        break;
+    }
     auto rs_id = rowset_id().to_string();
     rowset_value->AddMember("rowset_id", rapidjson::Value(rs_id.c_str(), allocator), allocator);
-    rowset_value->AddMember("index_storage_format", rapidjson::Value(format_str, allocator),
+    rowset_value->AddMember("index_storage_format", rapidjson::Value(format_str.c_str(), allocator),
                             allocator);
     rapidjson::Value segments(rapidjson::kArrayType);
     for (int seg_id = 0; seg_id < num_segments(); ++seg_id) {
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h
index d9e5080d2d584d..1e5e6f5d5cedd0 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h
@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include <CLucene.h>
+#include <CLucene.h> // IWYU pragma: keep
 #include <CLucene/analysis/CharFilter.h>
 
 #include <bitset>
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
index 113833d560fd06..8d480829a0cd37 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
@@ -30,8 +30,8 @@ namespace doris::segment_v2 {
 Status InvertedIndexFileReader::init(int32_t read_buffer_size) {
     if (!_inited) {
         _read_buffer_size = read_buffer_size;
-        if (_storage_format == InvertedIndexStorageFormatPB::V2) {
-            auto st = _init_from_v2(read_buffer_size);
+        if (_storage_format >= InvertedIndexStorageFormatPB::V2) {
+            auto st = _init_from(read_buffer_size);
             if (!st.ok()) {
                 return st;
             }
@@ -41,7 +41,7 @@ Status InvertedIndexFileReader::init(int32_t read_buffer_size) {
     return Status::OK();
 }
 
-Status InvertedIndexFileReader::_init_from_v2(int32_t read_buffer_size) {
+Status InvertedIndexFileReader::_init_from(int32_t read_buffer_size) {
     auto index_file_full_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix);
 
     std::unique_lock<std::shared_mutex> lock(_mutex); // Lock for writing
@@ -79,7 +79,7 @@ Status InvertedIndexFileReader::_init_from_v2(int32_t read_buffer_size) {
 
         // 3. read file
         int32_t version = _stream->readInt(); // Read version number
-        if (version == InvertedIndexStorageFormatPB::V2) {
+        if (version >= InvertedIndexStorageFormatPB::V2) {
             DCHECK(version == _storage_format);
             int32_t numIndices = _stream->readInt(); // Read number of indices
             ReaderFileEntry* entry = nullptr;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
index 3b7161c7643cef..443d40cfaf0d4f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
@@ -70,7 +70,7 @@ class InvertedIndexFileReader {
     int64_t get_inverted_file_size() const { return _stream == nullptr ? 0 : _stream->length(); }
 
 private:
-    Status _init_from_v2(int32_t read_buffer_size);
+    Status _init_from(int32_t read_buffer_size);
     Result<std::unique_ptr<DorisCompoundReader>> _open(int64_t index_id,
                                                        const std::string& index_suffix) const;
 
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
index bb373be5ee906a..4d6892aa78568f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
@@ -150,7 +150,7 @@ Status InvertedIndexFileWriter::close() {
         }
     } else {
         try {
-            RETURN_IF_ERROR(write_v2());
+            RETURN_IF_ERROR(write());
             for (const auto& entry : _indices_dirs) {
                 const auto& dir = entry.second;
                 // delete index path, which contains separated inverted index files
@@ -293,7 +293,7 @@ Status InvertedIndexFileWriter::write_v1() {
     return Status::OK();
 }
 
-Status InvertedIndexFileWriter::write_v2() {
+Status InvertedIndexFileWriter::write() {
     std::unique_ptr<lucene::store::Directory, DirectoryDeleter> out_dir = nullptr;
     std::unique_ptr<lucene::store::IndexOutput> compound_file_output = nullptr;
     ErrorContext error_context;
@@ -301,10 +301,10 @@ Status InvertedIndexFileWriter::write_v2() {
         // Calculate header length and initialize offset
         int64_t current_offset = headerLength();
         // Prepare file metadata
-        auto file_metadata = prepare_file_metadata_v2(current_offset);
+        auto file_metadata = prepare_file_metadata(current_offset);
 
         // Create output stream
-        auto result = create_output_stream_v2();
+        auto result = create_output_stream();
         out_dir = std::move(result.first);
         compound_file_output = std::move(result.second);
 
@@ -315,7 +315,7 @@ Status InvertedIndexFileWriter::write_v2() {
         write_index_headers_and_metadata(compound_file_output.get(), file_metadata);
 
         // Copy file data
-        copy_files_data_v2(compound_file_output.get(), file_metadata);
+        copy_files_data(compound_file_output.get(), file_metadata);
 
         _total_file_size = compound_file_output->getFilePointer();
         _file_info.set_index_size(_total_file_size);
@@ -470,7 +470,7 @@ void InvertedIndexFileWriter::write_header_and_data_v1(lucene::store::IndexOutpu
 
 std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
           std::unique_ptr<lucene::store::IndexOutput>>
-InvertedIndexFileWriter::create_output_stream_v2() {
+InvertedIndexFileWriter::create_output_stream() {
     io::Path index_path {InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)};
 
     auto* out_dir = DorisFSDirectoryFactory::getDirectory(_fs, index_path.parent_path().c_str());
@@ -486,15 +486,15 @@ InvertedIndexFileWriter::create_output_stream_v2() {
 
 void InvertedIndexFileWriter::write_version_and_indices_count(lucene::store::IndexOutput* output) {
     // Write the version number
-    output->writeInt(InvertedIndexStorageFormatPB::V2);
+    output->writeInt(_storage_format);
 
     // Write the number of indices
     const auto num_indices = static_cast<uint32_t>(_indices_dirs.size());
     output->writeInt(num_indices);
 }
 
-std::vector<InvertedIndexFileWriter::FileMetadata>
-InvertedIndexFileWriter::prepare_file_metadata_v2(int64_t& current_offset) {
+std::vector<InvertedIndexFileWriter::FileMetadata> InvertedIndexFileWriter::prepare_file_metadata(
+        int64_t& current_offset) {
     std::vector<FileMetadata> file_metadata;
 
     for (const auto& entry : _indices_dirs) {
@@ -546,8 +546,8 @@ void InvertedIndexFileWriter::write_index_headers_and_metadata(
     }
 }
 
-void InvertedIndexFileWriter::copy_files_data_v2(lucene::store::IndexOutput* output,
-                                                 const std::vector<FileMetadata>& file_metadata) {
+void InvertedIndexFileWriter::copy_files_data(lucene::store::IndexOutput* output,
+                                              const std::vector<FileMetadata>& file_metadata) {
     const int64_t buffer_length = 16384;
     uint8_t buffer[buffer_length];
 
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h
index ba42ffdceb1475..ab7cdbff152460 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h
@@ -71,7 +71,7 @@ class InvertedIndexFileWriter {
     Status delete_index(const TabletIndex* index_meta);
     Status initialize(InvertedIndexDirectoryMap& indices_dirs);
     virtual ~InvertedIndexFileWriter() = default;
-    Status write_v2();
+    Status write();
     Status write_v1();
     Status close();
     const InvertedIndexFileInfo* get_index_file_info() const {
@@ -122,7 +122,7 @@ class InvertedIndexFileWriter {
     // Helper functions specific to write_v2
     virtual std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
                       std::unique_ptr<lucene::store::IndexOutput>>
-    create_output_stream_v2();
+    create_output_stream();
     void write_version_and_indices_count(lucene::store::IndexOutput* output);
     struct FileMetadata {
         int64_t index_id;
@@ -141,11 +141,11 @@ class InvertedIndexFileWriter {
                   length(len),
                   directory(dir) {}
     };
-    std::vector<FileMetadata> prepare_file_metadata_v2(int64_t& current_offset);
+    std::vector<FileMetadata> prepare_file_metadata(int64_t& current_offset);
     virtual void write_index_headers_and_metadata(lucene::store::IndexOutput* output,
                                                   const std::vector<FileMetadata>& file_metadata);
-    void copy_files_data_v2(lucene::store::IndexOutput* output,
-                            const std::vector<FileMetadata>& file_metadata);
+    void copy_files_data(lucene::store::IndexOutput* output,
+                         const std::vector<FileMetadata>& file_metadata);
     Status _insert_directory_into_map(int64_t index_id, const std::string& index_suffix,
                                       std::shared_ptr<DorisFSDirectory> dir);
     // Member variables...
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 86a8f89e4c94e4..02edf2f1976e3e 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -212,6 +212,28 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
         (*field)->setOmitTermFreqAndPositions(
                 !(get_parser_phrase_support_string_from_properties(_index_meta->properties()) ==
                   INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES));
+        DBUG_EXECUTE_IF("InvertedIndexColumnWriterImpl::create_field_v3", {
+            if (_index_file_writer->get_storage_format() != InvertedIndexStorageFormatPB::V3) {
+                return Status::Error<doris::ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+                        "debug point: InvertedIndexColumnWriterImpl::create_field_v3 error");
+            }
+        })
+        if (_index_file_writer->get_storage_format() >= InvertedIndexStorageFormatPB::V3) {
+            (*field)->setIndexVersion(IndexVersion::kV3);
+            // Only effective in v3
+            std::string dict_compression =
+                    get_parser_dict_compression_from_properties(_index_meta->properties());
+            DBUG_EXECUTE_IF("InvertedIndexColumnWriterImpl::create_field_dic_compression", {
+                if (dict_compression != INVERTED_INDEX_PARSER_TRUE) {
+                    return Status::Error<doris::ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+                            "debug point: "
+                            "InvertedIndexColumnWriterImpl::create_field_dic_compression error");
+                }
+            })
+            if (dict_compression == INVERTED_INDEX_PARSER_TRUE) {
+                (*field)->updateFlag(FlagBits::DICT_COMPRESS);
+            }
+        }
         return Status::OK();
     }
 
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index fc9fc034b0bcd7..d1746836e231c2 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -203,6 +203,9 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
     case TInvertedIndexFileStorageFormat::V2:
         schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
         break;
+    case TInvertedIndexFileStorageFormat::V3:
+        schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3);
+        break;
     default:
         schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
         break;
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
index f3fb9763c9b84d..b8e11bece7b318 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
@@ -62,7 +62,6 @@ TEST_F(PhraseQueryTest, test_parser_info) {
         EXPECT_EQ(query_info.slop, res3);
         EXPECT_EQ(query_info.ordered, res4);
         EXPECT_EQ(query_info.additional_terms.size(), res5);
-        std::cout << "--- 1 ---: " << query_info.to_string() << std::endl;
     };
 
     // "english/history off.gif ~20+" sequential_opt = true
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp
index b454080434a008..41703d49d5e013 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp
@@ -506,7 +506,7 @@ TEST_F(InvertedIndexFileWriterTest, WriteV2ExceptionHandlingTest) {
     EXPECT_CALL(writer_mock, write_index_headers_and_metadata(::testing::_, ::testing::_))
             .WillOnce(::testing::Throw(CLuceneError(CL_ERR_IO, "Simulated exception", false)));
 
-    Status status = writer_mock.write_v2();
+    Status status = writer_mock.write();
     ASSERT_FALSE(status.ok());
     ASSERT_EQ(status.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR);
 }
@@ -523,7 +523,7 @@ class InvertedIndexFileWriterMockCreateOutputStreamV2 : public InvertedIndexFile
 
     MOCK_METHOD((std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
                            std::unique_ptr<lucene::store::IndexOutput>>),
-                create_output_stream_v2, (), (override));
+                create_output_stream, (), (override));
 };
 
 class InvertedIndexFileWriterMockCreateOutputStreamV1 : public InvertedIndexFileWriter {
@@ -622,7 +622,7 @@ TEST_F(InvertedIndexFileWriterTest, WriteV2OutputTest) {
     auto compound_file_output = std::unique_ptr<DorisFSDirectory::FSIndexOutputV2>(mock_output_v2);
     compound_file_output->init(file_writer.get());
 
-    EXPECT_CALL(writer_mock, create_output_stream_v2())
+    EXPECT_CALL(writer_mock, create_output_stream())
             .WillOnce(::testing::Invoke(
                     [&]() -> std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
                                        std::unique_ptr<lucene::store::IndexOutput>> {
@@ -680,7 +680,7 @@ TEST_F(InvertedIndexFileWriterTest, WriteV2OutputCloseErrorTest) {
     auto compound_file_output = std::unique_ptr<DorisFSDirectory::FSIndexOutputV2>(mock_output_v2);
     compound_file_output->init(file_writer.get());
 
-    EXPECT_CALL(writer_mock, create_output_stream_v2())
+    EXPECT_CALL(writer_mock, create_output_stream())
             .WillOnce(::testing::Invoke(
                     [&]() -> std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
                                        std::unique_ptr<lucene::store::IndexOutput>> {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
index abba2762d5664d..dd6a1a7612ab6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
@@ -54,6 +54,8 @@ public class InvertedIndexUtil {
 
     public static String INVERTED_INDEX_PARSER_STOPWORDS_KEY = "stopwords";
 
+    public static String INVERTED_INDEX_DICT_COMPRESSION_KEY = "dict_compression";
+
     public static String getInvertedIndexParser(Map<String, String> properties) {
         String parser = properties == null ? null : properties.get(INVERTED_INDEX_PARSER_KEY);
         // default is "none" if not set
@@ -157,7 +159,8 @@ public static void checkInvertedIndexProperties(Map<String, String> properties)
                 INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT,
                 INVERTED_INDEX_PARSER_IGNORE_ABOVE_KEY,
                 INVERTED_INDEX_PARSER_LOWERCASE_KEY,
-                INVERTED_INDEX_PARSER_STOPWORDS_KEY
+                INVERTED_INDEX_PARSER_STOPWORDS_KEY,
+                INVERTED_INDEX_DICT_COMPRESSION_KEY
         ));
 
         for (String key : properties.keySet()) {
@@ -174,6 +177,7 @@ public static void checkInvertedIndexProperties(Map<String, String> properties)
         String ignoreAbove = properties.get(INVERTED_INDEX_PARSER_IGNORE_ABOVE_KEY);
         String lowerCase = properties.get(INVERTED_INDEX_PARSER_LOWERCASE_KEY);
         String stopWords = properties.get(INVERTED_INDEX_PARSER_STOPWORDS_KEY);
+        String dictCompression = properties.get(INVERTED_INDEX_DICT_COMPRESSION_KEY);
 
         if (parser != null && !parser.matches("none|english|unicode|chinese|standard")) {
             throw new AnalysisException("Invalid inverted index 'parser' value: " + parser
@@ -221,5 +225,11 @@ public static void checkInvertedIndexProperties(Map<String, String> properties)
             throw new AnalysisException("Invalid inverted index 'stopWords' value: " + stopWords
                     + ", stopWords must be none");
         }
+
+        if (dictCompression != null && !dictCompression.matches("true|false")) {
+            throw new AnalysisException(
+                    "Invalid inverted index 'dict_compression' value: "
+                            + dictCompression + ", dict_compression must be true or false");
+        }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index b8a364f9449042..e14b4efb0d590d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -354,8 +354,12 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
         if (invertedIndexFileStorageFormat != null) {
             if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1) {
                 schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1);
-            } else {
+            } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V2) {
                 schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2);
+            } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V3) {
+                schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3);
+            } else {
+                throw new DdlException("invalid inverted index storage format");
             }
         }
         schemaBuilder.setRowStorePageSize(pageSize);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 5721db0c27e856..915b5d48f013b9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -1126,6 +1126,8 @@ public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFor
         } else {
             if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
                 return TInvertedIndexFileStorageFormat.V1;
+            } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V3")) {
+                return TInvertedIndexFileStorageFormat.V3;
             } else {
                 return TInvertedIndexFileStorageFormat.V2;
             }
@@ -1135,9 +1137,13 @@ public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFor
             return TInvertedIndexFileStorageFormat.V1;
         } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v2")) {
             return TInvertedIndexFileStorageFormat.V2;
+        } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v3")) {
+            return TInvertedIndexFileStorageFormat.V3;
         } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("default")) {
             if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
                 return TInvertedIndexFileStorageFormat.V1;
+            } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V3")) {
+                return TInvertedIndexFileStorageFormat.V3;
             } else {
                 return TInvertedIndexFileStorageFormat.V2;
             }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexUtilTest.java
new file mode 100644
index 00000000000000..a9be242cf3f744
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexUtilTest.java
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.common.AnalysisException;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class InvertedIndexUtilTest {
+    @Test
+    public void testCheckInvertedIndexProperties() throws AnalysisException {
+        Map<String, String> properties = new HashMap<>();
+        properties.put(InvertedIndexUtil.INVERTED_INDEX_DICT_COMPRESSION_KEY, "true");
+
+        InvertedIndexUtil.checkInvertedIndexProperties(properties);
+
+        properties.put(InvertedIndexUtil.INVERTED_INDEX_DICT_COMPRESSION_KEY, "invalid_value");
+        try {
+            InvertedIndexUtil.checkInvertedIndexProperties(properties);
+            Assertions.fail("Expected AnalysisException was not thrown");
+        } catch (AnalysisException e) {
+            Assertions.assertEquals(
+                    "errCode = 2, detailMessage = Invalid inverted index 'dict_compression' value: invalid_value, "
+                            + "dict_compression must be true or false",
+                    e.getMessage());
+        }
+    }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 041ca89bfc5bde..6d708aa0826ddf 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -28,6 +28,7 @@
 import org.apache.doris.common.util.PropertyAnalyzer;
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.resource.Tag;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
 import org.apache.doris.thrift.TStorageFormat;
 import org.apache.doris.thrift.TStorageMedium;
 
@@ -37,6 +38,7 @@
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
 import org.junit.rules.ExpectedException;
 
 import java.time.Instant;
@@ -236,4 +238,52 @@ public void testStoragePageSize() throws AnalysisException {
             Assert.assertTrue(e.getMessage().contains("Storage page size must be between 4KB and 10MB"));
         }
     }
+
+    @Test
+    public void testAnalyzeInvertedIndexFileStorageFormat() throws AnalysisException {
+        TInvertedIndexFileStorageFormat result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(null);
+        Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result);
+
+        Config.inverted_index_storage_format = "V1";
+        result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(new HashMap<>());
+        Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result);
+
+        Map<String, String> propertiesWithV1 = new HashMap<>();
+        propertiesWithV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v1");
+        result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV1);
+        Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result);
+
+        Map<String, String> propertiesWithV2 = new HashMap<>();
+        propertiesWithV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v2");
+        result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV2);
+        Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result);
+
+        Map<String, String> propertiesWithV3 = new HashMap<>();
+        propertiesWithV3.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v3");
+        result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV3);
+        Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result);
+
+        Config.inverted_index_storage_format = "V1";
+        Map<String, String> propertiesWithDefaultV1 = new HashMap<>();
+        propertiesWithDefaultV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "default");
+        result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV1);
+        Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result);
+
+        Config.inverted_index_storage_format = "V2";
+        Map<String, String> propertiesWithDefaultV2 = new HashMap<>();
+        propertiesWithDefaultV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "default");
+        result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV2);
+        Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result);
+
+        Map<String, String> propertiesWithUnknown = new HashMap<>();
+        propertiesWithUnknown.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "unknown_format");
+        try {
+            PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithUnknown);
+            Assertions.fail("Expected AnalysisException was not thrown");
+        } catch (AnalysisException e) {
+            Assertions.assertEquals(
+                    "errCode = 2, detailMessage = unknown inverted index storage format: unknown_format",
+                    e.getMessage());
+        }
+    }
 }
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 259f9f2861a11c..41f8727ed1b301 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -343,6 +343,7 @@ enum IndexType {
 enum InvertedIndexStorageFormatPB {
     V1 = 0;
     V2 = 1;
+    V3 = 2;
 }
 
 message TabletIndexPB {
diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index 1912f950587dbf..623da9ce067061 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -124,7 +124,8 @@ enum TStorageBackendType {
 enum TInvertedIndexFileStorageFormat {
     DEFAULT = 0, // Default format, unspecified storage method.
     V1 = 1,      // Index per idx: Each index is stored separately based on its identifier.
-    V2 = 2      // Segment id per idx: Indexes are organized based on segment identifiers, grouping indexes by their associated segment.
+    V2 = 2       // Segment id per idx: Indexes are organized based on segment identifiers, grouping indexes by their associated segment.
+    V3 = 3       // Position and dictionary compression
 }
 
 struct TScalarType {
diff --git a/regression-test/data/inverted_index_p0/test_inverted_index_v3.out b/regression-test/data/inverted_index_p0/test_inverted_index_v3.out
new file mode 100644
index 00000000000000..9dc20f3e0e0a85
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_inverted_index_v3.out
@@ -0,0 +1,25 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+238
+
+-- !sql --
+104
+
+-- !sql --
+104
+
+-- !sql --
+105
+
+-- !sql --
+238
+
+-- !sql --
+104
+
+-- !sql --
+104
+
+-- !sql --
+105
+
diff --git a/regression-test/suites/fault_injection_p0/test_inverted_index_v3_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_inverted_index_v3_fault_injection.groovy
new file mode 100644
index 00000000000000..98c0e110964b49
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_inverted_index_v3_fault_injection.groovy
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_inverted_index_v3_fault_injection", "nonConcurrent"){
+    def indexTbName1 = "test_inverted_index_v3_fault_injection"
+
+    sql "DROP TABLE IF EXISTS ${indexTbName1}"
+
+    sql """
+      CREATE TABLE ${indexTbName1} (
+      `@timestamp` int(11) NULL COMMENT "",
+      `clientip` varchar(20) NULL COMMENT "",
+      `request` text NULL COMMENT "",
+      `status` int(11) NULL COMMENT "",
+      `size` int(11) NULL COMMENT "",
+      INDEX clientip_idx (`clientip`) COMMENT '',
+      INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") COMMENT ''
+      ) ENGINE=OLAP
+      DUPLICATE KEY(`@timestamp`)
+      COMMENT "OLAP"
+      DISTRIBUTED BY RANDOM BUCKETS 1
+      PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1",
+      "inverted_index_storage_format" = "V3"
+      );
+    """
+
+    try {
+      GetDebugPoint().enableDebugPointForAllBEs("InvertedIndexColumnWriterImpl::create_field_v3")
+      
+      sql """ INSERT INTO ${indexTbName1} VALUES (1, '40.135.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+    } finally {
+      GetDebugPoint().disableDebugPointForAllBEs("InvertedIndexColumnWriterImpl::create_field_v3")
+    }
+
+    try {
+      GetDebugPoint().enableDebugPointForAllBEs("InvertedIndexColumnWriterImpl::create_field_v3")
+      GetDebugPoint().enableDebugPointForAllBEs("InvertedIndexColumnWriterImpl::create_field_dic_compression")
+
+      sql """ INSERT INTO ${indexTbName1} VALUES (2, '40.135.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+    } finally {
+      GetDebugPoint().disableDebugPointForAllBEs("InvertedIndexColumnWriterImpl::create_field_v3")
+      GetDebugPoint().disableDebugPointForAllBEs("InvertedIndexColumnWriterImpl::create_field_dic_compression")
+    }
+}
\ No newline at end of file
diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_v3.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_v3.groovy
new file mode 100644
index 00000000000000..ea7dd0b595f504
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_inverted_index_v3.groovy
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_inverted_index_v3", "p0"){
+    def indexTbName1 = "test_inverted_index_v3_1"
+    def indexTbName2 = "test_inverted_index_v3_2"
+
+    sql "DROP TABLE IF EXISTS ${indexTbName1}"
+    sql "DROP TABLE IF EXISTS ${indexTbName2}"
+
+    sql """
+      CREATE TABLE ${indexTbName1} (
+      `@timestamp` int(11) NULL COMMENT "",
+      `clientip` varchar(20) NULL COMMENT "",
+      `request` text NULL COMMENT "",
+      `status` int(11) NULL COMMENT "",
+      `size` int(11) NULL COMMENT "",
+      INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") COMMENT ''
+      ) ENGINE=OLAP
+      DUPLICATE KEY(`@timestamp`)
+      COMMENT "OLAP"
+      DISTRIBUTED BY RANDOM BUCKETS 1
+      PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1",
+      "inverted_index_storage_format" = "V2"
+      );
+    """
+
+    sql """
+      CREATE TABLE ${indexTbName2} (
+      `@timestamp` int(11) NULL COMMENT "",
+      `clientip` varchar(20) NULL COMMENT "",
+      `request` text NULL COMMENT "",
+      `status` int(11) NULL COMMENT "",
+      `size` int(11) NULL COMMENT "",
+      INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") COMMENT ''
+      ) ENGINE=OLAP
+      DUPLICATE KEY(`@timestamp`)
+      COMMENT "OLAP"
+      DISTRIBUTED BY RANDOM BUCKETS 1
+      PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1",
+      "inverted_index_storage_format" = "V3"
+      );
+    """
+
+    def load_httplogs_data = {table_name, label, read_flag, format_flag, file_name, ignore_failure=false,
+                        expected_succ_rows = -1, load_to_single_tablet = 'true' ->
+        
+        // load the json data
+        streamLoad {
+            table "${table_name}"
+            
+            // set http request header params
+            set 'label', label + "_" + UUID.randomUUID().toString()
+            set 'read_json_by_line', read_flag
+            set 'format', format_flag
+            file file_name // import json file
+            time 10000 // limit inflight 10s
+            if (expected_succ_rows >= 0) {
+                set 'max_filter_ratio', '1'
+            }
+
+            // if declared a check callback, the default check condition will ignore.
+            // So you must check all condition
+            check { result, exception, startTime, endTime ->
+		        if (ignore_failure && expected_succ_rows < 0) { return }
+                    if (exception != null) {
+                        throw exception
+                    }
+                    log.info("Stream load result: ${result}".toString())
+                    def json = parseJson(result)
+                    assertEquals("success", json.Status.toLowerCase())
+                    if (expected_succ_rows >= 0) {
+                        assertEquals(json.NumberLoadedRows, expected_succ_rows)
+                    } else {
+                        assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows)
+                        assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+                }
+            }
+        }
+    }
+
+    try {
+      load_httplogs_data.call(indexTbName1, indexTbName1, 'true', 'json', 'documents-1000.json')
+      load_httplogs_data.call(indexTbName2, indexTbName2, 'true', 'json', 'documents-1000.json')
+      
+      sql "sync"
+
+      qt_sql """ select count() from ${indexTbName1} where request match_any 'hm bg'; """
+      qt_sql """ select count() from ${indexTbName1} where request match_all 'hm bg'; """
+      qt_sql """ select count() from ${indexTbName1} where request match_phrase 'hm bg'; """
+      qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix 'hm bg'; """
+
+      qt_sql """ select count() from ${indexTbName2} where request match_any 'hm bg'; """
+      qt_sql """ select count() from ${indexTbName2} where request match_all 'hm bg'; """
+      qt_sql """ select count() from ${indexTbName2} where request match_phrase 'hm bg'; """
+      qt_sql """ select count() from ${indexTbName2} where request match_phrase_prefix 'hm bg'; """
+
+    } finally {
+    }
+}
\ No newline at end of file

From 7836e930fee56f4dbfd16a93ef8c8726d921670c Mon Sep 17 00:00:00 2001
From: GentleCold <gentlecold@qq.com>
Date: Wed, 4 Dec 2024 17:25:57 +0800
Subject: [PATCH 163/399] [Feat](Nereids) support show create repository
 command (#43902)

Issue Number: close #42724
migrate show create repository command to nereids
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  7 ++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/ShowCreateRepositoryCommand.java | 81 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../show_create_repository_command.groovy     | 77 ++++++++++++++++++
 6 files changed, 172 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateRepositoryCommand.java
 create mode 100644 regression-test/suites/nereids_p0/ddl/repository/show_create_repository_command.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index dcc87851196bfb..1fb5897066f021 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -230,6 +230,7 @@ supportedShowStatement
     | SHOW ALL? GRANTS                                                              #showGrants
     | SHOW GRANTS FOR userIdentify                                                  #showGrantsForUser
     | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL                                   #showLoadProfile
+    | SHOW CREATE REPOSITORY FOR identifier                                         #showCreateRepository
     | SHOW VIEW
         (FROM |IN) tableName=multipartIdentifier
         ((FROM | IN) database=identifier)?                                          #showView
@@ -300,7 +301,6 @@ unsupportedShowStatement
     | SHOW STORAGE POLICY (USING (FOR policy=identifierOrText)?)?                   #showStoragePolicy
     | SHOW STAGES                                                                   #showStages
     | SHOW STORAGE (VAULT | VAULTS)                                                 #showStorageVault
-    | SHOW CREATE REPOSITORY FOR identifier                                         #showCreateRepository
     | SHOW OPEN TABLES ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showOpenTables
     | SHOW TABLE STATUS ((FROM | IN) database=multipartIdentifier)? wildWhere?      #showTableStatus
     | SHOW FULL? TABLES ((FROM | IN) database=multipartIdentifier)? wildWhere?      #showTables
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 705acb4788fda1..9866b410c8fbb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -241,6 +241,7 @@
 import org.apache.doris.nereids.DorisParser.ShowCreateMTMVContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateMaterializedViewContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateProcedureContext;
+import org.apache.doris.nereids.DorisParser.ShowCreateRepositoryContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateTableContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateViewContext;
 import org.apache.doris.nereids.DorisParser.ShowDeleteContext;
@@ -539,6 +540,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMaterializedViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
@@ -4430,6 +4432,11 @@ public RefreshDatabaseCommand visitRefreshDatabase(RefreshDatabaseContext ctx) {
         throw new ParseException("Only one dot can be in the name: " + String.join(".", parts));
     }
 
+    @Override
+    public LogicalPlan visitShowCreateRepository(ShowCreateRepositoryContext ctx) {
+        return new ShowCreateRepositoryCommand(ctx.identifier().getText());
+    }
+
     public LogicalPlan visitShowLastInsert(ShowLastInsertContext ctx) {
         return new ShowLastInsertCommand();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 5d523ce3c1d425..969b51ee99afe4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -206,6 +206,7 @@ public enum PlanType {
     SHOW_CREATE_CATALOG_COMMAND,
     SHOW_CREATE_DATABASE_COMMAND,
     SHOW_CREATE_MATERIALIZED_VIEW_COMMAND,
+    SHOW_CREATE_REPOSITORY_COMMAND,
     SHOW_CREATE_TABLE_COMMAND,
     SHOW_CREATE_VIEW_COMMAND,
     SHOW_DELETE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateRepositoryCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateRepositoryCommand.java
new file mode 100644
index 00000000000000..8c27e8748fbb72
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateRepositoryCommand.java
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.backup.Repository;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.UserException;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * ShowCreateRepositoryCommand
+ */
+public class ShowCreateRepositoryCommand extends ShowCommand {
+    private static final ShowResultSetMetaData META_DATA =
+            ShowResultSetMetaData.builder()
+                            .addColumn(new Column("RepoName", ScalarType.createVarchar(128)))
+                            .addColumn(new Column("CreateStmt", ScalarType.createVarchar(65535)))
+                            .build();
+
+    private final String repoName;
+
+    public ShowCreateRepositoryCommand(String repoName) {
+        super(PlanType.SHOW_CREATE_REPOSITORY_COMMAND);
+        this.repoName = repoName;
+    }
+
+    private void validate() throws UserException {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR,
+                    PrivPredicate.ADMIN.getPrivs().toString());
+        }
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate();
+        List<List<String>> rows = Lists.newArrayList();
+        Repository repo = Env.getCurrentEnv().getBackupHandler().getRepoMgr().getRepo(repoName);
+        if (repo == null) {
+            throw new AnalysisException("repository not exist.");
+        }
+        rows.add(Lists.newArrayList(repoName, repo.getCreateStatement()));
+        return new ShowResultSet(META_DATA, rows);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowCreateRepositoryCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 23ceadbc3168f1..0d8c24086a6bb3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -87,6 +87,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMaterializedViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
@@ -354,6 +355,10 @@ default R visitRefreshCatalogCommand(RefreshCatalogCommand refreshCatalogCommand
         return visitCommand(refreshCatalogCommand, context);
     }
 
+    default R visitShowCreateRepositoryCommand(ShowCreateRepositoryCommand showCreateRepositoryCommand, C context) {
+        return visitCommand(showCreateRepositoryCommand, context);
+    }
+
     default R visitShowLastInsertCommand(ShowLastInsertCommand showLastInsertCommand, C context) {
         return visitCommand(showLastInsertCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/ddl/repository/show_create_repository_command.groovy b/regression-test/suites/nereids_p0/ddl/repository/show_create_repository_command.groovy
new file mode 100644
index 00000000000000..547cd8a6376db6
--- /dev/null
+++ b/regression-test/suites/nereids_p0/ddl/repository/show_create_repository_command.groovy
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("show_create_repository_command") {
+    String ak = getS3AK()
+    String sk = getS3SK()
+    String endpoint = getS3Endpoint()
+    String region = getS3Region()
+    String bucket = context.config.otherConfigs.get("s3BucketName");
+    String repoName = "test_show_create_repository"
+    String readOnlyRepoName = "test_show_create_read_only_repository"
+
+    //cloud-mode
+    if (isCloudMode()) {
+        return
+    }
+
+    try_sql "DROP REPOSITORY `${repoName}`"
+    try_sql "DROP REPOSITORY `${readOnlyRepoName}`"
+
+    // create S3 repo
+    sql """CREATE REPOSITORY `${repoName}`
+            WITH S3
+            ON LOCATION "s3://${bucket}/${repoName}"
+            PROPERTIES
+            (
+                "s3.endpoint" = "http://${endpoint}",
+                "s3.region" = "${region}",
+                "s3.access_key" = "${ak}",
+                "s3.secret_key" = "${sk}"
+            )"""
+
+    // create S3 read only repo
+    sql """CREATE READ ONLY REPOSITORY `${readOnlyRepoName}`
+            WITH S3
+            ON LOCATION "s3://${bucket}/${readOnlyRepoName}"
+            PROPERTIES
+            (
+                "s3.endpoint" = "http://${endpoint}",
+                "s3.region" = "${region}",
+                "s3.access_key" = "${ak}",
+                "s3.secret_key" = "${sk}"
+            )"""
+
+    def create_repo = checkNereidsExecuteWithResult("""SHOW CREATE REPOSITORY for `${repoName}`;""").toString();
+    assertTrue(create_repo.contains("${repoName}"))
+    assertTrue(create_repo.contains("s3://${bucket}/${repoName}"))
+
+    def create_read_only_repo = checkNereidsExecuteWithResult("""SHOW CREATE REPOSITORY for `${readOnlyRepoName}`;""").toString();
+    assertTrue(create_read_only_repo.contains("${readOnlyRepoName}"))
+    assertTrue(create_read_only_repo.contains("READ ONLY"))
+
+    sql """DROP REPOSITORY `${repoName}`;"""
+    sql """DROP REPOSITORY `${readOnlyRepoName}`;"""
+
+    try {
+        sql """SHOW CREATE REPOSITORY for `${repoName}`;"""
+    } catch (Exception e) {
+        assertTrue(e.getMessage().contains("repository not exist."))
+    }
+}

From 5b5cbf64afdb576dcb02ec30a0be477b604c27aa Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Wed, 4 Dec 2024 18:24:23 +0800
Subject: [PATCH 164/399] [Chore](join) add debug info for build_sink::close
 (#44974)

add debug info for build_sink::close
---
 be/src/exprs/runtime_filter.cpp              |  2 +-
 be/src/pipeline/dependency.cpp               |  9 ++++----
 be/src/pipeline/dependency.h                 | 22 +++++++++++++++++---
 be/src/pipeline/exec/hashjoin_build_sink.cpp | 13 +++++++++---
 4 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 606df9fe980571..34dfb8ff8babfb 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -1532,7 +1532,7 @@ std::string IRuntimeFilter::debug_string() const {
     return fmt::format(
             "RuntimeFilter: (id = {}, type = {}, is_broadcast: {}, ignored: {}, "
             "build_bf_cardinality: {}, dependency: {}, synced_size: {}, has_local_target: {}, "
-            "has_remote_target: {},error_msg: [{}]",
+            "has_remote_target: {}, error_msg: [{}]",
             _filter_id, to_string(_runtime_filter_type), _is_broadcast_join,
             _wrapper->_context->ignored, _wrapper->get_build_bf_cardinality(),
             _dependency ? _dependency->debug_string() : "none", _synced_size, _has_local_target,
diff --git a/be/src/pipeline/dependency.cpp b/be/src/pipeline/dependency.cpp
index 5fef018423df25..983429f15e2fd5 100644
--- a/be/src/pipeline/dependency.cpp
+++ b/be/src/pipeline/dependency.cpp
@@ -92,10 +92,11 @@ std::string Dependency::debug_string(int indentation_level) {
 
 std::string CountedFinishDependency::debug_string(int indentation_level) {
     fmt::memory_buffer debug_string_buffer;
-    fmt::format_to(debug_string_buffer,
-                   "{}{}: id={}, block_task={}, ready={}, _always_ready={}, count={}",
-                   std::string(indentation_level * 2, ' '), _name, _node_id, _blocked_task.size(),
-                   _ready, _always_ready, _counter);
+    fmt::format_to(
+            debug_string_buffer,
+            "{}{}: id={}, block_task={}, ready={}, _always_ready={}, count={}, _stack_set_ready={}",
+            std::string(indentation_level * 2, ' '), _name, _node_id, _blocked_task.size(), _ready,
+            _always_ready, _counter, _stack_set_ready);
     return fmt::to_string(debug_string_buffer);
 }
 
diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h
index f7712625d3e9a6..71ef2d8b8fd85f 100644
--- a/be/src/pipeline/dependency.h
+++ b/be/src/pipeline/dependency.h
@@ -34,6 +34,7 @@
 #include "pipeline/common/set_utils.h"
 #include "pipeline/exec/data_queue.h"
 #include "pipeline/exec/join/process_hash_table_probe.h"
+#include "util/stack_util.h"
 #include "vec/common/sort/partition_sorter.h"
 #include "vec/common/sort/sorter.h"
 #include "vec/core/block.h"
@@ -107,7 +108,7 @@ class Dependency : public std::enable_shared_from_this<Dependency> {
     // Which dependency current pipeline task is blocked by. `nullptr` if this dependency is ready.
     [[nodiscard]] virtual Dependency* is_blocked_by(PipelineTask* task = nullptr);
     // Notify downstream pipeline tasks this dependency is ready.
-    void set_ready();
+    virtual void set_ready();
     void set_ready_to_read() {
         DCHECK_EQ(_shared_state->source_deps.size(), 1) << debug_string();
         _shared_state->source_deps.front()->set_ready();
@@ -172,11 +173,26 @@ struct FakeSharedState final : public BasicSharedState {
     ENABLE_FACTORY_CREATOR(FakeSharedState)
 };
 
-class CountedFinishDependency final : public Dependency {
+class DependencyWithStack : public Dependency {
+public:
+    using SharedState = FakeSharedState;
+    DependencyWithStack(int id, int node_id, std::string name, bool ready = false)
+            : Dependency(id, node_id, name, ready) {}
+
+    void set_ready() override {
+        _stack_set_ready = get_stack_trace();
+        Dependency::set_ready();
+    }
+
+protected:
+    std::string _stack_set_ready;
+};
+
+class CountedFinishDependency final : public DependencyWithStack {
 public:
     using SharedState = FakeSharedState;
     CountedFinishDependency(int id, int node_id, std::string name)
-            : Dependency(id, node_id, name, true) {}
+            : DependencyWithStack(id, node_id, name, true) {}
 
     void add() {
         std::unique_lock<std::mutex> l(_mtx);
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index d6129e5af203d2..8115b87c8fe5ac 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -166,15 +166,22 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
                 SCOPED_TIMER(_runtime_filter_compute_timer);
                 _runtime_filter_slots->insert(block);
             }
+        } else if (p._shared_hashtable_controller && !p._shared_hash_table_context->signaled) {
+            throw Exception(ErrorCode::INTERNAL_ERROR,
+                            "build_sink::close meet error state, shared_hash_table_signaled: {}, "
+                            "complete_build_stage: {}",
+                            p._shared_hash_table_context->signaled,
+                            p._shared_hash_table_context->complete_build_stage);
         }
 
         SCOPED_TIMER(_publish_runtime_filter_timer);
         RETURN_IF_ERROR(_runtime_filter_slots->publish(state, !_should_build_hash_table));
     } catch (Exception& e) {
         return Status::InternalError(
-                "rf process meet error: {}, wake_up_by_downstream: {}, should_build_hash_table: {}",
-                e.to_string(), state->get_task()->wake_up_by_downstream(),
-                _should_build_hash_table);
+                "rf process meet error: {}, wake_up_by_downstream: {}, should_build_hash_table: "
+                "{}, _finish_dependency: {}",
+                e.to_string(), state->get_task()->wake_up_by_downstream(), _should_build_hash_table,
+                _finish_dependency->debug_string());
     }
     return Base::close(state, exec_status);
 }

From d58a972601683bb58a0956b1bb3af64f7bcd751d Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Wed, 4 Dec 2024 19:52:23 +0800
Subject: [PATCH 165/399] [feature](function) support
 orthogonal_bitmap_expr_calculate & orthogonal_bitmap_expr_calculate_count for
 nereids (#44991)

support orthogonal_bitmap_expr_calculate &
orthogonal_bitmap_expr_calculate_count for nereids
---
 .../aggregate_function_orthogonal_bitmap.h    |  2 +-
 .../catalog/BuiltinAggregateFunctions.java    | 12 ++-
 .../implementation/AggregateStrategies.java   | 14 ++-
 .../functions/agg/AggregateFunction.java      |  4 +
 .../functions/agg/AggregatePhase.java         | 23 +++++
 .../agg/OrthogonalBitmapExprCalculate.java    | 84 ++++++++++++++++++
 .../OrthogonalBitmapExprCalculateCount.java   | 86 +++++++++++++++++++
 .../trees/plans/logical/LogicalAggregate.java | 12 +++
 ...st_orthogonal_bitmap_expr_calculate.groovy | 55 ++++++++++++
 9 files changed, 286 insertions(+), 6 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregatePhase.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculate.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculateCount.java
 create mode 100644 regression-test/suites/nereids_function_p0/agg_function/test_orthogonal_bitmap_expr_calculate.groovy

diff --git a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h
index ed4b9675a76e6b..340ba5325fbe6a 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h
@@ -235,7 +235,7 @@ struct AggOrthBitmapExprCalBaseData {
         if (first_init) {
             DCHECK(argument_size > 1);
             const auto& col =
-                    assert_cast<const ColVecData&, TypeCheckOnRelease::DISABLE>(*columns[2]);
+                    assert_cast<const ColumnString&, TypeCheckOnRelease::DISABLE>(*columns[2]);
             std::string expr = col.get_data_at(row_num).to_string();
             bitmap_expr_cal.bitmap_calculation_init(expr);
             first_init = false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java
index 1db9cf6477e8fb..655a3dfce2985d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java
@@ -57,6 +57,8 @@
 import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctSum;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctSum0;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv;
+import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapExprCalculate;
+import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapExprCalculateCount;
 import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapIntersect;
 import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapIntersectCount;
 import org.apache.doris.nereids.trees.expressions.functions.agg.OrthogonalBitmapUnionCount;
@@ -124,7 +126,8 @@ public class BuiltinAggregateFunctions implements FunctionHelper {
             agg(HllUnion.class, "hll_raw_agg", "hll_union"),
             agg(HllUnionAgg.class, "hll_union_agg"),
             agg(IntersectCount.class, "intersect_count"),
-            agg(LinearHistogram.class, FunctionSet.LINEAR_HISTOGRAM),
+            agg(Kurt.class, "kurt", "kurt_pop", "kurtosis"),
+            agg(LinearHistogram.class, "linear_histogram"),
             agg(MapAgg.class, "map_agg"),
             agg(Max.class, "max"),
             agg(MaxBy.class, "max_by"),
@@ -135,6 +138,8 @@ public class BuiltinAggregateFunctions implements FunctionHelper {
             agg(MultiDistinctSum.class, "multi_distinct_sum"),
             agg(MultiDistinctSum0.class, "multi_distinct_sum0"),
             agg(Ndv.class, "approx_count_distinct", "ndv"),
+            agg(OrthogonalBitmapExprCalculate.class, "orthogonal_bitmap_expr_calculate"),
+            agg(OrthogonalBitmapExprCalculateCount.class, "orthogonal_bitmap_expr_calculate_count"),
             agg(OrthogonalBitmapIntersect.class, "orthogonal_bitmap_intersect"),
             agg(OrthogonalBitmapIntersectCount.class, "orthogonal_bitmap_intersect_count"),
             agg(OrthogonalBitmapUnionCount.class, "orthogonal_bitmap_union_count"),
@@ -148,6 +153,7 @@ public class BuiltinAggregateFunctions implements FunctionHelper {
             agg(Retention.class, "retention"),
             agg(SequenceCount.class, "sequence_count"),
             agg(SequenceMatch.class, "sequence_match"),
+            agg(Skew.class, "skew", "skew_pop", "skewness"),
             agg(Stddev.class, "stddev_pop", "stddev"),
             agg(StddevSamp.class, "stddev_samp"),
             agg(Sum.class, "sum"),
@@ -157,9 +163,7 @@ public class BuiltinAggregateFunctions implements FunctionHelper {
             agg(TopNWeighted.class, "topn_weighted"),
             agg(Variance.class, "var_pop", "variance_pop", "variance"),
             agg(VarianceSamp.class, "var_samp", "variance_samp"),
-            agg(WindowFunnel.class, "window_funnel"),
-            agg(Skew.class, "skew", "skew_pop", "skewness"),
-            agg(Kurt.class, "kurt", "kurt_pop", "kurtosis")
+            agg(WindowFunnel.class, "window_funnel")
     );
 
     public final Set<String> aggFuncNames = aggregateFunctions.stream()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
index 6cf243bbfeec7b..094882294fa6db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
@@ -47,6 +47,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregatePhase;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
 import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
@@ -294,31 +295,37 @@ public List<Rule> buildRules() {
             RuleType.ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT.build(
                 basePattern
                     .when(agg -> agg.getDistinctArguments().isEmpty())
+                    .when(agg -> agg.supportAggregatePhase(AggregatePhase.ONE))
                     .thenApplyMulti(ctx -> onePhaseAggregateWithoutDistinct(ctx.root, ctx.connectContext))
             ),
             RuleType.TWO_PHASE_AGGREGATE_WITHOUT_DISTINCT.build(
                 basePattern
                     .when(agg -> agg.getDistinctArguments().isEmpty())
+                    .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO))
                     .thenApplyMulti(ctx -> twoPhaseAggregateWithoutDistinct(ctx.root, ctx.connectContext))
             ),
             // RuleType.TWO_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI.build(
             //     basePattern
             //         .when(this::containsCountDistinctMultiExpr)
+            //         .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO))
             //         .thenApplyMulti(ctx -> twoPhaseAggregateWithCountDistinctMulti(ctx.root, ctx.cascadesContext))
             // ),
             RuleType.THREE_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI.build(
                 basePattern
                     .when(this::containsCountDistinctMultiExpr)
+                    .when(agg -> agg.supportAggregatePhase(AggregatePhase.THREE))
                     .thenApplyMulti(ctx -> threePhaseAggregateWithCountDistinctMulti(ctx.root, ctx.cascadesContext))
             ),
             RuleType.ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI.build(
                 basePattern
                     .when(agg -> agg.getDistinctArguments().size() == 1 && couldConvertToMulti(agg))
+                    .when(agg -> agg.supportAggregatePhase(AggregatePhase.ONE))
                     .thenApplyMulti(ctx -> onePhaseAggregateWithMultiDistinct(ctx.root, ctx.connectContext))
             ),
             RuleType.TWO_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI.build(
                 basePattern
                     .when(agg -> agg.getDistinctArguments().size() == 1 && couldConvertToMulti(agg))
+                    .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO))
                     .thenApplyMulti(ctx -> twoPhaseAggregateWithMultiDistinct(ctx.root, ctx.connectContext))
             ),
             RuleType.TWO_PHASE_AGGREGATE_WITH_MULTI_DISTINCT.build(
@@ -326,17 +333,20 @@ public List<Rule> buildRules() {
                     .when(agg -> agg.getDistinctArguments().size() > 1
                             && !containsCountDistinctMultiExpr(agg)
                             && couldConvertToMulti(agg))
+                    .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO))
                     .thenApplyMulti(ctx -> twoPhaseAggregateWithMultiDistinct(ctx.root, ctx.connectContext))
             ),
             // RuleType.TWO_PHASE_AGGREGATE_WITH_DISTINCT.build(
             //     basePattern
             //         .when(agg -> agg.getDistinctArguments().size() == 1)
+            //         .when(agg -> agg.supportAggregatePhase(AggregatePhase.TWO))
             //         .thenApplyMulti(ctx -> twoPhaseAggregateWithDistinct(ctx.root, ctx.connectContext))
             // ),
             RuleType.THREE_PHASE_AGGREGATE_WITH_DISTINCT.build(
                 basePattern
                     .when(agg -> agg.getDistinctArguments().size() == 1)
-                     .whenNot(agg -> agg.mustUseMultiDistinctAgg())
+                    .whenNot(agg -> agg.mustUseMultiDistinctAgg())
+                    .when(agg -> agg.supportAggregatePhase(AggregatePhase.THREE))
                     .thenApplyMulti(ctx -> threePhaseAggregateWithDistinct(ctx.root, ctx.connectContext))
             ),
             /*
@@ -361,6 +371,7 @@ && couldConvertToMulti(agg))
                     .when(agg -> agg.getDistinctArguments().size() == 1)
                     .when(agg -> agg.getGroupByExpressions().isEmpty())
                     .whenNot(agg -> agg.mustUseMultiDistinctAgg())
+                    .when(agg -> agg.supportAggregatePhase(AggregatePhase.FOUR))
                     .thenApplyMulti(ctx -> {
                         Function<List<Expression>, RequireProperties> secondPhaseRequireDistinctHash =
                                 groupByAndDistinct -> RequireProperties.of(
@@ -408,6 +419,7 @@ && couldConvertToMulti(agg))
                         }
                         return couldConvertToMulti(agg);
                     })
+                    .when(agg -> agg.supportAggregatePhase(AggregatePhase.FOUR))
                     .thenApplyMulti(ctx -> {
                         Function<List<Expression>, RequireProperties> secondPhaseRequireGroupByAndDistinctHash =
                                 groupByAndDistinct -> RequireProperties.of(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java
index 58b9d0274ddfc5..90df2f531da3fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java
@@ -131,6 +131,10 @@ public String toString() {
         return getName() + "(" + (distinct ? "DISTINCT " : "") + args + ")";
     }
 
+    public boolean supportAggregatePhase(AggregatePhase aggregatePhase) {
+        return true;
+    }
+
     public List<Expression> getDistinctArguments() {
         return distinct ? getArguments() : ImmutableList.of();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregatePhase.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregatePhase.java
new file mode 100644
index 00000000000000..9115e5890ad66c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregatePhase.java
@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.agg;
+
+/** AggregatePhase */
+public enum AggregatePhase {
+    ONE, TWO, THREE, FOUR
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculate.java
new file mode 100644
index 00000000000000..2575709fa318c2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculate.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.agg;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
+import org.apache.doris.nereids.types.BitmapType;
+import org.apache.doris.nereids.types.VarcharType;
+import org.apache.doris.nereids.types.coercion.CharacterType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/** OrthogonalBitmapExprCalculate */
+public class OrthogonalBitmapExprCalculate extends NotNullableAggregateFunction
+        implements OrthogonalBitmapFunction, ExplicitlyCastableSignature {
+
+    static final List<FunctionSignature> FUNCTION_SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BitmapType.INSTANCE)
+                    .varArgs(BitmapType.INSTANCE, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT)
+    );
+
+    /**
+     * constructor with 3 arguments.
+     */
+    public OrthogonalBitmapExprCalculate(
+            Expression bitmap, Expression filterColumn, VarcharLiteral inputString) {
+        super("orthogonal_bitmap_expr_calculate", ExpressionUtils.mergeArguments(bitmap, filterColumn, inputString));
+    }
+
+    /**
+     * constructor with 3 arguments.
+     */
+    public OrthogonalBitmapExprCalculate(boolean distinct,
+            Expression bitmap, Expression filterColumn, VarcharLiteral inputString) {
+        super("orthogonal_bitmap_expr_calculate", distinct,
+                ExpressionUtils.mergeArguments(bitmap, filterColumn, inputString));
+    }
+
+    @Override
+    public boolean supportAggregatePhase(AggregatePhase aggregatePhase) {
+        return aggregatePhase == AggregatePhase.TWO;
+    }
+
+    @Override
+    public Expression resultForEmptyInput() {
+        return new BitmapEmpty();
+    }
+
+    @Override
+    public OrthogonalBitmapExprCalculate withDistinctAndChildren(boolean distinct, List<Expression> children) {
+        Preconditions.checkArgument(children.size() == 3
+                && children.get(2).getDataType() instanceof CharacterType
+                && children.get(2).getDataType() instanceof VarcharType);
+        return new OrthogonalBitmapExprCalculate(
+                distinct, children.get(0), children.get(1), (VarcharLiteral) children.get(2));
+    }
+
+    @Override
+    public List<FunctionSignature> getSignatures() {
+        return FUNCTION_SIGNATURES;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculateCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculateCount.java
new file mode 100644
index 00000000000000..0ae69f9fe89c28
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapExprCalculateCount.java
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.agg;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
+import org.apache.doris.nereids.types.BigIntType;
+import org.apache.doris.nereids.types.BitmapType;
+import org.apache.doris.nereids.types.VarcharType;
+import org.apache.doris.nereids.types.coercion.CharacterType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/** OrthogonalBitmapExprCalculateCount */
+public class OrthogonalBitmapExprCalculateCount extends NotNullableAggregateFunction
+        implements OrthogonalBitmapFunction, ExplicitlyCastableSignature {
+
+    static final List<FunctionSignature> FUNCTION_SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BigIntType.INSTANCE)
+                    .varArgs(BitmapType.INSTANCE, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT)
+    );
+
+    /**
+     * constructor with 3 arguments.
+     */
+    public OrthogonalBitmapExprCalculateCount(
+            Expression bitmap, Expression filterColumn, VarcharLiteral inputString) {
+        super("orthogonal_bitmap_expr_calculate_count",
+                ExpressionUtils.mergeArguments(bitmap, filterColumn, inputString));
+    }
+
+    /**
+     * constructor with 3 arguments.
+     */
+    public OrthogonalBitmapExprCalculateCount(boolean distinct,
+            Expression bitmap, Expression filterColumn, VarcharLiteral inputString) {
+        super("orthogonal_bitmap_expr_calculate_count", distinct,
+                ExpressionUtils.mergeArguments(bitmap, filterColumn, inputString));
+    }
+
+    @Override
+    public boolean supportAggregatePhase(AggregatePhase aggregatePhase) {
+        return aggregatePhase == AggregatePhase.TWO;
+    }
+
+    @Override
+    public Expression resultForEmptyInput() {
+        return new BitmapEmpty();
+    }
+
+    @Override
+    public OrthogonalBitmapExprCalculateCount withDistinctAndChildren(boolean distinct, List<Expression> children) {
+        Preconditions.checkArgument(children.size() == 3
+                && children.get(2).getDataType() instanceof CharacterType
+                && children.get(2).getDataType() instanceof VarcharType);
+        return new OrthogonalBitmapExprCalculateCount(
+                distinct, children.get(0), children.get(1), (VarcharLiteral) children.get(2));
+    }
+
+    @Override
+    public List<FunctionSignature> getSignatures() {
+        return FUNCTION_SIGNATURES;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
index 2798b1aef0102b..bd68a74fe3e666 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
@@ -24,6 +24,8 @@
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregatePhase;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv;
 import org.apache.doris.nereids.trees.plans.Plan;
@@ -386,4 +388,14 @@ public void computeEqualSet(DataTrait.Builder builder) {
     public void computeFd(DataTrait.Builder builder) {
         builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
     }
+
+    /** supportAggregatePhase */
+    public boolean supportAggregatePhase(AggregatePhase aggregatePhase) {
+        for (AggregateFunction aggregateFunction : getAggregateFunctions()) {
+            if (!aggregateFunction.supportAggregatePhase(aggregatePhase)) {
+                return false;
+            }
+        }
+        return true;
+    }
 }
diff --git a/regression-test/suites/nereids_function_p0/agg_function/test_orthogonal_bitmap_expr_calculate.groovy b/regression-test/suites/nereids_function_p0/agg_function/test_orthogonal_bitmap_expr_calculate.groovy
new file mode 100644
index 00000000000000..b47315e96dbcec
--- /dev/null
+++ b/regression-test/suites/nereids_function_p0/agg_function/test_orthogonal_bitmap_expr_calculate.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_orthogonal_bitmap_expr_calculate") {
+    multi_sql """
+            drop table if exists test_orthogonal_bitmap_expr_calculate;
+
+            create table test_orthogonal_bitmap_expr_calculate(
+                id int,
+                tag int,
+                user_id bitmap bitmap_union
+            )
+            aggregate key(id, tag)
+            distributed by hash(id) buckets 1
+            properties(
+                'replication_num'='1'
+            );
+            
+            insert into test_orthogonal_bitmap_expr_calculate values
+            (1, 100, bitmap_from_string('1,2,3,4,5')),
+            (1, 200, bitmap_from_string('3,4,5,6,7'));
+
+            set enable_fallback_to_original_planner=false;
+            """
+
+    test {
+        sql """
+            select bitmap_to_string(orthogonal_bitmap_expr_calculate(user_id, tag, '(100&200)'))
+            from test_orthogonal_bitmap_expr_calculate
+            """
+        result([['3,4,5']])
+    }
+
+    test {
+        sql """
+            select orthogonal_bitmap_expr_calculate_count(user_id, tag, '(100&200)')
+            from test_orthogonal_bitmap_expr_calculate
+            """
+        result([[3L]])
+    }
+}

From d22bd83259923fadbdf135dc0e7063ca8b01943e Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Wed, 4 Dec 2024 22:15:26 +0800
Subject: [PATCH 166/399] [fix](cluster key) fix cluster key too many segments
 when schema change (#44972)

when schema change, the primary key index is not sorted, so the solution
is buffering and merging some blocks, and then write them to one segment
and start a new segment.
after this pr, the primary key index can be sorted in SegmentWriter.
---
 be/src/olap/rowset/segment_creator.cpp |  3 +--
 be/src/olap/schema_change.cpp          | 30 ++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/be/src/olap/rowset/segment_creator.cpp b/be/src/olap/rowset/segment_creator.cpp
index c2a4469d97f324..e0eb7534123a86 100644
--- a/be/src/olap/rowset/segment_creator.cpp
+++ b/be/src/olap/rowset/segment_creator.cpp
@@ -115,8 +115,7 @@ Status SegmentFlusher::close() {
 bool SegmentFlusher::need_buffering() {
     // buffering variants for schema change
     return _context.write_type == DataWriteType::TYPE_SCHEMA_CHANGE &&
-           (_context.tablet_schema->num_variant_columns() > 0 ||
-            !_context.tablet_schema->cluster_key_idxes().empty());
+           _context.tablet_schema->num_variant_columns() > 0;
 }
 
 Status SegmentFlusher::_add_rows(std::unique_ptr<segment_v2::SegmentWriter>& segment_writer,
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index ec291d8d2f0068..cd1b10d733a88c 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -198,6 +198,21 @@ class MultiBlockMerger {
                         pushed_row_refs.push_back(row_refs[i]);
                     }
                 }
+                if (!_tablet->tablet_schema()->cluster_key_idxes().empty()) {
+                    std::vector<uint32_t> ids;
+                    for (const auto& cid : _tablet->tablet_schema()->cluster_key_idxes()) {
+                        auto index = _tablet->tablet_schema()->field_index(cid);
+                        if (index == -1) {
+                            return Status::InternalError(
+                                    "could not find cluster key column with unique_id=" +
+                                    std::to_string(cid) + " in tablet schema");
+                        }
+                        ids.push_back(index);
+                    }
+                    // sort by cluster key
+                    std::stable_sort(pushed_row_refs.begin(), pushed_row_refs.end(),
+                                     ClusterKeyRowRefComparator(ids));
+                }
             }
 
             // update real inserted row number
@@ -249,6 +264,20 @@ class MultiBlockMerger {
         const size_t _num_columns;
     };
 
+    struct ClusterKeyRowRefComparator {
+        ClusterKeyRowRefComparator(std::vector<uint32_t> columns) : _columns(columns) {}
+
+        int compare(const RowRef& lhs, const RowRef& rhs) const {
+            return lhs.block->compare_at(lhs.position, rhs.position, &_columns, *rhs.block, -1);
+        }
+
+        bool operator()(const RowRef& lhs, const RowRef& rhs) const {
+            return compare(lhs, rhs) < 0;
+        }
+
+        const std::vector<uint32_t> _columns;
+    };
+
     BaseTabletSPtr _tablet;
     RowRefComparator _cmp;
     vectorized::Arena _arena;
@@ -1158,6 +1187,7 @@ Status SchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParams& sc
         }
 
         context.write_type = DataWriteType::TYPE_SCHEMA_CHANGE;
+        // TODO if support VerticalSegmentWriter, also need to handle cluster key primary key index
         auto result = _new_tablet->create_rowset_writer(context, false);
         if (!result.has_value()) {
             res = Status::Error<ROWSET_BUILDER_INIT>("create_rowset_writer failed, reason={}",

From f3c415dfaec698a443b657229a56be06923ed522 Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Thu, 5 Dec 2024 01:05:54 +0800
Subject: [PATCH 167/399] [fix](hudi) support reading hudi read optimized table
 with orc format (#44995)

### What problem does this PR solve?
Problem Summary:
When reading the hudi ro table, it will be pushed back from jni to the
native reader. However, this process will default the file format to
parquet, and does not consider the situation that the hudi table is
stored in orc format.

1. support reading hudi read optimized table with orc format
2. fix explain results of hudiScanNode when force_jni_reader=true
3. add cases about  timestamp with different timezones
---
 .../datasource/hudi/source/HudiScanNode.java  | 14 +++++--
 .../hudi/test_hudi_orc_tables.out             | 15 +++++++
 .../hudi/test_hudi_timestamp.out              | 31 ++++++++++++--
 .../hudi/test_hudi_orc_tables.groovy          | 41 +++++++++++++++++++
 .../hudi/test_hudi_timestamp.groovy           | 18 +++++++-
 5 files changed, 111 insertions(+), 8 deletions(-)
 create mode 100644 regression-test/data/external_table_p2/hudi/test_hudi_orc_tables.out
 create mode 100644 regression-test/suites/external_table_p2/hudi/test_hudi_orc_tables.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
index 28805aae63c1e3..a73a2065d0ffaf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
@@ -25,6 +25,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.FileFormatUtils;
 import org.apache.doris.common.util.LocationPath;
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.FileSplit;
@@ -247,8 +248,15 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
                     && !sessionVariable.isForceJniScanner()
                     && hudiSplit.getHudiDeltaLogs().isEmpty()) {
                 // no logs, is read optimize table, fallback to use native reader
-                // TODO: support read orc hudi table in native reader
-                rangeDesc.setFormatType(TFileFormatType.FORMAT_PARQUET);
+                String fileFormat = FileFormatUtils.getFileFormatBySuffix(hudiSplit.getDataFilePath())
+                        .orElse("Unknown");
+                if (fileFormat.equals("parquet")) {
+                    rangeDesc.setFormatType(TFileFormatType.FORMAT_PARQUET);
+                } else if (fileFormat.equals("orc")) {
+                    rangeDesc.setFormatType(TFileFormatType.FORMAT_ORC);
+                } else {
+                    throw new RuntimeException("Unsupported file format: " + fileFormat);
+                }
             }
             setHudiParams(rangeDesc, hudiSplit);
         }
@@ -495,7 +503,7 @@ private HudiSplit generateHudiSplit(FileSlice fileSlice, List<String> partitionV
         List<String> logs = fileSlice.getLogFiles().map(HoodieLogFile::getPath)
                 .map(StoragePath::toString)
                 .collect(Collectors.toList());
-        if (logs.isEmpty()) {
+        if (logs.isEmpty() && !sessionVariable.isForceJniScanner()) {
             noLogsSplitNum.incrementAndGet();
         }
 
diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_orc_tables.out b/regression-test/data/external_table_p2/hudi/test_hudi_orc_tables.out
new file mode 100644
index 00000000000000..9e28074dc9114e
--- /dev/null
+++ b/regression-test/data/external_table_p2/hudi/test_hudi_orc_tables.out
@@ -0,0 +1,15 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !cow --
+20241204190011744	20241204190011744_0_6	20241204190011744_0_0		a99e363a-6c10-40f3-a675-9117506d1a43-0_0-38-94_20241204190011744.orc	1	A
+20241204190011744	20241204190011744_0_7	20241204190011744_2_0		a99e363a-6c10-40f3-a675-9117506d1a43-0_0-38-94_20241204190011744.orc	3	C
+20241204190011744	20241204190011744_0_8	20241204190011744_4_0		a99e363a-6c10-40f3-a675-9117506d1a43-0_0-38-94_20241204190011744.orc	5	E
+20241204190011744	20241204190011744_0_9	20241204190011744_1_0		a99e363a-6c10-40f3-a675-9117506d1a43-0_0-38-94_20241204190011744.orc	2	B
+20241204190011744	20241204190011744_0_10	20241204190011744_3_0		a99e363a-6c10-40f3-a675-9117506d1a43-0_0-38-94_20241204190011744.orc	4	D
+
+-- !mor --
+20241204190002046	20241204190002046_0_11	20241204190002046_0_0		b1e68412-01d6-467f-b4c2-b4b18ec71346-0_0-30-75_20241204190002046.orc	1	A
+20241204190002046	20241204190002046_0_12	20241204190002046_2_0		b1e68412-01d6-467f-b4c2-b4b18ec71346-0_0-30-75_20241204190002046.orc	3	C
+20241204190002046	20241204190002046_0_13	20241204190002046_4_0		b1e68412-01d6-467f-b4c2-b4b18ec71346-0_0-30-75_20241204190002046.orc	5	E
+20241204190002046	20241204190002046_0_14	20241204190002046_1_0		b1e68412-01d6-467f-b4c2-b4b18ec71346-0_0-30-75_20241204190002046.orc	2	B
+20241204190002046	20241204190002046_0_15	20241204190002046_3_0		b1e68412-01d6-467f-b4c2-b4b18ec71346-0_0-30-75_20241204190002046.orc	4	D
+
diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_timestamp.out b/regression-test/data/external_table_p2/hudi/test_hudi_timestamp.out
index dc47ff86d90a8d..9bdb0f7cb7285f 100644
--- a/regression-test/data/external_table_p2/hudi/test_hudi_timestamp.out
+++ b/regression-test/data/external_table_p2/hudi/test_hudi_timestamp.out
@@ -1,6 +1,31 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
--- !timestamp --
+-- !timestamp1 --
 20241115015956800	20241115015956800_0_2	1		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	1	Alice	2024-10-25T08:00
-20241115015956800	20241115015956800_0_0	2		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	2	Bob	2024-10-25T09:30:00
-20241115015956800	20241115015956800_0_1	3		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	3	Charlie	2024-10-25T11:00:00
+20241115015956800	20241115015956800_0_0	2		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	2	Bob	2024-10-25T09:30
+20241115015956800	20241115015956800_0_1	3		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	3	Charlie	2024-10-25T11:00
+
+-- !timestamp2 --
+20241115015956800	20241115015956800_0_2	1		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	1	Alice	2024-10-25T23:00
+20241115015956800	20241115015956800_0_0	2		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	2	Bob	2024-10-26T00:30
+20241115015956800	20241115015956800_0_1	3		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	3	Charlie	2024-10-26T02:00
+
+-- !timestamp3 --
+20241115015956800	20241115015956800_0_2	1		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	1	Alice	2024-10-25T15:00
+20241115015956800	20241115015956800_0_0	2		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	2	Bob	2024-10-25T16:30
+20241115015956800	20241115015956800_0_1	3		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	3	Charlie	2024-10-25T18:00
+
+-- !timestamp1 --
+20241115015956800	20241115015956800_0_2	1		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	1	Alice	2024-10-25T08:00
+20241115015956800	20241115015956800_0_0	2		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	2	Bob	2024-10-25T09:30
+20241115015956800	20241115015956800_0_1	3		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	3	Charlie	2024-10-25T11:00
+
+-- !timestamp2 --
+20241115015956800	20241115015956800_0_2	1		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	1	Alice	2024-10-25T23:00
+20241115015956800	20241115015956800_0_0	2		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	2	Bob	2024-10-26T00:30
+20241115015956800	20241115015956800_0_1	3		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	3	Charlie	2024-10-26T02:00
+
+-- !timestamp3 --
+20241115015956800	20241115015956800_0_2	1		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	1	Alice	2024-10-25T15:00
+20241115015956800	20241115015956800_0_0	2		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	2	Bob	2024-10-25T16:30
+20241115015956800	20241115015956800_0_1	3		eec4913a-0d5f-4b8b-a0f5-934e252c2e45-0_0-7-14_20241115015956800.parquet	3	Charlie	2024-10-25T18:00
 
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_orc_tables.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_orc_tables.groovy
new file mode 100644
index 00000000000000..43638a23881e0e
--- /dev/null
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_orc_tables.groovy
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hudi_orc_tables", "p2,external,hudi,external_remote,external_remote_hudi") {
+    String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+        logger.info("disable hudi test")
+    }
+
+    String catalog_name = "test_hudi_orc_tables"
+    String props = context.config.otherConfigs.get("hudiEmrCatalog")
+    sql """drop catalog if exists ${catalog_name};"""
+    sql """
+        create catalog if not exists ${catalog_name} properties (
+            ${props}
+        );
+    """
+
+    sql """ switch ${catalog_name};"""
+    sql """ use regression_hudi;""" 
+    sql """ set enable_fallback_to_original_planner=false """
+    
+    qt_cow """ select * from  orc_hudi_table_cow; """
+    qt_mor """ select * from  orc_hudi_table_mor; """
+
+    sql """drop catalog if exists ${catalog_name};"""
+}
\ No newline at end of file
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy
index 36309322558f52..3d7bd40b2d54cf 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy
@@ -34,8 +34,22 @@ suite("test_hudi_timestamp", "p2,external,hudi,external_remote,external_remote_h
     sql """ use regression_hudi;""" 
     sql """ set enable_fallback_to_original_planner=false """
 
-    // TODO: fix hudi timezone issue and enable this
-    // qt_timestamp """ select * from hudi_table_with_timestamp order by id; """
+    def test_timestamp_different_timezones = {
+        sql """set time_zone = 'America/Los_Angeles';"""
+        qt_timestamp1 """ select * from hudi_table_with_timestamp order by id; """
+        sql """set time_zone = 'Asia/Shanghai';"""
+        qt_timestamp2 """ select * from hudi_table_with_timestamp order by id; """
+        sql """set time_zone = 'UTC';"""
+        qt_timestamp3 """ select * from hudi_table_with_timestamp order by id; """
+    }
+
+    // test native reader
+    test_timestamp_different_timezones()
+    sql """ set force_jni_scanner = true; """
+    // test jni reader
+    test_timestamp_different_timezones()
+    sql """ set force_jni_scanner = false; """
+
 
     sql """drop catalog if exists ${catalog_name};"""
 }

From abb39912fcf0bb20b86c5eb741f2d3caa813a2bd Mon Sep 17 00:00:00 2001
From: Sun Chenyang <sunchenyang@selectdb.com>
Date: Thu, 5 Dec 2024 09:43:59 +0800
Subject: [PATCH 168/399] [opt](test) Make the case stable (#44910)

Make the regression case stable
---
 .../test_index_compaction_dup_keys_arr.groovy              | 2 +-
 .../array_contains/test_index_compaction_null_arr.groovy   | 2 +-
 .../test_index_compaction_unique_keys_arr.groovy           | 2 +-
 .../test_index_compaction_empty_segments.groovy            | 7 ++++---
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_dup_keys_arr.groovy b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_dup_keys_arr.groovy
index ba30c9f044b7be..c7d41e68cdd112 100644
--- a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_dup_keys_arr.groovy
+++ b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_dup_keys_arr.groovy
@@ -17,7 +17,7 @@
 
 import org.codehaus.groovy.runtime.IOGroovyMethods
 
-suite("test_index_compaction_dup_keys_array", "array_contains_inverted_index") {
+suite("test_index_compaction_dup_keys_array", "array_contains_inverted_index, nonConcurrent") {
     // here some variable to control inverted index query
     sql """ set enable_profile=true"""
     sql """ set enable_pipeline_x_engine=true;"""
diff --git a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_null_arr.groovy b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_null_arr.groovy
index b33a197bd32170..dfc38998c4d4bc 100644
--- a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_null_arr.groovy
+++ b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_null_arr.groovy
@@ -17,7 +17,7 @@
 
 import org.codehaus.groovy.runtime.IOGroovyMethods
 
-suite("test_index_compaction_null_arr", "array_contains_inverted_index") {
+suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcurrent") {
     // here some variable to control inverted index query
     sql """ set enable_profile=true"""
     sql """ set enable_pipeline_x_engine=true;"""
diff --git a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
index e07618be6c1370..602e5d552598ea 100644
--- a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
+++ b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_unique_keys_arr.groovy
@@ -17,7 +17,7 @@
 
 import org.codehaus.groovy.runtime.IOGroovyMethods
 
-suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index") {
+suite("test_index_compaction_unique_keys_arr", "array_contains_inverted_index, nonConcurrent") {
     // here some variable to control inverted index query
     sql """ set enable_profile=true"""
     sql """ set enable_pipeline_x_engine=true;"""
diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_empty_segments.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_empty_segments.groovy
index 46126299bf5b88..1c70c9e8e5027b 100644
--- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_empty_segments.groovy
+++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_empty_segments.groovy
@@ -19,7 +19,7 @@ import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
 import org.awaitility.Awaitility
 
-suite("test_index_compaction_empty_segments", "p0") {
+suite("test_index_compaction_empty_segments", "p0, nonConcurrent") {
 
     def compaction_table_name = "test_index_compaction_empty_segments"
     def backendId_to_backendIP = [:]
@@ -82,8 +82,9 @@ suite("test_index_compaction_empty_segments", "p0") {
         });
     }
 
-    int afterSegmentCount = 0
+    
     for (def tablet in tablets) {
+        int afterSegmentCount = 0
         String tablet_id = tablet.TabletId
         (code, out, err) = curl("GET", tablet.CompactionStatus)
         logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
@@ -94,6 +95,6 @@ suite("test_index_compaction_empty_segments", "p0") {
             logger.info("rowset is: " + rowset)
             afterSegmentCount += Integer.parseInt(rowset.split(" ")[1])
         }
+        assertEquals(afterSegmentCount, 0)
     }
-    assertEquals(afterSegmentCount, 0)
 }

From 1b4eb99fd6ca0b5de0f6fa877aa7c7c5502535f1 Mon Sep 17 00:00:00 2001
From: Siyang Tang <tangsiyang@selectdb.com>
Date: Thu, 5 Dec 2024 10:08:19 +0800
Subject: [PATCH 169/399] [fix](regression-test) Fix potential multi FE desc
 table result not refresh after schema change (#44989)

---
 .../main/groovy/org/apache/doris/regression/suite/Suite.groovy  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 744edfe54cbc24..d622084c24bd82 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -770,6 +770,8 @@ class Suite implements GroovyInterceptable {
         runAction(new WaitForAction(context), actionSupplier)
         if (ObjectUtils.isNotEmpty(insertSql)){
             sql insertSql
+        } else {
+            sql "SYNC"
         }
         if (cleanOperator==true){
             if (ObjectUtils.isEmpty(tbName)) throw new RuntimeException("tbName cloud not be null")

From 00ee11e9f3a4018fc0eca686003eafc6ba9486fe Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Thu, 5 Dec 2024 10:17:02 +0800
Subject: [PATCH 170/399] [Fix](merge-on-write) Should clear
 `GetDeleteBitmapUpdateLockResponse` when geting delete bitmap update lock
 fail and retry (#44975)

Similar to https://github.com/apache/doris/pull/43261,
`GetDeleteBitmapUpdateLockResponse` should be cleared after
`get_delete_bitmap_update_lock` fails on MS. Otherwise BE may get staled
compaction cnts and wrongly skip to sync rowsets before
`update_delete_bitmap()` thus causing duplicate keys problem.
---
 cloud/src/meta-service/meta_service_helper.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h
index 5355ac0eb61ed2..e4ed7e2a231901 100644
--- a/cloud/src/meta-service/meta_service_helper.h
+++ b/cloud/src/meta-service/meta_service_helper.h
@@ -118,6 +118,12 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re
                   << " status=" << res->status().ShortDebugString()
                   << " tablet=" << res->tablet_id()
                   << " delete_bitmap_count=" << res->segment_delete_bitmaps_size();
+    } else if constexpr (std::is_same_v<Response, GetDeleteBitmapUpdateLockResponse>) {
+        if (res->status().code() != MetaServiceCode::OK) {
+            res->clear_base_compaction_cnts();
+            res->clear_cumulative_compaction_cnts();
+            res->clear_cumulative_points();
+        }
     } else if constexpr (std::is_same_v<Response, GetObjStoreInfoResponse> ||
                          std::is_same_v<Response, GetStageResponse>) {
         std::string debug_string = res->DebugString();

From bf4ba66ebdf6dd9b0b419ffec8cf6112fe9f5363 Mon Sep 17 00:00:00 2001
From: xzj7019 <xiongzhongjian@selectdb.com>
Date: Thu, 5 Dec 2024 10:28:26 +0800
Subject: [PATCH 171/399] [opt](nereids) support pushing down aggr distinct
 through join (#43380)

---
 .../doris/nereids/jobs/executor/Rewriter.java |   2 +
 .../apache/doris/nereids/rules/RuleType.java  |   1 +
 ...DownAggWithDistinctThroughJoinOneSide.java | 176 ++++++++++++
 .../trees/plans/logical/LogicalAggregate.java |   6 +
 ...n_count_distinct_through_join_one_side.out | 237 ++++++++++++++++
 ...own_min_distinct_through_join_one_side.out | 237 ++++++++++++++++
 ...own_sum_distinct_through_join_one_side.out | 231 ++++++++++++++++
 ...distinct_through_join_one_side_cust.groovy | 125 +++++++++
 ...ount_distinct_through_join_one_side.groovy | 257 ++++++++++++++++++
 ..._min_distinct_through_join_one_side.groovy | 256 +++++++++++++++++
 ..._sum_distinct_through_join_one_side.groovy | 252 +++++++++++++++++
 11 files changed, 1780 insertions(+)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java
 create mode 100644 regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.out
 create mode 100644 regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.out
 create mode 100644 regression-test/data/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.out
 create mode 100644 regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index fa00a4e9343325..dbaaf2a6b32244 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -113,6 +113,7 @@
 import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoin;
 import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOnPkFk;
 import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOneSide;
+import org.apache.doris.nereids.rules.rewrite.PushDownAggWithDistinctThroughJoinOneSide;
 import org.apache.doris.nereids.rules.rewrite.PushDownDistinctThroughJoin;
 import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject;
 import org.apache.doris.nereids.rules.rewrite.PushDownLimit;
@@ -344,6 +345,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
 
                 topic("Eager aggregation",
                         costBased(topDown(
+                                new PushDownAggWithDistinctThroughJoinOneSide(),
                                 new PushDownAggThroughJoinOneSide(),
                                 new PushDownAggThroughJoin()
                         )),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index c81f56f85b6add..e25050b012dc5f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -201,6 +201,7 @@ public enum RuleType {
     ELIMINATE_SORT(RuleTypeClass.REWRITE),
 
     PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE(RuleTypeClass.REWRITE),
+    PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE(RuleTypeClass.REWRITE),
     PUSH_DOWN_AGG_THROUGH_JOIN(RuleTypeClass.REWRITE),
     PUSH_DOWN_AGG_THROUGH_JOIN_ON_PKFK(RuleTypeClass.REWRITE),
     TRANSPOSE_LOGICAL_SEMI_JOIN_LOGICAL_JOIN(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java
new file mode 100644
index 00000000000000..3f9ad609744e21
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.Sets;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Push down agg function with distinct through join on only one side.
+ */
+public class PushDownAggWithDistinctThroughJoinOneSide implements RewriteRuleFactory {
+    @Override
+    public List<Rule> buildRules() {
+        return ImmutableList.of(
+                logicalAggregate(logicalProject(innerLogicalJoin()))
+                        .when(agg -> agg.child().isAllSlots())
+                        .when(agg -> agg.child().child().getOtherJoinConjuncts().isEmpty())
+                        .when(agg -> !agg.isGenerated())
+                        .whenNot(agg -> agg.getAggregateFunctions().isEmpty())
+                        .whenNot(agg -> agg.child()
+                                .child(0).children().stream().anyMatch(p -> p instanceof LogicalAggregate))
+                        .when(agg -> {
+                            Set<AggregateFunction> funcs = agg.getAggregateFunctions();
+                            if (funcs.size() > 1) {
+                                return false;
+                            } else {
+                                return funcs.stream()
+                                        .allMatch(f -> (f instanceof Min || f instanceof Max || f instanceof Sum
+                                                || f instanceof Count) && f.isDistinct()
+                                                && f.child(0) instanceof Slot);
+                            }
+                        })
+                        .thenApply(ctx -> {
+                            LogicalAggregate<LogicalProject<LogicalJoin<Plan, Plan>>> agg = ctx.root;
+                            return pushDownAggWithDistinct(agg, agg.child().child(), agg.child().getProjects());
+                        })
+                        .toRule(RuleType.PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)
+        );
+    }
+
+    private static LogicalAggregate<Plan> pushDownAggWithDistinct(LogicalAggregate<? extends Plan> agg,
+            LogicalJoin<Plan, Plan> join, List<NamedExpression> projects) {
+        Plan leftJoin = join.left();
+        Plan rightJoin = join.right();
+        List<Slot> leftJoinOutput = leftJoin.getOutput();
+        List<Slot> rightJoinOutput = rightJoin.getOutput();
+
+        List<AggregateFunction> leftFuncs = new ArrayList<>();
+        List<AggregateFunction> rightFuncs = new ArrayList<>();
+        Set<Slot> leftFuncSlotSet = new HashSet<>();
+        Set<Slot> rightFuncSlotSet = new HashSet<>();
+        Set<Slot> newAggOverJoinGroupByKeys = new HashSet<>();
+        for (AggregateFunction func : agg.getAggregateFunctions()) {
+            Slot slot = (Slot) func.child(0);
+            newAggOverJoinGroupByKeys.add(slot);
+            if (leftJoinOutput.contains(slot)) {
+                leftFuncs.add(func);
+                leftFuncSlotSet.add(slot);
+            } else if (rightJoinOutput.contains(slot)) {
+                rightFuncs.add(func);
+                rightFuncSlotSet.add(slot);
+            } else {
+                throw new IllegalStateException("Slot " + slot + " not found in join output");
+            }
+        }
+        boolean isLeftSideAggDistinct = !leftFuncs.isEmpty() && rightFuncs.isEmpty();
+        boolean isRightSideAggDistinct = leftFuncs.isEmpty() && !rightFuncs.isEmpty();
+        if (!isLeftSideAggDistinct && !isRightSideAggDistinct) {
+            return null;
+        }
+
+        Set<Slot> leftPushDownGroupBy = new HashSet<>();
+        Set<Slot> rightPushDownGroupBy = new HashSet<>();
+        for (Expression e : agg.getGroupByExpressions()) {
+            Slot slot = (Slot) e;
+            newAggOverJoinGroupByKeys.add(slot);
+            if (leftJoinOutput.contains(slot)) {
+                leftPushDownGroupBy.add(slot);
+            } else if (rightJoinOutput.contains(slot)) {
+                rightPushDownGroupBy.add(slot);
+            } else {
+                return null;
+            }
+        }
+        join.getHashJoinConjuncts().forEach(e -> e.getInputSlots().forEach(slot -> {
+            if (leftJoinOutput.contains(slot)) {
+                leftPushDownGroupBy.add(slot);
+            } else if (rightJoinOutput.contains(slot)) {
+                rightPushDownGroupBy.add(slot);
+            } else {
+                throw new IllegalStateException("Slot " + slot + " not found in join output");
+            }
+        }));
+
+        if (isLeftSideAggDistinct) {
+            leftPushDownGroupBy.add((Slot) leftFuncs.get(0).child(0));
+            Builder<NamedExpression> leftAggOutputBuilder = ImmutableList.<NamedExpression>builder()
+                    .addAll(leftPushDownGroupBy);
+            leftJoin = new LogicalAggregate<>(ImmutableList.copyOf(leftPushDownGroupBy),
+                    leftAggOutputBuilder.build(), join.left());
+        } else {
+            rightPushDownGroupBy.add((Slot) rightFuncs.get(0).child(0));
+            Builder<NamedExpression> rightAggOutputBuilder = ImmutableList.<NamedExpression>builder()
+                    .addAll(rightPushDownGroupBy);
+            rightJoin = new LogicalAggregate<>(ImmutableList.copyOf(rightPushDownGroupBy),
+                    rightAggOutputBuilder.build(), join.right());
+        }
+
+        Preconditions.checkState(leftJoin != join.left() || rightJoin != join.right(),
+                "not pushing down aggr with distinct through join on single side successfully");
+        Plan newJoin = join.withChildren(leftJoin, rightJoin);
+        LogicalAggregate<? extends Plan> newAggOverJoin = agg.withChildGroupByAndOutput(
+                ImmutableList.copyOf(newAggOverJoinGroupByKeys), projects, newJoin);
+
+        List<NamedExpression> newOutputExprs = new ArrayList<>();
+        for (NamedExpression ne : agg.getOutputExpressions()) {
+            if (ne instanceof Alias && ((Alias) ne).child() instanceof AggregateFunction) {
+                AggregateFunction func = (AggregateFunction) ((Alias) ne).child();
+                Slot slot = (Slot) func.child(0);
+                if (leftFuncSlotSet.contains(slot) || rightFuncSlotSet.contains(slot)) {
+                    Expression newFunc = discardDistinct(func);
+                    newOutputExprs.add((NamedExpression) ne.withChildren(newFunc));
+                } else {
+                    throw new IllegalStateException("Slot " + slot + " not found in join output");
+                }
+            } else {
+                newOutputExprs.add(ne);
+            }
+        }
+        return agg.withAggOutputChild(newOutputExprs, newAggOverJoin);
+    }
+
+    private static Expression discardDistinct(AggregateFunction func) {
+        Preconditions.checkState(func.isDistinct(), "current aggregation function is not distinct");
+        Set<Expression> aggChild = Sets.newLinkedHashSet(func.children());
+        return func.withDistinctAndChildren(false, ImmutableList.copyOf(aggChild));
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
index bd68a74fe3e666..d80b4f3166df86 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
@@ -273,6 +273,12 @@ public LogicalAggregate<Plan> withGroupByAndOutput(List<Expression> groupByExprL
                 hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), child());
     }
 
+    public LogicalAggregate<Plan> withChildGroupByAndOutput(List<Expression> groupByExprList,
+            List<NamedExpression> outputExpressionList, Plan newChild) {
+        return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated,
+                hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), newChild);
+    }
+
     public LogicalAggregate<Plan> withChildAndOutput(CHILD_TYPE child,
                                                        List<NamedExpression> outputExpressionList) {
         return new LogicalAggregate<>(groupByExpressions, outputExpressionList, normalized, ordinalIsResolved,
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.out
new file mode 100644
index 00000000000000..4a84ff5117c008
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.out
@@ -0,0 +1,237 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !groupby_pushdown_basic --
+1
+1
+1
+3
+
+-- !groupby_pushdown_left_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_right_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_full_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_left_semi_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_left_anti_join --
+
+-- !groupby_pushdown_complex_conditions --
+
+-- !groupby_pushdown_with_aggregate --
+1	1.0
+1	2.0
+1	3.0
+3	2.0
+
+-- !groupby_pushdown_subquery --
+
+-- !groupby_pushdown_outer_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_deep_subquery --
+
+-- !groupby_pushdown_having --
+
+-- !groupby_pushdown_mixed_aggregates --
+1	1
+1	2
+1	3
+3	6
+
+-- !groupby_pushdown_multi_table_join --
+1
+1
+1
+
+-- !groupby_pushdown_with_order_by --
+1
+1
+1
+3
+
+-- !groupby_pushdown_multiple_equal_conditions --
+1
+1
+1
+
+-- !groupby_pushdown_equal_conditions_with_aggregate --
+1	1
+2	1
+3	1
+
+-- !groupby_pushdown_equal_conditions_non_aggregate --
+a	1
+b	1
+c	1
+
+-- !groupby_pushdown_equal_conditions_non_aggregate_with_aggregate --
+a	1	1
+b	1	1
+c	1	1
+
+-- !groupby_pushdown_with_where_clause --
+
+-- !groupby_pushdown_varied_aggregates --
+1	1.5	1
+1	4.5	1
+1	7.5	1
+3	7.0	0
+
+-- !groupby_pushdown_with_order_by_limit --
+1
+1
+1
+3
+
+-- !groupby_pushdown_alias_multiple_equal_conditions --
+1
+1
+1
+
+-- !groupby_pushdown_complex_join_condition --
+
+-- !groupby_pushdown_function_processed_columns --
+0
+1
+1
+1
+
+-- !groupby_pushdown_nested_queries --
+
+-- !with_hint_groupby_pushdown_basic --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_left_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_right_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_full_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_left_semi_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_left_anti_join --
+
+-- !with_hint_groupby_pushdown_complex_conditions --
+
+-- !with_hint_groupby_pushdown_with_aggregate --
+1	1.0
+1	2.0
+1	3.0
+3	2.0
+
+-- !with_hint_groupby_pushdown_subquery --
+
+-- !with_hint_groupby_pushdown_outer_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_deep_subquery --
+
+-- !with_hint_groupby_pushdown_having --
+
+-- !with_hint_groupby_pushdown_mixed_aggregates --
+1	1
+1	2
+1	3
+3	6
+
+-- !with_hint_groupby_pushdown_multi_table_join --
+1
+1
+1
+
+-- !with_hint_groupby_pushdown_with_order_by --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_multiple_equal_conditions --
+1
+1
+1
+
+-- !with_hint_groupby_pushdown_equal_conditions_with_aggregate --
+1	1
+2	1
+3	1
+
+-- !with_hint_groupby_pushdown_equal_conditions_non_aggregate --
+a	1
+b	1
+c	1
+
+-- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate --
+a	1	1
+b	1	1
+c	1	1
+
+-- !with_hint_groupby_pushdown_with_where_clause --
+
+-- !with_hint_groupby_pushdown_varied_aggregates --
+1	1.5	1
+1	4.5	1
+1	7.5	1
+3	7.0	0
+
+-- !with_hint_groupby_pushdown_with_order_by_limit --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_alias_multiple_equal_conditions --
+1
+1
+1
+
+-- !with_hint_groupby_pushdown_complex_join_condition --
+
+-- !with_hint_groupby_pushdown_function_processed_columns --
+0
+1
+1
+1
+
+-- !with_hint_groupby_pushdown_nested_queries --
+
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.out
new file mode 100644
index 00000000000000..db15483c496eb4
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.out
@@ -0,0 +1,237 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !groupby_pushdown_basic --
+1
+1
+2
+3
+
+-- !groupby_pushdown_left_join --
+1
+1
+2
+3
+
+-- !groupby_pushdown_right_join --
+1
+1
+2
+3
+
+-- !groupby_pushdown_full_join --
+1
+1
+2
+3
+
+-- !groupby_pushdown_left_semi_join --
+1
+1
+2
+3
+
+-- !groupby_pushdown_left_anti_join --
+
+-- !groupby_pushdown_complex_conditions --
+
+-- !groupby_pushdown_with_aggregate --
+1	1.0
+1	2.0
+2	2.0
+3	3.0
+
+-- !groupby_pushdown_subquery --
+
+-- !groupby_pushdown_outer_join --
+1
+1
+2
+3
+
+-- !groupby_pushdown_deep_subquery --
+
+-- !groupby_pushdown_having --
+
+-- !groupby_pushdown_mixed_aggregates --
+1	1
+1	6
+2	2
+3	3
+
+-- !groupby_pushdown_multi_table_join --
+1
+2
+3
+
+-- !groupby_pushdown_with_order_by --
+1
+1
+2
+3
+
+-- !groupby_pushdown_multiple_equal_conditions --
+1
+2
+3
+
+-- !groupby_pushdown_equal_conditions_with_aggregate --
+1	1
+2	2
+3	3
+
+-- !groupby_pushdown_equal_conditions_non_aggregate --
+a	1
+b	2
+c	3
+
+-- !groupby_pushdown_equal_conditions_non_aggregate_with_aggregate --
+a	1	1
+b	2	2
+c	3	3
+
+-- !groupby_pushdown_with_where_clause --
+
+-- !groupby_pushdown_varied_aggregates --
+1	1.5	a
+1	7.0	\N
+2	4.5	b
+3	7.5	c
+
+-- !groupby_pushdown_with_order_by_limit --
+1
+1
+2
+3
+
+-- !groupby_pushdown_alias_multiple_equal_conditions --
+1
+2
+3
+
+-- !groupby_pushdown_complex_join_condition --
+
+-- !groupby_pushdown_function_processed_columns --
+\N
+1
+1
+1
+
+-- !groupby_pushdown_nested_queries --
+
+-- !with_hint_groupby_pushdown_basic --
+1
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_left_join --
+1
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_right_join --
+1
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_full_join --
+1
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_left_semi_join --
+1
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_left_anti_join --
+
+-- !with_hint_groupby_pushdown_complex_conditions --
+
+-- !with_hint_groupby_pushdown_with_aggregate --
+1	1.0
+1	2.0
+2	2.0
+3	3.0
+
+-- !with_hint_groupby_pushdown_subquery --
+
+-- !with_hint_groupby_pushdown_outer_join --
+1
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_deep_subquery --
+
+-- !with_hint_groupby_pushdown_having --
+
+-- !with_hint_groupby_pushdown_mixed_aggregates --
+1	1
+1	6
+2	2
+3	3
+
+-- !with_hint_groupby_pushdown_multi_table_join --
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_with_order_by --
+1
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_multiple_equal_conditions --
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_equal_conditions_with_aggregate --
+1	1
+2	2
+3	3
+
+-- !with_hint_groupby_pushdown_equal_conditions_non_aggregate --
+a	1
+b	2
+c	3
+
+-- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate --
+a	1	1
+b	2	2
+c	3	3
+
+-- !with_hint_groupby_pushdown_with_where_clause --
+
+-- !with_hint_groupby_pushdown_varied_aggregates --
+1	1.5	a
+1	7.0	\N
+2	4.5	b
+3	7.5	c
+
+-- !with_hint_groupby_pushdown_with_order_by_limit --
+1
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_alias_multiple_equal_conditions --
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_complex_join_condition --
+
+-- !with_hint_groupby_pushdown_function_processed_columns --
+\N
+1
+1
+1
+
+-- !with_hint_groupby_pushdown_nested_queries --
+
diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.out
new file mode 100644
index 00000000000000..bb8366176a7ba6
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.out
@@ -0,0 +1,231 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !groupby_pushdown_basic --
+1
+2
+3
+6
+
+-- !groupby_pushdown_left_join --
+1
+2
+3
+6
+
+-- !groupby_pushdown_right_join --
+1
+2
+3
+6
+
+-- !groupby_pushdown_full_join --
+1
+2
+3
+6
+
+-- !groupby_pushdown_left_semi_join --
+1
+2
+3
+6
+
+-- !groupby_pushdown_left_anti_join --
+
+-- !groupby_pushdown_complex_conditions --
+
+-- !groupby_pushdown_with_aggregate --
+1	1.0
+2	2.0
+3	3.0
+6	2.0
+
+-- !groupby_pushdown_subquery --
+
+-- !groupby_pushdown_outer_join --
+1
+2
+3
+6
+
+-- !groupby_pushdown_deep_subquery --
+
+-- !groupby_pushdown_having --
+
+-- !groupby_pushdown_mixed_aggregates --
+1	1
+2	2
+3	3
+6	6
+
+-- !groupby_pushdown_multi_table_join --
+1
+2
+3
+
+-- !groupby_pushdown_with_order_by --
+1
+2
+3
+6
+
+-- !groupby_pushdown_multiple_equal_conditions --
+1
+2
+3
+
+-- !groupby_pushdown_equal_conditions_with_aggregate --
+1	1
+2	2
+3	3
+
+-- !groupby_pushdown_equal_conditions_non_aggregate --
+a	1
+b	2
+c	3
+
+-- !groupby_pushdown_equal_conditions_non_aggregate_with_aggregate --
+a	1	1
+b	2	2
+c	3	3
+
+-- !groupby_pushdown_with_where_clause --
+
+-- !groupby_pushdown_varied_aggregates --
+1
+2
+3
+6
+
+-- !groupby_pushdown_with_order_by_limit --
+1
+2
+3
+6
+
+-- !groupby_pushdown_alias_multiple_equal_conditions --
+1
+2
+3
+
+-- !groupby_pushdown_complex_join_condition --
+
+-- !groupby_pushdown_function_processed_columns --
+1
+2
+3
+6
+
+-- !groupby_pushdown_nested_queries --
+
+-- !with_hint_groupby_pushdown_basic --
+1
+2
+3
+6
+
+-- !with_hint_groupby_pushdown_left_join --
+1
+2
+3
+6
+
+-- !with_hint_groupby_pushdown_right_join --
+1
+2
+3
+6
+
+-- !with_hint_groupby_pushdown_full_join --
+1
+2
+3
+6
+
+-- !with_hint_groupby_pushdown_left_semi_join --
+1
+2
+3
+6
+
+-- !with_hint_groupby_pushdown_left_anti_join --
+
+-- !with_hint_groupby_pushdown_complex_conditions --
+
+-- !with_hint_groupby_pushdown_with_aggregate --
+1	1.0
+2	2.0
+3	3.0
+6	2.0
+
+-- !with_hint_groupby_pushdown_subquery --
+
+-- !with_hint_groupby_pushdown_outer_join --
+1
+2
+3
+6
+
+-- !with_hint_groupby_pushdown_deep_subquery --
+
+-- !with_hint_groupby_pushdown_having --
+
+-- !with_hint_groupby_pushdown_mixed_aggregates --
+1	1
+2	2
+3	3
+6	6
+
+-- !with_hint_groupby_pushdown_multi_table_join --
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_with_order_by --
+1
+2
+3
+6
+
+-- !with_hint_groupby_pushdown_multiple_equal_conditions --
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_equal_conditions_with_aggregate --
+1	1
+2	2
+3	3
+
+-- !with_hint_groupby_pushdown_equal_conditions_non_aggregate --
+a	1
+b	2
+c	3
+
+-- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate --
+a	1	1
+b	2	2
+c	3	3
+
+-- !with_hint_groupby_pushdown_with_where_clause --
+
+-- !with_hint_groupby_pushdown_varied_aggregates --
+1
+2
+3
+6
+
+-- !with_hint_groupby_pushdown_with_order_by_limit --
+1
+2
+3
+6
+
+-- !with_hint_groupby_pushdown_alias_multiple_equal_conditions --
+1
+2
+3
+
+-- !with_hint_groupby_pushdown_complex_join_condition --
+
+-- !with_hint_groupby_pushdown_nested_queries --
+
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy
new file mode 100644
index 00000000000000..9134d66b76d751
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("push_down_aggr_distinct_through_join_one_side_cust") {
+    sql "SET enable_nereids_planner=true"
+    sql "set runtime_filter_mode=OFF"
+    sql "SET enable_fallback_to_original_planner=false"
+    sql "set DISABLE_NEREIDS_RULES='PRUNE_EMPTY_PARTITION, ELIMINATE_GROUP_BY_KEY_BY_UNIFORM'"
+
+    sql """
+        DROP TABLE IF EXISTS dwd_com_abtest_result_inc_ymds;
+	DROP TABLE IF EXISTS dwd_tracking_sensor_init_tmp_ymds;
+    """
+
+    sql """
+    CREATE TABLE `dwd_com_abtest_result_inc_ymds` (
+      `app_name` varchar(255) NULL,
+      `user_key` text NULL,
+      `group_name` text NULL,
+      `dt` date NOT NULL,
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`app_name`)
+    AUTO PARTITION BY RANGE (date_trunc(`dt`, 'day'))
+    (PARTITION p20240813000000 VALUES [('2024-08-13'), ('2024-08-14')),
+    PARTITION p20240814000000 VALUES [('2024-08-14'), ('2024-08-15')),
+    PARTITION p20240815000000 VALUES [('2024-08-15'), ('2024-08-16')),
+    PARTITION p20240816000000 VALUES [('2024-08-16'), ('2024-08-17')),
+    PARTITION p20240817000000 VALUES [('2024-08-17'), ('2024-08-18')),
+    PARTITION p20240818000000 VALUES [('2024-08-18'), ('2024-08-19')),
+    PARTITION p20240819000000 VALUES [('2024-08-19'), ('2024-08-20')))
+    DISTRIBUTED BY HASH(`app_name`) BUCKETS 1
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1",
+    "min_load_replica_num" = "-1",
+    "is_being_synced" = "false",
+    "storage_medium" = "hdd",
+    "storage_format" = "V2",
+    "inverted_index_storage_format" = "V2",
+    "light_schema_change" = "true",
+    "disable_auto_compaction" = "false",
+    "enable_single_replica_compaction" = "false",
+    "group_commit_interval_ms" = "10000",
+    "group_commit_data_bytes" = "134217728"
+    );
+
+    CREATE TABLE `dwd_tracking_sensor_init_tmp_ymds` (
+      `ip` varchar(20) NULL,
+      `gz_user_id` text NULL,
+      `dt` date NOT NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`ip`)
+    AUTO PARTITION BY RANGE (date_trunc(`dt`, 'day'))
+    (PARTITION p20240813000000 VALUES [('2024-08-13'), ('2024-08-14')),
+    PARTITION p20240814000000 VALUES [('2024-08-14'), ('2024-08-15')),
+    PARTITION p20240815000000 VALUES [('2024-08-15'), ('2024-08-16')),
+    PARTITION p20240816000000 VALUES [('2024-08-16'), ('2024-08-17')),
+    PARTITION p20240817000000 VALUES [('2024-08-17'), ('2024-08-18')),
+    PARTITION p20240818000000 VALUES [('2024-08-18'), ('2024-08-19')),
+    PARTITION p20240819000000 VALUES [('2024-08-19'), ('2024-08-20')))
+    DISTRIBUTED BY HASH(`ip`) BUCKETS 10
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1",
+    "min_load_replica_num" = "-1",
+    "is_being_synced" = "false",
+    "storage_medium" = "hdd",
+    "storage_format" = "V2",
+    "inverted_index_storage_format" = "V2",
+    "light_schema_change" = "true",
+    "disable_auto_compaction" = "false",
+    "enable_single_replica_compaction" = "false",
+    "group_commit_interval_ms" = "10000",
+    "group_commit_data_bytes" = "134217728"
+    );
+    """
+
+    explain {
+        sql("physical PLAN SELECT /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/" +
+                "COUNT(DISTINCT dwd_tracking_sensor_init_tmp_ymds.gz_user_id) AS a2c1a830_1," +
+                "dwd_com_abtest_result_inc_ymds.group_name AS ab1011d6," +
+                "dwd_tracking_sensor_init_tmp_ymds.dt AS ad466123 " +
+                "FROM dwd_tracking_sensor_init_tmp_ymds " +
+                "LEFT JOIN dwd_com_abtest_result_inc_ymds " +
+                "ON dwd_tracking_sensor_init_tmp_ymds.gz_user_id = dwd_com_abtest_result_inc_ymds.user_key " +
+                "AND dwd_tracking_sensor_init_tmp_ymds.dt = dwd_com_abtest_result_inc_ymds.dt " +
+                "WHERE dwd_tracking_sensor_init_tmp_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' " +
+                "AND dwd_com_abtest_result_inc_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' " +
+                "GROUP BY 2, 3 ORDER BY 3 asc limit 10000;");
+        contains"groupByExpr=[gz_user_id#1, dt#2]"
+        contains"groupByExpr=[gz_user_id#1, dt#2, group_name#5], outputExpr=[gz_user_id#1, dt#2, group_name#5]"
+        contains"[group_name#5, dt#2]"
+        contains"groupByExpr=[group_name#5, dt#2], outputExpr=[group_name#5, dt#2, count(partial_count(gz_user_id)#12) AS `a2c1a830_1`#7]"
+    }
+
+    explain {
+        sql("physical PLAN SELECT /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/" +
+                "COUNT(DISTINCT dwd_tracking_sensor_init_tmp_ymds.ip) AS a2c1a830_1," +
+                "dwd_com_abtest_result_inc_ymds.group_name AS ab1011d6," +
+                "dwd_tracking_sensor_init_tmp_ymds.dt AS ad466123 " +
+                "FROM dwd_tracking_sensor_init_tmp_ymds " +
+                "LEFT JOIN dwd_com_abtest_result_inc_ymds " +
+                "ON dwd_tracking_sensor_init_tmp_ymds.gz_user_id = dwd_com_abtest_result_inc_ymds.user_key " +
+                "AND dwd_tracking_sensor_init_tmp_ymds.dt = dwd_com_abtest_result_inc_ymds.dt " +
+                "WHERE dwd_tracking_sensor_init_tmp_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' " +
+                "AND dwd_com_abtest_result_inc_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' " +
+                "GROUP BY 2, 3 ORDER BY 3 asc limit 10000;");
+        contains"groupByExpr=[ip#0, gz_user_id#1, dt#2], outputExpr=[ip#0, gz_user_id#1, dt#2]"
+        contains"groupByExpr=[ip#0, dt#2, group_name#5], outputExpr=[ip#0, dt#2, group_name#5]"
+        contains"groupByExpr=[group_name#5, dt#2], outputExpr=[group_name#5, dt#2, partial_count(ip#0) AS `partial_count(ip)`#12]"
+        contains"groupByExpr=[group_name#5, dt#2], outputExpr=[group_name#5, dt#2, count(partial_count(ip)#12) AS `a2c1a830_1`#7]"
+    }
+}
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.groovy
new file mode 100644
index 00000000000000..65a6b15ceccf73
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.groovy
@@ -0,0 +1,257 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("push_down_count_distinct_through_join_one_side") {
+    sql "SET enable_nereids_planner=true"
+    sql "set runtime_filter_mode=OFF"
+    sql "SET enable_fallback_to_original_planner=false"
+    sql """
+        DROP TABLE IF EXISTS count_with_distinct_t;
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS count_with_distinct_t(
+      `id` int(32),
+      `score` int(64) NULL,
+      `name` varchar(64) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(id) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql "insert into count_with_distinct_t values (1, 1, 'a')"
+    sql "insert into count_with_distinct_t values (2, null, 'a')"
+    sql "insert into count_with_distinct_t values (3, 1, null)"
+    sql "insert into count_with_distinct_t values (4, 2, 'b')"
+    sql "insert into count_with_distinct_t values (5, null, 'b')"
+    sql "insert into count_with_distinct_t values (6, 2, null)"
+    sql "insert into count_with_distinct_t values (7, 3, 'c')"
+    sql "insert into count_with_distinct_t values (8, null, 'c')"
+    sql "insert into count_with_distinct_t values (9, 3, null)"
+    sql "insert into count_with_distinct_t values (10, null, null)"
+    sql "analyze table count_with_distinct_t with full with sync;"
+    
+    order_qt_groupby_pushdown_basic """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_left_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 left join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_right_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 right join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_full_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 full join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_left_semi_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 inner join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_left_anti_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 left anti join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_complex_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id and t1.name < t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score), avg(t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_subquery """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from count_with_distinct_t where score > 10) t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_outer_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 left join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_deep_subquery """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from count_with_distinct_t) count_with_distinct_t where score > 10) t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_having """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name having count(distinct t1.score) > 100;
+    """
+
+    order_qt_groupby_pushdown_mixed_aggregates """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score), sum(distinct t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_multi_table_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id join count_with_distinct_t t3 on t1.name = t3.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_with_order_by """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name order by t1.name;
+    """
+
+    order_qt_groupby_pushdown_multiple_equal_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_equal_conditions_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   sum(distinct t1.score), count(distinct t2.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_equal_conditions_non_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   t1.name, count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   t1.name, count(distinct t1.score), count(distinct t2.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+     order_qt_groupby_pushdown_with_where_clause """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and t1.score > 50 group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_varied_aggregates """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score), avg(t1.id), count(distinct t2.name) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_with_order_by_limit """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name order by count(distinct t1.score) limit 10;
+    """
+
+    order_qt_groupby_pushdown_alias_multiple_equal_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1_alias.score) from count_with_distinct_t t1_alias join count_with_distinct_t t2_alias on t1_alias.id = t2_alias.id and t1_alias.name = t2_alias.name group by t1_alias.name;
+    """
+
+    order_qt_groupby_pushdown_complex_join_condition """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id and t1.score = t2.score and t1.name <> t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_function_processed_columns """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct LENGTH(t1.name)) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_nested_queries """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from count_with_distinct_t where score > 20) t1 join (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from count_with_distinct_t where id < 100) t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_basic """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_left_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 left join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_right_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 right join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_full_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 full join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_left_semi_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 inner join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_left_anti_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 left anti join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_complex_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id and t1.name < t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score), avg(t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_subquery """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from count_with_distinct_t where score > 10) t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_outer_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 left join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_deep_subquery """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from count_with_distinct_t) count_with_distinct_t where score > 10) t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_having """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name having count(distinct t1.score) > 100;
+    """
+
+    order_qt_with_hint_groupby_pushdown_mixed_aggregates """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score), sum(distinct t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_multi_table_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id join count_with_distinct_t t3 on t1.name = t3.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_order_by """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name order by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_multiple_equal_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_equal_conditions_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   sum(distinct t1.score), count(distinct t2.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_equal_conditions_non_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   t1.name, count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   t1.name, count(distinct t1.score), count(distinct t2.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_where_clause """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and t1.score > 50 group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_varied_aggregates """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score), avg(t1.id), count(distinct t2.name) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_order_by_limit """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name order by count(distinct t1.score) limit 10;
+    """
+
+    order_qt_with_hint_groupby_pushdown_alias_multiple_equal_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1_alias.score) from count_with_distinct_t t1_alias join count_with_distinct_t t2_alias on t1_alias.id = t2_alias.id and t1_alias.name = t2_alias.name group by t1_alias.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_complex_join_condition """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id and t1.score = t2.score and t1.name <> t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_function_processed_columns """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct LENGTH(t1.name)) from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_nested_queries """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   count(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from count_with_distinct_t where score > 20) t1 join (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from count_with_distinct_t where id < 100) t2 on t1.id = t2.id group by t1.name;
+    """
+}
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.groovy
new file mode 100644
index 00000000000000..676becbcb6ecfa
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.groovy
@@ -0,0 +1,256 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("push_down_min_distinct_through_join_one_side") {
+    sql "SET enable_nereids_planner=true"
+    sql "set runtime_filter_mode=OFF"
+    sql "SET enable_fallback_to_original_planner=false"
+    sql """
+        DROP TABLE IF EXISTS min_with_distinct_t;
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS min_with_distinct_t(
+      `id` int(32),
+      `score` int(64) NULL,
+      `name` varchar(64) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(id) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql "insert into min_with_distinct_t values (1, 1, 'a')"
+    sql "insert into min_with_distinct_t values (2, null, 'a')"
+    sql "insert into min_with_distinct_t values (3, 1, null)"
+    sql "insert into min_with_distinct_t values (4, 2, 'b')"
+    sql "insert into min_with_distinct_t values (5, null, 'b')"
+    sql "insert into min_with_distinct_t values (6, 2, null)"
+    sql "insert into min_with_distinct_t values (7, 3, 'c')"
+    sql "insert into min_with_distinct_t values (8, null, 'c')"
+    sql "insert into min_with_distinct_t values (9, 3, null)"
+    sql "insert into min_with_distinct_t values (10, null, null)"
+    sql "analyze table min_with_distinct_t with sync;"
+    order_qt_groupby_pushdown_basic """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_left_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 left join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_right_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 right join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_full_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 full join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_left_semi_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 inner join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_left_anti_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 left anti join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_complex_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id and t1.name < t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score), avg(t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_subquery """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from min_with_distinct_t where score > 10) t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_outer_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 left join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_deep_subquery """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from min_with_distinct_t) min_with_distinct_t where score > 10) t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_having """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name having min(distinct t1.score) > 100;
+    """
+
+    order_qt_groupby_pushdown_mixed_aggregates """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score), sum(distinct t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_multi_table_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id join min_with_distinct_t t3 on t1.name = t3.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_with_order_by """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name order by t1.name;
+    """
+
+    order_qt_groupby_pushdown_multiple_equal_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_equal_conditions_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   sum(distinct t1.score), min(distinct t2.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_equal_conditions_non_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   t1.name, min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   t1.name, min(distinct t1.score), min(distinct t2.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+     order_qt_groupby_pushdown_with_where_clause """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id and t1.score > 50 group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_varied_aggregates """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score), avg(t1.id), min(distinct t2.name) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_with_order_by_limit """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name order by min(distinct t1.score) limit 10;
+    """
+
+    order_qt_groupby_pushdown_alias_multiple_equal_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1_alias.score) from min_with_distinct_t t1_alias join min_with_distinct_t t2_alias on t1_alias.id = t2_alias.id and t1_alias.name = t2_alias.name group by t1_alias.name;
+    """
+
+    order_qt_groupby_pushdown_complex_join_condition """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id and t1.score = t2.score and t1.name <> t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_function_processed_columns """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct LENGTH(t1.name)) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_nested_queries """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from min_with_distinct_t where score > 20) t1 join (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from min_with_distinct_t where id < 100) t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_basic """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_left_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 left join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_right_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 right join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_full_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 full join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_left_semi_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 inner join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_left_anti_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 left anti join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_complex_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id and t1.name < t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score), avg(t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_subquery """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from min_with_distinct_t where score > 10) t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_outer_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 left join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_deep_subquery """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from min_with_distinct_t) min_with_distinct_t where score > 10) t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_having """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name having min(distinct t1.score) > 100;
+    """
+
+    order_qt_with_hint_groupby_pushdown_mixed_aggregates """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score), sum(distinct t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_multi_table_join """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id join min_with_distinct_t t3 on t1.name = t3.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_order_by """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name order by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_multiple_equal_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_equal_conditions_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   sum(distinct t1.score), min(distinct t2.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_equal_conditions_non_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   t1.name, min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   t1.name, min(distinct t1.score), min(distinct t2.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_where_clause """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id and t1.score > 50 group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_varied_aggregates """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score), avg(t1.id), min(distinct t2.name) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_order_by_limit """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name order by min(distinct t1.score) limit 10;
+    """
+
+    order_qt_with_hint_groupby_pushdown_alias_multiple_equal_conditions """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1_alias.score) from min_with_distinct_t t1_alias join min_with_distinct_t t2_alias on t1_alias.id = t2_alias.id and t1_alias.name = t2_alias.name group by t1_alias.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_complex_join_condition """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from min_with_distinct_t t1 join min_with_distinct_t t2 on t1.id = t2.id and t1.score = t2.score and t1.name <> t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_function_processed_columns """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct LENGTH(t1.name)) from min_with_distinct_t t1, min_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_nested_queries """
+        select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   min(distinct t1.score) from (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from min_with_distinct_t where score > 20) t1 join (select /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/   * from min_with_distinct_t where id < 100) t2 on t1.id = t2.id group by t1.name;
+    """
+}
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.groovy
new file mode 100644
index 00000000000000..799d45ba413fb7
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.groovy
@@ -0,0 +1,252 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("push_down_sum_distinct_through_join_one_side") {
+    sql "SET enable_nereids_planner=true"
+    sql "set runtime_filter_mode=OFF"
+    sql "SET enable_fallback_to_original_planner=false"
+    sql """
+        DROP TABLE IF EXISTS sum_with_distinct_t;
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS sum_with_distinct_t(
+      `id` int(32),
+      `score` int(64) NULL,
+      `name` varchar(64) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(id) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql "insert into sum_with_distinct_t values (1, 1, 'a')"
+    sql "insert into sum_with_distinct_t values (2, null, 'a')"
+    sql "insert into sum_with_distinct_t values (3, 1, null)"
+    sql "insert into sum_with_distinct_t values (4, 2, 'b')"
+    sql "insert into sum_with_distinct_t values (5, null, 'b')"
+    sql "insert into sum_with_distinct_t values (6, 2, null)"
+    sql "insert into sum_with_distinct_t values (7, 3, 'c')"
+    sql "insert into sum_with_distinct_t values (8, null, 'c')"
+    sql "insert into sum_with_distinct_t values (9, 3, null)"
+    sql "insert into sum_with_distinct_t values (10, null, null)"
+    sql "analyze table sum_with_distinct_t with sync;"
+    order_qt_groupby_pushdown_basic """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_left_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 left join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_right_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 right join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_full_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 full join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_left_semi_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 inner join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_left_anti_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 left anti join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_complex_conditions """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id and t1.name < t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_with_aggregate """
+        select    sum(distinct t1.score), avg(t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_subquery """
+        select    sum(distinct t1.score) from (select    * from sum_with_distinct_t where score > 10) t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_outer_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 left join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_deep_subquery """
+        select    sum(distinct t1.score) from (select    * from (select    * from sum_with_distinct_t) sum_with_distinct_t where score > 10) t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_having """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id group by t1.name having sum(distinct t1.score) > 100;
+    """
+
+    order_qt_groupby_pushdown_mixed_aggregates """
+        select    sum(distinct t1.score), sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_multi_table_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id join sum_with_distinct_t t3 on t1.name = t3.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_with_order_by """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id group by t1.name order by t1.name;
+    """
+
+    order_qt_groupby_pushdown_multiple_equal_conditions """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_equal_conditions_with_aggregate """
+        select    sum(distinct t1.score), sum(distinct t2.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_equal_conditions_non_aggregate """
+        select    t1.name, sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate """
+        select    t1.name, sum(distinct t1.score), sum(distinct t2.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+     order_qt_groupby_pushdown_with_where_clause """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id and t1.score > 50 group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_varied_aggregates """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_with_order_by_limit """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id group by t1.name order by sum(distinct t1.score) limit 10;
+    """
+
+    order_qt_groupby_pushdown_alias_multiple_equal_conditions """
+        select    sum(distinct t1_alias.score) from sum_with_distinct_t t1_alias join sum_with_distinct_t t2_alias on t1_alias.id = t2_alias.id and t1_alias.name = t2_alias.name group by t1_alias.name;
+    """
+
+    order_qt_groupby_pushdown_complex_join_condition """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id and t1.score = t2.score and t1.name <> t2.name group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_function_processed_columns """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_groupby_pushdown_nested_queries """
+        select    sum(distinct t1.score) from (select    * from sum_with_distinct_t where score > 20) t1 join (select    * from sum_with_distinct_t where id < 100) t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_basic """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_left_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 left join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_right_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 right join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_full_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 full join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_left_semi_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 inner join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_left_anti_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 left anti join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_complex_conditions """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id and t1.name < t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_aggregate """
+        select    sum(distinct t1.score), avg(t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_subquery """
+        select    sum(distinct t1.score) from (select    * from sum_with_distinct_t where score > 10) t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_outer_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 left join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_deep_subquery """
+        select    sum(distinct t1.score) from (select    * from (select    * from sum_with_distinct_t) sum_with_distinct_t where score > 10) t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_having """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id group by t1.name having sum(distinct t1.score) > 100;
+    """
+
+    order_qt_with_hint_groupby_pushdown_mixed_aggregates """
+        select    sum(distinct t1.score), sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_multi_table_join """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id join sum_with_distinct_t t3 on t1.name = t3.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_order_by """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id group by t1.name order by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_multiple_equal_conditions """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_equal_conditions_with_aggregate """
+        select    sum(distinct t1.score), sum(distinct t2.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_equal_conditions_non_aggregate """
+        select    t1.name, sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate """
+        select    t1.name, sum(distinct t1.score), sum(distinct t2.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_where_clause """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id and t1.score > 50 group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_varied_aggregates """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_with_order_by_limit """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1, sum_with_distinct_t t2 where t1.id = t2.id group by t1.name order by sum(distinct t1.score) limit 10;
+    """
+
+    order_qt_with_hint_groupby_pushdown_alias_multiple_equal_conditions """
+        select    sum(distinct t1_alias.score) from sum_with_distinct_t t1_alias join sum_with_distinct_t t2_alias on t1_alias.id = t2_alias.id and t1_alias.name = t2_alias.name group by t1_alias.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_complex_join_condition """
+        select    sum(distinct t1.score) from sum_with_distinct_t t1 join sum_with_distinct_t t2 on t1.id = t2.id and t1.score = t2.score and t1.name <> t2.name group by t1.name;
+    """
+
+    order_qt_with_hint_groupby_pushdown_nested_queries """
+        select    sum(distinct t1.score) from (select    * from sum_with_distinct_t where score > 20) t1 join (select    * from sum_with_distinct_t where id < 100) t2 on t1.id = t2.id group by t1.name;
+    """
+}

From 397a6d0a49c0e10df954f77976e408e7abcc1b74 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Thu, 5 Dec 2024 11:00:14 +0800
Subject: [PATCH 172/399] =?UTF-8?q?[Improvement](queue)=20Return=20value?=
 =?UTF-8?q?=20of=20concurrent=20queue=20should=20be=20proce=E2=80=A6=20(#4?=
 =?UTF-8?q?4986)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ssed

Push items into concurrent queue will return false due to some unexpected error (e.g. poor memory available).
---
 be/src/pipeline/local_exchange/local_exchanger.h | 14 ++++++++++++--
 be/src/vec/exec/scan/scanner_context.cpp         |  7 ++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/be/src/pipeline/local_exchange/local_exchanger.h b/be/src/pipeline/local_exchange/local_exchanger.h
index f518e2649f89e6..4d699baa52fb8b 100644
--- a/be/src/pipeline/local_exchange/local_exchanger.h
+++ b/be/src/pipeline/local_exchange/local_exchanger.h
@@ -110,7 +110,11 @@ struct BlockQueue {
             : eos(other.eos.load()), data_queue(std::move(other.data_queue)) {}
     inline bool enqueue(BlockType const& item) {
         if (!eos) {
-            data_queue.enqueue(item);
+            if (!data_queue.enqueue(item)) [[unlikely]] {
+                throw Exception(ErrorCode::INTERNAL_ERROR,
+                                "Exception occurs in data queue [size = {}] of local exchange.",
+                                data_queue.size_approx());
+            }
             return true;
         }
         return false;
@@ -118,7 +122,11 @@ struct BlockQueue {
 
     inline bool enqueue(BlockType&& item) {
         if (!eos) {
-            data_queue.enqueue(std::move(item));
+            if (!data_queue.enqueue(std::move(item))) [[unlikely]] {
+                throw Exception(ErrorCode::INTERNAL_ERROR,
+                                "Exception occurs in data queue [size = {}] of local exchange.",
+                                data_queue.size_approx());
+            }
             return true;
         }
         return false;
@@ -186,6 +194,8 @@ struct BlockWrapper {
                         shared_state->exchanger->_free_block_limit *
                                 shared_state->exchanger->_num_sources) {
                 data_block.clear_column_data();
+                // Free blocks is used to improve memory efficiency. Failure during pushing back
+                // free block will not incur any bad result so just ignore the return value.
                 shared_state->exchanger->_free_blocks.enqueue(std::move(data_block));
             }
         }
diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp
index 65812cb428a8f8..688204bdff8ca7 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -65,7 +65,10 @@ ScannerContext::ScannerContext(
            _output_row_descriptor->tuple_descriptors().size() == 1);
     _query_id = _state->get_query_ctx()->query_id();
     ctx_id = UniqueId::gen_uid().to_string();
-    _scanners.enqueue_bulk(scanners.begin(), scanners.size());
+    if (!_scanners.enqueue_bulk(scanners.begin(), scanners.size())) [[unlikely]] {
+        throw Exception(ErrorCode::INTERNAL_ERROR,
+                        "Exception occurs during scanners initialization.");
+    };
     if (limit < 0) {
         limit = -1;
     }
@@ -250,6 +253,8 @@ void ScannerContext::return_free_block(vectorized::BlockUPtr block) {
         _block_memory_usage += block_size_to_reuse;
         _scanner_memory_used_counter->set(_block_memory_usage);
         block->clear_column_data();
+        // Free blocks is used to improve memory efficiency. Failure during pushing back
+        // free block will not incur any bad result so just ignore the return value.
         _free_blocks.enqueue(std::move(block));
     }
 }

From 011f616c6093939c3df1e59fd0b021d86935a76f Mon Sep 17 00:00:00 2001
From: koarz <66543806+koarz@users.noreply.github.com>
Date: Thu, 5 Dec 2024 11:02:17 +0800
Subject: [PATCH 173/399] [Enhancement] (nereids)implement
 showDatabaseIdCommand in nereids (#44192)

Issue Number: close #42742
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  7 ++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/ShowDatabaseIdCommand.java | 75 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../show/test_show_database_id_nereids.groovy | 20 +++++
 6 files changed, 109 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDatabaseIdCommand.java
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_database_id_nereids.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 1fb5897066f021..fd40e2c1400717 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -257,6 +257,7 @@ supportedShowStatement
     | SHOW FULL? TRIGGERS ((FROM | IN) database=multipartIdentifier)? wildWhere?    #showTriggers    
     | SHOW TABLET DIAGNOSIS tabletId=INTEGER_VALUE                                  #showDiagnoseTablet
     | SHOW FRONTENDS name=identifier?                                               #showFrontends 
+    | SHOW DATABASE databaseId=INTEGER_VALUE                                        #showDatabaseId
     | SHOW TABLE tableId=INTEGER_VALUE                                              #showTableId
     | SHOW TRASH (ON backend=STRING_LITERAL)?                                       #showTrash
     | SHOW WHITELIST                                                                #showWhitelist
@@ -311,7 +312,6 @@ unsupportedShowStatement
         LEFT_PAREN functionArguments? RIGHT_PAREN
         ((FROM | IN) database=multipartIdentifier)?                                 #showCreateFunction
     | SHOW (DATABASES | SCHEMAS) (FROM catalog=identifier)? wildWhere?              #showDatabases
-    | SHOW DATABASE databaseId=INTEGER_VALUE                                        #showDatabaseId
     | SHOW DATA TYPES                                                               #showDataTypes
     | SHOW CATALOGS wildWhere?                                                      #showCatalogs
     | SHOW CATALOG name=identifier                                                  #showCatalog
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 9866b410c8fbb0..9ed352b65e587a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -244,6 +244,7 @@
 import org.apache.doris.nereids.DorisParser.ShowCreateRepositoryContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateTableContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateViewContext;
+import org.apache.doris.nereids.DorisParser.ShowDatabaseIdContext;
 import org.apache.doris.nereids.DorisParser.ShowDeleteContext;
 import org.apache.doris.nereids.DorisParser.ShowDiagnoseTabletContext;
 import org.apache.doris.nereids.DorisParser.ShowDynamicPartitionContext;
@@ -543,6 +544,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowDatabaseIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
@@ -4739,6 +4741,11 @@ public LogicalPlan visitAlterRole(AlterRoleContext ctx) {
     }
 
     @Override
+    public LogicalPlan visitShowDatabaseId(ShowDatabaseIdContext ctx) {
+        long dbId = (ctx.databaseId != null) ? Long.parseLong(ctx.databaseId.getText()) : -1;
+        return new ShowDatabaseIdCommand(dbId);
+    }
+
     public LogicalPlan visitCreateRole(CreateRoleContext ctx) {
         String comment = ctx.STRING_LITERAL() == null ? "" : LogicalPlanBuilderAssistant.escapeBackSlash(
                 ctx.STRING_LITERAL().getText().substring(1, ctx.STRING_LITERAL().getText().length() - 1));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 969b51ee99afe4..aaa93d70c355ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -209,6 +209,7 @@ public enum PlanType {
     SHOW_CREATE_REPOSITORY_COMMAND,
     SHOW_CREATE_TABLE_COMMAND,
     SHOW_CREATE_VIEW_COMMAND,
+    SHOW_DATABASE_ID_COMMAND,
     SHOW_DELETE_COMMAND,
     SHOW_DIAGNOSE_TABLET_COMMAND,
     SHOW_DYNAMIC_PARTITION_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDatabaseIdCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDatabaseIdCommand.java
new file mode 100644
index 00000000000000..f183ea9df60ea3
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDatabaseIdCommand.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * show database id command
+ */
+public class ShowDatabaseIdCommand extends ShowCommand {
+    private final Long dbId;
+
+    /**
+     * constructor
+     */
+    public ShowDatabaseIdCommand(Long dbId) {
+        super(PlanType.SHOW_DATABASE_ID_COMMAND);
+        this.dbId = dbId;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check access first
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SHOW DATABASE");
+        }
+        List<List<String>> rows = Lists.newArrayList();
+        DatabaseIf database = ctx.getCurrentCatalog().getDbNullable(dbId);
+        if (database != null) {
+            List<String> row = new ArrayList<>();
+            row.add(database.getFullName());
+            rows.add(row);
+        }
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        builder.addColumn(new Column("DBName", ScalarType.createVarchar(30)));
+        return new ShowResultSet(builder.build(), rows);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowDatabaseIdCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 0d8c24086a6bb3..339bb4e4c6a5f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -90,6 +90,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowDatabaseIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDynamicPartitionCommand;
@@ -453,6 +454,10 @@ default R visitAlterRoleCommand(AlterRoleCommand alterRoleCommand, C context) {
         return visitCommand(alterRoleCommand, context);
     }
 
+    default R visitShowDatabaseIdCommand(ShowDatabaseIdCommand showDatabaseIdCommand, C context) {
+        return visitCommand(showDatabaseIdCommand, context);
+    }
+
     default R visitAlterWorkloadGroupCommand(AlterWorkloadGroupCommand alterWorkloadGroupCommand, C context) {
         return visitCommand(alterWorkloadGroupCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/show/test_show_database_id_nereids.groovy b/regression-test/suites/nereids_p0/show/test_show_database_id_nereids.groovy
new file mode 100644
index 00000000000000..6cc5eb1aac1005
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_database_id_nereids.groovy
@@ -0,0 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_database_id_nereids") {
+    checkNereidsExecute("show database 0;")
+}

From ab29448b9e7669339ed61aa80b7b3848056fd3a3 Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Thu, 5 Dec 2024 11:17:19 +0800
Subject: [PATCH 174/399] [enhance](mtmv)Optimize MTMV lock logic (#44967)

- `org.apache.doris.mtmv.MTMVPartitionUtil#getPartitionsIdsByNames ` If
the lock is no longer obtained and the partition does not exist, just
throw an exception
- `show create materialized view` Narrowing down the scope of the lock
- `OlapTable `adds a method called `getAndCopyPartitionItemWithoutLock`
that does not require obtaining a lock, for use in external scenarios
where it already exists
---
 .../java/org/apache/doris/catalog/Env.java    | 46 +++++++++++--------
 .../java/org/apache/doris/catalog/MTMV.java   |  2 +-
 .../org/apache/doris/catalog/OlapTable.java   | 20 ++++----
 .../apache/doris/mtmv/MTMVPartitionUtil.java  | 16 ++-----
 .../commands/info/ShowCreateMTMVInfo.java     | 13 ++----
 .../org/apache/doris/qe/ShowExecutor.java     | 12 ++---
 6 files changed, 52 insertions(+), 57 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 12eff06acbcbfa..b3544b4de55460 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3429,25 +3429,33 @@ public static void getSyncedDdlStmt(TableIf table, List<String> createTableStmt,
                 hidePassword, false, specificVersion, false, true);
     }
 
-    public static String getMTMVDdl(MTMV mtmv) {
-        StringBuilder sb = new StringBuilder("CREATE MATERIALIZED VIEW ");
-        sb.append(mtmv.getName());
-        addMTMVCols(mtmv, sb);
-        sb.append("\n");
-        sb.append(mtmv.getRefreshInfo());
-        addMTMVKeyInfo(mtmv, sb);
-        addTableComment(mtmv, sb);
-        addMTMVPartitionInfo(mtmv, sb);
-        DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
-        sb.append("\n").append(distributionInfo.toSql());
-        // properties
-        sb.append("\nPROPERTIES (\n");
-        addOlapTablePropertyInfo(mtmv, sb, false, false, null);
-        addMTMVPropertyInfo(mtmv, sb);
-        sb.append("\n)");
-        sb.append("\nAS ");
-        sb.append(mtmv.getQuerySql());
-        return sb.toString();
+    public static String getMTMVDdl(MTMV mtmv) throws AnalysisException {
+        if (!mtmv.tryReadLock(1, TimeUnit.MINUTES)) {
+            throw new AnalysisException(
+                    "get table read lock timeout, database=" + mtmv.getDBName() + ",table=" + mtmv.getName());
+        }
+        try {
+            StringBuilder sb = new StringBuilder("CREATE MATERIALIZED VIEW ");
+            sb.append(mtmv.getName());
+            addMTMVCols(mtmv, sb);
+            sb.append("\n");
+            sb.append(mtmv.getRefreshInfo());
+            addMTMVKeyInfo(mtmv, sb);
+            addTableComment(mtmv, sb);
+            addMTMVPartitionInfo(mtmv, sb);
+            DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
+            sb.append("\n").append(distributionInfo.toSql());
+            // properties
+            sb.append("\nPROPERTIES (\n");
+            addOlapTablePropertyInfo(mtmv, sb, false, false, null);
+            addMTMVPropertyInfo(mtmv, sb);
+            sb.append("\n)");
+            sb.append("\nAS ");
+            sb.append(mtmv.getQuerySql());
+            return sb.toString();
+        } finally {
+            mtmv.readUnlock();
+        }
     }
 
     private static void addMTMVKeyInfo(MTMV mtmv, StringBuilder sb) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index 955bfd4279fd5c..daf1aac333d653 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -392,7 +392,7 @@ public Pair<Map<String, Set<String>>, Map<String, String>> calculateDoublyPartit
         Map<String, String> baseToMv = Maps.newHashMap();
         Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs = MTMVPartitionUtil
                 .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties);
-        Map<String, PartitionItem> mvPartitionItems = getAndCopyPartitionItems();
+        Map<String, PartitionItem> mvPartitionItems = getAndCopyPartitionItemsWithoutLock();
         for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet()) {
             Set<String> basePartitionNames = relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
                     Sets.newHashSet());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index e1bc647d7cec5d..e4adf1f03963e9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -3326,19 +3326,23 @@ public Map<String, PartitionItem> getAndCopyPartitionItems() throws AnalysisExce
             throw new AnalysisException("get table read lock timeout, database=" + getDBName() + ",table=" + getName());
         }
         try {
-            Map<String, PartitionItem> res = Maps.newHashMap();
-            for (Entry<Long, PartitionItem> entry : getPartitionInfo().getIdToItem(false).entrySet()) {
-                Partition partition = idToPartition.get(entry.getKey());
-                if (partition != null) {
-                    res.put(partition.getName(), entry.getValue());
-                }
-            }
-            return res;
+            return getAndCopyPartitionItemsWithoutLock();
         } finally {
             readUnlock();
         }
     }
 
+    public Map<String, PartitionItem> getAndCopyPartitionItemsWithoutLock() throws AnalysisException {
+        Map<String, PartitionItem> res = Maps.newHashMap();
+        for (Entry<Long, PartitionItem> entry : getPartitionInfo().getIdToItem(false).entrySet()) {
+            Partition partition = idToPartition.get(entry.getKey());
+            if (partition != null) {
+                res.put(partition.getName(), entry.getValue());
+            }
+        }
+        return res;
+    }
+
     @Override
     public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
         return getPartitionColumns();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 8ba022de415006..55ce86e75706c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -172,18 +172,12 @@ public static Map<PartitionKeyDesc, Set<String>> generateRelatedPartitionDescs(M
     }
 
     public static List<Long> getPartitionsIdsByNames(MTMV mtmv, List<String> partitions) throws AnalysisException {
-        mtmv.readLock();
-        try {
-            List<Long> res = Lists.newArrayList();
-            for (String partitionName : partitions) {
-                Partition partition = mtmv.getPartitionOrAnalysisException(partitionName);
-                res.add(partition.getId());
-            }
-            return res;
-        } finally {
-            mtmv.readUnlock();
+        List<Long> res = Lists.newArrayList();
+        for (String partitionName : partitions) {
+            Partition partition = mtmv.getPartitionOrAnalysisException(partitionName);
+            res.add(partition.getId());
         }
-
+        return res;
     }
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java
index 588af2365741c5..b4fde11c3e1e64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java
@@ -74,18 +74,13 @@ public void analyze(ConnectContext ctx) {
      * @throws DdlException DdlException
      * @throws IOException IOException
      */
-    public void run(StmtExecutor executor) throws DdlException, IOException {
+    public void run(StmtExecutor executor) throws DdlException, IOException, org.apache.doris.common.AnalysisException {
         List<List<String>> rows = Lists.newArrayList();
         Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(mvName.getDb());
         MTMV mtmv = (MTMV) db.getTableOrDdlException(mvName.getTbl());
-        mtmv.readLock();
-        try {
-            String mtmvDdl = Env.getMTMVDdl(mtmv);
-            rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
-            executor.handleShowCreateMTMVStmt(rows);
-        } finally {
-            mtmv.readUnlock();
-        }
+        String mtmvDdl = Env.getMTMVDdl(mtmv);
+        rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
+        executor.handleShowCreateMTMVStmt(rows);
     }
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 9bda42139ef639..3dcb7069abf423 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -1184,15 +1184,9 @@ private void handleShowCreateMTMV() throws AnalysisException {
                 .getDbOrAnalysisException(showStmt.getDb());
         MTMV mtmv = (MTMV) db.getTableOrAnalysisException(showStmt.getTable());
         List<List<String>> rows = Lists.newArrayList();
-
-        mtmv.readLock();
-        try {
-            String mtmvDdl = Env.getMTMVDdl(mtmv);
-            rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
-            resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
-        } finally {
-            mtmv.readUnlock();
-        }
+        String mtmvDdl = Env.getMTMVDdl(mtmv);
+        rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
+        resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
     }
 
     // Describe statement

From 31729869c08c5be25f12b004b7c1577d13e89a3b Mon Sep 17 00:00:00 2001
From: Uniqueyou <134280716+wyxxxcat@users.noreply.github.com>
Date: Thu, 5 Dec 2024 12:06:35 +0800
Subject: [PATCH 175/399] [Test] add test for check show create table sql
 (#45002)

---
 .../check_before_quit.groovy                  | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/regression-test/suites/check_before_quit/check_before_quit.groovy b/regression-test/suites/check_before_quit/check_before_quit.groovy
index 7b097b58cd23c6..f507fddeaa583b 100644
--- a/regression-test/suites/check_before_quit/check_before_quit.groovy
+++ b/regression-test/suites/check_before_quit/check_before_quit.groovy
@@ -237,5 +237,31 @@ suite("check_before_quit", "nonConcurrent,p0") {
 
         Thread.sleep(2000)
     }
+
+    // check create table sql
+    List<List<Object>> allDataBases = sql "show databases;"
+    logger.info("show all databases: ${allDataBases}")
+
+    def num = allDataBases.size()
+
+    for (int i = 0; i < allDataBases.size(); i++) {
+        def db = allDataBases[i][0]
+        if (db == "__internal_schema" || db == "information_schema" || db == "mysql") {
+            continue
+        }
+        List<List<Object>> allTables = sql "show tables from ${db}"
+        logger.info("show all tabkes: ${allTables}")
+        for (int j = 0;j < allTables.size();j ++) {
+            def tbl = allTables[j][0]
+            def createTableSql = sql "show create table ${db}.${tbl}"
+            logger.info("create table sql info: ${createTableSql}")
+            sql "drop table if exists ${tbl}"
+            sql(createTableSql[0][1])
+            def createTableSqlResult = sql "show create table ${tbl}"
+            logger.info("create table sql result info: ${createTableSqlResult}")
+            assertEquals(createTableSqlResult, createTableSql)
+        }
+    }
+
     assertTrue(clear)
 }

From 8ea48afcb4d2e8052292cb4f62c55b262a7bbb8a Mon Sep 17 00:00:00 2001
From: seawinde <wusi@selectdb.com>
Date: Thu, 5 Dec 2024 12:07:50 +0800
Subject: [PATCH 176/399] [fix](mtmv) Fix refresh materialized view fail when
 mv def contains cte (#44857)

### What problem does this PR solve?

Related PR: #33988

Problem Summary:

if mv def contain cte and is partition mv, such as following:

    CREATE MATERIALIZED VIEW mv_name
        BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
        PARTITION BY (l_shipdate)
        DISTRIBUTED BY RANDOM BUCKETS 2
        PROPERTIES ('replication_num' = '1')
        AS
    WITH scan_data_cte as (
        select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey
        from (select * from lineitem where L_LINENUMBER > 1) t1
        left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY
    )
    SELECT *  FROM scan_data_cte;

if run refresh cmd as following, this would fail, throw exception `no
partition for this tuple`, this pr fix this

refresh materialized view mv_name partitions(p_20231210_20231211);

### Release note

Fix refresh materialized view fail when mv def contains cte
---
 .../commands/UpdateMvByPartitionCommand.java  |   9 +-
 .../mv/micro_test/micro_test_when_cte.out     | 128 +++++++++++
 .../mv/micro_test/micro_test_when_cte.groovy  | 204 ++++++++++++++++++
 3 files changed, 339 insertions(+), 2 deletions(-)
 create mode 100644 regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
index de284bd837748f..36cc0f95a77a8e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
@@ -246,11 +246,16 @@ public Plan visitLogicalCTE(LogicalCTE<? extends Plan> cte, PredicateAddContext
             if (predicates.isEmpty()) {
                 return cte;
             }
+            List<LogicalSubQueryAlias<Plan>> rewrittenSubQueryAlias = new ArrayList<>();
             for (LogicalSubQueryAlias<Plan> subQueryAlias : cte.getAliasQueries()) {
+                List<Plan> subQueryAliasChildren = new ArrayList<>();
                 this.virtualRelationNamePartSet.add(subQueryAlias.getQualifier());
-                subQueryAlias.children().forEach(subQuery -> subQuery.accept(this, predicates));
+                subQueryAlias.children().forEach(subQuery ->
+                        subQueryAliasChildren.add(subQuery.accept(this, predicates))
+                );
+                rewrittenSubQueryAlias.add(subQueryAlias.withChildren(subQueryAliasChildren));
             }
-            return super.visitLogicalCTE(cte, predicates);
+            return super.visitLogicalCTE(new LogicalCTE<>(rewrittenSubQueryAlias, cte.child()), predicates);
         }
 
         @Override
diff --git a/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out b/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out
new file mode 100644
index 00000000000000..8abaaf9adec0a8
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out
@@ -0,0 +1,128 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !query_0_after --
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+
+-- !query_mv_0 --
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+
+-- !query_mv_1 --
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+
+-- !query_0_after --
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+
+-- !query_mv_2 --
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+
+-- !query_mv_3 --
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-08	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-09	4	1	3
+2023-12-10	4	1	3
+2023-12-10	4	1	3
+2023-12-10	4	1	3
+2023-12-10	4	1	3
+2023-12-10	4	1	3
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-10	4	1	4
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-11	4	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+2023-12-12	6	2	3
+
diff --git a/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy b/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy
new file mode 100644
index 00000000000000..9e80ea966fc1a2
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy
@@ -0,0 +1,204 @@
+package mv.micro_test
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("micro_test_when_cte") {
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    sql "set runtime_filter_mode=OFF";
+    sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+
+    sql """
+    drop table if exists orders
+    """
+    sql """
+    CREATE TABLE IF NOT EXISTS orders  (
+      o_orderkey       INTEGER NOT NULL,
+      o_custkey        INTEGER NOT NULL,
+      o_orderstatus    CHAR(1) NOT NULL,
+      o_totalprice     DECIMALV3(15,2) NOT NULL,
+      o_orderdate      DATE NOT NULL,
+      o_orderpriority  CHAR(15) NOT NULL,  
+      o_clerk          CHAR(15) NOT NULL, 
+      o_shippriority   INTEGER NOT NULL,
+      O_COMMENT        VARCHAR(79) NOT NULL
+    )
+    DUPLICATE KEY(o_orderkey, o_custkey)
+    PARTITION BY RANGE(o_orderdate)(
+    FROM ('2023-12-01') TO ('2023-12-31') INTERVAL 1 DAY
+    )
+    DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    );
+    """
+
+    sql """
+    drop table if exists lineitem
+    """
+    sql"""
+    CREATE TABLE IF NOT EXISTS lineitem (
+      l_orderkey    INTEGER NOT NULL,
+      l_partkey     INTEGER NOT NULL,
+      l_suppkey     INTEGER NOT NULL,
+      l_linenumber  INTEGER NOT NULL,
+      l_quantity    DECIMALV3(15,2) NOT NULL,
+      l_extendedprice  DECIMALV3(15,2) NOT NULL,
+      l_discount    DECIMALV3(15,2) NOT NULL,
+      l_tax         DECIMALV3(15,2) NOT NULL,
+      l_returnflag  CHAR(1) NOT NULL,
+      l_linestatus  CHAR(1) NOT NULL,
+      l_shipdate    DATE NOT NULL,
+      l_commitdate  DATE NOT NULL,
+      l_receiptdate DATE NOT NULL,
+      l_shipinstruct CHAR(25) NOT NULL,
+      l_shipmode     CHAR(10) NOT NULL,
+      l_comment      VARCHAR(44) NOT NULL
+    )
+    DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber)
+    PARTITION BY RANGE(l_shipdate) 
+    (FROM ('2023-12-01') TO ('2023-12-31') INTERVAL 1 DAY)
+    DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    )
+    """
+
+    sql """
+    drop table if exists partsupp
+    """
+    sql """
+    CREATE TABLE IF NOT EXISTS partsupp (
+      ps_partkey     INTEGER NOT NULL,
+      ps_suppkey     INTEGER NOT NULL,
+      ps_availqty    INTEGER NOT NULL,
+      ps_supplycost  DECIMALV3(15,2)  NOT NULL,
+      ps_comment     VARCHAR(199) NOT NULL 
+    )
+    DUPLICATE KEY(ps_partkey, ps_suppkey)
+    DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3
+    PROPERTIES (
+      "replication_num" = "1"
+    )
+    """
+
+    sql """ insert into lineitem values
+    (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+    (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+    (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+    (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+    (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');
+    """
+
+    sql """
+    insert into orders values
+    (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+    (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi');  
+    """
+
+    sql """
+    insert into partsupp values
+    (2, 3, 9, 10.01, 'supply1'),
+    (2, 3, 10, 11.01, 'supply2');
+    """
+
+    sql """analyze table partsupp with sync"""
+    sql """analyze table lineitem with sync"""
+    sql """analyze table orders with sync"""
+    sql """alter table lineitem modify column l_comment set stats ('row_count'='5');"""
+    sql """alter table orders modify column O_COMMENT set stats ('row_count'='18');"""
+    sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');"""
+
+    def query_sql = """
+    WITH scan_data_cte as (
+        select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey
+        from (select * from lineitem where L_LINENUMBER > 1) t1
+        left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY
+    )
+    SELECT *  FROM scan_data_cte; 
+    """
+    def mv_sql = """
+    WITH scan_data_cte as (
+        select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey
+        from (select * from lineitem where L_LINENUMBER > 1) t1
+        left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY
+    )
+    SELECT *  FROM scan_data_cte; 
+    """
+    def mv_name = """mv_with_cte_test"""
+
+    // query directly
+    order_qt_query_0_after "${query_sql}"
+
+    // create and build complete mv
+    create_async_mv(db, mv_name, mv_sql)
+    // refresh mv complete
+    sql """refresh materialized view ${mv_name} complete"""
+    // query mv directly
+    waitingMTMVTaskFinishedByMvName(mv_name)
+    order_qt_query_mv_0 "select * from ${mv_name}"
+
+    // create and build partition mv
+    create_async_partition_mv(db, mv_name, mv_sql, "(l_shipdate)")
+
+    // refresh mv partly
+    sql """refresh materialized view ${mv_name} partitions(p_20231208_20231209)"""
+    // query mv directly
+    waitingMTMVTaskFinishedByMvName(mv_name)
+    order_qt_query_mv_1 "select * from ${mv_name}"
+
+    // query rewrite
+    mv_rewrite_success(mv_sql, mv_name)
+    order_qt_query_0_after "${query_sql}"
+
+    // DML
+    // base table insert into data when not partition table
+    sql """
+    insert into orders values
+    (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy');
+    """
+    sql """refresh materialized view ${mv_name} complete"""
+    // query mv directly
+    waitingMTMVTaskFinishedByMvName(mv_name)
+    order_qt_query_mv_2 "select * from ${mv_name}"
+
+    // base table insert into data when partition table
+    sql """
+    insert into lineitem values
+    (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy');
+    """
+    sql """refresh materialized view ${mv_name} partitions(p_20231210_20231211)"""
+    // query mv directly
+    waitingMTMVTaskFinishedByMvName(mv_name)
+    order_qt_query_mv_3 "select * from ${mv_name}"
+}

From d350e3c5f9ee6a3c936bd5b03f0cf82b14bd0fd4 Mon Sep 17 00:00:00 2001
From: seawinde <wusi@selectdb.com>
Date: Thu, 5 Dec 2024 12:08:50 +0800
Subject: [PATCH 177/399] [opt](mtmv) Optimize plan generate when create mtmv
 and use mtmv cache when collect table of mtmv (#44786)

Optimize plan generate when create mtmv and use mtmv cache when collect
table of mtmv
1. Reuse plans when creating materialized views to minimize plan
generation overhead.
2. During recursive base table resolution for MTMVs, prioritize MTMV
cache lookup. Fall back to real-time generation only when cache miss
occurs.
---
 .../java/org/apache/doris/mtmv/MTMVCache.java | 14 +++-
 .../org/apache/doris/mtmv/MTMVPlanUtil.java   | 31 +++-----
 .../mv/InitMaterializationContextHook.java    |  8 +--
 .../exploration/mv/MaterializedViewUtils.java |  2 +-
 .../plans/commands/info/CreateMTMVInfo.java   | 40 +++++------
 .../info/MTMVPartitionDefinition.java         | 71 +++++++------------
 .../trees/plans/visitor/TableCollector.java   | 34 +++++----
 .../nereids/trees/plans/PlanVisitorTest.java  | 14 ++--
 8 files changed, 94 insertions(+), 120 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
index 56061c75b9cee2..2895ad73e14297 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
@@ -51,14 +51,18 @@ public class MTMVCache {
     // The materialized view plan which should be optimized by the same rules to query
     // and will remove top sink and unused sort
     private final Plan logicalPlan;
-    // The original plan of mv def sql
+    // The original rewritten plan of mv def sql
     private final Plan originalPlan;
+    // The analyzed plan of mv def sql, which is used by tableCollector,should not be optimized by rbo
+    private final Plan analyzedPlan;
     private final Statistics statistics;
     private final StructInfo structInfo;
 
-    public MTMVCache(Plan logicalPlan, Plan originalPlan, Statistics statistics, StructInfo structInfo) {
+    public MTMVCache(Plan logicalPlan, Plan originalPlan, Plan analyzedPlan,
+            Statistics statistics, StructInfo structInfo) {
         this.logicalPlan = logicalPlan;
         this.originalPlan = originalPlan;
+        this.analyzedPlan = analyzedPlan;
         this.statistics = statistics;
         this.structInfo = structInfo;
     }
@@ -71,6 +75,10 @@ public Plan getOriginalPlan() {
         return originalPlan;
     }
 
+    public Plan getAnalyzedPlan() {
+        return analyzedPlan;
+    }
+
     public Statistics getStatistics() {
         return statistics;
     }
@@ -117,7 +125,7 @@ public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResu
         Optional<StructInfo> structInfoOptional = MaterializationContext.constructStructInfo(mvPlan, originPlan,
                 planner.getCascadesContext(),
                 new BitSet());
-        return new MTMVCache(mvPlan, originPlan, needCost
+        return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(), needCost
                 ? planner.getCascadesContext().getMemo().getRoot().getStatistics() : null,
                 structInfoOptional.orElseGet(() -> null));
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
index 576e87b44f8d3e..35c06e74d3cc80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
@@ -35,12 +35,10 @@
 import org.apache.doris.nereids.rules.RuleType;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
-import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
 import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
 import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.SessionVariable;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
@@ -104,31 +102,20 @@ private static void setCatalogAndDb(ConnectContext ctx, MTMV mtmv) {
     public static MTMVRelation generateMTMVRelation(MTMV mtmv, ConnectContext ctx) {
         // Should not make table without data to empty relation when analyze the related table,
         // so add disable rules
-        SessionVariable sessionVariable = ctx.getSessionVariable();
-        Set<String> tempDisableRules = sessionVariable.getDisableNereidsRuleNames();
-        sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES);
-        if (ctx.getStatementContext() != null) {
-            ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
-        }
-        Plan plan;
-        try {
-            plan = getPlanBySql(mtmv.getQuerySql(), ctx);
-        } finally {
-            sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules));
-            ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
-        }
-        return generateMTMVRelation(plan);
+        Plan plan = getAnalyzePlanBySql(mtmv.getQuerySql(), ctx);
+        return generateMTMVRelation(plan, ctx);
     }
 
-    public static MTMVRelation generateMTMVRelation(Plan plan) {
-        return new MTMVRelation(getBaseTables(plan, true), getBaseTables(plan, false), getBaseViews(plan));
+    public static MTMVRelation generateMTMVRelation(Plan plan, ConnectContext connectContext) {
+        return new MTMVRelation(getBaseTables(plan, true, connectContext),
+                getBaseTables(plan, false, connectContext), getBaseViews(plan));
     }
 
-    private static Set<BaseTableInfo> getBaseTables(Plan plan, boolean expand) {
+    private static Set<BaseTableInfo> getBaseTables(Plan plan, boolean expand, ConnectContext connectContext) {
         TableCollectorContext collectorContext =
                 new TableCollector.TableCollectorContext(
                         com.google.common.collect.Sets
-                                .newHashSet(TableType.values()), expand);
+                                .newHashSet(TableType.values()), expand, connectContext);
         plan.accept(TableCollector.INSTANCE, collectorContext);
         Set<TableIf> collectedTables = collectorContext.getCollectedTables();
         return transferTableIfToInfo(collectedTables);
@@ -146,7 +133,7 @@ private static Set<BaseTableInfo> transferTableIfToInfo(Set<TableIf> tables) {
         return result;
     }
 
-    private static Plan getPlanBySql(String querySql, ConnectContext ctx) {
+    private static Plan getAnalyzePlanBySql(String querySql, ConnectContext ctx) {
         List<StatementBase> statements;
         try {
             statements = new NereidsParser().parseSQL(querySql);
@@ -159,7 +146,7 @@ private static Plan getPlanBySql(String querySql, ConnectContext ctx) {
         ctx.setStatementContext(new StatementContext());
         try {
             NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
-            return planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.NONE);
+            return planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN);
         } finally {
             ctx.setStatementContext(original);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index f9ea00e178b5b7..4f8198e0b3c0bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -85,12 +85,12 @@ protected void doInitMaterializationContext(CascadesContext cascadesContext) {
             return;
         }
         // Only collect the table or mv which query use directly, to avoid useless mv partition in rewrite
-        TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false);
+        // Keep use one connection context when in query, if new connect context,
+        // the ConnectionContext.get() will change
+        TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false,
+                cascadesContext.getConnectContext());
         try {
             Plan rewritePlan = cascadesContext.getRewritePlan();
-            // Keep use one connection context when in query, if new connect context,
-            // the ConnectionContext.get() will change
-            collectorContext.setConnectContext(cascadesContext.getConnectContext());
             rewritePlan.accept(TableCollector.INSTANCE, collectorContext);
         } catch (Exception e) {
             LOG.warn(String.format("MaterializationContext init table collect fail, current queryId is %s",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index ee4b002007ea05..4c5703e27687e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -343,7 +343,7 @@ public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResu
                     ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute();
             return childContext.getRewritePlan();
         }, mvPlan, originPlan);
-        return new MTMVCache(mvPlan, originPlan,
+        return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(),
                 planner.getCascadesContext().getMemo().getRoot().getStatistics(), null);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index f9e298fd9d40d4..417ab807cfe4bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -256,9 +256,21 @@ public void analyzeQuery(ConnectContext ctx, Map<String, String> mvProperties) t
         NereidsPlanner planner = new NereidsPlanner(statementContext);
         // this is for expression column name infer when not use alias
         LogicalSink<Plan> logicalSink = new UnboundResultSink<>(logicalQuery);
-        // must disable constant folding by be, because be constant folding may return wrong type
-        ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_FOLD_CONSTANT_BY_BE, "false");
-        Plan plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
+        // Should not make table without data to empty relation when analyze the related table,
+        // so add disable rules
+        Set<String> tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames();
+        ctx.getSessionVariable().setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES);
+        ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
+        Plan plan;
+        try {
+            // must disable constant folding by be, because be constant folding may return wrong type
+            ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_FOLD_CONSTANT_BY_BE, "false");
+            plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
+        } finally {
+            // after operate, roll back the disable rules
+            ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules));
+            ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
+        }
         // can not contain VIEW or MTMV
         analyzeBaseTables(planner.getAnalyzedPlan());
         // can not contain Random function
@@ -269,8 +281,7 @@ public void analyzeQuery(ConnectContext ctx, Map<String, String> mvProperties) t
             throw new AnalysisException("can not contain invalid expression");
         }
         getRelation(planner);
-        this.mvPartitionInfo = mvPartitionDefinition
-                .analyzeAndTransferToMTMVPartitionInfo(planner, ctx, logicalQuery);
+        this.mvPartitionInfo = mvPartitionDefinition.analyzeAndTransferToMTMVPartitionInfo(planner, ctx);
         this.partitionDesc = generatePartitionDesc(ctx);
         getColumns(plan, ctx, mvPartitionInfo.getPartitionCol(), distribution);
         analyzeKeys();
@@ -315,24 +326,9 @@ private void analyzeKeys() {
         }
     }
 
+    // Should use analyzed plan for collect views and tables
     private void getRelation(NereidsPlanner planner) {
-        // Should not make table without data to empty relation when analyze the related table,
-        // so add disable rules
-        ConnectContext ctx = planner.getCascadesContext().getConnectContext();
-        SessionVariable sessionVariable = ctx.getSessionVariable();
-        Set<String> tempDisableRules = sessionVariable.getDisableNereidsRuleNames();
-        sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES);
-        if (ctx.getStatementContext() != null) {
-            ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
-        }
-        Plan plan;
-        try {
-            plan = planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.NONE);
-        } finally {
-            sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules));
-            ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
-        }
-        this.relation = MTMVPlanUtil.generateMTMVRelation(plan);
+        this.relation = MTMVPlanUtil.generateMTMVRelation(planner.getAnalyzedPlan(), planner.getConnectContext());
     }
 
     private PartitionDesc generatePartitionDesc(ConnectContext ctx) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
index c4117e8608e29d..a26a97f7240793 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
@@ -37,7 +37,6 @@
 import org.apache.doris.nereids.analyzer.UnboundFunction;
 import org.apache.doris.nereids.analyzer.UnboundSlot;
 import org.apache.doris.nereids.exceptions.AnalysisException;
-import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo;
 import org.apache.doris.nereids.trees.expressions.Cast;
@@ -45,11 +44,7 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
-import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.SessionVariable;
 
 import com.google.common.collect.Sets;
 
@@ -72,11 +67,9 @@ public class MTMVPartitionDefinition {
      *
      * @param planner planner
      * @param ctx ctx
-     * @param logicalQuery logicalQuery
      * @return MTMVPartitionInfo
      */
-    public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx,
-            LogicalPlan logicalQuery) {
+    public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx) {
         MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(partitionType);
         if (this.partitionType == MTMVPartitionType.SELF_MANAGE) {
             return mtmvPartitionInfo;
@@ -100,7 +93,7 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl
             timeUnit = null;
         }
         mtmvPartitionInfo.setPartitionCol(partitionColName);
-        RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, logicalQuery, partitionColName, timeUnit);
+        RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, partitionColName, timeUnit);
         mtmvPartitionInfo.setRelatedCol(relatedTableInfo.getColumn());
         mtmvPartitionInfo.setRelatedTable(relatedTableInfo.getTableInfo());
         if (relatedTableInfo.getPartitionExpression().isPresent()) {
@@ -125,47 +118,33 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl
         return mtmvPartitionInfo;
     }
 
-    private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, LogicalPlan
-            logicalQuery,
-            String partitionColName,
-            String timeUnit) {
+    // Should use rewritten plan without view and subQuery to get related partition table
+    private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx,
+            String partitionColName, String timeUnit) {
         CascadesContext cascadesContext = planner.getCascadesContext();
-        SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable();
-        Set<String> tempDisableRules = sessionVariable.getDisableNereidsRuleNames();
-        // Should not make table without data to empty relation when analyze the related table,
-        // so add disable rules
-        sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES);
-        cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
+
+        RelatedTableInfo relatedTableInfo = MaterializedViewUtils
+                .getRelatedTableInfo(partitionColName, timeUnit, planner.getRewrittenPlan(), cascadesContext);
+        if (!relatedTableInfo.isPctPossible()) {
+            throw new AnalysisException(String.format("Unable to find a suitable base table for partitioning,"
+                    + " the fail reason is %s", relatedTableInfo.getFailReason()));
+        }
+        MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo());
+        Set<String> partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
         try {
-            Plan mvRewrittenPlan =
-                    planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN);
-            RelatedTableInfo relatedTableInfo = MaterializedViewUtils
-                    .getRelatedTableInfo(partitionColName, timeUnit, mvRewrittenPlan, cascadesContext);
-            if (!relatedTableInfo.isPctPossible()) {
-                throw new AnalysisException(String.format("Unable to find a suitable base table for partitioning,"
-                        + " the fail reason is %s", relatedTableInfo.getFailReason()));
-            }
-            MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo());
-            Set<String> partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
-            try {
-                partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames(Optional.empty()));
-            } catch (DdlException e) {
-                throw new AnalysisException(e.getMessage(), e);
-            }
+            partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames(Optional.empty()));
+        } catch (DdlException e) {
+            throw new AnalysisException(e.getMessage(), e);
+        }
 
-            if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) {
-                throw new AnalysisException("error related column: " + relatedTableInfo.getColumn());
-            }
-            if (!(mtmvBaseRealtedTable instanceof HMSExternalTable)
-                    && partitionColumnNames.size() != 1) {
-                throw new AnalysisException("only hms table support multi column partition.");
-            }
-            return relatedTableInfo;
-        } finally {
-            // after operate, roll back the disable rules
-            sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules));
-            cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
+        if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) {
+            throw new AnalysisException("error related column: " + relatedTableInfo.getColumn());
+        }
+        if (!(mtmvBaseRealtedTable instanceof HMSExternalTable)
+                && partitionColumnNames.size() != 1) {
+            throw new AnalysisException("only hms table support multi column partition.");
         }
+        return relatedTableInfo;
     }
 
     private static List<Expr> convertToLegacyArguments(List<Expression> children) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java
index 2e2cdb810f0f72..27ff1e4b68c075 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java
@@ -20,8 +20,8 @@
 import org.apache.doris.catalog.MTMV;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.common.AnalysisException;
 import org.apache.doris.mtmv.MTMVCache;
-import org.apache.doris.mtmv.MTMVPlanUtil;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation;
@@ -70,13 +70,19 @@ public Plan visitPhysicalCatalogRelation(PhysicalCatalogRelation catalogRelation
     }
 
     private void expandMvAndCollect(MTMV mtmv, TableCollectorContext context) {
-        if (!context.isExpand()) {
+        if (!context.isExpandMaterializedView()) {
             return;
         }
         // Make sure use only one connection context when in query to avoid ConnectionContext.get() wrong
-        MTMVCache expandedMv = MTMVCache.from(mtmv, context.getConnectContext() == null
-                ? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext(), false);
-        expandedMv.getLogicalPlan().accept(this, context);
+        MTMVCache expandedMvCache;
+        try {
+            expandedMvCache = mtmv.getOrGenerateCache(context.getConnectContext());
+        } catch (AnalysisException exception) {
+            LOG.warn(String.format("expandMvAndCollect getOrGenerateCache fail, mtmv name is %s", mtmv.getName()),
+                    exception);
+            expandedMvCache = MTMVCache.from(mtmv, context.getConnectContext(), false);
+        }
+        expandedMvCache.getAnalyzedPlan().accept(this, context);
     }
 
     /**
@@ -87,12 +93,14 @@ public static final class TableCollectorContext {
         private final Set<TableIf> collectedTables = new HashSet<>();
         private final Set<TableType> targetTableTypes;
         // if expand the mv or not
-        private final boolean expand;
-        private ConnectContext connectContext;
+        private final boolean expandMaterializedView;
+        private final ConnectContext connectContext;
 
-        public TableCollectorContext(Set<TableType> targetTableTypes, boolean expand) {
+        public TableCollectorContext(Set<TableType> targetTableTypes, boolean expandMaterializedView,
+                ConnectContext connectContext) {
             this.targetTableTypes = targetTableTypes;
-            this.expand = expand;
+            this.expandMaterializedView = expandMaterializedView;
+            this.connectContext = connectContext;
         }
 
         public Set<TableIf> getCollectedTables() {
@@ -103,16 +111,12 @@ public Set<TableType> getTargetTableTypes() {
             return targetTableTypes;
         }
 
-        public boolean isExpand() {
-            return expand;
+        public boolean isExpandMaterializedView() {
+            return expandMaterializedView;
         }
 
         public ConnectContext getConnectContext() {
             return connectContext;
         }
-
-        public void setConnectContext(ConnectContext connectContext) {
-            this.connectContext = connectContext;
-        }
     }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java
index 60f6e19faab635..0c54f8fad5a107 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java
@@ -130,7 +130,7 @@ public void test1() {
                             Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Random);
                             // Check get tables
                             TableCollectorContext collectorContext = new TableCollector.TableCollectorContext(
-                                    Sets.newHashSet(TableType.OLAP), true);
+                                    Sets.newHashSet(TableType.OLAP), true, connectContext);
                             physicalPlan.accept(TableCollector.INSTANCE, collectorContext);
                             Set<String> expectedTables = new HashSet<>();
                             expectedTables.add("table1");
@@ -159,7 +159,7 @@ public void test2() {
                             Assertions.assertTrue(nondeterministicFunctionSet.get(1) instanceof Random);
                             // Check get tables
                             TableCollectorContext collectorContext = new TableCollector.TableCollectorContext(
-                                    Sets.newHashSet(TableType.OLAP), true);
+                                    Sets.newHashSet(TableType.OLAP), true, connectContext);
                             physicalPlan.accept(TableCollector.INSTANCE, collectorContext);
                             Set<String> expectedTables = new HashSet<>();
                             expectedTables.add("table1");
@@ -196,7 +196,7 @@ public BitSet getDisableNereidsRules() {
                             Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Uuid);
                             // Check get tables
                             TableCollectorContext collectorContext = new TableCollector.TableCollectorContext(
-                                    Sets.newHashSet(TableType.OLAP), true);
+                                    Sets.newHashSet(TableType.OLAP), true, connectContext);
                             physicalPlan.accept(TableCollector.INSTANCE, collectorContext);
                             Set<String> expectedTables = new HashSet<>();
                             expectedTables.add("table1");
@@ -210,7 +210,7 @@ public BitSet getDisableNereidsRules() {
 
                             TableCollectorContext collectorContextWithNoExpand =
                                     new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP),
-                                            false);
+                                            false, connectContext);
                             physicalPlan.accept(TableCollector.INSTANCE, collectorContextWithNoExpand);
                             Set<String> expectedTablesWithNoExpand = new HashSet<>();
                             expectedTablesWithNoExpand.add("table1");
@@ -222,7 +222,7 @@ public BitSet getDisableNereidsRules() {
                                     expectedTablesWithNoExpand);
 
                             TableCollectorContext mvCollectorContext = new TableCollector.TableCollectorContext(
-                                    Sets.newHashSet(TableType.MATERIALIZED_VIEW), true);
+                                    Sets.newHashSet(TableType.MATERIALIZED_VIEW), true, connectContext);
                             physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContext);
                             Set<String> expectedMvs = new HashSet<>();
                             expectedMvs.add("mv1");
@@ -234,7 +234,7 @@ public BitSet getDisableNereidsRules() {
 
                             TableCollectorContext mvCollectorContextWithNoExpand =
                                     new TableCollector.TableCollectorContext(
-                                    Sets.newHashSet(TableType.MATERIALIZED_VIEW), false);
+                                    Sets.newHashSet(TableType.MATERIALIZED_VIEW), false, connectContext);
                             physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContextWithNoExpand);
                             Set<String> expectedMvsWithNoExpand = new HashSet<>();
                             expectedMvsWithNoExpand.add("mv1");
@@ -246,7 +246,7 @@ public BitSet getDisableNereidsRules() {
 
                             TableCollectorContext allTableTypeWithExpand =
                                     new TableCollector.TableCollectorContext(
-                                            Sets.newHashSet(TableType.values()), true);
+                                            Sets.newHashSet(TableType.values()), true, connectContext);
                             physicalPlan.accept(TableCollector.INSTANCE, allTableTypeWithExpand);
                             // when collect in plan with expand, should collect table which is expended
                             Set<String> expectedTablesWithExpand = new HashSet<>();

From c477fd1b38ea3d09b19489aceb8e3adef8c0efe2 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Thu, 5 Dec 2024 14:05:58 +0800
Subject: [PATCH 178/399] [Fix](job)Fix CAS competition failure leading to
 message publishing failure. (#45018)

### What problem does this PR solve?

**tryPublish** returns false in case of CAS competition failure or when
the queue is full. Therefore, we need to first check if there is enough
capacity and then use the **publish** method for serial publishing.
---
 .../org/apache/doris/job/disruptor/TaskDisruptor.java | 11 ++++++++++-
 .../org/apache/doris/job/scheduler/JobScheduler.java  |  4 +++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java
index 6ca2924c593bc1..2b2e3df0418dd3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java
@@ -74,7 +74,16 @@ public void start() {
     public boolean publishEvent(Object... args) {
         try {
             RingBuffer<T> ringBuffer = disruptor.getRingBuffer();
-            return ringBuffer.tryPublishEvent(eventTranslator, args);
+            // Check if the RingBuffer has enough capacity to reserve 10 slots for tasks
+            // If there is insufficient capacity (less than 10 slots available)
+            // log a warning and drop the current task
+            if (!ringBuffer.hasAvailableCapacity(10)) {
+                LOG.warn("ring buffer has no available capacity,task will be dropped,"
+                        + "please check the task queue size.");
+                return false;
+            }
+            ringBuffer.publishEvent(eventTranslator, args);
+            return true;
         } catch (Exception e) {
             LOG.warn("Failed to publish event", e);
             // Handle the exception, e.g., retry or alert
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
index ea0c263a5ee1c3..7f8b39f1e66dc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
@@ -168,7 +168,9 @@ public void schedulerInstantJob(T job, TaskType taskType, C context) throws JobE
         for (AbstractTask task : tasks) {
             if (!taskDisruptorGroupManager.dispatchInstantTask(task, job.getJobType(),
                     job.getJobConfig())) {
-                throw new JobException(job.formatMsgWhenExecuteQueueFull(task.getTaskId()));
+                String errorMsg = job.formatMsgWhenExecuteQueueFull(task.getTaskId());
+                task.onFail(errorMsg);
+                throw new JobException(errorMsg);
 
             }
             log.info("dispatch instant job, job id is {}, job name is {}, task id is {}", job.getJobId(),

From 6a8ae7705a9641ce31aa1d9693044c8ddd54e394 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Thu, 5 Dec 2024 14:08:14 +0800
Subject: [PATCH 179/399] [Enchancement](runtime-filter) improvement for
 datetimev2 bloom filter hash method (#44924)

### What problem does this PR solve?
improvement for datetimev2 bloom filter hash method
In the past, datetimev2 use to_int64 to get a 64bit data and truncate to
32bit data, then use this 32bit data to build the bloom filter. this can
lead to poor performance and bad filterability.
---
 be/src/exprs/bloom_filter_func.h              | 169 ++++++++++++------
 be/src/exprs/create_predicate_function.h      |  25 ++-
 be/src/exprs/runtime_filter.cpp               |  17 +-
 be/src/exprs/runtime_filter.h                 |   2 +
 be/src/olap/bitmap_filter_predicate.h         |   2 +-
 be/src/olap/tablet_reader.cpp                 |  12 +-
 be/src/util/hash_util.hpp                     |   8 -
 be/src/vec/runtime/vdatetime_value.h          |   2 -
 .../bitmap_filter_column_predicate_test.cpp   |   2 +-
 .../org/apache/doris/qe/SessionVariable.java  |   6 +
 gensrc/thrift/PaloInternalService.thrift      |   2 +
 11 files changed, 153 insertions(+), 94 deletions(-)

diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h
index ff749420ad286e..4d221f7bfe8421 100644
--- a/be/src/exprs/bloom_filter_func.h
+++ b/be/src/exprs/bloom_filter_func.h
@@ -26,17 +26,41 @@
 #include "vec/common/string_ref.h"
 
 namespace doris {
+// there are problems with the implementation of the old datetimev2. for compatibility reason, we will keep this code temporary.
+struct fixed_len_to_uint32 {
+    template <typename T>
+    uint32_t operator()(T value) {
+        if constexpr (sizeof(T) <= sizeof(uint32_t)) {
+            if constexpr (std::is_same_v<T, DateV2Value<DateV2ValueType>>) {
+                return (uint32_t)value.to_int64();
+            } else {
+                return (uint32_t)value;
+            }
+        }
+        return std::hash<T>()(value);
+    }
+};
+
+struct fixed_len_to_uint32_v2 {
+    template <typename T>
+    uint32_t operator()(T value) {
+        if constexpr (sizeof(T) <= sizeof(uint32_t)) {
+            if constexpr (std::is_same_v<T, DateV2Value<DateV2ValueType>>) {
+                return (uint32_t)value.to_date_int_val();
+            } else {
+                return (uint32_t)value;
+            }
+        }
+        return std::hash<T>()(value);
+    }
+};
 
 class BloomFilterAdaptor {
 public:
-    BloomFilterAdaptor(bool null_aware = false) : _null_aware(null_aware) {
+    BloomFilterAdaptor(bool null_aware) : _null_aware(null_aware) {
         _bloom_filter = std::make_shared<doris::BlockBloomFilter>();
     }
 
-    static int64_t optimal_bit_num(int64_t expect_num, double fpp) {
-        return doris::segment_v2::BloomFilter::optimal_bit_num(expect_num, fpp) / 8;
-    }
-
     static BloomFilterAdaptor* create(bool null_aware) {
         return new BloomFilterAdaptor(null_aware);
     }
@@ -57,27 +81,23 @@ class BloomFilterAdaptor {
 
     size_t size() { return _bloom_filter->directory().size; }
 
-    template <typename T>
-    bool test(T data) const {
-        return _bloom_filter->find(data);
-    }
+    bool test(uint32_t data) const { return _bloom_filter->find(data); }
 
-    // test_element/find_element only used on vectorized engine
-    template <typename T>
+    template <typename fixed_len_to_uint32_method, typename T>
     bool test_element(T element) const {
         if constexpr (std::is_same_v<T, StringRef>) {
             return _bloom_filter->find(element);
         } else {
-            return _bloom_filter->find(HashUtil::fixed_len_to_uint32(element));
+            return _bloom_filter->find(fixed_len_to_uint32_method()(element));
         }
     }
 
-    template <typename T>
+    template <typename fixed_len_to_uint32_method, typename T>
     void add_element(T element) {
         if constexpr (std::is_same_v<T, StringRef>) {
             _bloom_filter->insert(element);
         } else {
-            _bloom_filter->insert(HashUtil::fixed_len_to_uint32(element));
+            _bloom_filter->insert(fixed_len_to_uint32_method()(element));
         }
     }
 
@@ -214,6 +234,8 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
 
     void set_contain_null_and_null_aware() { _bloom_filter->set_contain_null_and_null_aware(); }
 
+    void set_enable_fixed_len_to_uint32_v2() { _enable_fixed_len_to_uint32_v2 = true; }
+
     size_t get_size() const { return _bloom_filter ? _bloom_filter->size() : 0; }
 
     void light_copy(BloomFilterFuncBase* bloomfilter_func) {
@@ -221,6 +243,7 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
         _bloom_filter_alloced = other_func->_bloom_filter_alloced;
         _bloom_filter = other_func->_bloom_filter;
         _inited = other_func->_inited;
+        _enable_fixed_len_to_uint32_v2 |= other_func->_enable_fixed_len_to_uint32_v2;
     }
 
     virtual void insert(const void* data) = 0;
@@ -255,9 +278,10 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
     int64_t _runtime_bloom_filter_max_size;
     bool _build_bf_exactly = false;
     bool _bloom_filter_size_calculated_by_ndv = false;
+    bool _enable_fixed_len_to_uint32_v2 = false;
 };
 
-template <typename T, bool need_trim = false>
+template <typename fixed_len_to_uint32_method, typename T, bool need_trim = false>
 uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* data,
                          const uint8* nullmap, uint16_t* offsets, int number,
                          const bool is_parse_column) {
@@ -281,7 +305,8 @@ uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* dat
         if (nullmap == nullptr) {
             for (int i = 0; i < number; i++) {
                 uint16_t idx = offsets[i];
-                if (!bloom_filter.test_element(get_element(data, idx))) {
+                if (!bloom_filter.test_element<fixed_len_to_uint32_method>(
+                            get_element(data, idx))) {
                     continue;
                 }
                 offsets[new_size++] = idx;
@@ -294,7 +319,8 @@ uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* dat
                         continue;
                     }
                 } else {
-                    if (!bloom_filter.test_element(get_element(data, idx))) {
+                    if (!bloom_filter.test_element<fixed_len_to_uint32_method>(
+                                get_element(data, idx))) {
                         continue;
                     }
                 }
@@ -304,7 +330,7 @@ uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* dat
     } else {
         if (nullmap == nullptr) {
             for (int i = 0; i < number; i++) {
-                if (!bloom_filter.test_element(get_element(data, i))) {
+                if (!bloom_filter.test_element<fixed_len_to_uint32_method>(get_element(data, i))) {
                     continue;
                 }
                 offsets[new_size++] = i;
@@ -316,7 +342,8 @@ uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* dat
                         continue;
                     }
                 } else {
-                    if (!bloom_filter.test_element(get_element(data, i))) {
+                    if (!bloom_filter.test_element<fixed_len_to_uint32_method>(
+                                get_element(data, i))) {
                         continue;
                     }
                 }
@@ -327,16 +354,17 @@ uint16_t find_batch_olap(const BloomFilterAdaptor& bloom_filter, const char* dat
     return new_size;
 }
 
-template <class T>
+template <typename fixed_len_to_uint32_method, class T>
 struct CommonFindOp {
-    uint16_t find_batch_olap_engine(const BloomFilterAdaptor& bloom_filter, const char* data,
-                                    const uint8* nullmap, uint16_t* offsets, int number,
-                                    const bool is_parse_column) {
-        return find_batch_olap<T>(bloom_filter, data, nullmap, offsets, number, is_parse_column);
+    static uint16_t find_batch_olap_engine(const BloomFilterAdaptor& bloom_filter, const char* data,
+                                           const uint8* nullmap, uint16_t* offsets, int number,
+                                           const bool is_parse_column) {
+        return find_batch_olap<fixed_len_to_uint32_method, T>(bloom_filter, data, nullmap, offsets,
+                                                              number, is_parse_column);
     }
 
-    void insert_batch(BloomFilterAdaptor& bloom_filter, const vectorized::ColumnPtr& column,
-                      size_t start) const {
+    static void insert_batch(BloomFilterAdaptor& bloom_filter, const vectorized::ColumnPtr& column,
+                             size_t start) {
         const auto size = column->size();
         if (column->is_nullable()) {
             const auto* nullable = assert_cast<const vectorized::ColumnNullable*>(column.get());
@@ -348,7 +376,7 @@ struct CommonFindOp {
             const T* data = (T*)col.get_raw_data().data;
             for (size_t i = start; i < size; i++) {
                 if (!nullmap[i]) {
-                    bloom_filter.add_element(*(data + i));
+                    bloom_filter.add_element<fixed_len_to_uint32_method>(*(data + i));
                 } else {
                     bloom_filter.set_contain_null();
                 }
@@ -356,13 +384,13 @@ struct CommonFindOp {
         } else {
             const T* data = (T*)column->get_raw_data().data;
             for (size_t i = start; i < size; i++) {
-                bloom_filter.add_element(*(data + i));
+                bloom_filter.add_element<fixed_len_to_uint32_method>(*(data + i));
             }
         }
     }
 
-    void find_batch(const BloomFilterAdaptor& bloom_filter, const vectorized::ColumnPtr& column,
-                    uint8_t* results) const {
+    static void find_batch(const BloomFilterAdaptor& bloom_filter,
+                           const vectorized::ColumnPtr& column, uint8_t* results) {
         const T* __restrict data = nullptr;
         const uint8_t* __restrict nullmap = nullptr;
         if (column->is_nullable()) {
@@ -382,31 +410,32 @@ struct CommonFindOp {
         if (nullmap) {
             for (size_t i = 0; i < size; i++) {
                 if (!nullmap[i]) {
-                    results[i] = bloom_filter.test_element(data[i]);
+                    results[i] = bloom_filter.test_element<fixed_len_to_uint32_method>(data[i]);
                 } else {
                     results[i] = bloom_filter.contain_null();
                 }
             }
         } else {
             for (size_t i = 0; i < size; i++) {
-                results[i] = bloom_filter.test_element(data[i]);
+                results[i] = bloom_filter.test_element<fixed_len_to_uint32_method>(data[i]);
             }
         }
     }
 
-    void insert(BloomFilterAdaptor& bloom_filter, const void* data) const {
-        bloom_filter.add_element(*(T*)data);
+    static void insert(BloomFilterAdaptor& bloom_filter, const void* data) {
+        bloom_filter.add_element<fixed_len_to_uint32_method>(*(T*)data);
     }
 };
 
-struct StringFindOp : CommonFindOp<StringRef> {
+template <typename fixed_len_to_uint32_method>
+struct StringFindOp : CommonFindOp<fixed_len_to_uint32_method, StringRef> {
     static void insert_batch(BloomFilterAdaptor& bloom_filter, const vectorized::ColumnPtr& column,
                              size_t start) {
         auto _insert_batch_col_str = [&](const auto& col, const uint8_t* __restrict nullmap,
                                          size_t start, size_t size) {
             for (size_t i = start; i < size; i++) {
                 if (nullmap == nullptr || !nullmap[i]) {
-                    bloom_filter.add_element(col.get_data_at(i));
+                    bloom_filter.add_element<fixed_len_to_uint32_method>(col.get_data_at(i));
                 } else {
                     bloom_filter.set_contain_null();
                 }
@@ -451,20 +480,23 @@ struct StringFindOp : CommonFindOp<StringRef> {
             if (nullable->has_null()) {
                 for (size_t i = 0; i < col.size(); i++) {
                     if (!nullmap[i]) {
-                        results[i] = bloom_filter.test_element(col.get_data_at(i));
+                        results[i] = bloom_filter.test_element<fixed_len_to_uint32_method>(
+                                col.get_data_at(i));
                     } else {
                         results[i] = bloom_filter.contain_null();
                     }
                 }
             } else {
                 for (size_t i = 0; i < col.size(); i++) {
-                    results[i] = bloom_filter.test_element(col.get_data_at(i));
+                    results[i] = bloom_filter.test_element<fixed_len_to_uint32_method>(
+                            col.get_data_at(i));
                 }
             }
         } else {
             const auto& col = assert_cast<const vectorized::ColumnString*>(column.get());
             for (size_t i = 0; i < col->size(); i++) {
-                results[i] = bloom_filter.test_element(col->get_data_at(i));
+                results[i] =
+                        bloom_filter.test_element<fixed_len_to_uint32_method>(col->get_data_at(i));
             }
         }
     }
@@ -472,34 +504,35 @@ struct StringFindOp : CommonFindOp<StringRef> {
 
 // We do not need to judge whether data is empty, because null will not appear
 // when filer used by the storage engine
-struct FixedStringFindOp : public StringFindOp {
+template <typename fixed_len_to_uint32_method>
+struct FixedStringFindOp : public StringFindOp<fixed_len_to_uint32_method> {
     static uint16_t find_batch_olap_engine(const BloomFilterAdaptor& bloom_filter, const char* data,
                                            const uint8* nullmap, uint16_t* offsets, int number,
                                            const bool is_parse_column) {
-        return find_batch_olap<StringRef, true>(bloom_filter, data, nullmap, offsets, number,
-                                                is_parse_column);
+        return find_batch_olap<fixed_len_to_uint32_method, StringRef, true>(
+                bloom_filter, data, nullmap, offsets, number, is_parse_column);
     }
 };
 
-template <PrimitiveType type>
+template <typename fixed_len_to_uint32_method, PrimitiveType type>
 struct BloomFilterTypeTraits {
     using T = typename PrimitiveTypeTraits<type>::CppType;
-    using FindOp = CommonFindOp<T>;
+    using FindOp = CommonFindOp<fixed_len_to_uint32_method, T>;
 };
 
-template <>
-struct BloomFilterTypeTraits<TYPE_CHAR> {
-    using FindOp = FixedStringFindOp;
+template <typename fixed_len_to_uint32_method>
+struct BloomFilterTypeTraits<fixed_len_to_uint32_method, TYPE_CHAR> {
+    using FindOp = FixedStringFindOp<fixed_len_to_uint32_method>;
 };
 
-template <>
-struct BloomFilterTypeTraits<TYPE_VARCHAR> {
-    using FindOp = StringFindOp;
+template <typename fixed_len_to_uint32_method>
+struct BloomFilterTypeTraits<fixed_len_to_uint32_method, TYPE_VARCHAR> {
+    using FindOp = StringFindOp<fixed_len_to_uint32_method>;
 };
 
-template <>
-struct BloomFilterTypeTraits<TYPE_STRING> {
-    using FindOp = StringFindOp;
+template <typename fixed_len_to_uint32_method>
+struct BloomFilterTypeTraits<fixed_len_to_uint32_method, TYPE_STRING> {
+    using FindOp = StringFindOp<fixed_len_to_uint32_method>;
 };
 
 template <PrimitiveType type>
@@ -511,16 +544,28 @@ class BloomFilterFunc final : public BloomFilterFuncBase {
 
     void insert(const void* data) override {
         DCHECK(_bloom_filter != nullptr);
-        dummy.insert(*_bloom_filter, data);
+        if (_enable_fixed_len_to_uint32_v2) {
+            OpV2::insert(*_bloom_filter, data);
+        } else {
+            Op::insert(*_bloom_filter, data);
+        }
     }
 
     void insert_fixed_len(const vectorized::ColumnPtr& column, size_t start) override {
         DCHECK(_bloom_filter != nullptr);
-        dummy.insert_batch(*_bloom_filter, column, start);
+        if (_enable_fixed_len_to_uint32_v2) {
+            OpV2::insert_batch(*_bloom_filter, column, start);
+        } else {
+            Op::insert_batch(*_bloom_filter, column, start);
+        }
     }
 
     void find_fixed_len(const vectorized::ColumnPtr& column, uint8_t* results) override {
-        dummy.find_batch(*_bloom_filter, column, results);
+        if (_enable_fixed_len_to_uint32_v2) {
+            OpV2::find_batch(*_bloom_filter, column, results);
+        } else {
+            Op::find_batch(*_bloom_filter, column, results);
+        }
     }
 
     template <bool is_nullable>
@@ -542,12 +587,18 @@ class BloomFilterFunc final : public BloomFilterFuncBase {
 
     uint16_t find_fixed_len_olap_engine(const char* data, const uint8* nullmap, uint16_t* offsets,
                                         int number, bool is_parse_column) override {
-        return dummy.find_batch_olap_engine(*_bloom_filter, data, nullmap, offsets, number,
-                                            is_parse_column);
+        if (_enable_fixed_len_to_uint32_v2) {
+            return OpV2::find_batch_olap_engine(*_bloom_filter, data, nullmap, offsets, number,
+                                                is_parse_column);
+        } else {
+            return Op::find_batch_olap_engine(*_bloom_filter, data, nullmap, offsets, number,
+                                              is_parse_column);
+        }
     }
 
 private:
-    typename BloomFilterTypeTraits<type>::FindOp dummy;
+    using Op = typename BloomFilterTypeTraits<fixed_len_to_uint32, type>::FindOp;
+    using OpV2 = typename BloomFilterTypeTraits<fixed_len_to_uint32_v2, type>::FindOp;
 };
 
 } // namespace doris
diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h
index 44f39fb77f6d85..57a8b6376a9fed 100644
--- a/be/src/exprs/create_predicate_function.h
+++ b/be/src/exprs/create_predicate_function.h
@@ -232,7 +232,7 @@ inline auto create_bitmap_filter(PrimitiveType type) {
 template <PrimitiveType PT>
 ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
                                               const std::shared_ptr<BloomFilterFuncBase>& filter,
-                                              int be_exec_version, const TabletColumn*) {
+                                              const TabletColumn*) {
     std::shared_ptr<BloomFilterFuncBase> filter_olap;
     filter_olap.reset(create_bloom_filter(PT));
     filter_olap->light_copy(filter.get());
@@ -243,10 +243,10 @@ ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
 template <PrimitiveType PT>
 ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
                                               const std::shared_ptr<BitmapFilterFuncBase>& filter,
-                                              int be_exec_version, const TabletColumn*) {
+                                              const TabletColumn*) {
     if constexpr (PT == TYPE_TINYINT || PT == TYPE_SMALLINT || PT == TYPE_INT ||
                   PT == TYPE_BIGINT) {
-        return new BitmapFilterColumnPredicate<PT>(column_id, filter, be_exec_version);
+        return new BitmapFilterColumnPredicate<PT>(column_id, filter);
     } else {
         throw Exception(ErrorCode::INTERNAL_ERROR, "bitmap filter do not support type {}", PT);
     }
@@ -254,7 +254,7 @@ ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
 
 template <PrimitiveType PT>
 ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<HybridSetBase>& filter, int,
+                                              const std::shared_ptr<HybridSetBase>& filter,
                                               const TabletColumn* column = nullptr) {
     return create_in_list_predicate<PT, PredicateType::IN_LIST>(column_id, filter,
                                                                 column->length());
@@ -262,7 +262,7 @@ ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
 
 template <PrimitiveType PT>
 ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<FunctionFilter>& filter, int,
+                                              const std::shared_ptr<FunctionFilter>& filter,
                                               const TabletColumn* column = nullptr) {
     // currently only support like predicate
     if constexpr (PT == TYPE_CHAR) {
@@ -277,22 +277,19 @@ ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
 
 template <typename T>
 ColumnPredicate* create_column_predicate(uint32_t column_id, const std::shared_ptr<T>& filter,
-                                         FieldType type, int be_exec_version,
-                                         const TabletColumn* column = nullptr) {
+                                         FieldType type, const TabletColumn* column = nullptr) {
     switch (type) {
-#define M(NAME)                                                                                \
-    case FieldType::OLAP_FIELD_##NAME: {                                                       \
-        return create_olap_column_predicate<NAME>(column_id, filter, be_exec_version, column); \
+#define M(NAME)                                                               \
+    case FieldType::OLAP_FIELD_##NAME: {                                      \
+        return create_olap_column_predicate<NAME>(column_id, filter, column); \
     }
         APPLY_FOR_PRIMTYPE(M)
 #undef M
     case FieldType::OLAP_FIELD_TYPE_DECIMAL: {
-        return create_olap_column_predicate<TYPE_DECIMALV2>(column_id, filter, be_exec_version,
-                                                            column);
+        return create_olap_column_predicate<TYPE_DECIMALV2>(column_id, filter, column);
     }
     case FieldType::OLAP_FIELD_TYPE_BOOL: {
-        return create_olap_column_predicate<TYPE_BOOLEAN>(column_id, filter, be_exec_version,
-                                                          column);
+        return create_olap_column_predicate<TYPE_BOOLEAN>(column_id, filter, column);
     }
     default:
         return nullptr;
diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 34dfb8ff8babfb..8f297d7074ff12 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -740,6 +740,12 @@ class RuntimePredicateWrapper {
         return Status::OK();
     }
 
+    void set_enable_fixed_len_to_uint32_v2() {
+        if (is_bloomfilter()) {
+            _context->bloom_filter_func->set_enable_fixed_len_to_uint32_v2();
+        }
+    }
+
     // used by shuffle runtime filter
     // assign this filter by protobuf
     Status assign(const PBloomFilter* bloom_filter, butil::IOBufAsZeroCopyInputStream* data,
@@ -1357,6 +1363,8 @@ Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQue
     _expr_order = desc->expr_order;
     vectorized::VExprContextSPtr build_ctx;
     RETURN_IF_ERROR(vectorized::VExpr::create_expr_tree(desc->src_expr, build_ctx));
+    _enable_fixed_len_to_uint32_v2 = options->__isset.enable_fixed_len_to_uint32_v2 &&
+                                     options->enable_fixed_len_to_uint32_v2;
 
     RuntimeFilterParams params;
     params.filter_id = _filter_id;
@@ -1407,7 +1415,11 @@ Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQue
     }
 
     _wrapper = std::make_shared<RuntimePredicateWrapper>(&params);
-    return _wrapper->init(&params);
+    RETURN_IF_ERROR(_wrapper->init(&params));
+    if (_enable_fixed_len_to_uint32_v2) {
+        _wrapper->set_enable_fixed_len_to_uint32_v2();
+    }
+    return Status::OK();
 }
 
 Status IRuntimeFilter::serialize(PMergeFilterRequest* request, void** data, int* len) {
@@ -1604,6 +1616,9 @@ void IRuntimeFilter::update_filter(std::shared_ptr<RuntimePredicateWrapper> wrap
         wrapper->_column_return_type = _wrapper->_column_return_type;
     }
     _wrapper = wrapper;
+    if (_enable_fixed_len_to_uint32_v2) {
+        _wrapper->set_enable_fixed_len_to_uint32_v2();
+    }
     update_runtime_filter_type_to_profile(local_merge_time);
     signal();
 }
diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index 49232236179669..b0e82a75335cc5 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -424,6 +424,8 @@ class IRuntimeFilter {
 
     int64_t _synced_size = -1;
     std::shared_ptr<pipeline::CountedFinishDependency> _dependency;
+
+    bool _enable_fixed_len_to_uint32_v2 = false;
 };
 
 // avoid expose RuntimePredicateWrapper
diff --git a/be/src/olap/bitmap_filter_predicate.h b/be/src/olap/bitmap_filter_predicate.h
index 716c99927bf2d6..8d89c7a31fb271 100644
--- a/be/src/olap/bitmap_filter_predicate.h
+++ b/be/src/olap/bitmap_filter_predicate.h
@@ -37,7 +37,7 @@ class BitmapFilterColumnPredicate : public ColumnPredicate {
     using SpecificFilter = BitmapFilterFunc<T>;
 
     BitmapFilterColumnPredicate(uint32_t column_id,
-                                const std::shared_ptr<BitmapFilterFuncBase>& filter, int)
+                                const std::shared_ptr<BitmapFilterFuncBase>& filter)
             : ColumnPredicate(column_id),
               _filter(filter),
               _specific_filter(assert_cast<SpecificFilter*>(_filter.get())) {}
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index cfd5aa2068bf33..a78a52e5fd3e31 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -598,8 +598,7 @@ ColumnPredicate* TabletReader::_parse_to_predicate(
         return nullptr;
     }
     const TabletColumn& column = materialize_column(_tablet_schema->column(index));
-    return create_column_predicate(index, bloom_filter.second, column.type(),
-                                   _reader_context.runtime_state->be_exec_version(), &column);
+    return create_column_predicate(index, bloom_filter.second, column.type(), &column);
 }
 
 ColumnPredicate* TabletReader::_parse_to_predicate(
@@ -609,8 +608,7 @@ ColumnPredicate* TabletReader::_parse_to_predicate(
         return nullptr;
     }
     const TabletColumn& column = materialize_column(_tablet_schema->column(index));
-    return create_column_predicate(index, in_filter.second, column.type(),
-                                   _reader_context.runtime_state->be_exec_version(), &column);
+    return create_column_predicate(index, in_filter.second, column.type(), &column);
 }
 
 ColumnPredicate* TabletReader::_parse_to_predicate(
@@ -620,8 +618,7 @@ ColumnPredicate* TabletReader::_parse_to_predicate(
         return nullptr;
     }
     const TabletColumn& column = materialize_column(_tablet_schema->column(index));
-    return create_column_predicate(index, bitmap_filter.second, column.type(),
-                                   _reader_context.runtime_state->be_exec_version(), &column);
+    return create_column_predicate(index, bitmap_filter.second, column.type(), &column);
 }
 
 ColumnPredicate* TabletReader::_parse_to_predicate(const FunctionFilter& function_filter) {
@@ -631,8 +628,7 @@ ColumnPredicate* TabletReader::_parse_to_predicate(const FunctionFilter& functio
     }
     const TabletColumn& column = materialize_column(_tablet_schema->column(index));
     return create_column_predicate(index, std::make_shared<FunctionFilter>(function_filter),
-                                   column.type(), _reader_context.runtime_state->be_exec_version(),
-                                   &column);
+                                   column.type(), &column);
 }
 
 Status TabletReader::_init_delete_condition(const ReaderParams& read_params) {
diff --git a/be/src/util/hash_util.hpp b/be/src/util/hash_util.hpp
index d444daa8c68d11..bed8569afaca32 100644
--- a/be/src/util/hash_util.hpp
+++ b/be/src/util/hash_util.hpp
@@ -38,14 +38,6 @@ namespace doris {
 // Utility class to compute hash values.
 class HashUtil {
 public:
-    template <typename T>
-    static uint32_t fixed_len_to_uint32(T value) {
-        if constexpr (sizeof(T) <= sizeof(uint32_t)) {
-            return (uint32_t)value;
-        }
-        return std::hash<T>()(value);
-    }
-
     static uint32_t zlib_crc_hash(const void* data, uint32_t bytes, uint32_t hash) {
         return crc32(hash, (const unsigned char*)data, bytes);
     }
diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h
index dc7ba70b0b04ac..cfe9a368e83d4b 100644
--- a/be/src/vec/runtime/vdatetime_value.h
+++ b/be/src/vec/runtime/vdatetime_value.h
@@ -1275,8 +1275,6 @@ class DateV2Value {
         }
     }
 
-    operator int64_t() const { return to_int64(); }
-
     int64_t to_int64() const {
         if constexpr (is_datetime) {
             return (date_v2_value_.year_ * 10000L + date_v2_value_.month_ * 100 +
diff --git a/be/test/olap/bitmap_filter_column_predicate_test.cpp b/be/test/olap/bitmap_filter_column_predicate_test.cpp
index 2ad2bd124e3197..f3abef55f55ad7 100644
--- a/be/test/olap/bitmap_filter_column_predicate_test.cpp
+++ b/be/test/olap/bitmap_filter_column_predicate_test.cpp
@@ -42,7 +42,7 @@ class BitmapFilterColumnPredicateTest : public testing::Test {
     template <PrimitiveType type>
     BitmapFilterColumnPredicate<type> create_predicate(
             const std::shared_ptr<BitmapFilterFunc<type>>& filter) {
-        return BitmapFilterColumnPredicate<type>(0, filter, 0);
+        return BitmapFilterColumnPredicate<type>(0, filter);
     }
 
     const std::string kTestDir = "./ut_dir/bitmap_filter_column_predicate_test";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 374100598c8b75..d07a8f022d7092 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1179,6 +1179,11 @@ public enum IgnoreSplitType {
             "Ignore the rf when it encounters an error" })
     public boolean ignoreRuntimeFilterError = false;
 
+    @VariableMgr.VarAttr(name = "enable_fixed_len_to_uint32_v2", needForward = true, description = {
+            "使用新版本fixed_len_to_uint32_v2,对datetimev2类型bloom filter做了优化",
+            "Using the new version fixed_len_to_uint32_v2, the datetimev2 type bloom filter has been optimized" })
+    public boolean enableFixedLenToUint32V2 = true;
+
     @VariableMgr.VarAttr(name = RUNTIME_FILTER_MODE, needForward = true)
     private String runtimeFilterMode = "GLOBAL";
 
@@ -3984,6 +3989,7 @@ public TQueryOptions toThrift() {
         tResult.setOrcMaxMergeDistanceBytes(orcMaxMergeDistanceBytes);
         tResult.setOrcOnceMaxReadBytes(orcOnceMaxReadBytes);
         tResult.setIgnoreRuntimeFilterError(ignoreRuntimeFilterError);
+        tResult.setEnableFixedLenToUint32V2(enableFixedLenToUint32V2);
 
         return tResult;
     }
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index 9a0fd910d94387..745cb8f21fb627 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -358,6 +358,8 @@ struct TQueryOptions {
   140: optional i64 orc_max_merge_distance_bytes = 1048576;
 
   141: optional bool ignore_runtime_filter_error = false;
+  142: optional bool enable_fixed_len_to_uint32_v2 = false;
+
   // For cloud, to control if the content would be written into file cache
   // In write path, to control if the content would be written into file cache.
   // In read path, read from file cache or remote storage when execute query.

From 61df4e6baef77c25287875a5e4978118c15c8232 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Thu, 5 Dec 2024 14:08:47 +0800
Subject: [PATCH 180/399] [Bug](runtime-filter) set all task's
 wake_up_by_downstream before set dependency ready (#45003)

### What problem does this PR solve?
should_build_hash_table's instance close early and signal this instance
but function make_all_runnable still running and not set this instance's
wake_up_by_downstream to true
---
 be/src/pipeline/exec/hashjoin_build_sink.cpp | 17 +++++++++--------
 be/src/pipeline/pipeline.cpp                 |  7 ++++++-
 be/src/pipeline/pipeline_task.h              |  5 +++--
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index 8115b87c8fe5ac..ccd8757aba7f28 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -166,12 +166,10 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
                 SCOPED_TIMER(_runtime_filter_compute_timer);
                 _runtime_filter_slots->insert(block);
             }
-        } else if (p._shared_hashtable_controller && !p._shared_hash_table_context->signaled) {
-            throw Exception(ErrorCode::INTERNAL_ERROR,
-                            "build_sink::close meet error state, shared_hash_table_signaled: {}, "
-                            "complete_build_stage: {}",
-                            p._shared_hash_table_context->signaled,
-                            p._shared_hash_table_context->complete_build_stage);
+        } else if ((p._shared_hashtable_controller && !p._shared_hash_table_context->signaled) ||
+                   (p._shared_hash_table_context &&
+                    !p._shared_hash_table_context->complete_build_stage)) {
+            throw Exception(ErrorCode::INTERNAL_ERROR, "build_sink::close meet error state");
         }
 
         SCOPED_TIMER(_publish_runtime_filter_timer);
@@ -179,9 +177,12 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
     } catch (Exception& e) {
         return Status::InternalError(
                 "rf process meet error: {}, wake_up_by_downstream: {}, should_build_hash_table: "
-                "{}, _finish_dependency: {}",
+                "{}, _finish_dependency: {}, complete_build_stage: {}, shared_hash_table_signaled: "
+                "{}",
                 e.to_string(), state->get_task()->wake_up_by_downstream(), _should_build_hash_table,
-                _finish_dependency->debug_string());
+                _finish_dependency->debug_string(),
+                p._shared_hash_table_context && !p._shared_hash_table_context->complete_build_stage,
+                p._shared_hashtable_controller && !p._shared_hash_table_context->signaled);
     }
     return Base::close(state, exec_status);
 }
diff --git a/be/src/pipeline/pipeline.cpp b/be/src/pipeline/pipeline.cpp
index 96da754daa5d98..e4678b7dcf3a83 100644
--- a/be/src/pipeline/pipeline.cpp
+++ b/be/src/pipeline/pipeline.cpp
@@ -112,7 +112,12 @@ void Pipeline::make_all_runnable() {
     if (_sink->count_down_destination()) {
         for (auto* task : _tasks) {
             if (task) {
-                task->clear_blocking_state(true);
+                task->set_wake_up_by_downstream();
+            }
+        }
+        for (auto* task : _tasks) {
+            if (task) {
+                task->clear_blocking_state();
             }
         }
     }
diff --git a/be/src/pipeline/pipeline_task.h b/be/src/pipeline/pipeline_task.h
index 3b4627f589dc54..4bb062122c0c08 100644
--- a/be/src/pipeline/pipeline_task.h
+++ b/be/src/pipeline/pipeline_task.h
@@ -135,11 +135,12 @@ class PipelineTask {
     int task_id() const { return _index; };
     bool is_finalized() const { return _finalized; }
 
-    void clear_blocking_state(bool wake_up_by_downstream = false) {
+    void set_wake_up_by_downstream() { _wake_up_by_downstream = true; }
+
+    void clear_blocking_state() {
         _state->get_query_ctx()->get_execution_dependency()->set_always_ready();
         // We use a lock to assure all dependencies are not deconstructed here.
         std::unique_lock<std::mutex> lc(_dependency_lock);
-        _wake_up_by_downstream = _wake_up_by_downstream || wake_up_by_downstream;
         if (!_finalized) {
             _execution_dep->set_always_ready();
             for (auto* dep : _filter_dependencies) {

From 6b4b3cbf0ecc818909013db65030d7c645d719c7 Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Thu, 5 Dec 2024 14:15:59 +0800
Subject: [PATCH 181/399] [feat](mtmv)external table support partition rewrite
 (#44998)

### What problem does this PR solve?

Previously, transparent rewriting of the external table could only be
done as a whole or without rewriting.

Now supports partial partition rewriting and direct lookup of the base
table for some partitions.

### Release note

mtmv partition rewrite support external table
---
 .../doris/datasource/ExternalTable.java       |  2 +-
 .../datasource/hive/HMSExternalTable.java     |  2 +-
 .../maxcompute/MaxComputeExternalTable.java   |  2 +-
 .../mv/AbstractMaterializedViewRule.java      |  7 +-
 .../rules/exploration/mv/StructInfo.java      | 18 ++--
 .../commands/UpdateMvByPartitionCommand.java  | 16 ++--
 .../data/mtmv_p0/test_hive_rewrite_mtmv.out   | 31 +++++++
 .../mtmv_p0/test_hive_rewrite_mtmv.groovy     | 89 +++++++++++++++++++
 8 files changed, 148 insertions(+), 19 deletions(-)
 create mode 100644 regression-test/data/mtmv_p0/test_hive_rewrite_mtmv.out
 create mode 100644 regression-test/suites/mtmv_p0/test_hive_rewrite_mtmv.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index bd1e36e7bc968b..5451a219edfd3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -399,7 +399,7 @@ public SelectedPartitions initSelectedPartitions(Optional<MvccSnapshot> snapshot
      * @param snapshot if not support mvcc, ignore this
      * @return partitionName ==> PartitionItem
      */
-    protected Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
+    public Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
         return Collections.emptyMap();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 134ad362fa1eed..2115f47d777b80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -306,7 +306,7 @@ public boolean supportInternalPartitionPruned() {
     }
 
     @Override
-    protected Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
+    public Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
         return getNameToPartitionItems();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
index 0f748f59e927bc..dbbbcf2d6a1e5b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
@@ -92,7 +92,7 @@ public List<Column> getPartitionColumns() {
     }
 
     @Override
-    protected Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
+    public Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
         if (getPartitionColumns().isEmpty()) {
             return Collections.emptyMap();
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 8e9ef1eaa97b7a..e6f384502d620f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -461,17 +461,14 @@ protected Pair<Map<BaseTableInfo, Set<String>>, Map<BaseTableInfo, Set<String>>>
             return Pair.of(ImmutableMap.of(), ImmutableMap.of());
         }
         // Collect the mv related base table partitions which query used
-        Map<BaseTableInfo, Set<Partition>> queryUsedBaseTablePartitions = new LinkedHashMap<>();
+        Map<BaseTableInfo, Set<String>> queryUsedBaseTablePartitions = new LinkedHashMap<>();
         queryUsedBaseTablePartitions.put(relatedPartitionTable, new HashSet<>());
         queryPlan.accept(new StructInfo.QueryScanPartitionsCollector(), queryUsedBaseTablePartitions);
         // Bail out, not check invalid partition if not olap scan, support later
         if (queryUsedBaseTablePartitions.isEmpty()) {
             return Pair.of(ImmutableMap.of(), ImmutableMap.of());
         }
-        Set<String> queryUsedBaseTablePartitionNameSet = queryUsedBaseTablePartitions.get(relatedPartitionTable)
-                .stream()
-                .map(Partition::getName)
-                .collect(Collectors.toSet());
+        Set<String> queryUsedBaseTablePartitionNameSet = queryUsedBaseTablePartitions.get(relatedPartitionTable);
 
         Collection<Partition> mvValidPartitions = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv,
                 cascadesContext.getConnectContext(), System.currentTimeMillis(), false);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 5a84ab787d7bd5..3de48dc7ff6abf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -17,9 +17,9 @@
 
 package org.apache.doris.nereids.rules.exploration.mv;
 
-import org.apache.doris.catalog.Partition;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.mtmv.BaseTableInfo;
 import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.jobs.executor.Rewriter;
@@ -51,6 +51,8 @@
 import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand.PredicateAdder;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
 import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
@@ -731,22 +733,28 @@ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan,
      * Collect partitions on base table
      */
     public static class QueryScanPartitionsCollector extends DefaultPlanVisitor<Plan,
-            Map<BaseTableInfo, Set<Partition>>> {
+            Map<BaseTableInfo, Set<String>>> {
         @Override
         public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation,
-                Map<BaseTableInfo, Set<Partition>> targetTablePartitionMap) {
+                Map<BaseTableInfo, Set<String>> targetTablePartitionMap) {
             TableIf table = catalogRelation.getTable();
             BaseTableInfo relatedPartitionTable = new BaseTableInfo(table);
             if (!targetTablePartitionMap.containsKey(relatedPartitionTable)) {
                 return catalogRelation;
             }
+            Set<String> tablePartitions = targetTablePartitionMap.get(relatedPartitionTable);
             if (catalogRelation instanceof LogicalOlapScan) {
                 // Handle olap table
                 LogicalOlapScan logicalOlapScan = (LogicalOlapScan) catalogRelation;
-                Set<Partition> tablePartitions = targetTablePartitionMap.get(relatedPartitionTable);
                 for (Long partitionId : logicalOlapScan.getSelectedPartitionIds()) {
-                    tablePartitions.add(logicalOlapScan.getTable().getPartition(partitionId));
+                    tablePartitions.add(logicalOlapScan.getTable().getPartition(partitionId).getName());
                 }
+            } else if (catalogRelation instanceof LogicalFileScan
+                    && catalogRelation.getTable() instanceof ExternalTable
+                    && ((ExternalTable) catalogRelation.getTable()).supportInternalPartitionPruned()) {
+                LogicalFileScan logicalFileScan = (LogicalFileScan) catalogRelation;
+                SelectedPartitions selectedPartitions = logicalFileScan.getSelectedPartitions();
+                tablePartitions.addAll(selectedPartitions.selectedPartitions.keySet());
             } else {
                 // todo Support other type partition table
                 // Not support to partition check now when query external catalog table, support later.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
index 36cc0f95a77a8e..869c2d0b38bd4c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
@@ -28,6 +28,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.mtmv.BaseTableInfo;
 import org.apache.doris.mtmv.MTMVRelatedTableIf;
 import org.apache.doris.nereids.analyzer.UnboundRelation;
@@ -306,16 +307,19 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation,
                     MTMVRelatedTableIf targetTable = (MTMVRelatedTableIf) table;
                     for (String partitionName : filterTableEntry.getValue()) {
                         Partition partition = targetTable.getPartition(partitionName);
-                        if (!(targetTable instanceof OlapTable)) {
-                            // check partition is have data or not, only support olap table
-                            break;
-                        }
-                        if (!((OlapTable) targetTable).selectNonEmptyPartitionIds(
+                        if (targetTable instanceof OlapTable && !((OlapTable) targetTable).selectNonEmptyPartitionIds(
                                 Lists.newArrayList(partition.getId())).isEmpty()) {
-                            // Add filter only when partition has data
+                            // Add filter only when partition has data when olap table
                             partitionHasDataItems.add(
                                     ((OlapTable) targetTable).getPartitionInfo().getItem(partition.getId()));
                         }
+                        if (targetTable instanceof ExternalTable) {
+                            // Add filter only when partition has data when external table
+                            // TODO: 2024/12/4 real snapshot
+                            partitionHasDataItems.add(
+                                    ((ExternalTable) targetTable).getNameToPartitionItems(Optional.empty())
+                                            .get(partitionName));
+                        }
                     }
                     if (partitionHasDataItems.isEmpty()) {
                         predicates.setNeedAddFilter(false);
diff --git a/regression-test/data/mtmv_p0/test_hive_rewrite_mtmv.out b/regression-test/data/mtmv_p0/test_hive_rewrite_mtmv.out
new file mode 100644
index 00000000000000..452cff71e53e9c
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_hive_rewrite_mtmv.out
@@ -0,0 +1,31 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !refresh_one_partition --
+20230101	3
+
+-- !refresh_one_partition_rewrite --
+20230101	3
+20230102	3
+
+-- !refresh_complete --
+20230101	3
+20230102	3
+
+-- !refresh_all_partition_rewrite --
+20230101	3
+20230102	3
+
+-- !refresh_one_partition --
+20230101	3
+
+-- !refresh_one_partition_rewrite --
+20230101	3
+20230102	3
+
+-- !refresh_complete --
+20230101	3
+20230102	3
+
+-- !refresh_all_partition_rewrite --
+20230101	3
+20230102	3
+
diff --git a/regression-test/suites/mtmv_p0/test_hive_rewrite_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_rewrite_mtmv.groovy
new file mode 100644
index 00000000000000..f10d6bd65b4d3b
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_hive_rewrite_mtmv.groovy
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_rewrite_mtmv", "p0,external,hive,external_docker,external_docker_hive") {
+    String enabled = context.config.otherConfigs.get("enableHiveTest")
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+        logger.info("diable Hive test.")
+        return;
+    }
+    String suiteName = "test_hive_rewrite_mtmv"
+    String catalogName = "${suiteName}_catalog"
+    String mvName = "${suiteName}_mv"
+    String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+    sql """set materialized_view_rewrite_enable_contain_external_table=true;"""
+    String mvSql = "SELECT part_col,count(*) as num FROM ${catalogName}.`default`.mtmv_base1 group by part_col;";
+    for (String hivePrefix : ["hive2", "hive3"]) {
+        String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
+        sql """drop catalog if exists ${catalogName}"""
+        sql """create catalog if not exists ${catalogName} properties (
+            "type"="hms",
+            'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
+        );"""
+        sql """analyze table ${catalogName}.`default`.mtmv_base1 with sync"""
+        sql """alter table ${catalogName}.`default`.mtmv_base1 modify column part_col set stats ('row_count'='6');"""
+
+        sql """drop materialized view if exists ${mvName};"""
+        sql """
+            CREATE MATERIALIZED VIEW ${mvName}
+                BUILD DEFERRED REFRESH AUTO ON MANUAL
+                partition by(`part_col`)
+                DISTRIBUTED BY RANDOM BUCKETS 2
+                PROPERTIES ('replication_num' = '1')
+                AS
+                ${mvSql}
+            """
+        def showPartitionsResult = sql """show partitions from ${mvName}"""
+        logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+        assertTrue(showPartitionsResult.toString().contains("p_20230101"))
+        assertTrue(showPartitionsResult.toString().contains("p_20230102"))
+
+        // refresh one partitions
+        sql """
+                REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20230101);
+            """
+        waitingMTMVTaskFinishedByMvName(mvName)
+        order_qt_refresh_one_partition "SELECT * FROM ${mvName}"
+
+        def explainOnePartition = sql """ explain  ${mvSql} """
+        logger.info("explainOnePartition: " + explainOnePartition.toString())
+        assertTrue(explainOnePartition.toString().contains("VUNION"))
+        assertTrue(explainOnePartition.toString().contains("part_col[#4] = 20230102"))
+        order_qt_refresh_one_partition_rewrite "${mvSql}"
+
+        mv_rewrite_success("${mvSql}", "${mvName}")
+
+        //refresh complete
+        sql """
+                REFRESH MATERIALIZED VIEW ${mvName} complete
+            """
+        waitingMTMVTaskFinishedByMvName(mvName)
+        order_qt_refresh_complete "SELECT * FROM ${mvName}"
+
+        def explainAllPartition = sql """ explain  ${mvSql}; """
+        logger.info("explainAllPartition: " + explainAllPartition.toString())
+        assertTrue(explainAllPartition.toString().contains("VOlapScanNode"))
+        assertTrue(explainAllPartition.toString().contains("partitions=2/2"))
+        order_qt_refresh_all_partition_rewrite "${mvSql}"
+
+        mv_rewrite_success("${mvSql}", "${mvName}")
+
+        sql """drop materialized view if exists ${mvName};"""
+        sql """drop catalog if exists ${catalogName}"""
+    }
+}
+

From aa4318f438a920ef50f25695c95b1bbf14e3993a Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Thu, 5 Dec 2024 14:35:04 +0800
Subject: [PATCH 182/399] [minor](column) remove function
 "is_value_represented_by" (#44893)

### What problem does this PR solve?
[minor](column) remove function "is_value_represented_by"
---
 be/src/vec/data_types/data_type.h               | 12 ------------
 be/src/vec/data_types/data_type_bitmap.h        |  3 ---
 be/src/vec/data_types/data_type_hll.h           |  3 ---
 be/src/vec/data_types/data_type_number_base.cpp | 10 ----------
 be/src/vec/data_types/data_type_number_base.h   |  2 --
 be/src/vec/data_types/data_type_quantilestate.h |  3 ---
 6 files changed, 33 deletions(-)

diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
index 6c54241ea18a2a..e193856c05187d 100644
--- a/be/src/vec/data_types/data_type.h
+++ b/be/src/vec/data_types/data_type.h
@@ -151,16 +151,8 @@ class IDataType : private boost::noncopyable {
       */
     virtual bool is_value_represented_by_number() const { return false; }
 
-    /** Integers, Enums, Date, DateTime. Not nullable.
-      */
-    virtual bool is_value_represented_by_integer() const { return false; }
-
     virtual bool is_object() const { return false; }
 
-    /** Unsigned Integers, Date, DateTime. Not nullable.
-      */
-    virtual bool is_value_represented_by_unsigned_integer() const { return false; }
-
     /** Values are unambiguously identified by contents of contiguous memory region,
       *  that can be obtained by IColumn::get_data_at method.
       * Examples: numbers, Date, DateTime, String, FixedString,
@@ -386,10 +378,6 @@ inline bool is_not_decimal_but_comparable_to_decimal(const DataTypePtr& data_typ
     return which.is_int() || which.is_uint();
 }
 
-inline bool is_compilable_type(const DataTypePtr& data_type) {
-    return data_type->is_value_represented_by_number() && !is_decimal(data_type);
-}
-
 inline bool is_complex_type(const DataTypePtr& data_type) {
     WhichDataType which(data_type);
     return which.is_array() || which.is_map() || which.is_struct();
diff --git a/be/src/vec/data_types/data_type_bitmap.h b/be/src/vec/data_types/data_type_bitmap.h
index 24f1e6334431c1..ce0f327f5450a5 100644
--- a/be/src/vec/data_types/data_type_bitmap.h
+++ b/be/src/vec/data_types/data_type_bitmap.h
@@ -77,9 +77,6 @@ class DataTypeBitMap : public IDataType {
     bool should_align_right_in_pretty_formats() const override { return false; }
     bool text_can_contain_only_valid_utf8() const override { return true; }
     bool is_comparable() const override { return false; }
-    bool is_value_represented_by_number() const override { return false; }
-    bool is_value_represented_by_integer() const override { return false; }
-    bool is_value_represented_by_unsigned_integer() const override { return false; }
     // TODO:
     bool is_value_unambiguously_represented_in_contiguous_memory_region() const override {
         return true;
diff --git a/be/src/vec/data_types/data_type_hll.h b/be/src/vec/data_types/data_type_hll.h
index 2c4061cf7b4abd..6d8d085a1550f6 100644
--- a/be/src/vec/data_types/data_type_hll.h
+++ b/be/src/vec/data_types/data_type_hll.h
@@ -69,9 +69,6 @@ class DataTypeHLL : public IDataType {
     bool should_align_right_in_pretty_formats() const override { return false; }
     bool text_can_contain_only_valid_utf8() const override { return true; }
     bool is_comparable() const override { return false; }
-    bool is_value_represented_by_number() const override { return false; }
-    bool is_value_represented_by_integer() const override { return false; }
-    bool is_value_represented_by_unsigned_integer() const override { return false; }
     // TODO:
     bool is_value_unambiguously_represented_in_contiguous_memory_region() const override {
         return true;
diff --git a/be/src/vec/data_types/data_type_number_base.cpp b/be/src/vec/data_types/data_type_number_base.cpp
index da34d82031cda0..1afed3d7d1a394 100644
--- a/be/src/vec/data_types/data_type_number_base.cpp
+++ b/be/src/vec/data_types/data_type_number_base.cpp
@@ -313,16 +313,6 @@ MutableColumnPtr DataTypeNumberBase<T>::create_column() const {
     return ColumnVector<T>::create();
 }
 
-template <typename T>
-bool DataTypeNumberBase<T>::is_value_represented_by_integer() const {
-    return std::is_integral_v<T>;
-}
-
-template <typename T>
-bool DataTypeNumberBase<T>::is_value_represented_by_unsigned_integer() const {
-    return std::is_integral_v<T> && std::is_unsigned_v<T>;
-}
-
 /// Explicit template instantiations - to avoid code bloat in headers.
 template class DataTypeNumberBase<UInt8>;
 template class DataTypeNumberBase<UInt16>;
diff --git a/be/src/vec/data_types/data_type_number_base.h b/be/src/vec/data_types/data_type_number_base.h
index d5ddfe07e831ca..a73bd9951891a3 100644
--- a/be/src/vec/data_types/data_type_number_base.h
+++ b/be/src/vec/data_types/data_type_number_base.h
@@ -145,8 +145,6 @@ class DataTypeNumberBase : public IDataType {
     bool text_can_contain_only_valid_utf8() const override { return true; }
     bool is_comparable() const override { return true; }
     bool is_value_represented_by_number() const override { return true; }
-    bool is_value_represented_by_integer() const override;
-    bool is_value_represented_by_unsigned_integer() const override;
     bool is_value_unambiguously_represented_in_contiguous_memory_region() const override {
         return true;
     }
diff --git a/be/src/vec/data_types/data_type_quantilestate.h b/be/src/vec/data_types/data_type_quantilestate.h
index 5edfe123dca6df..9d34604395cd0d 100644
--- a/be/src/vec/data_types/data_type_quantilestate.h
+++ b/be/src/vec/data_types/data_type_quantilestate.h
@@ -72,9 +72,6 @@ class DataTypeQuantileState : public IDataType {
     bool should_align_right_in_pretty_formats() const override { return false; }
     bool text_can_contain_only_valid_utf8() const override { return true; }
     bool is_comparable() const override { return false; }
-    bool is_value_represented_by_number() const override { return false; }
-    bool is_value_represented_by_integer() const override { return false; }
-    bool is_value_represented_by_unsigned_integer() const override { return false; }
     // TODO:
     bool is_value_unambiguously_represented_in_contiguous_memory_region() const override {
         return true;

From f1f97589db3953fb33a99d95edeea5b84f848e6d Mon Sep 17 00:00:00 2001
From: starocean999 <lichi@selectdb.com>
Date: Thu, 5 Dec 2024 14:45:34 +0800
Subject: [PATCH 183/399] [fix](test)fix unstable test case (#44977)

---
 .../suites/nereids_p0/show/test_show_commands_nereids.groovy    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
index 84dcbc2323fa02..ee243b42d3209a 100644
--- a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
@@ -27,5 +27,5 @@ suite("test_show_commands_nereids") {
     checkNereidsExecute("""show whitelist;""")
     checkNereidsExecute("""show triggers;""")
     checkNereidsExecute("""show events;""")
-    checkNereidsExecute("""show load profile \"\\";""")
+    checkNereidsExecute("""show load profile "/";""")
 }

From c5366cdb8aae743d8d344c82ae3ff96cc8e24f67 Mon Sep 17 00:00:00 2001
From: lw112 <131352377+felixwluo@users.noreply.github.com>
Date: Thu, 5 Dec 2024 15:31:22 +0800
Subject: [PATCH 184/399] [fix](mtmv) Fixed an error in setting variables in
 nested materialize views (#44960)

Fixed an error in setting variables in nested materialized views
```
create database test;

use test;

CREATE TABLE tb1 (
        `id` bigint NOT NULL COMMENT '',
        `map_infos` map < int,
        varchar(65533) > NULL COMMENT ''
) ENGINE = OLAP UNIQUE KEY(`id`) COMMENT 'test' DISTRIBUTED BY HASH(`id`) BUCKETS 2 PROPERTIES (
        "replication_allocation" = "tag.location.default: 1",
        "min_load_replica_num" = "-1",
        "is_being_synced" = "false",
        "storage_medium" = "hdd",
        "storage_format" = "V2",
        "inverted_index_storage_format" = "V1",
        "enable_unique_key_merge_on_write" = "true",
        "light_schema_change" = "true",
        "disable_auto_compaction" = "false",
        "enable_single_replica_compaction" = "false",
        "group_commit_interval_ms" = "10000",
        "group_commit_data_bytes" = "134217728",
        "enable_mow_light_delete" = "false"
);

insert into
       tb1
select
        id,
        map_agg(a, b)
from
(
                select
                        123 id,
                        3 a,
                        '5' b
                union
                all
                select
                        123 id,
                        6 a,
                        '8' b
        ) aa
group by
        id;

 CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES (
        "replication_allocation" = "tag.location.default: 1",
        "min_load_replica_num" = "-1",
        "is_being_synced" = "false",
        "colocate_with" = "dwd_info_group",
        "storage_medium" = "hdd",
        "storage_format" = "V2",
        "inverted_index_storage_format" = "V1",
        "light_schema_change" = "true",
        "disable_auto_compaction" = "false",
        "enable_single_replica_compaction" = "false",
        "group_commit_interval_ms" = "10000",
        "group_commit_data_bytes" = "134217728",
        "enable_nondeterministic_function" = "true"
) AS
select
        /*+ SET_VAR(enable_force_spill = true) */
        cast(a.id as bigint) info_id,
        map_infos
from
        tb1 a;

CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES (
        "replication_allocation" = "tag.location.default: 1",
        "min_load_replica_num" = "-1",
        "is_being_synced" = "false",
        "colocate_with" = "dwd_info_group",
        "storage_medium" = "hdd",
        "storage_format" = "V2",
        "inverted_index_storage_format" = "V1",
        "light_schema_change" = "true",
        "disable_auto_compaction" = "false",
        "enable_single_replica_compaction" = "false",
        "group_commit_interval_ms" = "10000",
        "group_commit_data_bytes" = "134217728",
        "enable_nondeterministic_function" = "true"
) AS
select
        info_id,
        map_infos
from
        mv1 a;
```

Error
`java.lang.NullPointerException: Cannot invoke
"org.apache.doris.nereids.StatementContext.getConnectContext()" because
"context" is null`
---
 .../java/org/apache/doris/mtmv/MTMVCache.java |  5 +-
 .../test_mv_case/test_mv_case.out             |  3 +
 .../test_mv_case/test_mv_case.groovy          | 69 ++++++++++++++++++-
 3 files changed, 73 insertions(+), 4 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
index 2895ad73e14297..d3d7f1ad6ebbf5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
@@ -88,13 +88,14 @@ public StructInfo getStructInfo() {
     }
 
     public static MTMVCache from(MTMV mtmv, ConnectContext connectContext, boolean needCost) {
-        LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql());
         StatementContext mvSqlStatementContext = new StatementContext(connectContext,
                 new OriginStatement(mtmv.getQuerySql(), 0));
-        NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext);
         if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) {
             mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext);
         }
+        LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql());
+        NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext);
+
         // Can not convert to table sink, because use the same column from different table when self join
         // the out slot is wrong
         if (needCost) {
diff --git a/regression-test/data/correctness_p0/test_mv_case/test_mv_case.out b/regression-test/data/correctness_p0/test_mv_case/test_mv_case.out
index e87ee694dc1adb..bab672bd6abd07 100644
--- a/regression-test/data/correctness_p0/test_mv_case/test_mv_case.out
+++ b/regression-test/data/correctness_p0/test_mv_case/test_mv_case.out
@@ -5,3 +5,6 @@ cib2205045_1_1s	2023-06-10T03:56:33	{"DB1":168939,"DNT":"2023-06-10 03:56:33"}
 cib2205045_1_1s	2023-06-10T03:57:33	{"DB1":168939,"DNT":"2023-06-10 03:57:33"}
 cib2205045_1_1s	2023-06-10T03:58:33	{"DB1":168939,"DNT":"2023-06-10 03:58:33"}
 
+-- !select_mv --
+123	{3:"5", 6:"8"}
+
diff --git a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
index 0a87c8d2704d1c..0940f817a8de27 100644
--- a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
+++ b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
@@ -20,7 +20,7 @@ suite("test_mv_case") {
     sql """CREATE TABLE `test_table_aaa2` (
             `ordernum` varchar(65533) NOT NULL ,
             `dnt` datetime NOT NULL ,
-            `data` json NULL 
+            `data` json NULL
             ) ENGINE=OLAP
             DUPLICATE KEY(`ordernum`, `dnt`)
             COMMENT 'OLAP'
@@ -55,7 +55,7 @@ suite("test_mv_case") {
     createMV ("""create  materialized view  test_mv_view_t_mv as
                 select `day`, count(game_code)
                 from test_mv_view_t group by day;""")
-    sql """create view test_mv_view_t_view 
+    sql """create view test_mv_view_t_view
             as
             select `day`
                 from test_mv_view_t
@@ -69,4 +69,69 @@ suite("test_mv_case") {
         sql("""SELECT  * from test_mv_view_t_view where day='2024-04-15';""")
         notContains("mv_day")
     }
+
+    sql """ drop table if exists tb1 """
+    sql """ CREATE TABLE tb1 (
+        `id` bigint NOT NULL COMMENT '',
+        `map_infos` map < int,
+        varchar(65533) > NULL COMMENT ''
+        ) ENGINE = OLAP UNIQUE KEY(`id`) COMMENT 'test' DISTRIBUTED BY HASH(`id`) BUCKETS 2 PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1",
+        "min_load_replica_num" = "-1",
+        "is_being_synced" = "false",
+        "storage_medium" = "hdd",
+        "storage_format" = "V2",
+        "inverted_index_storage_format" = "V1",
+        "enable_unique_key_merge_on_write" = "true",
+        "light_schema_change" = "true",
+        "disable_auto_compaction" = "false",
+        "enable_single_replica_compaction" = "false",
+        "group_commit_interval_ms" = "10000",
+        "group_commit_data_bytes" = "134217728",
+        "enable_mow_light_delete" = "false"
+        )
+    """
+    sql """insert into tb1 select id,map_agg(a, b) from(select 123 id,3 a,'5' b union all select 123 id, 6 a, '8' b) aa group by id"""
+    createMV ("""CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1",
+        "min_load_replica_num" = "-1",
+        "is_being_synced" = "false",
+        "colocate_with" = "dwd_info_group",
+        "storage_medium" = "hdd",
+        "storage_format" = "V2",
+        "inverted_index_storage_format" = "V1",
+        "light_schema_change" = "true",
+        "disable_auto_compaction" = "false",
+        "enable_single_replica_compaction" = "false",
+        "group_commit_interval_ms" = "10000",
+        "group_commit_data_bytes" = "134217728",
+        "enable_nondeterministic_function" = "true"
+        ) AS
+        select
+        /*+ SET_VAR(enable_force_spill = true) */
+        cast(a.id as bigint) info_id,
+        map_infos
+        from
+        tb1 a;""")
+    createMV ("""CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1",
+        "min_load_replica_num" = "-1",
+        "is_being_synced" = "false",
+        "colocate_with" = "dwd_info_group",
+        "storage_medium" = "hdd",
+        "storage_format" = "V2",
+        "inverted_index_storage_format" = "V1",
+        "light_schema_change" = "true",
+        "disable_auto_compaction" = "false",
+        "enable_single_replica_compaction" = "false",
+        "group_commit_interval_ms" = "10000",
+        "group_commit_data_bytes" = "134217728",
+        "enable_nondeterministic_function" = "true"
+        ) AS
+        select
+        info_id,
+        map_infos
+        from
+        mv1 a;""")
+    qt_select_mv """ select * from mv2 """
 }

From 0f48cfd31ac749db9075b799ba6c815c401d2aeb Mon Sep 17 00:00:00 2001
From: hui lai <laihui@selectdb.com>
Date: Thu, 5 Dec 2024 15:42:26 +0800
Subject: [PATCH 185/399] [fix](routine load) replace heavy work pool with
 routine load thread pool for metadata fetching (#44907)

In production, we encountered an issue where the librdkafka consumer
stucked during destruction, causing the heavy work pool to become
saturated, which in turn made all heavy work pool-dependent
functionalities, such as querying, unusable. To mitigate this impact, we
replaced the heavy work pool with routine load threads for metadata
fetching.
---
 be/src/runtime/routine_load/routine_load_task_executor.h | 2 ++
 be/src/service/internal_service.cpp                      | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/be/src/runtime/routine_load/routine_load_task_executor.h b/be/src/runtime/routine_load/routine_load_task_executor.h
index 0e597d796c9f77..b1196f7824afac 100644
--- a/be/src/runtime/routine_load/routine_load_task_executor.h
+++ b/be/src/runtime/routine_load/routine_load_task_executor.h
@@ -73,6 +73,8 @@ class RoutineLoadTaskExecutor {
                                                  std::vector<PIntegerPair>* partition_offsets,
                                                  int timeout);
 
+    ThreadPool& get_thread_pool() { return *_thread_pool; }
+
 private:
     // execute the task
     void exec_task(std::shared_ptr<StreamLoadContext> ctx, DataConsumerPool* pool,
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index adcd07e7de7484..439f3f17faf00f 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -1240,7 +1240,10 @@ void PInternalService::report_stream_load_status(google::protobuf::RpcController
 void PInternalService::get_info(google::protobuf::RpcController* controller,
                                 const PProxyRequest* request, PProxyResult* response,
                                 google::protobuf::Closure* done) {
-    bool ret = _heavy_work_pool.try_offer([this, request, response, done]() {
+    bool ret = _exec_env->routine_load_task_executor()->get_thread_pool().submit_func([this,
+                                                                                       request,
+                                                                                       response,
+                                                                                       done]() {
         brpc::ClosureGuard closure_guard(done);
         // PProxyRequest is defined in gensrc/proto/internal_service.proto
         // Currently it supports 2 kinds of requests:

From 6b7c12fb805fb6f36297a3b6488a87d9f7f042f9 Mon Sep 17 00:00:00 2001
From: lide <lide@apache.org>
Date: Thu, 5 Dec 2024 16:48:29 +0800
Subject: [PATCH 186/399] [enhancement] Support TCP Keep-Alive on MySQL
 connections (#43904)

The default behavior of Options.KEEP_ALIVE is typically disabled,
meaning that TCP keep-alive is not enabled by default. This means that
if you do not explicitly enable it, the connection will not send
keep-alive probes when idle.

Co-authored-by: derenli <derenli@tencent.com>
---
 .../src/main/java/org/apache/doris/common/Config.java |  4 ++++
 .../main/java/org/apache/doris/mysql/MysqlServer.java | 11 +++++++----
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 2ef391bb7ab390..247c61ecd365ec 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -425,6 +425,10 @@ public class Config extends ConfigBase {
                     + "`/proc/sys/net/core/somaxconn` at the same time"})
     public static int mysql_nio_backlog_num = 1024;
 
+    @ConfField(description = {"是否启用 mysql 连接中的 TCP keep alive，默认禁用",
+            "Whether to enable TCP Keep-Alive for MySQL connections, disabled by default"})
+    public static boolean mysql_nio_enable_keep_alive = false;
+
     @ConfField(description = {"thrift client 的连接超时时间，单位是毫秒。0 表示不设置超时时间。",
             "The connection timeout of thrift client, in milliseconds. 0 means no timeout."})
     public static int thrift_client_timeout_ms = 0;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlServer.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlServer.java
index 5f70e3000b9507..e7a888cdd249b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlServer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlServer.java
@@ -68,14 +68,17 @@ public MysqlServer(int port, ConnectScheduler connectScheduler) {
     // return true if success, otherwise false
     public boolean start() {
         try {
+            OptionMap optionMap = OptionMap.builder()
+                    .set(Options.TCP_NODELAY, true)
+                    .set(Options.BACKLOG, Config.mysql_nio_backlog_num)
+                    .set(Options.KEEP_ALIVE, Config.mysql_nio_enable_keep_alive)
+                    .getMap();
             if (FrontendOptions.isBindIPV6()) {
                 server = xnioWorker.createStreamConnectionServer(new InetSocketAddress("::0", port), acceptListener,
-                    OptionMap.create(Options.TCP_NODELAY, true, Options.BACKLOG, Config.mysql_nio_backlog_num));
-
+                    optionMap);
             } else {
                 server = xnioWorker.createStreamConnectionServer(new InetSocketAddress(port), acceptListener,
-                    OptionMap.create(Options.TCP_NODELAY, true, Options.BACKLOG, Config.mysql_nio_backlog_num));
-
+                    optionMap);
             }
             server.resumeAccepts();
             running = true;

From ecd8af2192131ec64b22f0530792e5c5e93d9d65 Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Thu, 5 Dec 2024 17:14:04 +0800
Subject: [PATCH 187/399] [fix](be ut) fix BUILD FAIL in be-ut (#45044)

### What problem does this PR solve?
The following two PRs:
1. Deletes the function is_value_represented_by_unsigned_integer.
https://github.com/apache/doris/pull/44893
2. Calls this function in the BEUT.
https://github.com/apache/doris/pull/44642
---
 be/test/vec/data_types/common_data_type_test.h | 5 +----
 be/test/vec/data_types/data_type_ip_test.cpp   | 4 +---
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/be/test/vec/data_types/common_data_type_test.h b/be/test/vec/data_types/common_data_type_test.h
index 0480137ac7f0e4..36abc3402e2156 100644
--- a/be/test/vec/data_types/common_data_type_test.h
+++ b/be/test/vec/data_types/common_data_type_test.h
@@ -45,7 +45,7 @@
 //         have_maximum_size_of_value, get_maximum_size_of_value_in_memory, get_size_of_value_in_memory
 //         get_precision, get_scale
 //         get_field
-//         is_null_literal, is_value_represented_by_number, is_value_represented_by_integer, is_value_represented_by_unsigned_integer, is_value_unambiguously_represented_in_contiguous_memory_region, is_value_unambiguously_represented_in_fixed_size_contiguous_memory_region
+//         is_null_literal, is_value_represented_by_number, is_value_unambiguously_represented_in_contiguous_memory_region, is_value_unambiguously_represented_in_fixed_size_contiguous_memory_region
 // 2. datatype creation with column: create_column, create_column_const (size_t size, const Field &field), create_column_const_with_default_value (size_t size),  get_uncompressed_serialized_bytes (const IColumn &column, int be_exec_version)
 // 3. serde related: get_serde (int nesting_level=1)
 //          to_string (const IColumn &column, size_t row_num, BufferWritable &ostr), to_string (const IColumn &column, size_t row_num), to_string_batch (const IColumn &column, ColumnString &column_to), from_string (ReadBuffer &rb, IColumn *column)
@@ -114,7 +114,6 @@ class CommonDataTypeTest : public ::testing::Test {
         size_t scale = -1;
         bool is_null_literal = true;
         bool is_value_represented_by_number = false;
-        bool is_value_represented_by_unsigned_integer = false;
         PColumnMeta* pColumnMeta = nullptr;
         DataTypeSerDeSPtr serde = nullptr;
         //        bool is_value_unambiguously_represented_in_contiguous_memory_region = false;
@@ -146,8 +145,6 @@ class CommonDataTypeTest : public ::testing::Test {
         ASSERT_EQ(data_type->is_null_literal(), meta_info.is_null_literal);
         ASSERT_EQ(data_type->is_value_represented_by_number(),
                   meta_info.is_value_represented_by_number);
-        ASSERT_EQ(data_type->is_value_represented_by_unsigned_integer(),
-                  meta_info.is_value_represented_by_unsigned_integer);
         //        ASSERT_EQ(data_type->is_value_unambiguously_represented_in_contiguous_memory_region(), meta_info.is_value_unambiguously_represented_in_contiguous_memory_region);
     }
 
diff --git a/be/test/vec/data_types/data_type_ip_test.cpp b/be/test/vec/data_types/data_type_ip_test.cpp
index 91ae064e4473a3..72a340b9e145b8 100644
--- a/be/test/vec/data_types/data_type_ip_test.cpp
+++ b/be/test/vec/data_types/data_type_ip_test.cpp
@@ -46,7 +46,7 @@
 //         text_can_contain_only_valid_utf8
 //         have_maximum_size_of_value, get_maximum_size_of_value_in_memory, get_size_of_value_in_memory
 //         get_precision, get_scale
-//         is_null_literal, is_value_represented_by_number, is_value_represented_by_integer, is_value_represented_by_unsigned_integer, is_value_unambiguously_represented_in_contiguous_memory_region
+//         is_null_literal, is_value_represented_by_number, is_value_unambiguously_represented_in_contiguous_memory_region
 // 2. datatype creation with column : create_column, create_column_const (size_t size, const Field &field), create_column_const_with_default_value (size_t size), get_uncompressed_serialized_bytes (const IColumn &column, int be_exec_version)
 // 3. serde related: get_serde (int nesting_level=1)
 //          to_string (const IColumn &column, size_t row_num, BufferWritable &ostr), to_string (const IColumn &column, size_t row_num), to_string_batch (const IColumn &column, ColumnString &column_to), from_string (ReadBuffer &rb, IColumn *column)
@@ -95,7 +95,6 @@ TEST_F(DataTypeIPTest, MetaInfoTest) {
             .scale = size_t(-1),
             .is_null_literal = false,
             .is_value_represented_by_number = true,
-            .is_value_represented_by_unsigned_integer = true,
             .pColumnMeta = col_meta.get()
             //                .is_value_unambiguously_represented_in_contiguous_memory_region = true
     };
@@ -116,7 +115,6 @@ TEST_F(DataTypeIPTest, MetaInfoTest) {
             .scale = size_t(-1),
             .is_null_literal = false,
             .is_value_represented_by_number = true,
-            .is_value_represented_by_unsigned_integer = true,
             .pColumnMeta = col_meta6.get()
             //                .is_value_unambiguously_represented_in_contiguous_memory_region = true
     };

From c1fea2d4bc01ce3c25f3d6b913aa11da994cc6e5 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Thu, 5 Dec 2024 19:18:00 +0800
Subject: [PATCH 188/399] [fix](arrow-flight-sql) Fix query result is empty and
 not return query error message (#45023)

### What problem does this PR solve?

After the query failed, Doris Arrow Flight SQL returned an empty result
without returning the query error message.
---
 be/src/runtime/buffer_control_block.cpp       | 20 ++++++++++++++-----
 .../arrow_flight_batch_reader.cpp             |  2 +-
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/be/src/runtime/buffer_control_block.cpp b/be/src/runtime/buffer_control_block.cpp
index 1ed2836f8eb616..8c1ae79955f317 100644
--- a/be/src/runtime/buffer_control_block.cpp
+++ b/be/src/runtime/buffer_control_block.cpp
@@ -292,6 +292,9 @@ Status BufferControlBlock::get_arrow_batch(std::shared_ptr<vectorized::Block>* r
         _arrow_data_arrival.wait_for(l, std::chrono::milliseconds(20));
     }
 
+    if (!_status.ok()) {
+        return _status;
+    }
     if (_is_cancelled) {
         return Status::Cancelled(fmt::format("Cancelled ()", print_id(_fragment_id)));
     }
@@ -311,9 +314,12 @@ Status BufferControlBlock::get_arrow_batch(std::shared_ptr<vectorized::Block>* r
 
     // normal path end
     if (_is_close) {
+        if (!_status.ok()) {
+            return _status;
+        }
         std::stringstream ss;
         _profile.pretty_print(&ss);
-        VLOG_NOTICE << fmt::format(
+        LOG(INFO) << fmt::format(
                 "BufferControlBlock finished, fragment_id={}, is_close={}, is_cancelled={}, "
                 "packet_num={}, peak_memory_usage={}, profile={}",
                 print_id(_fragment_id), _is_close, _is_cancelled, _packet_num,
@@ -321,7 +327,7 @@ Status BufferControlBlock::get_arrow_batch(std::shared_ptr<vectorized::Block>* r
         return Status::OK();
     }
     return Status::InternalError(
-            fmt::format("Get Arrow Batch Abnormal Ending ()", print_id(_fragment_id)));
+            fmt::format("Get Arrow Batch Abnormal Ending (), ()", print_id(_fragment_id), _status));
 }
 
 void BufferControlBlock::get_arrow_batch(GetArrowResultBatchCtx* ctx) {
@@ -354,10 +360,14 @@ void BufferControlBlock::get_arrow_batch(GetArrowResultBatchCtx* ctx) {
 
     // normal path end
     if (_is_close) {
+        if (!_status.ok()) {
+            ctx->on_failure(_status);
+            return;
+        }
         ctx->on_close(_packet_num);
         std::stringstream ss;
         _profile.pretty_print(&ss);
-        VLOG_NOTICE << fmt::format(
+        LOG(INFO) << fmt::format(
                 "BufferControlBlock finished, fragment_id={}, is_close={}, is_cancelled={}, "
                 "packet_num={}, peak_memory_usage={}, profile={}",
                 print_id(_fragment_id), _is_close, _is_cancelled, _packet_num,
@@ -391,8 +401,8 @@ Status BufferControlBlock::find_arrow_schema(std::shared_ptr<arrow::Schema>* arr
     if (_is_close) {
         return Status::RuntimeError(fmt::format("Closed ()", print_id(_fragment_id)));
     }
-    return Status::InternalError(
-            fmt::format("Get Arrow Schema Abnormal Ending ()", print_id(_fragment_id)));
+    return Status::InternalError(fmt::format("Get Arrow Schema Abnormal Ending (), ()",
+                                             print_id(_fragment_id), _status));
 }
 
 Status BufferControlBlock::close(const TUniqueId& id, Status exec_status) {
diff --git a/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp b/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
index e935aff996d55e..c24fcb73384494 100644
--- a/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
+++ b/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
@@ -56,7 +56,7 @@ arrow::Status ArrowFlightBatchReaderBase::_return_invalid_status(const std::stri
 }
 
 ArrowFlightBatchReaderBase::~ArrowFlightBatchReaderBase() {
-    VLOG_NOTICE << fmt::format(
+    LOG(INFO) << fmt::format(
             "ArrowFlightBatchReader finished, packet_seq={}, result_addr={}:{}, finistId={}, "
             "convert_arrow_batch_timer={}, deserialize_block_timer={}, peak_memory_usage={}",
             _packet_seq, _statement->result_addr.hostname, _statement->result_addr.port,

From 4a26eb7bcf02fe38fed206ae95e7c2650ede43d8 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Thu, 5 Dec 2024 19:18:23 +0800
Subject: [PATCH 189/399] [fix](arrow-flight-sql) Revert `arrow_flight_sql`
 group from regression test (#44992)

### What problem does this PR solve?

After pipeline `P0 Regression (Doris Regression)` executes
`./run-regression-test.sh --run -g p0`, it will execute
`./run-regression-test.sh -g arrow_flight_sql` to run regression-test
again, can use `jdbc:arrow-flight-sql` to run all Suites whose group
contains arrow_flight_sql.
---
 .../nereids_p0/show/test_show_create_table_nereids.groovy       | 2 +-
 .../suites/nereids_p0/show/test_show_create_view.groovy         | 2 +-
 .../suites/query_p0/aggregate/aggregate_count1.groovy           | 2 +-
 .../suites/query_p0/aggregate/select_distinct.groovy            | 2 +-
 .../casesensetive_column/join_with_column_casesensetive.groovy  | 2 +-
 regression-test/suites/query_p0/cast/test_cast.groovy           | 2 +-
 regression-test/suites/query_p0/except/test_query_except.groovy | 2 +-
 .../suites/query_p0/group_concat/test_group_concat.groovy       | 2 +-
 .../suites/query_p0/grouping_sets/test_grouping_sets1.groovy    | 2 +-
 regression-test/suites/query_p0/having/having.groovy            | 2 +-
 regression-test/suites/query_p0/intersect/test_intersect.groovy | 2 +-
 regression-test/suites/query_p0/join/test_join2.groovy          | 2 +-
 regression-test/suites/query_p0/join/test_left_join1.groovy     | 2 +-
 .../suites/query_p0/join/test_nestedloop_outer_join.groovy      | 2 +-
 .../suites/query_p0/join/test_partitioned_hash_join.groovy      | 2 +-
 .../suites/query_p0/lateral_view/lateral_view.groovy            | 2 +-
 .../suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy       | 2 +-
 .../suites/query_p0/literal_view/lietral_test.groovy            | 2 +-
 .../suites/query_p0/operator/test_set_operator.groovy           | 2 +-
 .../suites/query_p0/operator/test_sort_operator.groovy          | 2 +-
 .../suites/query_p0/session_variable/test_default_limit.groovy  | 2 +-
 .../query_p0/show/test_show_create_materialized_view.groovy     | 2 +-
 .../suites/query_p0/show/test_show_create_table.groovy          | 2 +-
 .../aggregate_functions/test_aggregate_all_functions.groovy     | 2 +-
 .../sql_functions/case_function/test_case_function_null.groovy  | 2 +-
 .../sql_functions/hash_functions/test_hash_function.groovy      | 2 +-
 .../sql_functions/ip_functions/test_ip_functions.groovy         | 2 +-
 .../sql_functions/json_function/test_query_json_insert.groovy   | 2 +-
 .../sql_functions/json_functions/test_json_function.groovy      | 2 +-
 .../query_p0/sql_functions/math_functions/test_conv.groovy      | 2 +-
 .../search_functions/test_multi_string_search.groovy            | 2 +-
 .../sql_functions/spatial_functions/test_gis_function.groovy    | 2 +-
 .../sql_functions/string_functions/test_string_function.groovy  | 2 +-
 .../query_p0/sql_functions/table_function/explode_split.groovy  | 2 +-
 .../suites/query_p0/sql_functions/test_alias_function.groovy    | 2 +-
 .../suites/query_p0/sql_functions/test_predicate.groovy         | 2 +-
 .../width_bucket_fuctions/test_width_bucket_function.groovy     | 2 +-
 regression-test/suites/query_p0/subquery/test_subquery2.groovy  | 2 +-
 regression-test/suites/query_p0/test_data_type_marks.groovy     | 2 +-
 regression-test/suites/query_p0/test_dict_with_null.groovy      | 2 +-
 regression-test/suites/query_p0/test_orderby_nullliteral.groovy | 2 +-
 regression-test/suites/query_p0/test_select_constant.groovy     | 2 +-
 .../suites/query_p0/test_select_with_predicate_like.groovy      | 2 +-
 .../suites/query_p0/test_select_with_predicate_prune.groovy     | 2 +-
 .../suites/query_p0/type_inference/test_largeint.groovy         | 2 +-
 .../suites/query_p0/with/test_with_and_two_phase_agg.groovy     | 2 +-
 46 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/regression-test/suites/nereids_p0/show/test_show_create_table_nereids.groovy b/regression-test/suites/nereids_p0/show/test_show_create_table_nereids.groovy
index 8c9b42af41a8a6..53f2242d1fa7f6 100644
--- a/regression-test/suites/nereids_p0/show/test_show_create_table_nereids.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_create_table_nereids.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_show_create_table_nereids", "query") {
+suite("test_show_create_table_nereids", "query,arrow_flight_sql") {
     String tb_name = "tb_show_create_table";
     try {  
         sql """drop table if exists ${tb_name} """
diff --git a/regression-test/suites/nereids_p0/show/test_show_create_view.groovy b/regression-test/suites/nereids_p0/show/test_show_create_view.groovy
index be3e691f6dfc38..65ed4758a9ae66 100644
--- a/regression-test/suites/nereids_p0/show/test_show_create_view.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_create_view.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_show_create_view", "query") {
+suite("test_show_create_view", "query,arrow_flight_sql") {
     String view_name = "view_show_create_view";
     String table_name = "table_for_view_test";
     try {
diff --git a/regression-test/suites/query_p0/aggregate/aggregate_count1.groovy b/regression-test/suites/query_p0/aggregate/aggregate_count1.groovy
index cf657cc8ef3e3d..3971f304e38646 100644
--- a/regression-test/suites/query_p0/aggregate/aggregate_count1.groovy
+++ b/regression-test/suites/query_p0/aggregate/aggregate_count1.groovy
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-suite("aggregate_count1", "query") {
+suite("aggregate_count1", "query,arrow_flight_sql") {
     sql """ DROP TABLE IF EXISTS aggregate_count1 """
     sql """create table if not exists aggregate_count1 (
                 name varchar(128),
diff --git a/regression-test/suites/query_p0/aggregate/select_distinct.groovy b/regression-test/suites/query_p0/aggregate/select_distinct.groovy
index 6456158bdadb0d..2d6a8679d87ed8 100644
--- a/regression-test/suites/query_p0/aggregate/select_distinct.groovy
+++ b/regression-test/suites/query_p0/aggregate/select_distinct.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("select_distinct") {
+suite("select_distinct", "arrow_flight_sql") {
     sql """DROP TABLE IF EXISTS decimal_a;"""
     sql """DROP TABLE IF EXISTS decimal_b;"""
     sql """DROP TABLE IF EXISTS decimal_c;"""
diff --git a/regression-test/suites/query_p0/casesensetive_column/join_with_column_casesensetive.groovy b/regression-test/suites/query_p0/casesensetive_column/join_with_column_casesensetive.groovy
index 45499fc6f248a2..8bd3b19088a486 100644
--- a/regression-test/suites/query_p0/casesensetive_column/join_with_column_casesensetive.groovy
+++ b/regression-test/suites/query_p0/casesensetive_column/join_with_column_casesensetive.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("join_with_column_casesensetive") {
+suite("join_with_column_casesensetive", "arrow_flight_sql") {
     def tables=["ad_order_data_v1","ad_order_data"]
 
     for (String table in tables) {
diff --git a/regression-test/suites/query_p0/cast/test_cast.groovy b/regression-test/suites/query_p0/cast/test_cast.groovy
index 947d61bc828861..dae669e2965f04 100644
--- a/regression-test/suites/query_p0/cast/test_cast.groovy
+++ b/regression-test/suites/query_p0/cast/test_cast.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite('test_cast') {
+suite('test_cast', "arrow_flight_sql") {
     def date = "date '2020-01-01'"
     def datev2 = "datev2 '2020-01-01'"
     def datetime = "timestamp '2020-01-01 12:34:45'"
diff --git a/regression-test/suites/query_p0/except/test_query_except.groovy b/regression-test/suites/query_p0/except/test_query_except.groovy
index 1a2aa742d2910d..410e24f89b92de 100644
--- a/regression-test/suites/query_p0/except/test_query_except.groovy
+++ b/regression-test/suites/query_p0/except/test_query_except.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_query_except") {
+suite("test_query_except", "arrow_flight_sql") {
     // test query except, depend on query_test_data_load.groovy
     sql "use test_query_db"
     qt_select_except1 """
diff --git a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
index 5054dc2ee3a91d..522d66ed64b30b 100644
--- a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
+++ b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_group_concat", "query,p0") {
+suite("test_group_concat", "query,p0,arrow_flight_sql") {
     qt_select """
                 SELECT group_concat(k6) FROM test_query_db.test where k6='false'
               """
diff --git a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets1.groovy b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets1.groovy
index 1f12de6628a5eb..f8180b0ab43846 100644
--- a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets1.groovy
+++ b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets1.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_grouping_sets1") {
+suite("test_grouping_sets1", "arrow_flight_sql") {
     qt_select """
         select 
             col1
diff --git a/regression-test/suites/query_p0/having/having.groovy b/regression-test/suites/query_p0/having/having.groovy
index fb32b3834af4d6..bbad236c973870 100644
--- a/regression-test/suites/query_p0/having/having.groovy
+++ b/regression-test/suites/query_p0/having/having.groovy
@@ -19,7 +19,7 @@
 // /testing/trino-product-tests/src/main/resources/sql-tests/testcases/aggregate
 // and modified by Doris.
 
-suite("having", "query,p0") {
+suite("having", "query,p0,arrow_flight_sql") {
     sql """DROP TABLE IF EXISTS supplier"""
     sql """CREATE TABLE `supplier` (
             `s_suppkey` int(11) NOT NULL,
diff --git a/regression-test/suites/query_p0/intersect/test_intersect.groovy b/regression-test/suites/query_p0/intersect/test_intersect.groovy
index 1c007b95d7d07d..7919bec324b876 100644
--- a/regression-test/suites/query_p0/intersect/test_intersect.groovy
+++ b/regression-test/suites/query_p0/intersect/test_intersect.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_intersect") {
+suite("test_intersect", "arrow_flight_sql") {
     qt_select """
                 SELECT * FROM (SELECT k1 FROM test_query_db.baseall
                     INTERSECT SELECT k1 FROM test_query_db.test) a ORDER BY k1
diff --git a/regression-test/suites/query_p0/join/test_join2.groovy b/regression-test/suites/query_p0/join/test_join2.groovy
index 6125b9a873f77e..9158133948f754 100644
--- a/regression-test/suites/query_p0/join/test_join2.groovy
+++ b/regression-test/suites/query_p0/join/test_join2.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_join2", "query,p0") {
+suite("test_join2", "query,p0,arrow_flight_sql") {
     def DBname = "regression_test_join2"
     def TBname1 = "J1_TBL"
     def TBname2 = "J2_TBL"
diff --git a/regression-test/suites/query_p0/join/test_left_join1.groovy b/regression-test/suites/query_p0/join/test_left_join1.groovy
index d4cbeeee65eda2..104adab4a850d0 100644
--- a/regression-test/suites/query_p0/join/test_left_join1.groovy
+++ b/regression-test/suites/query_p0/join/test_left_join1.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_left_join1", "query,p0") {
+suite("test_left_join1", "query,p0,arrow_flight_sql") {
 
     def tableName = "test_left_join1"
     sql """drop table if exists ${tableName}"""
diff --git a/regression-test/suites/query_p0/join/test_nestedloop_outer_join.groovy b/regression-test/suites/query_p0/join/test_nestedloop_outer_join.groovy
index ad19e554690ee7..f99dfa042446e9 100644
--- a/regression-test/suites/query_p0/join/test_nestedloop_outer_join.groovy
+++ b/regression-test/suites/query_p0/join/test_nestedloop_outer_join.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_nestedloop_outer_join", "query_p0") {
+suite("test_nestedloop_outer_join", "query_p0,arrow_flight_sql") {
     def tbl1 = "test_nestedloop_outer_join1"
     def tbl2 = "test_nestedloop_outer_join2"
 
diff --git a/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy b/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
index cbe09ec527ffbc..676cdd06274a68 100644
--- a/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
+++ b/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_partitioned_hash_join", "query,p0") {
+suite("test_partitioned_hash_join", "query,p0,arrow_flight_sql") {
     sql "drop table if exists test_partitioned_hash_join_l"
     sql "drop table if exists test_partitioned_hash_join_r"
     sql """ create table test_partitioned_hash_join_l (
diff --git a/regression-test/suites/query_p0/lateral_view/lateral_view.groovy b/regression-test/suites/query_p0/lateral_view/lateral_view.groovy
index a24623590cd0af..bfe6ca76872ea8 100644
--- a/regression-test/suites/query_p0/lateral_view/lateral_view.groovy
+++ b/regression-test/suites/query_p0/lateral_view/lateral_view.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("lateral_view") {
+suite("lateral_view", "arrow_flight_sql") {
     sql """ DROP TABLE IF EXISTS `test_explode_bitmap` """
 	sql """
 		CREATE TABLE `test_explode_bitmap` (
diff --git a/regression-test/suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy b/regression-test/suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy
index da0c7231f425d1..caa75ac7be38ed 100644
--- a/regression-test/suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy
+++ b/regression-test/suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_offset_in_subquery_with_join", "query") {
+suite("test_offset_in_subquery_with_join", "query,arrow_flight_sql") {
     // define a sql table
     def testTable = "test_offset_in_subquery_with_join"
 
diff --git a/regression-test/suites/query_p0/literal_view/lietral_test.groovy b/regression-test/suites/query_p0/literal_view/lietral_test.groovy
index 6e9d51f0a0d610..27b82c16247a73 100644
--- a/regression-test/suites/query_p0/literal_view/lietral_test.groovy
+++ b/regression-test/suites/query_p0/literal_view/lietral_test.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("literal_view_test") {
+suite("literal_view_test", "arrow_flight_sql") {
 
     sql """DROP TABLE IF EXISTS table1"""
 
diff --git a/regression-test/suites/query_p0/operator/test_set_operator.groovy b/regression-test/suites/query_p0/operator/test_set_operator.groovy
index cb05e18b3e870b..7d6219585e4c4c 100644
--- a/regression-test/suites/query_p0/operator/test_set_operator.groovy
+++ b/regression-test/suites/query_p0/operator/test_set_operator.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_set_operators", "query,p0") {
+suite("test_set_operators", "query,p0,arrow_flight_sql") {
 
     sql """
         DROP TABLE IF EXISTS t1;
diff --git a/regression-test/suites/query_p0/operator/test_sort_operator.groovy b/regression-test/suites/query_p0/operator/test_sort_operator.groovy
index 24a2b8ef73a424..d76daff01f6fcc 100644
--- a/regression-test/suites/query_p0/operator/test_sort_operator.groovy
+++ b/regression-test/suites/query_p0/operator/test_sort_operator.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_sort_operator", "query,p0") {
+suite("test_sort_operator", "query,p0,arrow_flight_sql") {
 
     sql """
         DROP TABLE IF EXISTS dim_org_ful;
diff --git a/regression-test/suites/query_p0/session_variable/test_default_limit.groovy b/regression-test/suites/query_p0/session_variable/test_default_limit.groovy
index edda5d51790c56..2ce3b647142ae4 100644
--- a/regression-test/suites/query_p0/session_variable/test_default_limit.groovy
+++ b/regression-test/suites/query_p0/session_variable/test_default_limit.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite('test_default_limit') {
+suite('test_default_limit', "arrow_flight_sql") {
     sql 'drop table if exists baseall'
     sql 'drop table if exists bigtable'
 
diff --git a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
index 7c5edcc81a2e8e..9550a7fec3dbd2 100644
--- a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
+++ b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_show_create_materialized_view", "query") {
+suite("test_show_create_materialized_view", "query,arrow_flight_sql") {
     String tableName = "table_for_mv_test";
     String mvName = "mv_show_create_materialized_view";
     try {  
diff --git a/regression-test/suites/query_p0/show/test_show_create_table.groovy b/regression-test/suites/query_p0/show/test_show_create_table.groovy
index 6325cbe319fd88..1e3fc7ff5cb527 100644
--- a/regression-test/suites/query_p0/show/test_show_create_table.groovy
+++ b/regression-test/suites/query_p0/show/test_show_create_table.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_show_create_table", "query") {
+suite("test_show_create_table", "query,arrow_flight_sql") {
     String tb_name = "tb_show_create_table";
     try {  
         sql """drop table if exists ${tb_name} """
diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy
index 86a951d7ac33c6..cdab9472e27dbd 100644
--- a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy
+++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_aggregate_all_functions") {
+suite("test_aggregate_all_functions", "arrow_flight_sql") {
 
     sql "set batch_size = 4096"
     
diff --git a/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy b/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
index 41ef1625080806..a91c86b5f48908 100644
--- a/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
+++ b/regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_case_function_null", "query,p0") {
+suite("test_case_function_null", "query,p0,arrow_flight_sql") {
     sql """ drop table if exists case_null0 """
     sql """ create table case_null0 (
                 `c0` decimalv3(17, 1) NULL,
diff --git a/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy b/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy
index 590ccd10821f61..d547e9fb287d71 100644
--- a/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_hash_function") {
+suite("test_hash_function", "arrow_flight_sql") {
     sql "set batch_size = 4096;"
     sql "set enable_profile = true;"
 
diff --git a/regression-test/suites/query_p0/sql_functions/ip_functions/test_ip_functions.groovy b/regression-test/suites/query_p0/sql_functions/ip_functions/test_ip_functions.groovy
index 03e9788a58a3b8..5373217503a018 100644
--- a/regression-test/suites/query_p0/sql_functions/ip_functions/test_ip_functions.groovy
+++ b/regression-test/suites/query_p0/sql_functions/ip_functions/test_ip_functions.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_ip_functions") {
+suite("test_ip_functions", "arrow_flight_sql") {
     sql "set batch_size = 4096;"
 
     qt_sql "SELECT ipv4_num_to_string(-1);"
diff --git a/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_insert.groovy b/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_insert.groovy
index c885e3ae3431f3..b5865034538a11 100644
--- a/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_insert.groovy
+++ b/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_insert.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_query_json_insert", "query") {
+suite("test_query_json_insert", "query,arrow_flight_sql") {
     qt_sql "select json_insert('{\"a\": 1, \"b\": [2, 3]}', '\$', null);"
     qt_sql "select json_insert('{\"k\": [1, 2]}', '\$.k[0]', null, '\$.[1]', null);"
     def tableName = "test_query_json_insert"
diff --git a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy
index aa0deec96f46a2..4bd88bf131e727 100644
--- a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_json_function") {
+suite("test_json_function", "arrow_flight_sql") {
     sql "set batch_size = 4096;"
 
     qt_sql "SELECT get_json_double('{\"k1\":1.3, \"k2\":\"2\"}', \"\$.k1\");"
diff --git a/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy b/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy
index 6c4867174d11ac..3a74abfe9c8b22 100644
--- a/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy
+++ b/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_conv") {
+suite("test_conv", "arrow_flight_sql") {
     qt_select "SELECT CONV(15,10,2)"
 
     sql """ drop table if exists test_conv; """
diff --git a/regression-test/suites/query_p0/sql_functions/search_functions/test_multi_string_search.groovy b/regression-test/suites/query_p0/sql_functions/search_functions/test_multi_string_search.groovy
index 061665d3b9da6e..f1487d283dfcdf 100644
--- a/regression-test/suites/query_p0/sql_functions/search_functions/test_multi_string_search.groovy
+++ b/regression-test/suites/query_p0/sql_functions/search_functions/test_multi_string_search.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_multi_string_search") {
+suite("test_multi_string_search", "arrow_flight_sql") {
     def table_name = "test_multi_string_search_strings"
 
     sql """ DROP TABLE IF EXISTS ${table_name} """
diff --git a/regression-test/suites/query_p0/sql_functions/spatial_functions/test_gis_function.groovy b/regression-test/suites/query_p0/sql_functions/spatial_functions/test_gis_function.groovy
index e98e11ba7e6888..f76cb44cb4ad4b 100644
--- a/regression-test/suites/query_p0/sql_functions/spatial_functions/test_gis_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/spatial_functions/test_gis_function.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_gis_function") {
+suite("test_gis_function", "arrow_flight_sql") {
     sql "set batch_size = 4096;"
 
     qt_sql "SELECT ST_AsText(ST_Point(24.7, 56.7));"
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
index f5d32653c818b5..6e18fb57eeb4cf 100644
--- a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_string_function") {
+suite("test_string_function", "arrow_flight_sql") {
     sql "set batch_size = 4096;"
 
     qt_sql "select elt(0, \"hello\", \"doris\");"
diff --git a/regression-test/suites/query_p0/sql_functions/table_function/explode_split.groovy b/regression-test/suites/query_p0/sql_functions/table_function/explode_split.groovy
index b7dd4d640799fb..53db931c03bb03 100644
--- a/regression-test/suites/query_p0/sql_functions/table_function/explode_split.groovy
+++ b/regression-test/suites/query_p0/sql_functions/table_function/explode_split.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("explode_split") {
+suite("explode_split", "arrow_flight_sql") {
     def tableName = "test_lv_str"
 
     sql """ DROP TABLE IF EXISTS ${tableName} """
diff --git a/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy b/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy
index 8e0e94fa2df805..095ec89e220f1b 100644
--- a/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite('test_alias_function') {
+suite('test_alias_function', "arrow_flight_sql") {
     sql '''
         CREATE ALIAS FUNCTION IF NOT EXISTS f1(DATETIMEV2(3), INT)
             with PARAMETER (datetime1, int1) as date_trunc(days_sub(datetime1, int1), 'day')'''
diff --git a/regression-test/suites/query_p0/sql_functions/test_predicate.groovy b/regression-test/suites/query_p0/sql_functions/test_predicate.groovy
index 20b3c179ad5c01..6cca6b62c9960b 100644
--- a/regression-test/suites/query_p0/sql_functions/test_predicate.groovy
+++ b/regression-test/suites/query_p0/sql_functions/test_predicate.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_predicate") {
+suite("test_predicate", "arrow_flight_sql") {
     sql """drop table if exists t1;"""
     sql """
             create table t1 (
diff --git a/regression-test/suites/query_p0/sql_functions/width_bucket_fuctions/test_width_bucket_function.groovy b/regression-test/suites/query_p0/sql_functions/width_bucket_fuctions/test_width_bucket_function.groovy
index d0862a580ca600..1a455da92446f8 100644
--- a/regression-test/suites/query_p0/sql_functions/width_bucket_fuctions/test_width_bucket_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/width_bucket_fuctions/test_width_bucket_function.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_width_bucket_function") {
+suite("test_width_bucket_function", "arrow_flight_sql") {
     qt_sql "select width_bucket(1, 2, 3, 2)"
     qt_sql "select width_bucket(null, 2, 3, 2)"
     qt_sql "select width_bucket(6, 2, 6, 4)"
diff --git a/regression-test/suites/query_p0/subquery/test_subquery2.groovy b/regression-test/suites/query_p0/subquery/test_subquery2.groovy
index e572459cc72fe3..a14a44fa152b97 100644
--- a/regression-test/suites/query_p0/subquery/test_subquery2.groovy
+++ b/regression-test/suites/query_p0/subquery/test_subquery2.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_subquery2") {
+suite("test_subquery2", "arrow_flight_sql") {
     
     sql """DROP TABLE IF EXISTS subquerytest2"""
     
diff --git a/regression-test/suites/query_p0/test_data_type_marks.groovy b/regression-test/suites/query_p0/test_data_type_marks.groovy
index 79803d98723313..51fb7c9614e488 100644
--- a/regression-test/suites/query_p0/test_data_type_marks.groovy
+++ b/regression-test/suites/query_p0/test_data_type_marks.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_data_type_marks") {
+suite("test_data_type_marks", "arrow_flight_sql") {
     def tbName = "org"
     sql "DROP TABLE IF EXISTS ${tbName}"
     sql """
diff --git a/regression-test/suites/query_p0/test_dict_with_null.groovy b/regression-test/suites/query_p0/test_dict_with_null.groovy
index b3738bb68aa1ba..83d253fa4d1b04 100644
--- a/regression-test/suites/query_p0/test_dict_with_null.groovy
+++ b/regression-test/suites/query_p0/test_dict_with_null.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("dict_with_null", "query") {
+suite("dict_with_null", "query,arrow_flight_sql") {
     def tableName = "test_dict_with_null"
     sql "DROP TABLE IF EXISTS ${tableName}"
     sql """
diff --git a/regression-test/suites/query_p0/test_orderby_nullliteral.groovy b/regression-test/suites/query_p0/test_orderby_nullliteral.groovy
index fe11c778af0b98..e806060c8bcb1c 100644
--- a/regression-test/suites/query_p0/test_orderby_nullliteral.groovy
+++ b/regression-test/suites/query_p0/test_orderby_nullliteral.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("orderby_nullliteral", "query") {
+suite("orderby_nullliteral", "query,arrow_flight_sql") {
 
     def tableName = "test_orderby_nullliteral"
     sql "DROP TABLE IF EXISTS ${tableName}"
diff --git a/regression-test/suites/query_p0/test_select_constant.groovy b/regression-test/suites/query_p0/test_select_constant.groovy
index 6015e19576c690..68f0a28a20e853 100644
--- a/regression-test/suites/query_p0/test_select_constant.groovy
+++ b/regression-test/suites/query_p0/test_select_constant.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_select_constant") {
+suite("test_select_constant", "arrow_flight_sql") {
     qt_select1 'select 100, "test", date("2021-01-02");'
     qt_select_geo1 'SELECT ST_AsText(ST_Point(123.12345678901234567890,89.1234567890));'
 }
diff --git a/regression-test/suites/query_p0/test_select_with_predicate_like.groovy b/regression-test/suites/query_p0/test_select_with_predicate_like.groovy
index 9491c4271ca530..0d01f1b958a11c 100644
--- a/regression-test/suites/query_p0/test_select_with_predicate_like.groovy
+++ b/regression-test/suites/query_p0/test_select_with_predicate_like.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_select_with_predicate_like") {
+suite("test_select_with_predicate_like", "arrow_flight_sql") {
     def tables=["test_basic_agg"]
 
     for (String table in tables) {
diff --git a/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy b/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy
index 768e04b4c327b5..ccd1b9160fb148 100644
--- a/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy
+++ b/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_select_with_predicate_prune") {
+suite("test_select_with_predicate_prune", "arrow_flight_sql") {
     sql """
         drop table if exists `test_select_with_predicate_prune`;
     """
diff --git a/regression-test/suites/query_p0/type_inference/test_largeint.groovy b/regression-test/suites/query_p0/type_inference/test_largeint.groovy
index d5cbfa4b479838..161359cfa97e72 100644
--- a/regression-test/suites/query_p0/type_inference/test_largeint.groovy
+++ b/regression-test/suites/query_p0/type_inference/test_largeint.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_largeint") {
+suite("test_largeint", "arrow_flight_sql") {
     def tbName = "test_largeint"
     sql "DROP TABLE IF EXISTS ${tbName}"
     sql """
diff --git a/regression-test/suites/query_p0/with/test_with_and_two_phase_agg.groovy b/regression-test/suites/query_p0/with/test_with_and_two_phase_agg.groovy
index 99164a999c557e..d563ef1630517d 100644
--- a/regression-test/suites/query_p0/with/test_with_and_two_phase_agg.groovy
+++ b/regression-test/suites/query_p0/with/test_with_and_two_phase_agg.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_with_and_two_phase_agg") {
+suite("test_with_and_two_phase_agg", "arrow_flight_sql") {
     def tableName = "test_with_and_two_phase_agg_table"
     sql """ DROP TABLE IF EXISTS ${tableName} """
     sql """

From 2160f0541d9815e0dade3bf2c61a6ec3a147ab31 Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Thu, 5 Dec 2024 20:33:50 +0800
Subject: [PATCH 190/399] [fix](cluster key) rename cluster_key_idxes to
 cluster_key_uids (#44981)

Problem Summary:

the `cluster_key_idxes` is column unique ids, not position of column in
tablet schema, so rename it
---
 be/src/cloud/cloud_tablet.cpp                    |  2 +-
 be/src/cloud/pb_convert.cpp                      |  8 ++++----
 be/src/olap/base_tablet.cpp                      |  6 +++---
 be/src/olap/compaction.cpp                       |  8 ++++----
 be/src/olap/memtable.cpp                         |  4 ++--
 be/src/olap/merger.cpp                           | 10 +++++-----
 be/src/olap/rowset/beta_rowset_writer.cpp        |  2 +-
 be/src/olap/rowset/segment_v2/segment.cpp        |  4 ++--
 .../olap/rowset/segment_v2/segment_iterator.cpp  |  4 ++--
 be/src/olap/rowset/segment_v2/segment_writer.cpp | 10 +++++-----
 .../segment_v2/vertical_segment_writer.cpp       | 16 ++++++++--------
 be/src/olap/schema_change.cpp                    |  4 ++--
 be/src/olap/tablet_meta.cpp                      |  4 ++--
 be/src/olap/tablet_reader.cpp                    | 11 +++++++++--
 be/src/olap/tablet_schema.cpp                    | 16 ++++++++--------
 be/src/olap/tablet_schema.h                      |  4 ++--
 be/src/vec/olap/vertical_block_reader.cpp        |  2 +-
 be/test/olap/delta_writer_cluster_key_test.cpp   |  4 ++--
 .../doris/alter/CloudSchemaChangeJobV2.java      |  6 +++---
 .../apache/doris/alter/SchemaChangeJobV2.java    | 12 ++++++------
 .../java/org/apache/doris/backup/RestoreJob.java | 12 ++++++------
 .../java/org/apache/doris/catalog/OlapTable.java |  8 ++++----
 .../cloud/datasource/CloudInternalCatalog.java   | 14 +++++++-------
 .../apache/doris/datasource/InternalCatalog.java | 14 +++++++-------
 .../org/apache/doris/master/ReportHandler.java   | 10 +++++-----
 .../org/apache/doris/task/CreateReplicaTask.java | 12 ++++++------
 gensrc/proto/olap_file.proto                     |  4 ++--
 gensrc/thrift/AgentService.thrift                |  2 +-
 .../org/apache/doris/regression/Config.groovy    |  2 ++
 29 files changed, 112 insertions(+), 103 deletions(-)

diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index c27353be5235d0..86cf23e39cc335 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -776,7 +776,7 @@ Status CloudTablet::calc_delete_bitmap_for_compaction(
 
     std::unique_ptr<std::map<RowsetSharedPtr, RowLocationPairList>> location_map;
     if (config::enable_rowid_conversion_correctness_check &&
-        tablet_schema()->cluster_key_idxes().empty()) {
+        tablet_schema()->cluster_key_uids().empty()) {
         location_map = std::make_unique<std::map<RowsetSharedPtr, RowLocationPairList>>();
         LOG(INFO) << "Location Map inited succ for tablet:" << tablet_id();
     }
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index bff7d8388d30d8..e655ceacf2f08d 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -324,7 +324,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB
     out->set_store_row_column(in.store_row_column());
     out->set_enable_single_replica_compaction(in.enable_single_replica_compaction());
     out->set_skip_write_index_on_load(in.skip_write_index_on_load());
-    out->mutable_cluster_key_idxes()->CopyFrom(in.cluster_key_idxes());
+    out->mutable_cluster_key_uids()->CopyFrom(in.cluster_key_uids());
     out->set_is_dynamic_schema(in.is_dynamic_schema());
     out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
     out->set_inverted_index_storage_format(in.inverted_index_storage_format());
@@ -353,7 +353,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in)
     out->set_store_row_column(in.store_row_column());
     out->set_enable_single_replica_compaction(in.enable_single_replica_compaction());
     out->set_skip_write_index_on_load(in.skip_write_index_on_load());
-    out->mutable_cluster_key_idxes()->Swap(in.mutable_cluster_key_idxes());
+    out->mutable_cluster_key_uids()->Swap(in.mutable_cluster_key_uids());
     out->set_is_dynamic_schema(in.is_dynamic_schema());
     out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
     out->set_inverted_index_storage_format(in.inverted_index_storage_format());
@@ -395,7 +395,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB
     out->set_store_row_column(in.store_row_column());
     out->set_enable_single_replica_compaction(in.enable_single_replica_compaction());
     out->set_skip_write_index_on_load(in.skip_write_index_on_load());
-    out->mutable_cluster_key_idxes()->CopyFrom(in.cluster_key_idxes());
+    out->mutable_cluster_key_uids()->CopyFrom(in.cluster_key_uids());
     out->set_is_dynamic_schema(in.is_dynamic_schema());
     out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
     out->set_inverted_index_storage_format(in.inverted_index_storage_format());
@@ -425,7 +425,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in)
     out->set_store_row_column(in.store_row_column());
     out->set_enable_single_replica_compaction(in.enable_single_replica_compaction());
     out->set_skip_write_index_on_load(in.skip_write_index_on_load());
-    out->mutable_cluster_key_idxes()->Swap(in.mutable_cluster_key_idxes());
+    out->mutable_cluster_key_uids()->Swap(in.mutable_cluster_key_uids());
     out->set_is_dynamic_schema(in.is_dynamic_schema());
     out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
     out->set_inverted_index_storage_format(in.inverted_index_storage_format());
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 104dc44ebec8ae..9fc253d5a8d53e 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -376,7 +376,7 @@ Status BaseTablet::calc_delete_bitmap_between_segments(
         seq_col_length = _tablet_meta->tablet_schema()->column(seq_col_idx).length() + 1;
     }
     size_t rowid_length = 0;
-    if (!_tablet_meta->tablet_schema()->cluster_key_idxes().empty()) {
+    if (!_tablet_meta->tablet_schema()->cluster_key_uids().empty()) {
         rowid_length = PrimaryKeyIndexReader::ROW_ID_LENGTH;
     }
 
@@ -461,7 +461,7 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest
         seq_col_length = schema->column(schema->sequence_col_idx()).length() + 1;
     }
     size_t rowid_length = 0;
-    if (with_rowid && !schema->cluster_key_idxes().empty()) {
+    if (with_rowid && !schema->cluster_key_uids().empty()) {
         rowid_length = PrimaryKeyIndexReader::ROW_ID_LENGTH;
     }
     Slice key_without_seq =
@@ -654,7 +654,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
             Slice key = Slice(index_column->get_data_at(i).data, index_column->get_data_at(i).size);
             RowLocation loc;
             // calculate row id
-            if (!_tablet_meta->tablet_schema()->cluster_key_idxes().empty()) {
+            if (!_tablet_meta->tablet_schema()->cluster_key_uids().empty()) {
                 size_t seq_col_length = 0;
                 if (_tablet_meta->tablet_schema()->has_sequence_col()) {
                     seq_col_length =
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 98d7787550d197..8c45c20f799427 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -191,14 +191,14 @@ Status Compaction::merge_input_rowsets() {
         SCOPED_TIMER(_merge_rowsets_latency_timer);
         // 1. Merge segment files and write bkd inverted index
         if (_is_vertical) {
-            if (!_tablet->tablet_schema()->cluster_key_idxes().empty()) {
+            if (!_tablet->tablet_schema()->cluster_key_uids().empty()) {
                 RETURN_IF_ERROR(update_delete_bitmap());
             }
             res = Merger::vertical_merge_rowsets(_tablet, compaction_type(), *_cur_tablet_schema,
                                                  input_rs_readers, _output_rs_writer.get(),
                                                  get_avg_segment_rows(), way_num, &_stats);
         } else {
-            if (!_tablet->tablet_schema()->cluster_key_idxes().empty()) {
+            if (!_tablet->tablet_schema()->cluster_key_uids().empty()) {
                 return Status::InternalError(
                         "mow table with cluster keys does not support non vertical compaction");
             }
@@ -966,7 +966,7 @@ Status CompactionMixin::modify_rowsets() {
         }
         std::unique_ptr<std::map<RowsetSharedPtr, RowLocationPairList>> location_map;
         if (config::enable_rowid_conversion_correctness_check &&
-            tablet()->tablet_schema()->cluster_key_idxes().empty()) {
+            tablet()->tablet_schema()->cluster_key_uids().empty()) {
             location_map = std::make_unique<std::map<RowsetSharedPtr, RowLocationPairList>>();
             LOG(INFO) << "Location Map inited succ for tablet:" << _tablet->tablet_id();
         }
@@ -983,7 +983,7 @@ Status CompactionMixin::modify_rowsets() {
         if (missed_rows) {
             missed_rows_size = missed_rows->size();
             std::size_t merged_missed_rows_size = _stats.merged_rows;
-            if (!_tablet->tablet_meta()->tablet_schema()->cluster_key_idxes().empty()) {
+            if (!_tablet->tablet_meta()->tablet_schema()->cluster_key_uids().empty()) {
                 merged_missed_rows_size += _stats.filtered_rows;
             }
             if (_tablet->tablet_state() == TABLET_RUNNING &&
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index 5db3d89378bcc2..765f67a07c7884 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -350,7 +350,7 @@ Status MemTable::_sort_by_cluster_keys() {
     }
     Tie tie = Tie(0, mutable_block.rows());
 
-    for (auto cid : _tablet_schema->cluster_key_idxes()) {
+    for (auto cid : _tablet_schema->cluster_key_uids()) {
         auto index = _tablet_schema->field_index(cid);
         if (index == -1) {
             return Status::InternalError("could not find cluster key column with unique_id=" +
@@ -619,7 +619,7 @@ Status MemTable::_to_block(std::unique_ptr<vectorized::Block>* res) {
         (_skip_bitmap_col_idx == -1) ? _aggregate<true, false>() : _aggregate<true, true>();
     }
     if (_keys_type == KeysType::UNIQUE_KEYS && _enable_unique_key_mow &&
-        !_tablet_schema->cluster_key_idxes().empty()) {
+        !_tablet_schema->cluster_key_uids().empty()) {
         if (_partial_update_mode != UniqueKeyUpdateModePB::UPSERT) {
             return Status::InternalError(
                     "Partial update for mow with cluster keys is not supported");
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index b207cc4c5ad22b..975aaa0bca3de5 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -86,7 +86,7 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type,
         merge_tablet_schema->merge_dropped_columns(*del_pred_rs->tablet_schema());
     }
     reader_params.tablet_schema = merge_tablet_schema;
-    if (!tablet->tablet_schema()->cluster_key_idxes().empty()) {
+    if (!tablet->tablet_schema()->cluster_key_uids().empty()) {
         reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap();
     }
 
@@ -173,8 +173,8 @@ void Merger::vertical_split_columns(const TabletSchema& tablet_schema,
         if (delete_sign_idx != -1) {
             key_columns.emplace_back(delete_sign_idx);
         }
-        if (!tablet_schema.cluster_key_idxes().empty()) {
-            for (const auto& cid : tablet_schema.cluster_key_idxes()) {
+        if (!tablet_schema.cluster_key_uids().empty()) {
+            for (const auto& cid : tablet_schema.cluster_key_uids()) {
                 auto idx = tablet_schema.field_index(cid);
                 DCHECK(idx >= 0) << "could not find cluster key column with unique_id=" << cid
                                  << " in tablet schema, table_id=" << tablet_schema.table_id();
@@ -186,7 +186,7 @@ void Merger::vertical_split_columns(const TabletSchema& tablet_schema,
             // cluster key unique ids: [3, 1, 4]
             // the key_columns should be [0, 1, 3, 5]
             // the key_group_cluster_key_idxes should be [2, 1, 3]
-            for (const auto& cid : tablet_schema.cluster_key_idxes()) {
+            for (const auto& cid : tablet_schema.cluster_key_uids()) {
                 auto idx = tablet_schema.field_index(cid);
                 for (auto i = 0; i < key_columns.size(); ++i) {
                     if (idx == key_columns[i]) {
@@ -261,7 +261,7 @@ Status Merger::vertical_compact_one_group(
 
     reader_params.tablet_schema = merge_tablet_schema;
     bool has_cluster_key = false;
-    if (!tablet->tablet_schema()->cluster_key_idxes().empty()) {
+    if (!tablet->tablet_schema()->cluster_key_uids().empty()) {
         reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap();
         has_cluster_key = true;
     }
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
index 198b4e8595ed20..475a04df4911fd 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -581,7 +581,7 @@ Status BetaRowsetWriter::_segcompaction_if_necessary() {
     Status status = Status::OK();
     // if not doing segcompaction, just check segment number
     if (!config::enable_segcompaction || !_context.enable_segcompaction ||
-        !_context.tablet_schema->cluster_key_idxes().empty() ||
+        !_context.tablet_schema->cluster_key_uids().empty() ||
         _context.tablet_schema->num_variant_columns() > 0) {
         return _check_segment_number_limit(_num_segment);
     }
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index d2c2920d4e6f74..513c0be4f8cd14 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -966,7 +966,7 @@ Status Segment::lookup_row_key(const Slice& key, const TabletSchema* latest_sche
                                std::string* encoded_seq_value, OlapReaderStatistics* stats) {
     RETURN_IF_ERROR(load_pk_index_and_bf());
     bool has_seq_col = latest_schema->has_sequence_col();
-    bool has_rowid = !latest_schema->cluster_key_idxes().empty();
+    bool has_rowid = !latest_schema->cluster_key_uids().empty();
     size_t seq_col_length = 0;
     if (has_seq_col) {
         seq_col_length = latest_schema->column(latest_schema->sequence_col_idx()).length() + 1;
@@ -1076,7 +1076,7 @@ Status Segment::read_key_by_rowid(uint32_t row_id, std::string* key) {
     RETURN_IF_ERROR(iter->next_batch(&num_read, index_column));
     CHECK(num_read == 1);
     // trim row id
-    if (_tablet_schema->cluster_key_idxes().empty()) {
+    if (_tablet_schema->cluster_key_uids().empty()) {
         *key = index_column->get_data_at(0).to_string();
     } else {
         Slice sought_key =
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 4ee73547c117e9..f5f46e938650e1 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -377,7 +377,7 @@ Status SegmentIterator::_lazy_init() {
     _row_bitmap.addRange(0, _segment->num_rows());
     // z-order can not use prefix index
     if (_segment->_tablet_schema->sort_type() != SortType::ZORDER &&
-        _segment->_tablet_schema->cluster_key_idxes().empty()) {
+        _segment->_tablet_schema->cluster_key_uids().empty()) {
         RETURN_IF_ERROR(_get_row_ranges_by_keys());
     }
     RETURN_IF_ERROR(_get_row_ranges_by_column_conditions());
@@ -1193,7 +1193,7 @@ Status SegmentIterator::_lookup_ordinal_from_pk_index(const RowCursor& key, bool
     bool has_seq_col = _segment->_tablet_schema->has_sequence_col();
     // Used to get key range from primary key index,
     // for mow with cluster key table, we should get key range from short key index.
-    DCHECK(_segment->_tablet_schema->cluster_key_idxes().empty());
+    DCHECK(_segment->_tablet_schema->cluster_key_uids().empty());
 
     // if full key is exact_match, the primary key without sequence column should also the same
     if (has_seq_col && !exact_match) {
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index c6c9664be4b232..8641687a0069b8 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -103,7 +103,7 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id,
                 << ", table_id=" << _tablet_schema->table_id()
                 << ", num_key_columns=" << _num_sort_key_columns
                 << ", num_short_key_columns=" << _num_short_key_columns
-                << ", cluster_key_columns=" << _tablet_schema->cluster_key_idxes().size();
+                << ", cluster_key_columns=" << _tablet_schema->cluster_key_uids().size();
     }
     for (size_t cid = 0; cid < _num_sort_key_columns; ++cid) {
         const auto& column = _tablet_schema->column(cid);
@@ -125,8 +125,8 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id,
             // cluster keys
             _key_coders.clear();
             _key_index_size.clear();
-            _num_sort_key_columns = _tablet_schema->cluster_key_idxes().size();
-            for (auto cid : _tablet_schema->cluster_key_idxes()) {
+            _num_sort_key_columns = _tablet_schema->cluster_key_uids().size();
+            for (auto cid : _tablet_schema->cluster_key_uids()) {
                 const auto& column = _tablet_schema->column_by_uid(cid);
                 _key_coders.push_back(get_key_coder(column.type()));
                 _key_index_size.push_back(column.index_length());
@@ -788,7 +788,7 @@ Status SegmentWriter::append_block(const vectorized::Block* block, size_t row_po
                                                         seq_column, num_rows, true));
             // 2. generate short key index (use cluster key)
             key_columns.clear();
-            for (const auto& cid : _tablet_schema->cluster_key_idxes()) {
+            for (const auto& cid : _tablet_schema->cluster_key_uids()) {
                 // find cluster key index in tablet schema
                 auto cluster_key_index = _tablet_schema->field_index(cid);
                 if (cluster_key_index == -1) {
@@ -1290,7 +1290,7 @@ inline bool SegmentWriter::_is_mow() {
 }
 
 inline bool SegmentWriter::_is_mow_with_cluster_key() {
-    return _is_mow() && !_tablet_schema->cluster_key_idxes().empty();
+    return _is_mow() && !_tablet_schema->cluster_key_uids().empty();
 }
 } // namespace segment_v2
 } // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index ce16e2d502b622..f40da898a28742 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -109,7 +109,7 @@ VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32
                 << ", table_id=" << _tablet_schema->table_id()
                 << ", num_key_columns=" << _num_sort_key_columns
                 << ", num_short_key_columns=" << _num_short_key_columns
-                << ", cluster_key_columns=" << _tablet_schema->cluster_key_idxes().size();
+                << ", cluster_key_columns=" << _tablet_schema->cluster_key_uids().size();
     }
     for (size_t cid = 0; cid < _num_sort_key_columns; ++cid) {
         const auto& column = _tablet_schema->column(cid);
@@ -131,8 +131,8 @@ VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32
             // cluster keys
             _key_coders.clear();
             _key_index_size.clear();
-            _num_sort_key_columns = _tablet_schema->cluster_key_idxes().size();
-            for (auto cid : _tablet_schema->cluster_key_idxes()) {
+            _num_sort_key_columns = _tablet_schema->cluster_key_uids().size();
+            for (auto cid : _tablet_schema->cluster_key_uids()) {
                 const auto& column = _tablet_schema->column_by_uid(cid);
                 _key_coders.push_back(get_key_coder(column.type()));
                 _key_index_size.push_back(column.index_length());
@@ -1149,9 +1149,9 @@ Status VerticalSegmentWriter::write_batch() {
             }
             auto column_unique_id = _tablet_schema->column(cid).unique_id();
             if (_is_mow_with_cluster_key() &&
-                std::find(_tablet_schema->cluster_key_idxes().begin(),
-                          _tablet_schema->cluster_key_idxes().end(),
-                          column_unique_id) != _tablet_schema->cluster_key_idxes().end()) {
+                std::find(_tablet_schema->cluster_key_uids().begin(),
+                          _tablet_schema->cluster_key_uids().end(),
+                          column_unique_id) != _tablet_schema->cluster_key_uids().end()) {
                 cid_to_column[column_unique_id] = column;
             }
             RETURN_IF_ERROR(_column_writers[cid]->append(column->get_nullmap(), column->get_data(),
@@ -1213,7 +1213,7 @@ Status VerticalSegmentWriter::_generate_key_index(
                                                     data.num_rows, true));
         // 2. generate short key index (use cluster key)
         std::vector<vectorized::IOlapColumnDataAccessor*> short_key_columns;
-        for (const auto& cid : _tablet_schema->cluster_key_idxes()) {
+        for (const auto& cid : _tablet_schema->cluster_key_uids()) {
             short_key_columns.push_back(cid_to_column[cid]);
         }
         RETURN_IF_ERROR(_generate_short_key_index(short_key_columns, data.num_rows, short_key_pos));
@@ -1572,7 +1572,7 @@ inline bool VerticalSegmentWriter::_is_mow() {
 }
 
 inline bool VerticalSegmentWriter::_is_mow_with_cluster_key() {
-    return _is_mow() && !_tablet_schema->cluster_key_idxes().empty();
+    return _is_mow() && !_tablet_schema->cluster_key_uids().empty();
 }
 } // namespace segment_v2
 } // namespace doris
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index cd1b10d733a88c..cdb637b1c42647 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -198,9 +198,9 @@ class MultiBlockMerger {
                         pushed_row_refs.push_back(row_refs[i]);
                     }
                 }
-                if (!_tablet->tablet_schema()->cluster_key_idxes().empty()) {
+                if (!_tablet->tablet_schema()->cluster_key_uids().empty()) {
                     std::vector<uint32_t> ids;
-                    for (const auto& cid : _tablet->tablet_schema()->cluster_key_idxes()) {
+                    for (const auto& cid : _tablet->tablet_schema()->cluster_key_uids()) {
                         auto index = _tablet->tablet_schema()->field_index(cid);
                         if (index == -1) {
                             return Status::InternalError(
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index d1746836e231c2..9f905918a66e9a 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -219,8 +219,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
         schema->set_sort_type(SortType::LEXICAL);
     }
     schema->set_sort_col_num(tablet_schema.sort_col_num);
-    for (const auto& i : tablet_schema.cluster_key_idxes) {
-        schema->add_cluster_key_idxes(i);
+    for (const auto& i : tablet_schema.cluster_key_uids) {
+        schema->add_cluster_key_uids(i);
     }
     tablet_meta_pb.set_in_restore_mode(false);
 
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index a78a52e5fd3e31..a83e0bfdbf4c30 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -464,9 +464,16 @@ Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) {
     // UNIQUE_KEYS will compare all keys as before
     if (_tablet_schema->keys_type() == DUP_KEYS || (_tablet_schema->keys_type() == UNIQUE_KEYS &&
                                                     _tablet->enable_unique_key_merge_on_write())) {
-        if (!_tablet_schema->cluster_key_idxes().empty()) {
+        if (!_tablet_schema->cluster_key_uids().empty()) {
+            if (read_params.read_orderby_key_num_prefix_columns >
+                _tablet_schema->cluster_key_uids().size()) {
+                return Status::Error<ErrorCode::INTERNAL_ERROR>(
+                        "read_orderby_key_num_prefix_columns={} > cluster_keys.size()={}",
+                        read_params.read_orderby_key_num_prefix_columns,
+                        _tablet_schema->cluster_key_uids().size());
+            }
             for (uint32_t i = 0; i < read_params.read_orderby_key_num_prefix_columns; i++) {
-                auto cid = _tablet_schema->cluster_key_idxes()[i];
+                auto cid = _tablet_schema->cluster_key_uids()[i];
                 auto index = _tablet_schema->field_index(cid);
                 if (index < 0) {
                     return Status::Error<ErrorCode::INTERNAL_ERROR>(
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 47179ce19b23e7..3ec5d22166477f 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -973,10 +973,10 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac
     _indexes.clear();
     _field_name_to_index.clear();
     _field_id_to_index.clear();
-    _cluster_key_idxes.clear();
+    _cluster_key_uids.clear();
     clear_column_cache_handlers();
-    for (const auto& i : schema.cluster_key_idxes()) {
-        _cluster_key_idxes.push_back(i);
+    for (const auto& i : schema.cluster_key_uids()) {
+        _cluster_key_uids.push_back(i);
     }
     for (auto& column_pb : schema.column()) {
         TabletColumnPtr column;
@@ -1124,10 +1124,10 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version
     _sequence_col_idx = -1;
     _version_col_idx = -1;
     _skip_bitmap_col_idx = -1;
-    _cluster_key_idxes.clear();
+    _cluster_key_uids.clear();
     clear_column_cache_handlers();
-    for (const auto& i : ori_tablet_schema._cluster_key_idxes) {
-        _cluster_key_idxes.push_back(i);
+    for (const auto& i : ori_tablet_schema._cluster_key_uids) {
+        _cluster_key_uids.push_back(i);
     }
     for (auto& column : index->columns) {
         if (column->is_key()) {
@@ -1235,8 +1235,8 @@ void TabletSchema::reserve_extracted_columns() {
 }
 
 void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
-    for (const auto& i : _cluster_key_idxes) {
-        tablet_schema_pb->add_cluster_key_idxes(i);
+    for (const auto& i : _cluster_key_uids) {
+        tablet_schema_pb->add_cluster_key_uids(i);
     }
     tablet_schema_pb->set_keys_type(_keys_type);
     for (const auto& col : _cols) {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 9a0cd53f7b1786..c813d6f0ef8722 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -349,7 +349,7 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
     const std::vector<TabletColumnPtr>& columns() const;
     size_t num_columns() const { return _num_columns; }
     size_t num_key_columns() const { return _num_key_columns; }
-    const std::vector<uint32_t>& cluster_key_idxes() const { return _cluster_key_idxes; }
+    const std::vector<uint32_t>& cluster_key_uids() const { return _cluster_key_uids; }
     size_t num_null_columns() const { return _num_null_columns; }
     size_t num_short_key_columns() const { return _num_short_key_columns; }
     size_t num_rows_per_row_block() const { return _num_rows_per_row_block; }
@@ -548,7 +548,7 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
     size_t _num_columns = 0;
     size_t _num_variant_columns = 0;
     size_t _num_key_columns = 0;
-    std::vector<uint32_t> _cluster_key_idxes;
+    std::vector<uint32_t> _cluster_key_uids;
     size_t _num_null_columns = 0;
     size_t _num_short_key_columns = 0;
     size_t _num_rows_per_row_block = 0;
diff --git a/be/src/vec/olap/vertical_block_reader.cpp b/be/src/vec/olap/vertical_block_reader.cpp
index 369bf04459a8f2..8df4e38e455881 100644
--- a/be/src/vec/olap/vertical_block_reader.cpp
+++ b/be/src/vec/olap/vertical_block_reader.cpp
@@ -237,7 +237,7 @@ Status VerticalBlockReader::init(const ReaderParams& read_params,
         _next_block_func = &VerticalBlockReader::_direct_next_block;
         break;
     case KeysType::UNIQUE_KEYS:
-        if (tablet()->tablet_meta()->tablet_schema()->cluster_key_idxes().empty()) {
+        if (tablet()->tablet_meta()->tablet_schema()->cluster_key_uids().empty()) {
             _next_block_func = &VerticalBlockReader::_unique_key_next_block;
             if (_filter_delete) {
                 _delete_filter_column = ColumnUInt8::create();
diff --git a/be/test/olap/delta_writer_cluster_key_test.cpp b/be/test/olap/delta_writer_cluster_key_test.cpp
index 9c3e64109c4440..6c4b4d367a3d21 100644
--- a/be/test/olap/delta_writer_cluster_key_test.cpp
+++ b/be/test/olap/delta_writer_cluster_key_test.cpp
@@ -116,8 +116,8 @@ static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t s
     request->tablet_schema.__set_sequence_col_idx(4);
     request->__set_storage_format(TStorageFormat::V2);
     request->__set_enable_unique_key_merge_on_write(enable_mow);
-    request->tablet_schema.cluster_key_idxes.push_back(1);
-    request->tablet_schema.cluster_key_idxes.push_back(0);
+    request->tablet_schema.cluster_key_uids.push_back(1);
+    request->tablet_schema.cluster_key_uids.push_back(0);
 
     TColumn k1;
     k1.column_name = "k1";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index 92a74e46b922be..f765dd1bb7fdca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -242,9 +242,9 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio
 
                 short shadowShortKeyColumnCount = indexShortKeyMap.get(shadowIdxId);
                 List<Column> shadowSchema = indexSchemaMap.get(shadowIdxId);
-                List<Integer> clusterKeyIndexes = null;
+                List<Integer> clusterKeyUids = null;
                 if (shadowIdxId == tbl.getBaseIndexId() || isShadowIndexOfBase(shadowIdxId, tbl)) {
-                    clusterKeyIndexes = OlapTable.getClusterKeyIndexes(shadowSchema);
+                    clusterKeyUids = OlapTable.getClusterKeyUids(shadowSchema);
                 }
                 int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash;
                 int shadowSchemaVersion = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaVersion;
@@ -277,7 +277,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio
                                             tbl.getEnableMowLightDelete(),
                                             tbl.getInvertedIndexFileStorageFormat(),
                                             tbl.rowStorePageSize(),
-                                            tbl.variantEnableFlattenNested(), clusterKeyIndexes,
+                                            tbl.variantEnableFlattenNested(), clusterKeyUids,
                                             tbl.storagePageSize());
                     requestBuilder.addTabletMetas(builder);
                 } // end for rollupTablets
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 58e3e05e214a9d..a624dc3c733374 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -287,9 +287,9 @@ protected void createShadowIndexReplica() throws AlterCancelException {
 
                     short shadowShortKeyColumnCount = indexShortKeyMap.get(shadowIdxId);
                     List<Column> shadowSchema = indexSchemaMap.get(shadowIdxId);
-                    List<Integer> clusterKeyIndexes = null;
+                    List<Integer> clusterKeyUids = null;
                     if (shadowIdxId == tbl.getBaseIndexId() || isShadowIndexOfBase(shadowIdxId, tbl)) {
-                        clusterKeyIndexes = OlapTable.getClusterKeyIndexes(shadowSchema);
+                        clusterKeyUids = OlapTable.getClusterKeyUids(shadowSchema);
                     }
                     int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash;
                     long originIndexId = indexIdMap.get(shadowIdxId);
@@ -340,10 +340,10 @@ protected void createShadowIndexReplica() throws AlterCancelException {
                             }
                             createReplicaTask.setInvertedIndexFileStorageFormat(tbl
                                                     .getInvertedIndexFileStorageFormat());
-                            if (!CollectionUtils.isEmpty(clusterKeyIndexes)) {
-                                createReplicaTask.setClusterKeyIndexes(clusterKeyIndexes);
-                                LOG.info("table: {}, partition: {}, index: {}, tablet: {}, cluster key indexes: {}",
-                                        tableId, partitionId, shadowIdxId, shadowTabletId, clusterKeyIndexes);
+                            if (!CollectionUtils.isEmpty(clusterKeyUids)) {
+                                createReplicaTask.setClusterKeyUids(clusterKeyUids);
+                                LOG.info("table: {}, partition: {}, index: {}, tablet: {}, cluster key uids: {}",
+                                        tableId, partitionId, shadowIdxId, shadowTabletId, clusterKeyUids);
                             }
                             batchTask.addTask(createReplicaTask);
                         } // end for rollupReplicas
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index f1abb0c9e632bc..6dfd02b3a42648 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1312,9 +1312,9 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc
             MaterializedIndexMeta indexMeta = localTbl.getIndexMetaByIndexId(restoredIdx.getId());
             List<Index> indexes = restoredIdx.getId() == localTbl.getBaseIndexId()
                                     ? localTbl.getCopiedIndexes() : null;
-            List<Integer> clusterKeyIndexes = null;
+            List<Integer> clusterKeyUids = null;
             if (indexMeta.getIndexId() == localTbl.getBaseIndexId() || localTbl.isShadowIndex(indexMeta.getIndexId())) {
-                clusterKeyIndexes = OlapTable.getClusterKeyIndexes(indexMeta.getSchema());
+                clusterKeyUids = OlapTable.getClusterKeyUids(indexMeta.getSchema());
             }
             for (Tablet restoreTablet : restoredIdx.getTablets()) {
                 TabletRef baseTabletRef = tabletBases == null ? null : tabletBases.get(restoreTablet.getId());
@@ -1363,11 +1363,11 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc
                         LOG.info("set base tablet {} for replica {} in restore job {}, tablet id={}",
                                 baseTabletRef.tabletId, restoreReplica.getId(), jobId, restoreTablet.getId());
                     }
-                    if (!CollectionUtils.isEmpty(clusterKeyIndexes)) {
-                        task.setClusterKeyIndexes(clusterKeyIndexes);
-                        LOG.info("table: {}, partition: {}, index: {}, tablet: {}, cluster key indexes: {}",
+                    if (!CollectionUtils.isEmpty(clusterKeyUids)) {
+                        task.setClusterKeyUids(clusterKeyUids);
+                        LOG.info("table: {}, partition: {}, index: {}, tablet: {}, cluster key uids: {}",
                                 localTbl.getId(), restorePart.getId(), restoredIdx.getId(), restoreTablet.getId(),
-                                clusterKeyIndexes);
+                                clusterKeyUids);
                     }
                     batchTask.addTask(task);
                 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index e4adf1f03963e9..c1d62a4be0d4d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -3292,14 +3292,14 @@ private static List<Long> getVisibleVersionFromMeta(List<Long> dbIds, List<Long>
         }
     }
 
-    public static List<Integer> getClusterKeyIndexes(List<Column> columns) {
-        Map<Integer, Integer> clusterKeyIndexes = new TreeMap<>();
+    public static List<Integer> getClusterKeyUids(List<Column> columns) {
+        Map<Integer, Integer> clusterKeyUids = new TreeMap<>();
         for (Column column : columns) {
             if (column.isClusterKey()) {
-                clusterKeyIndexes.put(column.getClusterKeyId(), column.getUniqueId());
+                clusterKeyUids.put(column.getClusterKeyId(), column.getUniqueId());
             }
         }
-        return clusterKeyIndexes.isEmpty() ? null : new ArrayList<>(clusterKeyIndexes.values());
+        return clusterKeyUids.isEmpty() ? null : new ArrayList<>(clusterKeyUids.values());
     }
 
     public long getVisibleVersionTime() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index e14b4efb0d590d..fb0df9e488b3c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -156,10 +156,10 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa
             } else {
                 indexes = Lists.newArrayList();
             }
-            List<Integer> clusterKeyIndexes = null;
+            List<Integer> clusterKeyUids = null;
             if (indexId == tbl.getBaseIndexId()) {
-                // only base and shadow index need cluster key indexes
-                clusterKeyIndexes = OlapTable.getClusterKeyIndexes(columns);
+                // only base and shadow index need cluster key unique column ids
+                clusterKeyUids = OlapTable.getClusterKeyUids(columns);
             }
             Cloud.CreateTabletsRequest.Builder requestBuilder = Cloud.CreateTabletsRequest.newBuilder();
             List<String> rowStoreColumns =
@@ -180,7 +180,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa
                         tbl.getEnableMowLightDelete(),
                         tbl.getInvertedIndexFileStorageFormat(),
                         tbl.rowStorePageSize(),
-                        tbl.variantEnableFlattenNested(), clusterKeyIndexes,
+                        tbl.variantEnableFlattenNested(), clusterKeyUids,
                         tbl.storagePageSize());
                 requestBuilder.addTabletMetas(builder);
             }
@@ -231,7 +231,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
             Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction,
             List<Integer> rowStoreColumnUniqueIds, boolean enableMowLightDelete,
             TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat, long pageSize,
-            boolean variantEnableFlattenNested, List<Integer> clusterKeyIdxes,
+            boolean variantEnableFlattenNested, List<Integer> clusterKeyUids,
             long storagePageSize) throws DdlException {
         OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder();
         builder.setTableId(tableId);
@@ -365,8 +365,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
         schemaBuilder.setRowStorePageSize(pageSize);
         schemaBuilder.setStoragePageSize(storagePageSize);
         schemaBuilder.setEnableVariantFlattenNested(variantEnableFlattenNested);
-        if (!CollectionUtils.isEmpty(clusterKeyIdxes)) {
-            schemaBuilder.addAllClusterKeyIdxes(clusterKeyIdxes);
+        if (!CollectionUtils.isEmpty(clusterKeyUids)) {
+            schemaBuilder.addAllClusterKeyUids(clusterKeyUids);
         }
 
         OlapFile.TabletSchemaCloudPB schema = schemaBuilder.build();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 65f77fd039137d..212ee89fcea84c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2151,10 +2151,10 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa
             short shortKeyColumnCount = indexMeta.getShortKeyColumnCount();
             TStorageType storageType = indexMeta.getStorageType();
             List<Column> schema = indexMeta.getSchema();
-            List<Integer> clusterKeyIndexes = null;
+            List<Integer> clusterKeyUids = null;
             if (indexId == tbl.getBaseIndexId()) {
-                // only base and shadow index need cluster key indexes
-                clusterKeyIndexes = OlapTable.getClusterKeyIndexes(schema);
+                // only base and shadow index need cluster key unique column ids
+                clusterKeyUids = OlapTable.getClusterKeyUids(schema);
             }
             KeysType keysType = indexMeta.getKeysType();
             List<Index> indexes = indexId == tbl.getBaseIndexId() ? tbl.getCopiedIndexes() : null;
@@ -2188,10 +2188,10 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa
 
                     task.setStorageFormat(tbl.getStorageFormat());
                     task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat());
-                    if (!CollectionUtils.isEmpty(clusterKeyIndexes)) {
-                        task.setClusterKeyIndexes(clusterKeyIndexes);
-                        LOG.info("table: {}, partition: {}, index: {}, tablet: {}, cluster key indexes: {}",
-                                tbl.getId(), partitionId, indexId, tabletId, clusterKeyIndexes);
+                    if (!CollectionUtils.isEmpty(clusterKeyUids)) {
+                        task.setClusterKeyUids(clusterKeyUids);
+                        LOG.info("table: {}, partition: {}, index: {}, tablet: {}, cluster key uids: {}",
+                                tbl.getId(), partitionId, indexId, tabletId, clusterKeyUids);
                     }
                     batchTask.addTask(task);
                     // add to AgentTaskQueue for handling finish report.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 06047e2cf16682..08e4844863a1b7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -981,13 +981,13 @@ private static void deleteFromMeta(ListMultimap<Long, Long> tabletDeleteFromMeta
                                     createReplicaTask.setInvertedIndexFileStorageFormat(olapTable
                                                                 .getInvertedIndexFileStorageFormat());
                                     if (indexId == olapTable.getBaseIndexId() || olapTable.isShadowIndex(indexId)) {
-                                        List<Integer> clusterKeyIndexes = OlapTable.getClusterKeyIndexes(
+                                        List<Integer> clusterKeyUids = OlapTable.getClusterKeyUids(
                                                 indexMeta.getSchema());
-                                        if (!CollectionUtils.isEmpty(clusterKeyIndexes)) {
-                                            createReplicaTask.setClusterKeyIndexes(clusterKeyIndexes);
+                                        if (!CollectionUtils.isEmpty(clusterKeyUids)) {
+                                            createReplicaTask.setClusterKeyUids(clusterKeyUids);
                                             LOG.info("table: {}, partition: {}, index: {}, tablet: {}, "
-                                                            + "cluster key indexes: {}", tableId, partitionId, indexId,
-                                                    tabletId, clusterKeyIndexes);
+                                                            + "cluster key uids: {}", tableId, partitionId, indexId,
+                                                    tabletId, clusterKeyUids);
                                         }
                                     }
                                     createReplicaBatchTask.addTask(createReplicaTask);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 94b73ef286475a..90e709c96c3e5a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -125,7 +125,7 @@ public class CreateReplicaTask extends AgentTask {
     private boolean storeRowColumn;
 
     private BinlogConfig binlogConfig;
-    private List<Integer> clusterKeyIndexes;
+    private List<Integer> clusterKeyUids;
 
     private Map<Object, Object> objectPool;
     private List<Integer> rowStoreColumnUniqueIds;
@@ -276,8 +276,8 @@ public void setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat in
         this.invertedIndexFileStorageFormat = invertedIndexFileStorageFormat;
     }
 
-    public void setClusterKeyIndexes(List<Integer> clusterKeyIndexes) {
-        this.clusterKeyIndexes = clusterKeyIndexes;
+    public void setClusterKeyUids(List<Integer> clusterKeyUids) {
+        this.clusterKeyUids = clusterKeyUids;
     }
 
     public TCreateTabletReq toThrift() {
@@ -337,10 +337,10 @@ public TCreateTabletReq toThrift() {
         tSchema.setSequenceColIdx(sequenceCol);
         tSchema.setVersionColIdx(versionCol);
         tSchema.setRowStoreColCids(rowStoreColumnUniqueIds);
-        if (!CollectionUtils.isEmpty(clusterKeyIndexes)) {
-            tSchema.setClusterKeyIdxes(clusterKeyIndexes);
+        if (!CollectionUtils.isEmpty(clusterKeyUids)) {
+            tSchema.setClusterKeyUids(clusterKeyUids);
             if (LOG.isDebugEnabled()) {
-                LOG.debug("cluster key index={}, table_id={}, tablet_id={}", clusterKeyIndexes, tableId, tabletId);
+                LOG.debug("cluster key uids={}, table_id={}, tablet_id={}", clusterKeyUids, tableId, tabletId);
             }
         }
         if (CollectionUtils.isNotEmpty(indexes)) {
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 41f8727ed1b301..2c378fe2d458e0 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -392,7 +392,7 @@ message TabletSchemaPB {
     repeated string partial_update_input_columns = 21; // deprecated
     optional bool enable_single_replica_compaction = 22 [default=false];
     optional bool skip_write_index_on_load = 23 [default=false];
-    repeated int32 cluster_key_idxes = 24;
+    repeated int32 cluster_key_uids = 24;
     optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1];
     // column unique ids for row store columns
     repeated int32 row_store_column_unique_ids = 26;
@@ -425,7 +425,7 @@ message TabletSchemaCloudPB {
     reserved 21; // deprecated partial_update_input_columns
     optional bool enable_single_replica_compaction = 22 [default=false];
     optional bool skip_write_index_on_load = 23 [default=false];
-    repeated int32 cluster_key_idxes = 24;
+    repeated int32 cluster_key_uids = 24;
     optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1];
     // column unique ids for row store columns
     repeated int32 row_store_column_unique_ids = 26;
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index fdbf4483bf8c4c..7bb24ba8d4d9da 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -44,7 +44,7 @@ struct TTabletSchema {
     16: optional bool store_row_column = false
     17: optional bool enable_single_replica_compaction = false
     18: optional bool skip_write_index_on_load = false
-    19: optional list<i32> cluster_key_idxes
+    19: optional list<i32> cluster_key_uids
     // col unique id for row store column
     20: optional list<i32> row_store_col_cids
     21: optional i64 row_store_page_size = 16384
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index aca1596f863769..246558eaf06d07 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -1032,6 +1032,8 @@ class Config {
             List<String> excludeCases = ["test_table_properties", "test_create_table"
                 , "test_default_hll", "test_default_pi", "test_default_bitmap_empty"
                 , "test_full_compaction", "test_full_compaction_by_table_id"
+                // schema change
+                , "test_alter_muti_modify_column"
                 // partial update
                 , "txn_insert", "test_update_schema_change", "test_generated_column_update", "test_nested_type_with_rowstore", "test_partial_update_generated_column", "nereids_partial_update_native_insert_stmt"
                 , "partial_update", "nereids_update_on_current_timestamp", "update_on_current_timestamp", "nereids_delete_mow_partial_update", "delete_mow_partial_update", "test_unique_table_auto_inc"

From 5ce7604934b364216ea8ff41c984feee903a502b Mon Sep 17 00:00:00 2001
From: seawinde <wusi@selectdb.com>
Date: Thu, 5 Dec 2024 23:13:19 +0800
Subject: [PATCH 191/399] [fix](mtmv) Fix rewrite fail by materialized view
 when filter or join condition has alias (#44779)

### What problem does this PR solve?

Related PR: #27922

Problem Summary:
query and mv def are as following,` partsupp.public_col as public_col `
is alias, this would cause rewritting fail by materialized view with
msg, the graph logic between query and view is different.

      select
      o_custkey,
      o_orderdate,
      o_shippriority,
      o_comment,
      o_orderkey,
      orders.public_col as col1,
      l_orderkey,
      l_partkey,
      l_suppkey,
      lineitem.public_col as col2,
      ps_partkey,
      ps_suppkey,
      partsupp.public_col as col3,
      partsupp.public_col * 2 as col4,
      o_orderkey + l_orderkey + ps_partkey * 2,
      sum(
        o_orderkey + l_orderkey + ps_partkey * 2
      ),
      count() as count_all
    from
      (
        select
          o_custkey,
          o_orderdate,
          o_shippriority,
          o_comment,
          o_orderkey,
          orders.public_col as public_col
        from
          orders
      ) orders
      left join (
        select
          l_orderkey,
          l_partkey,
          l_suppkey,
          lineitem.public_col as public_col
        from
          lineitem
        where
          lineitem.public_col is null
          or lineitem.public_col <> 1
      ) lineitem on l_orderkey = o_orderkey
      inner join (
        select
          ps_partkey,
          ps_suppkey,
          partsupp.public_col as public_col
        from
          partsupp
      ) partsupp on ps_partkey = o_orderkey
    where
      lineitem.public_col is null
      or lineitem.public_col <> 1
      and o_orderkey = 2
    group by
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14;

### Release note

Fix rewrite fail by materialized view when filter or join condition has
alias
---
 .../joinorder/hypergraph/HyperElement.java    |  27 +
 .../jobs/joinorder/hypergraph/edge/Edge.java  |   4 +-
 .../hypergraph/node/AbstractNode.java         |   8 +-
 .../exploration/mv/HyperGraphComparator.java  | 103 +++-
 .../mv/LogicalCompatibilityContext.java       |  74 ++-
 .../rules/exploration/mv/StructInfo.java      | 107 ++--
 .../aggregate_without_roll_up.out             |  24 +
 .../aggregate_without_roll_up.groovy          | 463 +++++++++++++++++-
 .../range_datetime_part_up_rewrite.groovy     |  20 +-
 9 files changed, 738 insertions(+), 92 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java
new file mode 100644
index 00000000000000..6d8d7c6326c0d9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.jobs.joinorder.hypergraph;
+
+/**
+ * This is the common base class for all
+ * */
+public interface HyperElement {
+
+    // Get the references nodes
+    long getReferenceNodes();
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
index 7698d881c661aa..f75ed83250119a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.jobs.joinorder.hypergraph.edge;
 
 import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
@@ -32,7 +33,7 @@
 /**
  * Edge in HyperGraph
  */
-public abstract class Edge {
+public abstract class Edge implements HyperElement {
     private final int index;
     private final double selectivity;
 
@@ -192,6 +193,7 @@ public boolean isSub(Edge edge) {
         return LongBitmap.isSubset(getReferenceNodes(), otherBitmap);
     }
 
+    @Override
     public long getReferenceNodes() {
         return LongBitmap.newBitmapUnion(leftExtendedNodes, rightExtendedNodes);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java
index a4a64e0449deee..686576de771d94 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java
@@ -17,6 +17,7 @@
 
 package org.apache.doris.nereids.jobs.joinorder.hypergraph.node;
 
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
@@ -33,7 +34,7 @@
 /**
  * HyperGraph Node.
  */
-public class AbstractNode {
+public class AbstractNode implements HyperElement {
     protected final int index;
     protected final List<JoinEdge> joinEdges;
     protected final List<FilterEdge> filterEdges;
@@ -65,6 +66,11 @@ public List<Edge> getEdges() {
                 .build();
     }
 
+    @Override
+    public long getReferenceNodes() {
+        return getNodeMap();
+    }
+
     public int getIndex() {
         return index;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
index d4594583c314c2..22282a2351627b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
@@ -19,12 +19,14 @@
 
 import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.ConflictRulesMaker;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
+import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition;
 import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -51,6 +53,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
@@ -79,9 +82,9 @@ public class HyperGraphComparator {
     private final Map<Edge, List<? extends Expression>> pullUpViewExprWithEdge = new HashMap<>();
     private final LogicalCompatibilityContext logicalCompatibilityContext;
     // this records the slots which needs to reject null
-    // the key is the target join which should reject null, the value is a pair, the first value of the pair is the
-    // join type, the second value is also a pair which left represents the slots in the left of join that should
-    // reject null, right represents the slots in the right of join that should reject null.
+    // the key is the view join edge which should reject null, the value is a pair, the first value of the pair is the
+    // query join type, the second value is also a pair which left represents the slots in the left of view join that
+    // should reject null, right represents the slots in the right of view join that should reject null.
     private final Map<JoinEdge, Pair<JoinType, Pair<Set<Slot>, Set<Slot>>>> inferredViewEdgeWithCond = new HashMap<>();
     private List<JoinEdge> viewJoinEdgesAfterInferring;
     private List<FilterEdge> viewFilterEdgesAfterInferring;
@@ -249,9 +252,17 @@ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode view) {
         }
         int size = queryExprSetList.size();
         for (int i = 0; i < size; i++) {
-            Set<Expression> mappingQueryExprSet = queryExprSetList.get(i).stream()
-                    .map(logicalCompatibilityContext::getViewNodeExprFromQuery)
-                    .collect(Collectors.toSet());
+            Set<Expression> queryExpressions = queryExprSetList.get(i);
+            Set<Expression> mappingQueryExprSet = new HashSet<>();
+            for (Expression queryExpression : queryExpressions) {
+                Optional<Expression> mappingViewExprByQueryExpr = getMappingViewExprByQueryExpr(queryExpression, query,
+                        this.logicalCompatibilityContext,
+                        ExpressionPosition.NODE);
+                if (!mappingViewExprByQueryExpr.isPresent()) {
+                    return false;
+                }
+                mappingQueryExprSet.add(mappingViewExprByQueryExpr.get());
+            }
             if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) {
                 return false;
             }
@@ -407,7 +418,10 @@ private Map<Edge, Edge> constructQueryToViewJoinMapWithExpr() {
             if (edgeMap.containsKey(entry.getValue())) {
                 continue;
             }
-            Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(entry.getKey());
+            Expression viewExpr = getMappingViewExprByQueryExpr(entry.getKey(),
+                    entry.getValue(),
+                    logicalCompatibilityContext,
+                    ExpressionPosition.JOIN_EDGE).orElse(null);
             if (viewExprToEdge.containsKey(viewExpr)) {
                 edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
             }
@@ -441,15 +455,19 @@ private Map<Edge, Edge> constructQueryToViewFilterMapWithExpr() {
 
         HashMap<Edge, Edge> queryToViewEdgeMap = new HashMap<>();
         for (Entry<Expression, Collection<Edge>> entry : queryExprToEdge.asMap().entrySet()) {
-            Expression queryExprViewBased = logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey());
-            if (queryExprViewBased == null) {
-                continue;
-            }
-            Collection<Edge> viewEdges = viewExprToEdge.get(queryExprViewBased);
-            if (viewEdges.isEmpty()) {
-                continue;
-            }
+            Expression queryExprViewBased = null;
             for (Edge queryEdge : entry.getValue()) {
+                queryExprViewBased = getMappingViewExprByQueryExpr(entry.getKey(),
+                        queryEdge,
+                        logicalCompatibilityContext,
+                        ExpressionPosition.FILTER_EDGE).orElse(null);
+                if (queryExprViewBased == null) {
+                    continue;
+                }
+                Collection<Edge> viewEdges = viewExprToEdge.get(queryExprViewBased);
+                if (viewEdges.isEmpty()) {
+                    continue;
+                }
                 for (Edge viewEdge : viewEdges) {
                     if (!isSubTreeNodesEquals(queryEdge, viewEdge, logicalCompatibilityContext)) {
                         // Such as query filter edge is <{1} --FILTER-- {}> but view filter edge is
@@ -514,17 +532,17 @@ private boolean compareEdgeWithNode(Edge query, Edge view) {
     }
 
     private boolean compareFilterEdgeWithNode(FilterEdge query, FilterEdge view) {
-        return rewriteQueryNodeMap(query.getReferenceNodes()) == view.getReferenceNodes();
+        return getViewNodesByQuery(query.getReferenceNodes()) == view.getReferenceNodes();
     }
 
     private boolean compareJoinEdgeWithNode(JoinEdge query, JoinEdge view) {
         boolean res = false;
         if (query.getJoinType().swap() == view.getJoinType()) {
-            res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getRightExtendedNodes()
-                    && rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getLeftExtendedNodes();
+            res |= getViewNodesByQuery(query.getLeftExtendedNodes()) == view.getRightExtendedNodes()
+                    && getViewNodesByQuery(query.getRightExtendedNodes()) == view.getLeftExtendedNodes();
         }
-        res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes()
-                && rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getRightExtendedNodes();
+        res |= getViewNodesByQuery(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes()
+                && getViewNodesByQuery(query.getRightExtendedNodes()) == view.getRightExtendedNodes();
         return res;
     }
 
@@ -547,8 +565,8 @@ private boolean compareJoinEdgeOrInfer(JoinEdge query, JoinEdge view) {
     }
 
     private boolean tryInferEdge(JoinEdge query, JoinEdge view) {
-        if (rewriteQueryNodeMap(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes()
-                || rewriteQueryNodeMap(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) {
+        if (getViewNodesByQuery(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes()
+                || getViewNodesByQuery(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) {
             return false;
         }
         if (!query.getJoinType().equals(view.getJoinType())) {
@@ -569,7 +587,7 @@ private boolean tryInferEdge(JoinEdge query, JoinEdge view) {
         return true;
     }
 
-    private long rewriteQueryNodeMap(long bitmap) {
+    private long getViewNodesByQuery(long bitmap) {
         long newBitmap = LongBitmap.newBitmap();
         for (int i : LongBitmap.getIterator(bitmap)) {
             int newIdx = getQueryToViewNodeIdMap().getOrDefault(i, 0);
@@ -578,6 +596,35 @@ private long rewriteQueryNodeMap(long bitmap) {
         return newBitmap;
     }
 
+    private Optional<Expression> getMappingViewExprByQueryExpr(Expression queryExpression,
+            HyperElement queryExpressionBelongedHyperElement,
+            LogicalCompatibilityContext context,
+            ExpressionPosition expressionPosition) {
+        Expression queryShuttledExpr;
+        Collection<Pair<Expression, HyperElement>> viewExpressions;
+        if (ExpressionPosition.JOIN_EDGE.equals(expressionPosition)) {
+            queryShuttledExpr = context.getQueryJoinShuttledExpr(queryExpression);
+            viewExpressions = context.getViewJoinExprFromQuery(queryShuttledExpr);
+        } else if (ExpressionPosition.FILTER_EDGE.equals(expressionPosition)) {
+            queryShuttledExpr = context.getQueryFilterShuttledExpr(queryExpression);
+            viewExpressions = context.getViewFilterExprFromQuery(queryShuttledExpr);
+        } else {
+            queryShuttledExpr = context.getQueryNodeShuttledExpr(queryExpression);
+            viewExpressions = context.getViewNodeExprFromQuery(queryShuttledExpr);
+        }
+        if (viewExpressions.size() == 1) {
+            return Optional.of(viewExpressions.iterator().next().key());
+        }
+        long queryReferenceNodes = queryExpressionBelongedHyperElement.getReferenceNodes();
+        long viewReferenceNodes = getViewNodesByQuery(queryReferenceNodes);
+        for (Pair<Expression, HyperElement> viewExpressionPair : viewExpressions) {
+            if (viewExpressionPair.value().getReferenceNodes() == viewReferenceNodes) {
+                return Optional.of(viewExpressionPair.key());
+            }
+        }
+        return Optional.empty();
+    }
+
     private void compareJoinEdgeWithExpr(Edge query, Edge view) {
         Set<? extends Expression> queryExprSet = query.getExpressionSet();
         Set<? extends Expression> viewExprSet = view.getExpressionSet();
@@ -585,7 +632,10 @@ private void compareJoinEdgeWithExpr(Edge query, Edge view) {
         Set<Expression> exprMappedOfView = new HashSet<>();
         List<Expression> residualQueryExpr = new ArrayList<>();
         for (Expression queryExpr : queryExprSet) {
-            Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(queryExpr);
+            Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr,
+                    query,
+                    logicalCompatibilityContext,
+                    ExpressionPosition.JOIN_EDGE).orElse(null);
             if (viewExprSet.contains(viewExpr)) {
                 exprMappedOfView.add(viewExpr);
             } else {
@@ -604,7 +654,10 @@ private void compareFilterEdgeWithExpr(Edge query, Edge view) {
         Set<Expression> exprMappedOfView = new HashSet<>();
         List<Expression> residualQueryExpr = new ArrayList<>();
         for (Expression queryExpr : queryExprSet) {
-            Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(queryExpr);
+            Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr,
+                    query,
+                    logicalCompatibilityContext,
+                    ExpressionPosition.FILTER_EDGE).orElse(null);
             if (viewExprSet.contains(viewExpr)) {
                 exprMappedOfView.add(viewExpr);
             } else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
index ca13c9701dabc2..77ab37873d06b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
@@ -17,6 +17,8 @@
 
 package org.apache.doris.nereids.rules.exploration.mv;
 
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
 import org.apache.doris.nereids.memo.GroupExpression;
 import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition;
@@ -36,8 +38,10 @@
 import com.google.common.base.Suppliers;
 import com.google.common.collect.BiMap;
 import com.google.common.collect.HashBiMap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
 
-import java.util.HashMap;
+import java.util.Collection;
 import java.util.Map;
 import java.util.function.Supplier;
 
@@ -48,11 +52,15 @@ public class LogicalCompatibilityContext {
     private final BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping;
     private final BiMap<Integer, Integer> queryToViewNodeIDMapping;
     private final ObjectId planNodeId;
-    private final Supplier<BiMap<Expression, Expression>> queryToViewJoinEdgeExpressionMappingSupplier;
-    private final Supplier<BiMap<Expression, Expression>> queryToViewNodeExpressionMappingSupplier;
-    private final Supplier<BiMap<Expression, Expression>> queryToViewFilterEdgeExpressionMappingSupplier;
-    @Deprecated
-    private BiMap<Expression, Expression> queryToViewAllExpressionMapping;
+    private final Supplier<Multimap<Expression, Pair<Expression, HyperElement>>>
+            queryToViewJoinEdgeExpressionMappingSupplier;
+    private final Supplier<Map<Expression, Expression>> queryToQueryShuttledJoinExpressionMappingSupplier;
+    private final Supplier<Multimap<Expression, Pair<Expression, HyperElement>>>
+            queryToViewNodeExpressionMappingSupplier;
+    private final Supplier<Map<Expression, Expression>> queryToQueryShuttledNodeExpressionMappingSupplier;
+    private final Supplier<Multimap<Expression, Pair<Expression, HyperElement>>>
+            queryToViewFilterEdgeExpressionMappingSupplier;
+    private final Supplier<Map<Expression, Expression>> queryToQueryShuttledFilterExpressionMappingSupplier;
 
     /**
      * LogicalCompatibilityContext
@@ -66,16 +74,25 @@ private LogicalCompatibilityContext(BiMap<StructInfoNode, StructInfoNode> queryT
                         queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE),
                         viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE)));
 
+        this.queryToQueryShuttledJoinExpressionMappingSupplier = Suppliers.memoize(
+                () -> queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.JOIN_EDGE));
+
         this.queryToViewNodeExpressionMappingSupplier =
                 Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping,
                         queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE),
                         viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE)));
 
+        this.queryToQueryShuttledNodeExpressionMappingSupplier = Suppliers.memoize(
+                () -> queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.NODE));
+
         this.queryToViewFilterEdgeExpressionMappingSupplier =
                 Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping,
                         queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE),
                         viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE)));
 
+        this.queryToQueryShuttledFilterExpressionMappingSupplier = Suppliers.memoize(
+                () -> queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.FILTER_EDGE));
+
         this.queryToViewNodeMapping = queryToViewNodeMapping;
         this.queryToViewNodeIDMapping = HashBiMap.create();
         queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex()));
@@ -92,18 +109,30 @@ public BiMap<Integer, Integer> getQueryToViewNodeIDMapping() {
         return queryToViewNodeIDMapping;
     }
 
-    public Expression getViewJoinExprFromQuery(Expression queryJoinExpr) {
+    public Collection<Pair<Expression, HyperElement>> getViewJoinExprFromQuery(Expression queryJoinExpr) {
         return queryToViewJoinEdgeExpressionMappingSupplier.get().get(queryJoinExpr);
     }
 
-    public Expression getViewFilterExprFromQuery(Expression queryJoinExpr) {
+    public Expression getQueryJoinShuttledExpr(Expression queryJoinExpr) {
+        return queryToQueryShuttledJoinExpressionMappingSupplier.get().get(queryJoinExpr);
+    }
+
+    public Collection<Pair<Expression, HyperElement>> getViewFilterExprFromQuery(Expression queryJoinExpr) {
         return queryToViewFilterEdgeExpressionMappingSupplier.get().get(queryJoinExpr);
     }
 
-    public Expression getViewNodeExprFromQuery(Expression queryJoinExpr) {
+    public Expression getQueryFilterShuttledExpr(Expression queryFilterExpr) {
+        return queryToQueryShuttledFilterExpressionMappingSupplier.get().get(queryFilterExpr);
+    }
+
+    public Collection<Pair<Expression, HyperElement>> getViewNodeExprFromQuery(Expression queryJoinExpr) {
         return queryToViewNodeExpressionMappingSupplier.get().get(queryJoinExpr);
     }
 
+    public Expression getQueryNodeShuttledExpr(Expression queryNodeExpr) {
+        return queryToQueryShuttledNodeExpressionMappingSupplier.get().get(queryNodeExpr);
+    }
+
     /**
      * Generate logical compatibility context,
      * this make expression mapping between query and view by relation and the slot in relation mapping
@@ -134,24 +163,31 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping,
                 viewStructInfo);
     }
 
-    private static BiMap<Expression, Expression> generateExpressionMapping(
+    /**
+     * The result is multimap
+     * the key is shuttled query expr
+     * the value is original view expr collection
+     * */
+    private static Multimap<Expression, Pair<Expression, HyperElement>> generateExpressionMapping(
             Map<SlotReference, SlotReference> viewToQuerySlotMapping,
-            Map<Expression, Expression> queryShuttledExprToExprMap,
-            Map<Expression, Expression> viewShuttledExprToExprMap) {
-        final Map<Expression, Expression> viewEdgeToConjunctsMapQueryBased = new HashMap<>();
-        BiMap<Expression, Expression> queryToViewEdgeMapping = HashBiMap.create();
+            Multimap<Expression, Pair<Expression, HyperElement>> queryShuttledExprToExprMap,
+            Multimap<Expression, Pair<Expression, HyperElement>> viewShuttledExprToExprMap) {
+        Multimap<Expression, Pair<Expression, HyperElement>> queryToViewEdgeMapping = HashMultimap.create();
         if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap == null
                 || queryShuttledExprToExprMap.isEmpty() || viewShuttledExprToExprMap.isEmpty()) {
             return queryToViewEdgeMapping;
         }
+        final Multimap<Expression, Pair<Expression, HyperElement>> viewShuttledExprToExprMapQueryBased =
+                HashMultimap.create();
         viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> {
-            viewEdgeToConjunctsMapQueryBased.put(
+            viewShuttledExprToExprMapQueryBased.put(
                     orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), expr);
         });
-        queryShuttledExprToExprMap.forEach((exprSet, edge) -> {
-            Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(orderSlotAsc(exprSet));
-            if (viewExpr != null) {
-                queryToViewEdgeMapping.put(edge, viewExpr);
+        queryShuttledExprToExprMap.forEach((shuttledExpr, expr) -> {
+            Collection<Pair<Expression, HyperElement>> viewExpressions = viewShuttledExprToExprMapQueryBased.get(
+                    orderSlotAsc(shuttledExpr));
+            if (viewExpressions != null) {
+                queryToViewEdgeMapping.putAll(shuttledExpr, viewExpressions);
             }
         });
         return queryToViewEdgeMapping;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 3de48dc7ff6abf..365360e06b096c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -23,10 +23,10 @@
 import org.apache.doris.mtmv.BaseTableInfo;
 import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.jobs.executor.Rewriter;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
-import org.apache.doris.nereids.memo.Group;
 import org.apache.doris.nereids.memo.GroupExpression;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.TableQueryOperatorChecker;
 import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
@@ -65,12 +65,15 @@
 import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
+import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 
 import java.util.ArrayList;
 import java.util.BitSet;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -113,9 +116,23 @@ public class StructInfo {
     // split predicates is shuttled
     private SplitPredicate splitPredicate;
     private EquivalenceClass equivalenceClass;
-    // Key is the expression shuttled and the value is the origin expression
+    // For value of Map, the key is the position of expression
+    // the value is the expressions and the hyper element of expression pair
+    // Key of pair is the expression shuttled and the value is the origin expression and the hyper element it belonged
+    // Sometimes origin expressions are different and shuttled expression is same
+    // Such as origin expressions are l_partkey#0 > 1 and l_partkey#10 > 1 and shuttled expression is l_partkey#10 > 1
     // this is for building LogicalCompatibilityContext later.
-    private final Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap;
+    private final Map<ExpressionPosition, Multimap<Expression, Pair<Expression, HyperElement>>>
+            shuttledExpressionsToExpressionsMap;
+    // For value of Map, the key is the position of expression
+    // the value is the original expression and shuttled expression map
+    // Such as origin expressions are l_partkey#0 > 1 and shuttled expression is l_partkey#10 > 1
+    // the map would be {ExpressionPosition.FILTER, {
+    //     l_partkey#0 > 1 : l_partkey#10 > 1
+    // }}
+    // this is for building LogicalCompatibilityContext later.
+    private final Map<ExpressionPosition, Map<Expression, Expression>> expressionToShuttledExpressionToMap;
+
     // Record the exprId and the corresponding expr map, this is used by expression shuttled
     private final Map<ExprId, Expression> namedExprIdAndExprMapping;
     private final List<? extends Expression> planOutputShuttledExpressions;
@@ -127,7 +144,9 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG
             Plan bottomPlan, List<CatalogRelation> relations,
             Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
             @Nullable Predicates predicates,
-            Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
+            Map<ExpressionPosition, Multimap<Expression, Pair<Expression, HyperElement>>>
+                    shuttledExpressionsToExpressionsMap,
+            Map<ExpressionPosition, Map<Expression, Expression>> expressionToShuttledExpressionToMap,
             Map<ExprId, Expression> namedExprIdAndExprMapping,
             BitSet tableIdSet,
             SplitPredicate splitPredicate,
@@ -146,6 +165,7 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG
         this.splitPredicate = splitPredicate;
         this.equivalenceClass = equivalenceClass;
         this.shuttledExpressionsToExpressionsMap = shuttledExpressionsToExpressionsMap;
+        this.expressionToShuttledExpressionToMap = expressionToShuttledExpressionToMap;
         this.namedExprIdAndExprMapping = namedExprIdAndExprMapping;
         this.planOutputShuttledExpressions = planOutputShuttledExpressions;
     }
@@ -156,7 +176,8 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG
     public StructInfo withPredicates(Predicates predicates) {
         return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan,
                 this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates,
-                this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, this.tableBitSet,
+                this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap,
+                this.namedExprIdAndExprMapping, this.tableBitSet,
                 null, null, this.planOutputShuttledExpressions);
     }
 
@@ -166,13 +187,16 @@ public StructInfo withPredicates(Predicates predicates) {
     public StructInfo withTableBitSet(BitSet tableBitSet) {
         return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan,
                 this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, this.predicates,
-                this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, tableBitSet,
+                this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap,
+                this.namedExprIdAndExprMapping, tableBitSet,
                 this.splitPredicate, this.equivalenceClass, this.planOutputShuttledExpressions);
     }
 
     private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
             Plan topPlan,
-            Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
+            Map<ExpressionPosition, Multimap<Expression, Pair<Expression, HyperElement>>>
+                    shuttledExpressionsToExpressionsMap,
+            Map<ExpressionPosition, Map<Expression, Expression>> expressionToShuttledExpressionToMap,
             Map<ExprId, Expression> namedExprIdAndExprMapping,
             List<CatalogRelation> relations,
             Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
@@ -200,8 +224,9 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
                     structInfoNode.getPlan().accept(ExpressionLineageReplacer.INSTANCE, replaceContext);
                     // Replace expressions by expression map
                     List<Expression> replacedExpressions = replaceContext.getReplacedExpressions();
-                    putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap,
-                            ExpressionPosition.NODE, replacedExpressions.get(0), expression);
+                    putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap,
+                            expressionToShuttledExpressionToMap,
+                            ExpressionPosition.NODE, replacedExpressions.get(0), expression, node);
                     // Record this, will be used in top level expression shuttle later, see the method
                     // ExpressionLineageReplacer#visitGroupPlan
                     namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap());
@@ -227,8 +252,10 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
             // Replace expressions by expression map
             List<Expression> replacedExpressions = replaceContext.getReplacedExpressions();
             for (int i = 0; i < replacedExpressions.size(); i++) {
-                putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap,
-                        ExpressionPosition.JOIN_EDGE, replacedExpressions.get(i), joinConjunctExpressions.get(i));
+                putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap,
+                        expressionToShuttledExpressionToMap,
+                        ExpressionPosition.JOIN_EDGE, replacedExpressions.get(i), joinConjunctExpressions.get(i),
+                        edge);
             }
             // Record this, will be used in top level expression shuttle later, see the method
             // ExpressionLineageReplacer#visitGroupPlan
@@ -240,10 +267,11 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
             filterExpressions.forEach(predicate -> {
                 // this is used for LogicalCompatibilityContext
                 ExpressionUtils.extractConjunction(predicate).forEach(expr ->
-                        putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap,
+                        putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap,
+                                expressionToShuttledExpressionToMap,
                                 ExpressionPosition.FILTER_EDGE,
                                 ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan, new BitSet()),
-                                predicate));
+                                predicate, filterEdge));
             });
         });
         return true;
@@ -315,11 +343,13 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable
         // collect struct info fromGraph
         List<CatalogRelation> relationList = new ArrayList<>();
         Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap = new LinkedHashMap<>();
-        Map<ExpressionPosition, Map<Expression, Expression>> shuttledHashConjunctsToConjunctsMap =
-                new LinkedHashMap<>();
+        Map<ExpressionPosition, Multimap<Expression, Pair<Expression, HyperElement>>>
+                shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>();
         Map<ExprId, Expression> namedExprIdAndExprMapping = new LinkedHashMap<>();
         BitSet tableBitSet = new BitSet();
+        Map<ExpressionPosition, Map<Expression, Expression>> expressionToShuttledExpressionToMap = new HashMap<>();
         boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan, shuttledHashConjunctsToConjunctsMap,
+                expressionToShuttledExpressionToMap,
                 namedExprIdAndExprMapping,
                 relationList,
                 relationIdStructInfoNodeMap,
@@ -341,19 +371,11 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable
                 ExpressionUtils.shuttleExpressionWithLineage(originalPlan.getOutput(), originalPlan, new BitSet());
         return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan,
                 relationList, relationIdStructInfoNodeMap, predicates, shuttledHashConjunctsToConjunctsMap,
+                expressionToShuttledExpressionToMap,
                 namedExprIdAndExprMapping, tableBitSet, null, null,
                 planOutputShuttledExpressions);
     }
 
-    /**
-     * Build Struct info from group.
-     * Maybe return multi structInfo when original plan already be rewritten by mv
-     */
-    public static StructInfo of(Group group) {
-        // TODO build graph from original plan and get relations and predicates from graph
-        return null;
-    }
-
     public List<CatalogRelation> getRelations() {
         return relations;
     }
@@ -410,21 +432,36 @@ public Map<RelationId, StructInfoNode> getRelationIdStructInfoNodeMap() {
         return relationIdStructInfoNodeMap;
     }
 
-    public Map<ExpressionPosition, Map<Expression, Expression>> getShuttledExpressionsToExpressionsMap() {
+    public Map<ExpressionPosition, Multimap<Expression, Pair<Expression, HyperElement>>>
+            getShuttledExpressionsToExpressionsMap() {
         return shuttledExpressionsToExpressionsMap;
     }
 
-    private static void putShuttledExpressionsToExpressionsMap(
-            Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
+    public Map<ExpressionPosition, Map<Expression, Expression>> getExpressionToShuttledExpressionToMap() {
+        return expressionToShuttledExpressionToMap;
+    }
+
+    private static void putShuttledExpressionToExpressionsMap(
+            Map<ExpressionPosition, Multimap<Expression, Pair<Expression, HyperElement>>>
+                    shuttledExpressionsToExpressionsMap,
+            Map<ExpressionPosition, Map<Expression, Expression>> expressionPositionToExpressionToMap,
             ExpressionPosition expressionPosition,
-            Expression key, Expression value) {
-        Map<Expression, Expression> expressionExpressionMap = shuttledExpressionsToExpressionsMap.get(
-                expressionPosition);
-        if (expressionExpressionMap == null) {
-            expressionExpressionMap = new LinkedHashMap<>();
-            shuttledExpressionsToExpressionsMap.put(expressionPosition, expressionExpressionMap);
-        }
-        expressionExpressionMap.put(key, value);
+            Expression shuttledExpression, Expression originalExpression, HyperElement valueBelongedElement) {
+        Multimap<Expression, Pair<Expression, HyperElement>> shuttledExpressionToExpressionMap =
+                shuttledExpressionsToExpressionsMap.get(expressionPosition);
+        if (shuttledExpressionToExpressionMap == null) {
+            shuttledExpressionToExpressionMap = HashMultimap.create();
+            shuttledExpressionsToExpressionsMap.put(expressionPosition, shuttledExpressionToExpressionMap);
+        }
+        shuttledExpressionToExpressionMap.put(shuttledExpression, Pair.of(originalExpression, valueBelongedElement));
+
+        Map<Expression, Expression> originalExprToShuttledExprMap =
+                expressionPositionToExpressionToMap.get(expressionPosition);
+        if (originalExprToShuttledExprMap == null) {
+            originalExprToShuttledExprMap = new HashMap<>();
+            expressionPositionToExpressionToMap.put(expressionPosition, originalExprToShuttledExprMap);
+        }
+        originalExprToShuttledExprMap.put(originalExpression, shuttledExpression);
     }
 
     public List<? extends Expression> getExpressions() {
diff --git a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
index 5c9df6b7f92256..c400e078daff1a 100644
--- a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
+++ b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
@@ -323,3 +323,27 @@ c	3	6	c,c,c	5.333333333333333	mi	3	2
 1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
 1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
 
+-- !query29_0_before --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
+-- !query29_0_after --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
+-- !query30_0_before --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
+-- !query30_0_after --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
+-- !query31_0_before --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
+-- !query31_0_after --
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	\N	\N	8	8	1
+1	2023-12-09	1	yy	2	2	2	4	3	\N	2	3	1	2	8	8	1
+
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
index 3af41bda69065f..356b96267a88f5 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
@@ -45,11 +45,9 @@ suite("aggregate_without_roll_up") {
       "replication_num" = "1"
     );
     """
-
     sql """
     drop table if exists lineitem
     """
-
     sql"""
     CREATE TABLE IF NOT EXISTS lineitem (
       l_orderkey    INTEGER NOT NULL,
@@ -76,11 +74,9 @@ suite("aggregate_without_roll_up") {
       "replication_num" = "1"
     )
     """
-
     sql """
     drop table if exists partsupp
     """
-
     sql """
     CREATE TABLE IF NOT EXISTS partsupp (
       ps_partkey     INTEGER NOT NULL,
@@ -1517,4 +1513,463 @@ suite("aggregate_without_roll_up") {
     order_qt_query28_0_after "${query28_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv28_0"""
 
+
+
+    // query and mv has the same filter but position is different, should rewrite successfully
+    def mv29_0 = """
+      select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey, 
+      orders.public_col as col1, 
+      l_orderkey, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey + l_orderkey + ps_partkey * 2, 
+      sum(
+        o_orderkey + l_orderkey + ps_partkey * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          lineitem.public_col is null 
+          or lineitem.public_col <> 1
+      ) lineitem on l_orderkey = o_orderkey 
+      inner join (
+        select 
+          ps_partkey, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on ps_partkey = o_orderkey
+    where 
+      lineitem.public_col is null 
+      or lineitem.public_col <> 1 
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    def query29_0 = """
+      select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey, 
+      orders.public_col as col1, 
+      l_orderkey, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey + l_orderkey + ps_partkey * 2, 
+      sum(
+        o_orderkey + l_orderkey + ps_partkey * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          lineitem.public_col is null 
+          or lineitem.public_col <> 1
+      ) lineitem on l_orderkey = o_orderkey 
+      inner join (
+        select 
+          ps_partkey, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on ps_partkey = o_orderkey
+    where 
+      lineitem.public_col is null 
+      or lineitem.public_col <> 1 
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    order_qt_query29_0_before "${query29_0}"
+    async_mv_rewrite_success(db, mv29_0, query29_0, "mv29_0")
+    order_qt_query29_0_after "${query29_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0"""
+
+
+    // query and mv has the same filter but position is different, should rewrite successfully
+    // mv join condition has alias
+    def mv30_0 = """
+      select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey_alias, 
+      orders.public_col as col1, 
+      l_orderkey_alias, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey_alias, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2, 
+      sum(
+        o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey as o_orderkey_alias, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey as l_orderkey_alias, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          lineitem.public_col is null 
+          or lineitem.public_col <> 1
+      ) lineitem on lineitem.l_orderkey_alias = orders.o_orderkey_alias 
+      inner join (
+        select 
+          ps_partkey as ps_partkey_alias, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on partsupp.ps_partkey_alias = orders.o_orderkey_alias
+    where 
+      lineitem.public_col is null 
+      or lineitem.public_col <> 1 
+      and o_orderkey_alias = 2
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    def query30_0 = """
+      select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey, 
+      orders.public_col as col1, 
+      l_orderkey, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey + l_orderkey + ps_partkey * 2, 
+      sum(
+        o_orderkey + l_orderkey + ps_partkey * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          lineitem.public_col is null 
+          or lineitem.public_col <> 1
+      ) lineitem on l_orderkey = o_orderkey 
+      inner join (
+        select 
+          ps_partkey, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on ps_partkey = o_orderkey
+    where 
+      lineitem.public_col is null 
+      or lineitem.public_col <> 1 
+      and o_orderkey = 2
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    order_qt_query30_0_before "${query30_0}"
+    async_mv_rewrite_success(db, mv30_0, query30_0, "mv30_0")
+    order_qt_query30_0_after "${query30_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv30_0"""
+
+
+    // query and mv has the same filter but position is different, should rewrite successfully
+    // query join condition has alias
+    def mv31_0 = """
+      select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey, 
+      orders.public_col as col1, 
+      l_orderkey, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey + l_orderkey + ps_partkey * 2, 
+      sum(
+        o_orderkey + l_orderkey + ps_partkey * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          lineitem.public_col is null 
+          or lineitem.public_col <> 1
+      ) lineitem on l_orderkey = o_orderkey 
+      inner join (
+        select 
+          ps_partkey, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on ps_partkey = o_orderkey
+    where 
+      lineitem.public_col is null 
+      or lineitem.public_col <> 1 
+      and o_orderkey = 2
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    def query31_0 = """
+select 
+      o_custkey, 
+      o_orderdate, 
+      o_shippriority, 
+      o_comment, 
+      o_orderkey_alias, 
+      orders.public_col as col1, 
+      l_orderkey_alias, 
+      l_partkey, 
+      l_suppkey, 
+      lineitem.public_col as col2, 
+      ps_partkey_alias, 
+      ps_suppkey, 
+      partsupp.public_col as col3, 
+      partsupp.public_col * 2 as col4, 
+      o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2, 
+      sum(
+        o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2
+      ), 
+      count() as count_all 
+    from 
+      (
+        select 
+          o_custkey, 
+          o_orderdate, 
+          o_shippriority, 
+          o_comment, 
+          o_orderkey as o_orderkey_alias, 
+          orders.public_col as public_col 
+        from 
+          orders
+      ) orders 
+      left join (
+        select 
+          l_orderkey as l_orderkey_alias, 
+          l_partkey, 
+          l_suppkey, 
+          lineitem.public_col as public_col 
+        from 
+          lineitem 
+        where 
+          lineitem.public_col is null 
+          or lineitem.public_col <> 1
+      ) lineitem on lineitem.l_orderkey_alias = orders.o_orderkey_alias 
+      inner join (
+        select 
+          ps_partkey as ps_partkey_alias, 
+          ps_suppkey, 
+          partsupp.public_col as public_col 
+        from 
+          partsupp
+      ) partsupp on partsupp.ps_partkey_alias = orders.o_orderkey_alias
+    where 
+      lineitem.public_col is null 
+      or lineitem.public_col <> 1 
+      and o_orderkey_alias = 2
+    group by 
+      1, 
+      2, 
+      3, 
+      4, 
+      5, 
+      6, 
+      7, 
+      8, 
+      9, 
+      10, 
+      11, 
+      12, 
+      13, 
+      14;
+    """
+    order_qt_query31_0_before "${query31_0}"
+    async_mv_rewrite_success(db, mv31_0, query31_0, "mv31_0")
+    order_qt_query31_0_after "${query31_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv31_0"""
 }
diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
index f8e601e64f5819..140a91edd7ca96 100644
--- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
@@ -168,7 +168,7 @@ suite("mtmv_range_datetime_part_up_rewrite") {
     for (int i = 0; i < mv_name_list.size(); i++) {
         def job_name = getJobName(db, mv_name_list[i])
         waitingMTMVTaskFinished(job_name)
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
@@ -178,13 +178,15 @@ suite("mtmv_range_datetime_part_up_rewrite") {
         (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-29 03:00:00')"""
     sql """alter table lineitem_range_datetime_union modify column l_comment set stats ('row_count'='8');"""
     for (int i = 0; i < mv_name_list.size(); i++) {
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        // both mv should rewrite success
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
     for (int i = 0; i < mv_name_list.size(); i++) {
         sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        // both mv should rewrite success
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
@@ -192,25 +194,29 @@ suite("mtmv_range_datetime_part_up_rewrite") {
         (3, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-29 03:00:00');"""
     sql """alter table lineitem_range_datetime_union modify column l_comment set stats ('row_count'='9');"""
     for (int i = 0; i < mv_name_list.size(); i++) {
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        // both mv should rewrite success
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
     for (int i = 0; i < mv_name_list.size(); i++) {
         sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        // both mv should rewrite success
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
     sql """ALTER TABLE lineitem_range_datetime_union DROP PARTITION IF EXISTS p4 FORCE"""
     for (int i = 0; i < mv_name_list.size(); i++) {
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        // both mv should rewrite success
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 
     for (int i = 0; i < mv_name_list.size(); i++) {
         sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
-        mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+        // both mv should rewrite success
+        mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
         compare_res(query_stmt_list[i] + " order by 1,2,3")
     }
 

From c1e4ebe5c0be76df40387fbbb9618ccdc81fe1e8 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Fri, 6 Dec 2024 02:00:06 +0800
Subject: [PATCH 192/399] [Chore](Job)Add scheduling logs for periodic jobs.
 (#44843)

### What problem does this PR solve?
To facilitate troubleshooting and ensure the scheduler is functioning
correctly.
---
 .../apache/doris/job/executor/TimerJobSchedulerTask.java  | 2 ++
 .../java/org/apache/doris/job/scheduler/JobScheduler.java | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/TimerJobSchedulerTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/TimerJobSchedulerTask.java
index 65a9cf2e091164..4269fa0d8f3c52 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/executor/TimerJobSchedulerTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/TimerJobSchedulerTask.java
@@ -48,6 +48,8 @@ public void run(Timeout timeout) {
                 log.warn("dispatch timer job failed, queue maybe full. job id is {}, job name is {}",
                         this.job.getJobId(), this.job.getJobName() + getMsgWhenExecuteQueueFull());
             }
+            log.info("dispatch timer job success, job id is {}, job name is {}", this.job.getJobId(),
+                    this.job.getJobName());
         } catch (Exception e) {
             log.warn("dispatch timer job error, task id is {}", this.job.getJobId(), e);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
index 7f8b39f1e66dc7..921f333791cb44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
@@ -146,6 +146,11 @@ public void close() throws IOException {
     private void cycleTimerJobScheduler(T job, long startTimeWindowMs) {
         List<Long> delaySeconds = job.getJobConfig().getTriggerDelayTimes(System.currentTimeMillis(),
                 startTimeWindowMs, latestBatchSchedulerTimerTaskTimeMs);
+        if (CollectionUtils.isEmpty(delaySeconds)) {
+            log.info("skip job {} scheduler timer job, delay seconds is empty", job.getJobName());
+            return;
+        }
+        log.info("job {} scheduler timer job, delay seconds size is {}", job.getJobName(), delaySeconds.size());
         if (CollectionUtils.isNotEmpty(delaySeconds)) {
             delaySeconds.forEach(delaySecond -> {
                 TimerJobSchedulerTask<T> timerJobSchedulerTask = new TimerJobSchedulerTask<>(timerJobDisruptor, job);
@@ -188,6 +193,8 @@ private void executeTimerJobIdsWithinLastTenMinutesWindow() {
             this.latestBatchSchedulerTimerTaskTimeMs = System.currentTimeMillis();
         }
         this.latestBatchSchedulerTimerTaskTimeMs += BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS;
+        log.info("execute timer job ids within last ten minutes window, last time window is {}",
+                TimeUtils.longToTimeString(lastTimeWindowMs));
         if (jobMap.isEmpty()) {
             return;
         }
@@ -209,6 +216,7 @@ private void clearEndJob(T job) {
         }
         try {
             Env.getCurrentEnv().getJobManager().unregisterJob(job.getJobId());
+            log.info("clear finish job, job id is {}, job name is {}", job.getJobId(), job.getJobName());
         } catch (JobException e) {
             log.error("clear finish job error, job id is {}", job.getJobId(), e);
         }

From 613b1ef8beb68954f2af442680018514ca367af7 Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Fri, 6 Dec 2024 09:31:17 +0800
Subject: [PATCH 193/399] [test] fix partition prune unstable cases (#44976)

Related PR: #44722

Problem Summary:
ENABLE_FOLD_CONSTANT_BY_BE is fuzzy, when ENABLE_FOLD_CONSTANT_BY_BE is
true, some cases will fail.
---
 .../partition_prune/int_as_date_partition_col.groovy             | 1 +
 .../partition_prune/one_col_range_partition.groovy               | 1 +
 .../partition_prune/varchar_as_date_partition_col.groovy         | 1 +
 3 files changed, 3 insertions(+)

diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy
index 5e65e6445ef5e5..9d4e2a16490512 100644
--- a/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/int_as_date_partition_col.groovy
@@ -16,6 +16,7 @@
 // under the License.
 
 suite("int_as_date_partition_col") {
+    sql "set ENABLE_FOLD_CONSTANT_BY_BE=false"
     sql "drop table if exists partition_int"
     sql """CREATE TABLE partition_int(a int, dt int) PARTITION BY range(dt) (
             partition p20240101 values less than ("20240101"),
diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy
index e8b7621e6d7ff7..4d1e3eb223b1a7 100644
--- a/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/one_col_range_partition.groovy
@@ -16,6 +16,7 @@
 // under the License.
 
 suite("one_col_range_partition") {
+    sql "set ENABLE_FOLD_CONSTANT_BY_BE=false"
     sql "drop table if exists one_col_range_partition_date"
     sql """
     create table one_col_range_partition_date(a int, dt datetime, d date, c varchar(100)) duplicate key(a)
diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy
index 0cfe4963f27b2a..a5b3a4375c12bc 100644
--- a/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/varchar_as_date_partition_col.groovy
@@ -16,6 +16,7 @@
 // under the License.
 
 suite("varchar_as_date_partition_col") {
+    sql "set ENABLE_FOLD_CONSTANT_BY_BE=false"
     sql"""drop table if exists partition_varchar;"""
     sql """CREATE TABLE partition_varchar(a int, dt varchar(10), rdt datetime) PARTITION BY list(dt) (
     partition p20240101 values in ("20240101","20240102"),

From da9b818b9156e09b19db02b28521310e56704c58 Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Fri, 6 Dec 2024 10:33:19 +0800
Subject: [PATCH 194/399] [enchement](mc)mc catalog append netowrk config
 (#44194)

### What problem does this PR solve?

Since there may be some network problems when accessing maxcompute, in
order to provide a better user experience, we introduce some parameters
to help users better adapt to the network.

You can add these parameters when creating a catalog:
`mc.connect_timeout` :  Connection timeout, default is 10s.
`mc.read_timeout` : Read timeout,default is 120s.
`mc.retry_count` : Number of retries after failure,default is 4.

### Release note

MaxCompute catalog add three parameters : `mc.connect_timeout` ,
`mc.read_timeout`,`mc.retry_count` help users better adapt to the
network between MaxCompute and Doris.
---
 .../format/table/max_compute_jni_reader.cpp   |  6 +-
 .../maxcompute/MaxComputeJniScanner.java      | 47 ++++++++++++---
 .../maxcompute/MaxComputeExternalCatalog.java | 57 +++++++++++++++++++
 .../maxcompute/source/MaxComputeScanNode.java | 13 +++++
 .../property/constants/MCProperties.java      |  8 +++
 gensrc/thrift/PlanNodes.thrift                |  4 ++
 .../test_external_catalog_maxcompute.groovy   |  5 +-
 7 files changed, 131 insertions(+), 9 deletions(-)

diff --git a/be/src/vec/exec/format/table/max_compute_jni_reader.cpp b/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
index d1a71fd1a2f9d9..665e19b6bcebd9 100644
--- a/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
@@ -77,7 +77,11 @@ MaxComputeJniReader::MaxComputeJniReader(const MaxComputeTableDescriptor* mc_des
             {"start_offset", std::to_string(_range.start_offset)},
             {"split_size", std::to_string(_range.size)},
             {"required_fields", required_fields.str()},
-            {"columns_types", columns_types.str()}};
+            {"columns_types", columns_types.str()},
+
+            {"connect_timeout", std::to_string(_max_compute_params.connect_timeout)},
+            {"read_timeout", std::to_string(_max_compute_params.read_timeout)},
+            {"retry_count", std::to_string(_max_compute_params.retry_times)}};
     _jni_connector = std::make_unique<JniConnector>(
             "org/apache/doris/maxcompute/MaxComputeJniScanner", params, column_names);
 }
diff --git a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
index 6cbed70adc7d46..d6325bdae4673a 100644
--- a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
+++ b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
@@ -25,6 +25,7 @@
 import com.aliyun.odps.account.AliyunAccount;
 import com.aliyun.odps.table.configuration.CompressionCodec;
 import com.aliyun.odps.table.configuration.ReaderOptions;
+import com.aliyun.odps.table.configuration.RestOptions;
 import com.aliyun.odps.table.enviroment.Credentials;
 import com.aliyun.odps.table.enviroment.EnvironmentSettings;
 import com.aliyun.odps.table.read.SplitReader;
@@ -67,6 +68,10 @@ public class MaxComputeJniScanner extends JniScanner {
     private static final String SCAN_SERIALIZER = "scan_serializer";
     private static final String TIME_ZONE = "time_zone";
 
+    private static final String CONNECT_TIMEOUT = "connect_timeout";
+    private static final String READ_TIMEOUT = "read_timeout";
+    private static final String RETRY_COUNT  = "retry_count";
+
     private enum SplitType {
         BYTE_SIZE,
         ROW_OFFSET
@@ -136,16 +141,40 @@ public MaxComputeJniScanner(int batchSize, Map<String, String> params) {
         Credentials credentials = Credentials.newBuilder().withAccount(odps.getAccount())
                 .withAppAccount(odps.getAppAccount()).build();
 
+
+        int connectTimeout = 10; // 10s
+        if (!Strings.isNullOrEmpty(params.get(CONNECT_TIMEOUT))) {
+            connectTimeout = Integer.parseInt(params.get(CONNECT_TIMEOUT));
+        }
+
+        int readTimeout = 120; // 120s
+        if (!Strings.isNullOrEmpty(params.get(READ_TIMEOUT))) {
+            readTimeout =  Integer.parseInt(params.get(READ_TIMEOUT));
+        }
+
+        int retryTimes = 4; // 4 times
+        if (!Strings.isNullOrEmpty(params.get(RETRY_COUNT))) {
+            retryTimes = Integer.parseInt(params.get(RETRY_COUNT));
+        }
+
+        RestOptions restOptions = RestOptions.newBuilder()
+                .withConnectTimeout(connectTimeout)
+                .withReadTimeout(readTimeout)
+                .withRetryTimes(retryTimes).build();
+
         settings = EnvironmentSettings.newBuilder()
                 .withCredentials(credentials)
                 .withServiceEndpoint(odps.getEndpoint())
                 .withQuotaName(quota)
+                .withRestOptions(restOptions)
                 .build();
 
         try {
             scan = (TableBatchReadSession) deserialize(scanSerializer);
         } catch (Exception e) {
-            LOG.info("deserialize TableBatchReadSession failed.", e);
+            String errorMsg = "Failed to deserialize table batch read session.";
+            LOG.warn(errorMsg, e);
+            throw new IllegalArgumentException(errorMsg, e);
         }
     }
 
@@ -176,11 +205,11 @@ public void open() throws IOException {
                     .withReuseBatch(true)
                     .build());
 
-        } catch (IOException e) {
-            LOG.info("createArrowReader failed.", e);
         } catch (Exception e) {
+            String errorMsg = "MaxComputeJniScanner Failed to open table batch read session.";
+            LOG.warn(errorMsg, e);
             close();
-            throw new IOException(e.getMessage(), e);
+            throw new IOException(errorMsg, e);
         }
     }
 
@@ -215,8 +244,9 @@ private int readVectors(int expectedRows) throws IOException {
                     break;
                 }
             } catch (Exception e) {
-                LOG.info("currentSplitReader hasNext fail", e);
-                break;
+                String errorMsg = "MaxComputeJniScanner readVectors hasNext fail";
+                LOG.warn(errorMsg, e);
+                throw new IOException(e.getMessage(), e);
             }
 
             try {
@@ -241,7 +271,10 @@ private int readVectors(int expectedRows) throws IOException {
                 }
                 curReadRows += batchRows;
             } catch (Exception e) {
-                throw new RuntimeException("Fail to read arrow data, reason: " + e.getMessage(), e);
+                String errorMsg = String.format("MaxComputeJniScanner Fail to read arrow data. "
+                        + "curReadRows = {}, expectedRows = {}", curReadRows, expectedRows);
+                LOG.warn(errorMsg, e);
+                throw new RuntimeException(errorMsg, e);
             }
         }
         return curReadRows;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
index e6cd77103dbc3b..06c1e55dcf6f4b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
@@ -33,6 +33,7 @@
 import com.aliyun.odps.account.Account;
 import com.aliyun.odps.account.AliyunAccount;
 import com.aliyun.odps.security.SecurityManager;
+import com.aliyun.odps.table.configuration.RestOptions;
 import com.aliyun.odps.table.configuration.SplitOptions;
 import com.aliyun.odps.table.enviroment.Credentials;
 import com.aliyun.odps.table.enviroment.EnvironmentSettings;
@@ -71,6 +72,10 @@ public class MaxComputeExternalCatalog extends ExternalCatalog {
     private long splitRowCount;
     private long splitByteSize;
 
+    private int connectTimeout;
+    private int readTimeout;
+    private int retryTimes;
+
     private static final Map<String, ZoneId> REGION_ZONE_MAP;
     private static final List<String> REQUIRED_PROPERTIES = ImmutableList.of(
             MCProperties.PROJECT,
@@ -178,6 +183,17 @@ protected void initLocalObjectsImpl() {
                     .build();
         }
 
+        connectTimeout = Integer.parseInt(
+                props.getOrDefault(MCProperties.CONNECT_TIMEOUT, MCProperties.DEFAULT_CONNECT_TIMEOUT));
+        readTimeout = Integer.parseInt(
+                props.getOrDefault(MCProperties.READ_TIMEOUT, MCProperties.DEFAULT_READ_TIMEOUT));
+        retryTimes = Integer.parseInt(
+                props.getOrDefault(MCProperties.RETRY_COUNT, MCProperties.DEFAULT_RETRY_COUNT));
+
+        RestOptions restOptions = RestOptions.newBuilder()
+                .withConnectTimeout(connectTimeout)
+                .withReadTimeout(readTimeout)
+                .withRetryTimes(retryTimes).build();
 
         CloudCredential credential = MCProperties.getCredential(props);
         accessKey = credential.getAccessKey();
@@ -196,6 +212,7 @@ protected void initLocalObjectsImpl() {
                 .withCredentials(credentials)
                 .withServiceEndpoint(odps.getEndpoint())
                 .withQuotaName(quota)
+                .withRestOptions(restOptions)
                 .build();
     }
 
@@ -304,6 +321,21 @@ public String getDefaultProject() {
         return defaultProject;
     }
 
+    public int getRetryTimes() {
+        makeSureInitialized();
+        return retryTimes;
+    }
+
+    public int getConnectTimeout() {
+        makeSureInitialized();
+        return connectTimeout;
+    }
+
+    public int getReadTimeout() {
+        makeSureInitialized();
+        return readTimeout;
+    }
+
     public ZoneId getProjectDateTimeZone() {
         makeSureInitialized();
 
@@ -385,6 +417,31 @@ public void checkProperties() throws DdlException {
                     + MCProperties.SPLIT_ROW_COUNT + "must be an integer");
         }
 
+
+        try {
+            connectTimeout = Integer.parseInt(
+                    props.getOrDefault(MCProperties.CONNECT_TIMEOUT, MCProperties.DEFAULT_CONNECT_TIMEOUT));
+            readTimeout = Integer.parseInt(
+                    props.getOrDefault(MCProperties.READ_TIMEOUT, MCProperties.DEFAULT_READ_TIMEOUT));
+            retryTimes = Integer.parseInt(
+                    props.getOrDefault(MCProperties.RETRY_COUNT, MCProperties.DEFAULT_RETRY_COUNT));
+            if (connectTimeout <= 0) {
+                throw new DdlException(MCProperties.CONNECT_TIMEOUT + " must be greater than 0");
+            }
+
+            if (readTimeout <= 0) {
+                throw new DdlException(MCProperties.READ_TIMEOUT + " must be greater than 0");
+            }
+
+            if (retryTimes <= 0) {
+                throw new DdlException(MCProperties.RETRY_COUNT + " must be greater than 0");
+            }
+
+        } catch (NumberFormatException e) {
+            throw new DdlException("property " + MCProperties.CONNECT_TIMEOUT + "/"
+                    + MCProperties.READ_TIMEOUT + "/" + MCProperties.RETRY_COUNT + "must be an integer");
+        }
+
         CloudCredential credential = MCProperties.getCredential(props);
         if (!credential.isWhole()) {
             throw new DdlException("Max-Compute credential properties '"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index e177e9d8b7c88c..4ad971a5c64789 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -89,6 +89,10 @@ public class MaxComputeScanNode extends FileQueryScanNode {
     private static final LocationPath ROW_OFFSET_PATH = new LocationPath("/row_offset", Maps.newHashMap());
     private static final LocationPath BYTE_SIZE_PATH = new LocationPath("/byte_size", Maps.newHashMap());
 
+    private int connectTimeout;
+    private int readTimeout;
+    private int retryTimes;
+
     @Setter
     private SelectedPartitions selectedPartitions = null;
 
@@ -127,6 +131,11 @@ private void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeS
         fileDesc.setPartitionSpec("deprecated");
         fileDesc.setTableBatchReadSession(maxComputeSplit.scanSerialize);
         fileDesc.setSessionId(maxComputeSplit.getSessionId());
+
+        fileDesc.setReadTimeout(readTimeout);
+        fileDesc.setConnectTimeout(connectTimeout);
+        fileDesc.setRetryTimes(retryTimes);
+
         tableFormatFileDesc.setMaxComputeParams(fileDesc);
         rangeDesc.setTableFormatParams(tableFormatFileDesc);
         rangeDesc.setPath("[ " + maxComputeSplit.getStart() + " , " + maxComputeSplit.getLength() + " ]");
@@ -477,6 +486,10 @@ public List<Split> getSplits() throws UserException {
 
             MaxComputeExternalCatalog mcCatalog = (MaxComputeExternalCatalog) table.getCatalog();
 
+            readTimeout = mcCatalog.getReadTimeout();
+            connectTimeout = mcCatalog.getConnectTimeout();
+            retryTimes = mcCatalog.getRetryTimes();
+
             if (mcCatalog.getSplitStrategy().equals(MCProperties.SPLIT_BY_BYTE_SIZE_STRATEGY)) {
 
                 for (com.aliyun.odps.table.read.split.InputSplit split : assigner.getAllSplits()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
index 20a77574fc7820..efbd01c14777de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
@@ -56,6 +56,14 @@ public class MCProperties extends BaseProperties {
     public static final String SPLIT_ROW_COUNT = "mc.split_row_count";
     public static final String DEFAULT_SPLIT_ROW_COUNT = "1048576"; // 256 * 4096
 
+    public static final String CONNECT_TIMEOUT = "mc.connect_timeout";
+    public static final String READ_TIMEOUT = "mc.read_timeout";
+    public static final String RETRY_COUNT = "mc.retry_count";
+
+    public static final String DEFAULT_CONNECT_TIMEOUT = "10"; // 10s
+    public static final String DEFAULT_READ_TIMEOUT = "120"; // 120s
+    public static final String DEFAULT_RETRY_COUNT = "4"; // 4 times
+
     public static CloudCredential getCredential(Map<String, String> props) {
         return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN);
     }
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 77d9c0ef672feb..bd8c43622d1f60 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -353,6 +353,10 @@ struct TMaxComputeFileDesc {
     1: optional string partition_spec // deprecated 
     2: optional string session_id 
     3: optional string table_batch_read_session
+    // for mc network configuration
+    4: optional i32 connect_timeout
+    5: optional i32 read_timeout
+    6: optional i32 retry_times
 }
 
 struct THudiFileDesc {
diff --git a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
index 3f0929b59ea96a..81133270fb6e60 100644
--- a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
+++ b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
@@ -388,7 +388,10 @@ suite("test_external_catalog_maxcompute", "p2,external,maxcompute,external_remot
         order_qt_multi_partition_q6 """ select max(pt), yy, mm from multi_partitions where yy = '2023' and mm='08' group by yy, mm order by yy, mm; """
         order_qt_multi_partition_q7 """ select count(*) from multi_partitions where yy < '2023' or dd < '03'; """
         order_qt_multi_partition_q8 """ select count(*) from multi_partitions where pt>=3; """
-        order_qt_multi_partition_q9 """ select city,mnt,gender,finished_time,order_rate,cut_date,create_time,pt, yy, mm, dd from multi_partitions where pt >= 2 and pt < 4 and finished_time is not null; """
+        
+        //`finished_time is not null` => com.aliyun.odps.OdpsException: ODPS-0010000:System internal error - fuxi job failed, caused by: timestamp_ntz
+        // order_qt_multi_partition_q9 """ select city,mnt,gender,finished_time,order_rate,cut_date,create_time,pt, yy, mm, dd from multi_partitions where pt >= 2 and pt < 4 and finished_time is not null; """
+
         order_qt_multi_partition_q10 """ select pt, yy, mm, dd from multi_partitions where pt >= 2 and create_time > '2023-08-03 03:11:00' order by pt, yy, mm, dd; """
 
 

From 08d1e76308ae8dc0e3c2bb1ce24137e3916be7b9 Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Fri, 6 Dec 2024 10:42:49 +0800
Subject: [PATCH 195/399] [Fix](merge-on-write) Add defensive check before
 partial update (#44687)

### What problem does this PR solve?

as stated in https://github.com/apache/doris/pull/44692, Some user can
create Duplicate keys table with `enable_unique_key_merge_on_write=true`
successfully in version 1.2. After they upgrade to higher version, the
incorrect table property still remains. Some logic in BE treat such
table as an unique key mow table, makes a wrong query plan and BE may do
partial update on duplicate table, which causes BE core.
This PR add checks before doing partial update and return error when
some preconditions are not met to prevent coredump.
---
 .../olap/rowset/segment_v2/segment_writer.cpp | 39 +++++++++++--
 .../olap/rowset/segment_v2/segment_writer.h   |  1 +
 .../segment_v2/vertical_segment_writer.cpp    | 56 ++++++++++++++++---
 .../segment_v2/vertical_segment_writer.h      |  1 +
 4 files changed, 83 insertions(+), 14 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 8641687a0069b8..fe465f98a2aad2 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -545,6 +545,39 @@ Status SegmentWriter::probe_key_for_mow(
     return Status::OK();
 }
 
+Status SegmentWriter::partial_update_preconditions_check(size_t row_pos) {
+    if (!_is_mow()) {
+        auto msg = fmt::format(
+                "Can only do partial update on merge-on-write unique table, but found: "
+                "keys_type={}, _opts.enable_unique_key_merge_on_write={}, tablet_id={}",
+                _tablet_schema->keys_type(), _opts.enable_unique_key_merge_on_write,
+                _tablet->tablet_id());
+        DCHECK(false) << msg;
+        return Status::InternalError<false>(msg);
+    }
+    if (_opts.rowset_ctx->partial_update_info == nullptr) {
+        auto msg =
+                fmt::format("partial_update_info should not be nullptr, please check, tablet_id={}",
+                            _tablet->tablet_id());
+        DCHECK(false) << msg;
+        return Status::InternalError<false>(msg);
+    }
+    if (!_opts.rowset_ctx->partial_update_info->is_fixed_partial_update()) {
+        auto msg = fmt::format(
+                "in fixed partial update code, but update_mode={}, please check, tablet_id={}",
+                _opts.rowset_ctx->partial_update_info->update_mode(), _tablet->tablet_id());
+        DCHECK(false) << msg;
+        return Status::InternalError<false>(msg);
+    }
+    if (row_pos != 0) {
+        auto msg = fmt::format("row_pos should be 0, but found {}, tablet_id={}", row_pos,
+                               _tablet->tablet_id());
+        DCHECK(false) << msg;
+        return Status::InternalError<false>(msg);
+    }
+    return Status::OK();
+}
+
 // for partial update, we should do following steps to fill content of block:
 // 1. set block data to data convertor, and get all key_column's converted slice
 // 2. get pk of input block, and read missing columns
@@ -562,11 +595,7 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block*
                             block->columns(), _tablet_schema->num_key_columns(),
                             _tablet_schema->num_columns()));
     }
-    DCHECK(_is_mow());
-
-    DCHECK(_opts.rowset_ctx->partial_update_info);
-    DCHECK(_opts.rowset_ctx->partial_update_info->is_fixed_partial_update());
-    DCHECK(row_pos == 0);
+    RETURN_IF_ERROR(partial_update_preconditions_check(row_pos));
 
     // find missing column cids
     const auto& missing_cids = _opts.rowset_ctx->partial_update_info->missing_cids;
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h
index a1b7491a669673..60300383d7287d 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.h
+++ b/be/src/olap/rowset/segment_v2/segment_writer.h
@@ -105,6 +105,7 @@ class SegmentWriter {
                              const std::function<void(const RowLocation& loc)>& found_cb,
                              const std::function<Status()>& not_found_cb,
                              PartialUpdateStats& stats);
+    Status partial_update_preconditions_check(size_t row_pos);
     Status append_block_with_partial_content(const vectorized::Block* block, size_t row_pos,
                                              size_t num_rows);
     Status append_block_with_variant_subcolumns(vectorized::Block& data);
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index f40da898a28742..0846b0fc1186a8 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -418,6 +418,51 @@ Status VerticalSegmentWriter::_probe_key_for_mow(
     return Status::OK();
 }
 
+Status VerticalSegmentWriter::_partial_update_preconditions_check(size_t row_pos,
+                                                                  bool is_flexible_update) {
+    if (!_is_mow()) {
+        auto msg = fmt::format(
+                "Can only do partial update on merge-on-write unique table, but found: "
+                "keys_type={}, _opts.enable_unique_key_merge_on_write={}, tablet_id={}",
+                _tablet_schema->keys_type(), _opts.enable_unique_key_merge_on_write,
+                _tablet->tablet_id());
+        DCHECK(false) << msg;
+        return Status::InternalError<false>(msg);
+    }
+    if (_opts.rowset_ctx->partial_update_info == nullptr) {
+        auto msg =
+                fmt::format("partial_update_info should not be nullptr, please check, tablet_id={}",
+                            _tablet->tablet_id());
+        DCHECK(false) << msg;
+        return Status::InternalError<false>(msg);
+    }
+    if (!is_flexible_update) {
+        if (!_opts.rowset_ctx->partial_update_info->is_fixed_partial_update()) {
+            auto msg = fmt::format(
+                    "in fixed partial update code, but update_mode={}, please check, tablet_id={}",
+                    _opts.rowset_ctx->partial_update_info->update_mode(), _tablet->tablet_id());
+            DCHECK(false) << msg;
+            return Status::InternalError<false>(msg);
+        }
+    } else {
+        if (!_opts.rowset_ctx->partial_update_info->is_flexible_partial_update()) {
+            auto msg = fmt::format(
+                    "in flexible partial update code, but update_mode={}, please check, "
+                    "tablet_id={}",
+                    _opts.rowset_ctx->partial_update_info->update_mode(), _tablet->tablet_id());
+            DCHECK(false) << msg;
+            return Status::InternalError<false>(msg);
+        }
+    }
+    if (row_pos != 0) {
+        auto msg = fmt::format("row_pos should be 0, but found {}, tablet_id={}", row_pos,
+                               _tablet->tablet_id());
+        DCHECK(false) << msg;
+        return Status::InternalError<false>(msg);
+    }
+    return Status::OK();
+}
+
 // for partial update, we should do following steps to fill content of block:
 // 1. set block data to data convertor, and get all key_column's converted slice
 // 2. get pk of input block, and read missing columns
@@ -427,11 +472,7 @@ Status VerticalSegmentWriter::_probe_key_for_mow(
 // 3. set columns to data convertor and then write all columns
 Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& data,
                                                                  vectorized::Block& full_block) {
-    DCHECK(_is_mow());
-    DCHECK(_opts.rowset_ctx->partial_update_info != nullptr);
-    DCHECK(_opts.rowset_ctx->partial_update_info->is_fixed_partial_update());
-    DCHECK(data.row_pos == 0);
-
+    RETURN_IF_ERROR(_partial_update_preconditions_check(data.row_pos, false));
     // create full block and fill with input columns
     full_block = _tablet_schema->create_block();
     const auto& including_cids = _opts.rowset_ctx->partial_update_info->update_cids;
@@ -580,10 +621,7 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da
 
 Status VerticalSegmentWriter::_append_block_with_flexible_partial_content(
         RowsInBlock& data, vectorized::Block& full_block) {
-    DCHECK(_is_mow());
-    DCHECK(_opts.rowset_ctx->partial_update_info != nullptr);
-    DCHECK(_opts.rowset_ctx->partial_update_info->is_flexible_partial_update());
-    DCHECK(data.row_pos == 0);
+    RETURN_IF_ERROR(_partial_update_preconditions_check(data.row_pos, true));
 
     // data.block has the same schema with full_block
     DCHECK(data.block->columns() == _tablet_schema->num_columns());
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h
index 951e9c2e2838c3..8cec6ed4d1abd6 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h
@@ -175,6 +175,7 @@ class VerticalSegmentWriter {
                               const std::function<void(const RowLocation& loc)>& found_cb,
                               const std::function<Status()>& not_found_cb,
                               PartialUpdateStats& stats);
+    Status _partial_update_preconditions_check(size_t row_pos, bool is_flexible_update);
     Status _append_block_with_partial_content(RowsInBlock& data, vectorized::Block& full_block);
     Status _append_block_with_flexible_partial_content(RowsInBlock& data,
                                                        vectorized::Block& full_block);

From 5ff94a1149dea8c736873c716a490e3bab59bd39 Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Fri, 6 Dec 2024 10:48:26 +0800
Subject: [PATCH 196/399] [improvement](information_schema)Support show default
 value in information_schema. (#44849)

### What problem does this PR solve?

The COLUMN_DEFAULT column in INFORMATION_SCHEMA.COLUMNS table is always
NULL, this PR set the correct default value for COLUMN_DEFAULT.

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None
---
 .../schema_scanner/schema_columns_scanner.cpp | 14 +++++++++++-
 .../doris/service/FrontendServiceImpl.java    |  4 ++++
 gensrc/thrift/FrontendService.thrift          |  1 +
 .../account_p0/test_information_schema.out    |  7 ++++++
 .../account_p0/test_information_schema.groovy | 22 +++++++++++++++++++
 5 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp
index 8325a7f5dc4f2d..b60dfc3d203f89 100644
--- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp
@@ -450,7 +450,19 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) {
         RETURN_IF_ERROR(fill_dest_column_for_range(block, 4, datas));
     }
     // COLUMN_DEFAULT
-    { RETURN_IF_ERROR(fill_dest_column_for_range(block, 5, null_datas)); }
+    {
+        std::vector<StringRef> strs(columns_num);
+        for (int i = 0; i < columns_num; ++i) {
+            if (_desc_result.columns[i].columnDesc.__isset.defaultValue) {
+                strs[i] = StringRef(_desc_result.columns[i].columnDesc.defaultValue.c_str(),
+                                    _desc_result.columns[i].columnDesc.defaultValue.length());
+                datas[i] = strs.data() + i;
+            } else {
+                datas[i] = nullptr;
+            }
+        }
+        RETURN_IF_ERROR(fill_dest_column_for_range(block, 5, datas));
+    }
     // IS_NULLABLE
     {
         StringRef str_yes = StringRef("YES", 3);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index cf9e8e82ce112f..1ad8d733ddea07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -972,6 +972,10 @@ private TColumnDesc getColumnDesc(Column column) {
             }
             desc.setChildren(children);
         }
+        String defaultValue = column.getDefaultValue();
+        if (defaultValue != null) {
+            desc.setDefaultValue(defaultValue);
+        }
         return desc;
     }
 
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 246cc999562934..916885028ad4a0 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -53,6 +53,7 @@ struct TColumnDesc {
   6: optional bool isAllowNull
   7: optional string columnKey
   8: optional list<TColumnDesc> children
+  9: optional string defaultValue
 }
 
 // A column definition; used by CREATE TABLE and DESCRIBE <table> statements. A column
diff --git a/regression-test/data/account_p0/test_information_schema.out b/regression-test/data/account_p0/test_information_schema.out
index 77d5f6dccd50e8..e4f0ed09d0e1c7 100644
--- a/regression-test/data/account_p0/test_information_schema.out
+++ b/regression-test/data/account_p0/test_information_schema.out
@@ -17,3 +17,10 @@
 -- !sql --
 DUP
 
+-- !default --
+id	largeint	YES	\N
+name	varchar(20)	YES	无
+age	smallint(6)	YES	0
+address	varchar(100)	YES	beijing
+date	datetime	YES	20240101
+
diff --git a/regression-test/suites/account_p0/test_information_schema.groovy b/regression-test/suites/account_p0/test_information_schema.groovy
index dcbc0c3532838e..fc94ad4b9bc2fa 100644
--- a/regression-test/suites/account_p0/test_information_schema.groovy
+++ b/regression-test/suites/account_p0/test_information_schema.groovy
@@ -87,4 +87,26 @@ suite("test_information_schema") {
         def dbName = dbPrefix + i.toString()
         sql "DROP DATABASE `${dbName}`"
     }
+
+    def dbName = dbPrefix + "default"
+    def tableName = tablePrefix + "default"
+    sql "CREATE DATABASE IF NOT EXISTS `${dbName}`"
+    sql "USE `${dbName}`"
+    sql """drop table if exists `${tableName}`"""
+    sql """
+        CREATE TABLE `${tableName}` (
+          `id` largeint NULL COMMENT '用户ID',
+          `name` varchar(20) NULL DEFAULT "无" COMMENT '用户姓名',
+          `age` smallint NULL DEFAULT "0" COMMENT '用户年龄',
+          `address` varchar(100) NULL DEFAULT "beijing" COMMENT '用户所在地区',
+          `date` datetime NULL DEFAULT "20240101" COMMENT '数据导入时间'
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`, `name`)
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1")
+    """
+    qt_default "SELECT COLUMN_NAME as field,COLUMN_TYPE as type,IS_NULLABLE as isNullable, COLUMN_DEFAULT as defaultValue FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '${tableName}' AND TABLE_SCHEMA = '${dbName}'"
+    sql "DROP DATABASE `${dbName}`"
 }
+

From 682be10a549801a1d70764cc0f25512738fa1c41 Mon Sep 17 00:00:00 2001
From: hui lai <laihui@selectdb.com>
Date: Fri, 6 Dec 2024 11:07:26 +0800
Subject: [PATCH 197/399] [fix](routine load) fix invalid EOF tasks scheduling
 delay (#44949)

Related PR: https://github.com/apache/doris/pull/39975
In the above PR, we introduce delay schedule EOF tasks to avoid too many
small transactions, but it is invalid, the pr fix the issue.
---
 .../apache/doris/load/routineload/RoutineLoadTaskScheduler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
index 8afc35411b55f5..d40a6705626c84 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
@@ -106,7 +106,7 @@ private void process() throws UserException, InterruptedException {
             if (routineLoadTaskInfo.getIsEof()) {
                 RoutineLoadJob routineLoadJob = routineLoadManager.getJob(routineLoadTaskInfo.getJobId());
                 if (System.currentTimeMillis() - routineLoadTaskInfo.getLastScheduledTime()
-                        < routineLoadJob.getMaxBatchIntervalS()) {
+                        < routineLoadJob.getMaxBatchIntervalS() * 1000) {
                     needScheduleTasksQueue.addLast(routineLoadTaskInfo);
                     return;
                 }

From de2ce48b992d12237f2d3aeb6b0461f6770e16c3 Mon Sep 17 00:00:00 2001
From: Tiewei Fang <fangtiewei@selectdb.com>
Date: Fri, 6 Dec 2024 13:02:58 +0800
Subject: [PATCH 198/399] [fix](Export) Fix the problem of exporting stuck
 (#44944)

### What problem does this PR solve?
Problem Summary:

The `disruptor` will be witing when the ringbuffer does not have enough
capacity. At the same time, `addExportJobAndRegisterTask` will not
release the lock of `ExportMgr`. This prevents other methods from
obtaining the lock.
---
 .../java/org/apache/doris/load/ExportMgr.java | 30 +++++++++----------
 .../scheduler/disruptor/TaskDisruptor.java    | 11 ++++---
 .../manager/TransientTaskManager.java         |  4 +--
 3 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index eddd5fb27eeef1..94ae436ee6d87d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -108,26 +108,24 @@ public void addExportJobAndRegisterTask(ExportJob job) throws Exception {
                 }
             }
             unprotectAddJob(job);
-            // delete existing files
-            if (Config.enable_delete_existing_files && Boolean.parseBoolean(job.getDeleteExistingFiles())) {
-                if (job.getBrokerDesc() == null) {
-                    throw new AnalysisException("Local file system does not support delete existing files");
-                }
-                String fullPath = job.getExportPath();
-                BrokerUtil.deleteDirectoryWithFileSystem(fullPath.substring(0, fullPath.lastIndexOf('/') + 1),
-                        job.getBrokerDesc());
-            }
             Env.getCurrentEnv().getEditLog().logExportCreate(job);
-            // ATTN: Must add task after edit log, otherwise the job may finish before adding job.
-            job.getCopiedTaskExecutors().forEach(executor -> {
-                Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(executor);
-            });
-            LOG.info("add export job. {}", job);
-
         } finally {
             writeUnlock();
         }
-
+        // delete existing files
+        if (Config.enable_delete_existing_files && Boolean.parseBoolean(job.getDeleteExistingFiles())) {
+            if (job.getBrokerDesc() == null) {
+                throw new AnalysisException("Local file system does not support delete existing files");
+            }
+            String fullPath = job.getExportPath();
+            BrokerUtil.deleteDirectoryWithFileSystem(fullPath.substring(0, fullPath.lastIndexOf('/') + 1),
+                    job.getBrokerDesc());
+        }
+        // ATTN: Must add task after edit log, otherwise the job may finish before adding job.
+        for (int i = 0; i < job.getCopiedTaskExecutors().size(); i++) {
+            Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(job.getCopiedTaskExecutors().get(i));
+        }
+        LOG.info("add export job. {}", job);
     }
 
     public void cancelExportJob(CancelExportStmt stmt) throws DdlException, AnalysisException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java
index 345b31d6bc2537..8144ca22ea22eb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java
@@ -20,6 +20,7 @@
 import org.apache.doris.common.Config;
 import org.apache.doris.common.CustomThreadFactory;
 import org.apache.doris.scheduler.constants.TaskType;
+import org.apache.doris.scheduler.exception.JobException;
 
 import com.lmax.disruptor.EventTranslatorThreeArg;
 import com.lmax.disruptor.LiteTimeoutBlockingWaitStrategy;
@@ -119,15 +120,17 @@ public void tryPublish(Long jobId, Long taskId, TaskType taskType) {
      *
      * @param taskId task id
      */
-    public void tryPublishTask(Long taskId) {
+    public void tryPublishTask(Long taskId) throws JobException {
         if (isClosed) {
             log.info("tryPublish failed, disruptor is closed, taskId: {}", taskId);
             return;
         }
-        try {
+        // We reserve two slots in the ring buffer
+        // to prevent it from becoming stuck due to competition between producers and consumers.
+        if (disruptor.getRingBuffer().hasAvailableCapacity(2)) {
             disruptor.publishEvent(TRANSLATOR, taskId, 0L, TaskType.TRANSIENT_TASK);
-        } catch (Exception e) {
-            log.warn("tryPublish failed, taskId: {}", taskId, e);
+        } else {
+            throw new JobException("There is not enough available capacity in the RingBuffer.");
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java
index 7461399c8eb0c5..de501d3e0c2ffd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java
@@ -21,7 +21,6 @@
 import org.apache.doris.scheduler.exception.JobException;
 import org.apache.doris.scheduler.executor.TransientTaskExecutor;
 
-import lombok.Setter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -41,7 +40,6 @@ public class TransientTaskManager {
      * disruptor is used to handle task
      * disruptor will start a thread pool to handle task
      */
-    @Setter
     private TaskDisruptor disruptor;
 
     public TransientTaskManager() {
@@ -56,7 +54,7 @@ public TransientTaskExecutor getMemoryTaskExecutor(Long taskId) {
         return taskExecutorMap.get(taskId);
     }
 
-    public Long addMemoryTask(TransientTaskExecutor executor) {
+    public Long addMemoryTask(TransientTaskExecutor executor) throws JobException {
         Long taskId = executor.getId();
         taskExecutorMap.put(taskId, executor);
         disruptor.tryPublishTask(taskId);

From 0f74869d59e2198923caa2e4dc098c09ade4448c Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchi@selectdb.com>
Date: Fri, 6 Dec 2024 13:53:59 +0800
Subject: [PATCH 199/399] [opt](paimon)Remove unnecessary fields (#44906)

### What problem does this PR solve?

Related PR: #44274

Problem Summary:

We have already set the serialization table in `TFileScanRangeParams`,
so there is no need to repeat it for each split.
---
 .../apache/doris/datasource/paimon/source/PaimonScanNode.java | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index bf917804fb7b6d..5009ec3c9049fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -183,9 +183,7 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit)
         fileDesc.setDbId(((PaimonExternalTable) source.getTargetTable()).getDbId());
         fileDesc.setTblId(source.getTargetTable().getId());
         fileDesc.setLastUpdateTime(source.getTargetTable().getUpdateTime());
-        fileDesc.setPaimonTable(encodeObjectToString(source.getPaimonTable()));
-        // The hadoop conf should be same with
-        // PaimonExternalCatalog.createCatalog()#getConfiguration()
+        // The hadoop conf should be same with PaimonExternalCatalog.createCatalog()#getConfiguration()
         fileDesc.setHadoopConf(source.getCatalog().getCatalogProperty().getHadoopProperties());
         Optional<DeletionFile> optDeletionFile = paimonSplit.getDeletionFile();
         if (optDeletionFile.isPresent()) {

From 4512cb05a7678529f5ca80d12aee7a57ca6b298f Mon Sep 17 00:00:00 2001
From: Kaijie Chen <chenkaijie@selectdb.com>
Date: Fri, 6 Dec 2024 14:00:06 +0800
Subject: [PATCH 200/399] [fix](runtime_profile) fix race condition in
 to_thrift (#45047)

### What problem does this PR solve?

Fix race condition in `RuntimeProfile::to_thrift()`.

```
#6  0x000055bce5a78bbf in std::__throw_length_error (__s=0x55bca1eb7880 <str> "vector::reserve") at ../../../../../libstdc++-v3/src/c++11/functexcept.cc:82
#7  0x000055bcafbbbc8f in std::vector<doris::TRuntimeProfileNode, std::allocator<doris::TRuntimeProfileNode> >::reserve (this=this@entry=0x7f2e69c39f48, __n=<optimized out>)
    at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/vector.tcc:70
#8  0x000055bcafbb6e34 in doris::RuntimeProfile::to_thrift (this=<optimized out>, nodes=0x7f2e69c39f48) at /root/doris/be/src/util/runtime_profile.cpp:577
#9  0x000055bcafbb7780 in doris::RuntimeProfile::to_thrift (this=<optimized out>, nodes=0x7f2e69c39f48) at /root/doris/be/src/util/runtime_profile.cpp:612
#10 0x000055bcafbb7780 in doris::RuntimeProfile::to_thrift (this=<optimized out>, nodes=0x7f2e69c39f48) at /root/doris/be/src/util/runtime_profile.cpp:612
#11 0x000055bcafbb7780 in doris::RuntimeProfile::to_thrift (this=<optimized out>, nodes=0x7f2e69c39f48) at /root/doris/be/src/util/runtime_profile.cpp:612
#12 0x000055bcaf32ee52 in doris::LoadChannel::_report_profile (this=this@entry=0x6150116fca80, response=response@entry=0x61201b768340)
```
---
 be/src/util/runtime_profile.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp
index 8b9e97bfc9cb54..1df4d8b55c278e 100644
--- a/be/src/util/runtime_profile.cpp
+++ b/be/src/util/runtime_profile.cpp
@@ -574,8 +574,6 @@ void RuntimeProfile::to_thrift(TRuntimeProfileTree* tree) {
 }
 
 void RuntimeProfile::to_thrift(std::vector<TRuntimeProfileNode>* nodes) {
-    nodes->reserve(nodes->size() + _children.size());
-
     int index = nodes->size();
     nodes->push_back(TRuntimeProfileNode());
     TRuntimeProfileNode& node = (*nodes)[index];
@@ -602,10 +600,13 @@ void RuntimeProfile::to_thrift(std::vector<TRuntimeProfileNode>* nodes) {
 
     ChildVector children;
     {
+        // _children may be modified during to_thrift(),
+        // so we have to lock and copy _children to avoid race condition
         std::lock_guard<std::mutex> l(_children_lock);
         children = _children;
     }
     node.num_children = children.size();
+    nodes->reserve(nodes->size() + children.size());
 
     for (int i = 0; i < children.size(); ++i) {
         int child_idx = nodes->size();

From 1a03fed419975db54d417b71bc457667ab3184db Mon Sep 17 00:00:00 2001
From: Gavin Chou <gavin@selectdb.com>
Date: Fri, 6 Dec 2024 14:40:39 +0800
Subject: [PATCH 201/399] [fix](recycler) Process self-defined domain names for
 s3 storage vault (#45072)

---
 cloud/src/recycler/s3_accessor.cpp   |  3 +-
 cloud/src/recycler/s3_accessor.h     |  1 +
 cloud/src/recycler/s3_obj_client.cpp |  1 +
 cloud/test/s3_accessor_test.cpp      | 68 ++++++++++++++++++++++++++++
 4 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp
index 2c983a5fa0617b..1aca88d2d1161d 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -205,6 +205,7 @@ std::optional<S3Conf> S3Conf::from_obj_store_info(const ObjectStoreInfoPB& obj_i
     s3_conf.region = obj_info.region();
     s3_conf.bucket = obj_info.bucket();
     s3_conf.prefix = obj_info.prefix();
+    s3_conf.use_virtual_addressing = !obj_info.use_path_style();
 
     return s3_conf;
 }
@@ -289,7 +290,7 @@ int S3Accessor::init() {
         auto s3_client = std::make_shared<Aws::S3::S3Client>(
                 std::move(aws_cred), std::move(aws_config),
                 Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
-                true /* useVirtualAddressing */);
+                conf_.use_virtual_addressing /* useVirtualAddressing */);
         obj_client_ = std::make_shared<S3ObjClient>(std::move(s3_client), conf_.endpoint);
         return 0;
     }
diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h
index 6886ee5e7c5640..e9640b5693a1f5 100644
--- a/cloud/src/recycler/s3_accessor.h
+++ b/cloud/src/recycler/s3_accessor.h
@@ -69,6 +69,7 @@ struct S3Conf {
     std::string region;
     std::string bucket;
     std::string prefix;
+    bool use_virtual_addressing {true};
 
     enum Provider : uint8_t {
         S3,
diff --git a/cloud/src/recycler/s3_obj_client.cpp b/cloud/src/recycler/s3_obj_client.cpp
index 53fa821c7e5503..c8dcdad18d7115 100644
--- a/cloud/src/recycler/s3_obj_client.cpp
+++ b/cloud/src/recycler/s3_obj_client.cpp
@@ -284,6 +284,7 @@ ObjectStorageResponse S3ObjClient::delete_object(ObjectStoragePathRef path) {
         SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency);
         return s3_client_->DeleteObject(request);
     });
+    TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_object", &outcome);
     if (!outcome.IsSuccess()) {
         LOG_WARNING("failed to delete object")
                 .tag("endpoint", endpoint_)
diff --git a/cloud/test/s3_accessor_test.cpp b/cloud/test/s3_accessor_test.cpp
index 0dd51b749d86e2..c19f5f6a1dfdfb 100644
--- a/cloud/test/s3_accessor_test.cpp
+++ b/cloud/test/s3_accessor_test.cpp
@@ -17,8 +17,10 @@
 
 #include "recycler/s3_accessor.h"
 
+#include <aws/s3/S3Client.h>
 #include <aws/s3/model/ListObjectsV2Request.h>
 #include <butil/guid.h>
+#include <gen_cpp/cloud.pb.h>
 #include <gtest/gtest.h>
 
 #include <azure/storage/blobs/blob_options.hpp>
@@ -320,4 +322,70 @@ TEST(S3AccessorTest, gcs) {
     test_s3_accessor(*accessor);
 }
 
+TEST(S3AccessorTest, path_style_test) {
+    ObjectStoreInfoPB obj_info;
+    obj_info.set_prefix("doris-debug-instance-prefix");
+    obj_info.set_provider(ObjectStoreInfoPB_Provider_S3);
+    obj_info.set_ak("dummy_ak");
+    obj_info.set_sk("dummy_sk");
+    obj_info.set_endpoint("dummy-bucket");
+    obj_info.set_region("cn-north-1");
+    obj_info.set_bucket("dummy-bucket");
+    config::max_s3_client_retry = 0;
+
+    auto* sp = SyncPoint::get_instance();
+    sp->enable_processing();
+    std::vector<SyncPoint::CallbackGuard> guards;
+
+    std::string base_domain = "s3.cn-north-1.amazonaws.com.cn";
+    std::string domain_ip = "54.222.51.71"; // first ip of base_domain
+    // to test custom_domain,  add ${domain_ip} ${custom_domain} to /etc/hosts
+    // otherwise the related cases will fail
+    std::string custom_domain = "gavin.s3.aws.com";
+    // clang-format off
+    // http code 403 means there is nothing wrong the given domain in objinfo
+    //                 domain, use_path_style, http_code
+    std::vector<std::tuple<std::string, bool, int>> inputs {
+        {base_domain               , false , 403}, // works
+        {base_domain               , true  , 403}, // works
+        {"http://" + base_domain   , false , 403}, // works
+        {"http://" + base_domain   , true  , 403}, // works
+        {"https://" + base_domain  , false , 403}, // works
+        {"https://" + base_domain  , true  , 403}, // works
+        {"http://" + domain_ip     , false , 301}, // works, ip with virtual addressing
+        {"http://" + domain_ip     , true  , 301}, // works, ip with path style
+        {custom_domain             , false , -1} , // custom_domain could not resolve with virtual addressing
+        {custom_domain             , true  , 403}, // custom_domain working with path style
+        {"http://" + custom_domain , false , -1} , // custom_domain could not resolve with virtual addressing
+        {"https://" + custom_domain, true  , -1},  // certificate issue, custom_domain does not attached with any certs
+        // {"https://54.222.51.71" , false , -1} , // certificate issue
+        // {"https://54.222.51.71" , true  , -1} , // certificate issue
+    };
+
+    int case_idx = 0;
+    sp->set_call_back("S3ObjClient::delete_object",
+            [&case_idx, &inputs](auto&& args) {
+                auto* res = try_any_cast<Aws::S3::Model::DeleteObjectOutcome*>(args[0]);
+                EXPECT_EQ(std::get<2>(inputs[case_idx]), static_cast<int>(res->GetError().GetResponseCode())) << "<<<<<<<<<<<<<<<<<<<<< " << case_idx;
+                case_idx++;
+            },
+            &guards.emplace_back());
+    // clang-format on
+
+    for (auto& i : inputs) {
+        obj_info.set_endpoint(std::get<0>(i));
+        obj_info.set_use_path_style(std::get<1>(i));
+        auto s3_conf = S3Conf::from_obj_store_info(obj_info);
+        EXPECT_EQ(s3_conf->use_virtual_addressing, !obj_info.use_path_style()) << case_idx;
+        std::shared_ptr<S3Accessor> accessor;
+        int ret = S3Accessor::create(*s3_conf, &accessor);
+        EXPECT_EQ(ret, 0) << "<<<<<<<<<<<<<<<<<<<<< " << case_idx;
+        ret = accessor->init();
+        EXPECT_EQ(ret, 0) << "<<<<<<<<<<<<<<<<<<<<< " << case_idx;
+        // this function call will trigger syncpoint callback to increment case_idx
+        accessor->delete_file("abc"); // try to delete a nonexisted file, ignore the result
+        // EXPECT_EQ(ret, exp) << "<<<<<<<<<<<<<<<<<<<<< " << case_idx << " domain " << std::get<0>(i);
+    }
+}
+
 } // namespace doris::cloud

From 6eca02307df0fb89bf1612b704934994567bb4ee Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Fri, 6 Dec 2024 14:43:16 +0800
Subject: [PATCH 202/399] [fix](editlog) Fix replay BatchDropInfo (#45077)

introduced by #44677

The new field `indexNameMap` does not exist in the former version of the
persisted metadata, and should be skipped during replaying.
---
 .../apache/doris/persist/BatchDropInfo.java   |  4 ++++
 .../org/apache/doris/persist/DropInfo.java    |  2 +-
 .../org/apache/doris/persist/EditLog.java     | 22 +++++++++++++------
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/BatchDropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/BatchDropInfo.java
index 260ad316d3cc24..8e4e9b9af8e0d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/BatchDropInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/BatchDropInfo.java
@@ -86,6 +86,10 @@ public Set<Long> getIndexIdSet() {
         return indexIdSet;
     }
 
+    public boolean hasIndexNameMap() {
+        return indexNameMap != null;
+    }
+
     public Map<Long, String> getIndexNameMap() {
         return indexNameMap;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
index 69994caf23d5dc..db0688bd6ad267 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
@@ -52,7 +52,7 @@ public DropInfo() {
 
     public DropInfo(long dbId, long tableId, String tableName, boolean isView, boolean forceDrop,
             long recycleTime) {
-        this(dbId, tableId, tableName, -1, "", isView, forceDrop, recycleTime);
+        this(dbId, tableId, tableName, -1L, "", isView, forceDrop, recycleTime);
     }
 
     public DropInfo(long dbId, long tableId, String tableName, long indexId, String indexName, boolean isView,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index 7d1f2127eecaaf..f1377e9daebfc4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -347,13 +347,21 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) {
                 }
                 case OperationType.OP_BATCH_DROP_ROLLUP: {
                     BatchDropInfo batchDropInfo = (BatchDropInfo) journal.getData();
-                    for (Map.Entry<Long, String> entry : batchDropInfo.getIndexNameMap().entrySet()) {
-                        long indexId = entry.getKey();
-                        String indexName = entry.getValue();
-                        DropInfo info = new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(),
-                                        batchDropInfo.getTableName(), indexId, indexName, false, false, 0);
-                        env.getMaterializedViewHandler().replayDropRollup(info, env);
-                        env.getBinlogManager().addDropRollup(info, logId);
+                    if (batchDropInfo.hasIndexNameMap()) {
+                        for (Map.Entry<Long, String> entry : batchDropInfo.getIndexNameMap().entrySet()) {
+                            long indexId = entry.getKey();
+                            String indexName = entry.getValue();
+                            DropInfo info = new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(),
+                                            batchDropInfo.getTableName(), indexId, indexName, false, false, 0);
+                            env.getMaterializedViewHandler().replayDropRollup(info, env);
+                            env.getBinlogManager().addDropRollup(info, logId);
+                        }
+                    } else {
+                        for (Long indexId : batchDropInfo.getIndexIdSet()) {
+                            DropInfo info = new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(),
+                                    batchDropInfo.getTableName(), indexId, "", false, false, 0);
+                            env.getMaterializedViewHandler().replayDropRollup(info, env);
+                        }
                     }
                     break;
                 }

From 7082222560ecbe48fa4e547cd20e5a89874f9e8b Mon Sep 17 00:00:00 2001
From: zclllhhjj <zhaochangle@selectdb.com>
Date: Fri, 6 Dec 2024 15:11:01 +0800
Subject: [PATCH 203/399] [Opt](function) Speed up function string and add
 regression (#45062)

Speed up function string and add regression
Related PR: https://github.com/apache/doris/pull/44919
---
 be/src/vec/functions/function_string.cpp      | 19 ++++++++++---------
 .../scalar_function/S.groovy                  |  1 +
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/be/src/vec/functions/function_string.cpp b/be/src/vec/functions/function_string.cpp
index 89737d03453ae3..c434d344daa40a 100644
--- a/be/src/vec/functions/function_string.cpp
+++ b/be/src/vec/functions/function_string.cpp
@@ -888,16 +888,17 @@ struct StringSpace {
                          ColumnString::Offsets& res_offsets) {
         res_offsets.resize(data.size());
         size_t input_size = res_offsets.size();
-        std::vector<char, Allocator_<char>> buffer;
+        // sample to get approximate best reserve size
+        if (input_size > 4) {
+            res_data.reserve(((data[0] + data[input_size >> 1] + data[input_size >> 2] +
+                               data[input_size - 1]) >>
+                              2) *
+                             input_size);
+        }
         for (size_t i = 0; i < input_size; ++i) {
-            buffer.clear();
-            if (data[i] > 0) {
-                buffer.resize(data[i]);
-                for (size_t j = 0; j < data[i]; ++j) {
-                    buffer[j] = ' ';
-                }
-                StringOP::push_value_string(std::string_view(buffer.data(), buffer.size()), i,
-                                            res_data, res_offsets);
+            if (data[i] > 0) [[likely]] {
+                res_data.resize_fill(res_data.size() + data[i], ' ');
+                res_offsets[i] = res_data.size();
             } else {
                 StringOP::push_empty_string(i, res_data, res_offsets);
             }
diff --git a/regression-test/suites/nereids_function_p0/scalar_function/S.groovy b/regression-test/suites/nereids_function_p0/scalar_function/S.groovy
index 98a8685a88840b..ea0d80d8adce85 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/S.groovy
+++ b/regression-test/suites/nereids_function_p0/scalar_function/S.groovy
@@ -126,6 +126,7 @@ suite("nereids_scalar_fn_S") {
 
 	sql "select space(10) from fn_test order by kint"
 	sql "select space(10) from fn_test_not_nullable order by kint"
+	sql """select k from (select length(space(number)) k from numbers("number" = "10"))t;""" // before #44919 will crash
 	qt_sql_split_part_Varchar_Varchar_Integer "select split_part(kvchrs1, ' ', 1) from fn_test order by kvchrs1"
 	qt_sql_split_part_Varchar_Varchar_Integer_notnull "select split_part(kvchrs1, ' ', 1) from fn_test_not_nullable order by kvchrs1"
 	qt_sql_split_part_String_String_Integer "select split_part(kstr, ' ', 1) from fn_test order by kstr"

From 0b718e863c8ec07d7d2351ea22e7a983947dded6 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Fri, 6 Dec 2024 12:52:42 +0530
Subject: [PATCH 204/399] [Enhancement] (nereids)implement
 DropRepositoryCommand in nereids (#45004)

Issue Number: close #42617
---
 .../org/apache/doris/nereids/DorisParser.g4   |  3 +-
 .../apache/doris/backup/BackupHandler.java    |  7 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  7 ++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/AlterRoleCommand.java      |  3 -
 .../plans/commands/DropRepositoryCommand.java | 67 +++++++++++++++++++
 .../plans/commands/ShowGrantsCommand.java     |  3 -
 .../commands/ShowPartitionIdCommand.java      |  3 -
 .../trees/plans/commands/ShowProcCommand.java |  4 --
 .../plans/commands/ShowRolesCommand.java      |  4 --
 .../plans/commands/ShowTableIdCommand.java    |  3 -
 .../trees/plans/visitor/CommandVisitor.java   | 13 ++--
 .../show_repositories_command.groovy          |  2 +-
 13 files changed, 93 insertions(+), 27 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropRepositoryCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index fd40e2c1400717..f229a1b2f4a7b1 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -216,6 +216,8 @@ supportedDropStatement
     | DROP FILE name=STRING_LITERAL
         ((FROM | IN) database=identifier)? properties=propertyClause            #dropFile
     | DROP WORKLOAD POLICY (IF EXISTS)? name=identifierOrText                   #dropWorkloadPolicy
+    | DROP REPOSITORY name=identifier                                           #dropRepository
+
     ;
 
 supportedShowStatement
@@ -684,7 +686,6 @@ unsupportedDropStatement
         functionIdentifier LEFT_PAREN functionArguments? RIGHT_PAREN            #dropFunction
     | DROP TABLE (IF EXISTS)? name=multipartIdentifier FORCE?                   #dropTable
     | DROP VIEW (IF EXISTS)? name=multipartIdentifier                           #dropView
-    | DROP REPOSITORY name=identifier                                           #dropRepository
     | DROP INDEX (IF EXISTS)? name=identifier ON tableName=multipartIdentifier  #dropIndex
     | DROP RESOURCE (IF EXISTS)? name=identifierOrText                          #dropResource
     | DROP ROW POLICY (IF EXISTS)? policyName=identifier
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index d70544add98747..6a12eee3a78cb3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -281,9 +281,14 @@ public void alterRepository(AlterRepositoryStmt stmt) throws DdlException {
 
     // handle drop repository stmt
     public void dropRepository(DropRepositoryStmt stmt) throws DdlException {
+        dropRepository(stmt.getRepoName());
+    }
+
+    // handle drop repository stmt
+    public void dropRepository(String repoName) throws DdlException {
         tryLock();
         try {
-            Repository repo = repoMgr.getRepo(stmt.getRepoName());
+            Repository repo = repoMgr.getRepo(repoName);
             if (repo == null) {
                 ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Repository does not exist");
             }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 9ed352b65e587a..71ea972c4ab2a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -118,6 +118,7 @@
 import org.apache.doris.nereids.DorisParser.DropFileContext;
 import org.apache.doris.nereids.DorisParser.DropMTMVContext;
 import org.apache.doris.nereids.DorisParser.DropProcedureContext;
+import org.apache.doris.nereids.DorisParser.DropRepositoryContext;
 import org.apache.doris.nereids.DorisParser.DropRoleContext;
 import org.apache.doris.nereids.DorisParser.DropSqlBlockRuleContext;
 import org.apache.doris.nereids.DorisParser.DropUserContext;
@@ -508,6 +509,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DropJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropProcedureCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropRepositoryCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropUserCommand;
@@ -4867,6 +4869,11 @@ public LogicalPlan visitDropFile(DropFileContext ctx) {
         return new DropFileCommand(stripQuotes(ctx.name.getText()), dbName, properties);
     }
 
+    @Override
+    public LogicalPlan visitDropRepository(DropRepositoryContext ctx) {
+        return new DropRepositoryCommand(stripQuotes(ctx.name.getText()));
+    }
+
     @Override
     public LogicalPlan visitDropSqlBlockRule(DropSqlBlockRuleContext ctx) {
         return new DropSqlBlockRuleCommand(visitIdentifierSeq(ctx.identifierSeq()), ctx.EXISTS() != null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index aaa93d70c355ac..bc8ccf83093b25 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -168,6 +168,7 @@ public enum PlanType {
     CREATE_PROCEDURE_COMMAND,
     DROP_PROCEDURE_COMMAND,
     DROP_ROLE_COMMAND,
+    DROP_REPOSITOORY_COMMAND,
     SHOW_PROCEDURE_COMMAND,
     SHOW_CREATE_PROCEDURE_COMMAND,
     CREATE_VIEW_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoleCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoleCommand.java
index fe8d0fd5db95a1..9f2d4ba22745ea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoleCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoleCommand.java
@@ -27,14 +27,11 @@
 import org.apache.doris.qe.StmtExecutor;
 
 import com.google.common.base.Strings;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 /**
  * alter role command
  */
 public class AlterRoleCommand extends AlterCommand {
-    public static final Logger LOG = LogManager.getLogger(AlterRoleCommand.class);
     private final String role;
     private final String comment;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropRepositoryCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropRepositoryCommand.java
new file mode 100644
index 00000000000000..4fef49c39937ed
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropRepositoryCommand.java
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * drop repository command
+ */
+public class DropRepositoryCommand extends DropCommand {
+    public static final Logger LOG = LogManager.getLogger(DropRepositoryCommand.class);
+    private final String repoName;
+
+    /**
+     * constructor
+     */
+    public DropRepositoryCommand(String repoName) {
+        super(PlanType.DROP_REPOSITOORY_COMMAND);
+        this.repoName = repoName;
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+        Env.getCurrentEnv().getBackupHandler().dropRepository(repoName);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitDropRepositoryCommand(this, context);
+    }
+
+    @Override
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
+        LOG.info("DropRepositoryCommand not supported in cloud mode");
+        throw new DdlException("Unsupported operation");
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowGrantsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowGrantsCommand.java
index f877f42b83599f..6d52d374fa7ae4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowGrantsCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowGrantsCommand.java
@@ -34,8 +34,6 @@
 import org.apache.doris.qe.StmtExecutor;
 
 import com.google.common.base.Preconditions;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import java.util.List;
 
@@ -43,7 +41,6 @@
  * show grants command
  */
 public class ShowGrantsCommand extends ShowCommand {
-    public static final Logger LOG = LogManager.getLogger(ShowGrantsCommand.class);
     private static final ShowResultSetMetaData META_DATA;
     private final boolean isAll;
     private UserIdentity userIdent; // if not given will update with self.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommand.java
index fb7e5cdf968330..47977a8e896a37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommand.java
@@ -35,8 +35,6 @@
 import org.apache.doris.qe.StmtExecutor;
 
 import com.google.common.collect.Lists;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -45,7 +43,6 @@
  * show partition command
  */
 public class ShowPartitionIdCommand extends ShowCommand {
-    public static final Logger LOG = LogManager.getLogger(ShowPartitionIdCommand.class);
     private final long partitionId;
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcCommand.java
index da2fb38c4d8d80..fc865a07457ab6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcCommand.java
@@ -35,16 +35,12 @@
 import org.apache.doris.qe.ShowResultSetMetaData;
 import org.apache.doris.qe.StmtExecutor;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
 import java.util.List;
 
 /**
  * show proc command
  */
 public class ShowProcCommand extends ShowCommand {
-    public static final Logger LOG = LogManager.getLogger(ShowProcCommand.class);
     private final String path;
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowRolesCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowRolesCommand.java
index 7cbb5f934cabba..4ca4c051cdbf8f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowRolesCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowRolesCommand.java
@@ -30,16 +30,12 @@
 import org.apache.doris.qe.ShowResultSetMetaData;
 import org.apache.doris.qe.StmtExecutor;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
 import java.util.List;
 
 /**
  * show roles command
  */
 public class ShowRolesCommand extends ShowCommand {
-    public static final Logger LOG = LogManager.getLogger(ShowRolesCommand.class);
     private static final ShowResultSetMetaData META_DATA;
 
     static {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableIdCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableIdCommand.java
index 3acd6c7992c4df..59e1834215a40f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableIdCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableIdCommand.java
@@ -33,8 +33,6 @@
 import org.apache.doris.qe.StmtExecutor;
 
 import com.google.common.collect.Lists;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -43,7 +41,6 @@
  * show table id command
  */
 public class ShowTableIdCommand extends ShowCommand {
-    public static final Logger LOG = LogManager.getLogger(ShowTableIdCommand.class);
     private final long tableId;
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 339bb4e4c6a5f0..77846ff3ad5328 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -55,6 +55,7 @@
 import org.apache.doris.nereids.trees.plans.commands.DropJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropProcedureCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropRepositoryCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropUserCommand;
@@ -506,12 +507,16 @@ default R visitShowLoadProfileCommand(ShowLoadProfileCommand showLoadProfileComm
         return visitCommand(showLoadProfileCommand, context);
     }
 
-    default R visitAlterSqlBlockRuleCommand(AlterSqlBlockRuleCommand dropRoleCommand, C context) {
-        return visitCommand(dropRoleCommand, context);
+    default R visitAlterSqlBlockRuleCommand(AlterSqlBlockRuleCommand cmd, C context) {
+        return visitCommand(cmd, context);
     }
 
-    default R visitCreateSqlBlockRuleCommand(CreateSqlBlockRuleCommand dropRoleCommand, C context) {
-        return visitCommand(dropRoleCommand, context);
+    default R visitCreateSqlBlockRuleCommand(CreateSqlBlockRuleCommand cmd, C context) {
+        return visitCommand(cmd, context);
+    }
+
+    default R visitDropRepositoryCommand(DropRepositoryCommand cmd, C context) {
+        return visitCommand(cmd, context);
     }
 
     default R visitCreateRoleCommand(CreateRoleCommand createRoleCommand, C context) {
diff --git a/regression-test/suites/nereids_p0/ddl/repository/show_repositories_command.groovy b/regression-test/suites/nereids_p0/ddl/repository/show_repositories_command.groovy
index 43fadf3b1a86d8..5d3520109002d3 100644
--- a/regression-test/suites/nereids_p0/ddl/repository/show_repositories_command.groovy
+++ b/regression-test/suites/nereids_p0/ddl/repository/show_repositories_command.groovy
@@ -47,7 +47,7 @@ suite("show_repositories_command") {
     def show_repo = checkNereidsExecuteWithResult("""SHOW REPOSITORIES;""").toString();
     assertTrue(show_repo.contains("${repoName}"))
 
-    sql """DROP REPOSITORY `${repoName}`;"""
+    checkNereidsExecute("DROP REPOSITORY `${repoName}`;")
 
     def show_repo_after_drop = checkNereidsExecuteWithResult("""SHOW REPOSITORIES;""").toString();
     assertFalse(show_repo_after_drop.contains("${repoName}"))

From 689bd3783d0d6138d17fe0e7ba6628ea7a110580 Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Fri, 6 Dec 2024 15:25:47 +0800
Subject: [PATCH 205/399] [fix](hudi)Add hudi catalog read partition table
 partition prune (#44669)

### What problem does this PR solve?
Problem Summary:
In the previous PR #41218, some partition pruning logic was changed,
which caused the hudi partition pruning to fail. This PR is to fix this
problem.

### Release note

[fix](hudi) fix hudi partition prune issue
---
 .../datasource/hive/HMSExternalTable.java     |  26 +-
 .../doris/datasource/hudi/HudiUtils.java      |  50 +++
 .../datasource/hudi/source/HudiScanNode.java  |  71 ++--
 .../translator/PhysicalPlanTranslator.java    |   2 +
 .../trees/plans/logical/LogicalHudiScan.java  |   2 +-
 .../hudi/test_hudi_partition_prune.out        | 357 ++++++++++++++++++
 .../hudi/test_hudi_partition_prune.groovy     | 333 ++++++++++++++++
 7 files changed, 792 insertions(+), 49 deletions(-)
 create mode 100644 regression-test/data/external_table_p2/hudi/test_hudi_partition_prune.out
 create mode 100644 regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 2115f47d777b80..7584b5b392feb5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -17,6 +17,7 @@
 
 package org.apache.doris.datasource.hive;
 
+import org.apache.doris.analysis.TableSnapshot;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.ListPartitionItem;
@@ -31,6 +32,7 @@
 import org.apache.doris.common.DdlException;
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.SchemaCacheValue;
+import org.apache.doris.datasource.TablePartitionValues;
 import org.apache.doris.datasource.hudi.HudiUtils;
 import org.apache.doris.datasource.iceberg.IcebergUtils;
 import org.apache.doris.datasource.mvcc.MvccSnapshot;
@@ -41,6 +43,7 @@
 import org.apache.doris.mtmv.MTMVSnapshotIf;
 import org.apache.doris.mtmv.MTMVTimestampSnapshot;
 import org.apache.doris.nereids.exceptions.NotSupportedException;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
 import org.apache.doris.qe.GlobalVariable;
 import org.apache.doris.statistics.AnalysisInfo;
 import org.apache.doris.statistics.BaseAnalysisTask;
@@ -302,7 +305,28 @@ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
 
     @Override
     public boolean supportInternalPartitionPruned() {
-        return getDlaType() == DLAType.HIVE;
+        return getDlaType() == DLAType.HIVE || getDlaType() == DLAType.HUDI;
+    }
+
+    public SelectedPartitions initHudiSelectedPartitions(Optional<TableSnapshot> tableSnapshot) {
+        if (getDlaType() != DLAType.HUDI) {
+            return SelectedPartitions.NOT_PRUNED;
+        }
+
+        if (getPartitionColumns().isEmpty()) {
+            return SelectedPartitions.NOT_PRUNED;
+        }
+        TablePartitionValues tablePartitionValues = HudiUtils.getPartitionValues(tableSnapshot, this);
+
+        Map<Long, PartitionItem> idToPartitionItem = tablePartitionValues.getIdToPartitionItem();
+        Map<Long, String> idToNameMap = tablePartitionValues.getPartitionIdToNameMap();
+
+        Map<String, PartitionItem> nameToPartitionItems = Maps.newHashMapWithExpectedSize(idToPartitionItem.size());
+        for (Entry<Long, PartitionItem> entry : idToPartitionItem.entrySet()) {
+            nameToPartitionItems.put(idToNameMap.get(entry.getKey()), entry.getValue());
+        }
+
+        return new SelectedPartitions(nameToPartitionItems.size(), nameToPartitionItems, false);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
index c98d994a28a08f..0f38abafaa4d98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java
@@ -17,24 +17,35 @@
 
 package org.apache.doris.datasource.hudi;
 
+import org.apache.doris.analysis.TableSnapshot;
 import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.MapType;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.StructField;
 import org.apache.doris.catalog.StructType;
 import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.TablePartitionValues;
+import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper;
+import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor;
 
 import org.apache.avro.LogicalType;
 import org.apache.avro.LogicalTypes;
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Field;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
 import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.util.Option;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 public class HudiUtils {
@@ -231,4 +242,43 @@ private static Type handleUnionType(Schema avroSchema) {
         }
         return Type.UNSUPPORTED;
     }
+
+    public static TablePartitionValues getPartitionValues(Optional<TableSnapshot> tableSnapshot,
+            HMSExternalTable hmsTable) {
+        TablePartitionValues partitionValues = new TablePartitionValues();
+        if (hmsTable.getPartitionColumns().isEmpty()) {
+            //isn't partition table.
+            return partitionValues;
+        }
+
+        HoodieTableMetaClient hudiClient = HiveMetaStoreClientHelper.getHudiClient(hmsTable);
+        HudiCachedPartitionProcessor processor = (HudiCachedPartitionProcessor) Env.getCurrentEnv()
+                .getExtMetaCacheMgr().getHudiPartitionProcess(hmsTable.getCatalog());
+        boolean useHiveSyncPartition = hmsTable.useHiveSyncPartition();
+
+        if (tableSnapshot.isPresent()) {
+            if (tableSnapshot.get().getType() == TableSnapshot.VersionType.VERSION) {
+                // Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`";
+                return partitionValues;
+            }
+            String queryInstant = tableSnapshot.get().getTime().replaceAll("[-: ]", "");
+
+            partitionValues =
+                    HiveMetaStoreClientHelper.ugiDoAs(
+                            HiveMetaStoreClientHelper.getConfiguration(hmsTable),
+                            () -> processor.getSnapshotPartitionValues(
+                                    hmsTable, hudiClient, queryInstant, useHiveSyncPartition));
+        } else {
+            HoodieTimeline timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants();
+            Option<HoodieInstant> snapshotInstant = timeline.lastInstant();
+            if (!snapshotInstant.isPresent()) {
+                return partitionValues;
+            }
+            partitionValues =
+                    HiveMetaStoreClientHelper.ugiDoAs(
+                            HiveMetaStoreClientHelper.getConfiguration(hmsTable),
+                            () -> processor.getPartitionValues(hmsTable, hudiClient, useHiveSyncPartition));
+        }
+        return partitionValues;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
index a73a2065d0ffaf..b2cad8ab710178 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
@@ -21,6 +21,7 @@
 import org.apache.doris.analysis.TableSnapshot;
 import org.apache.doris.analysis.TupleDescriptor;
 import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ListPartitionItem;
 import org.apache.doris.catalog.PartitionItem;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
@@ -30,12 +31,10 @@
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.FileSplit;
 import org.apache.doris.datasource.TableFormatType;
-import org.apache.doris.datasource.TablePartitionValues;
 import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper;
 import org.apache.doris.datasource.hive.HivePartition;
 import org.apache.doris.datasource.hive.source.HiveScanNode;
 import org.apache.doris.datasource.hudi.HudiUtils;
-import org.apache.doris.planner.ListPartitionPrunerV2;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.SessionVariable;
@@ -70,7 +69,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -286,50 +284,29 @@ private boolean canUseNativeReader() {
         return !sessionVariable.isForceJniScanner() && isCowTable;
     }
 
-    private List<HivePartition> getPrunedPartitions(
-            HoodieTableMetaClient metaClient, Option<String> snapshotTimestamp) throws AnalysisException {
+    private List<HivePartition> getPrunedPartitions(HoodieTableMetaClient metaClient) {
         List<Type> partitionColumnTypes = hmsTable.getPartitionColumnTypes();
         if (!partitionColumnTypes.isEmpty()) {
-            HudiCachedPartitionProcessor processor = (HudiCachedPartitionProcessor) Env.getCurrentEnv()
-                    .getExtMetaCacheMgr().getHudiPartitionProcess(hmsTable.getCatalog());
-            TablePartitionValues partitionValues;
-            if (snapshotTimestamp.isPresent()) {
-                partitionValues = processor.getSnapshotPartitionValues(
-                        hmsTable, metaClient, snapshotTimestamp.get(), useHiveSyncPartition);
-            } else {
-                partitionValues = processor.getPartitionValues(hmsTable, metaClient, useHiveSyncPartition);
-            }
-            if (partitionValues != null) {
-                // 2. prune partitions by expr
-                partitionValues.readLock().lock();
-                try {
-                    Map<Long, PartitionItem> idToPartitionItem = partitionValues.getIdToPartitionItem();
-                    this.totalPartitionNum = idToPartitionItem.size();
-                    ListPartitionPrunerV2 pruner = new ListPartitionPrunerV2(idToPartitionItem,
-                            hmsTable.getPartitionColumns(), columnNameToRange,
-                            partitionValues.getUidToPartitionRange(),
-                            partitionValues.getRangeToId(),
-                            partitionValues.getSingleColumnRangeMap(),
-                            true);
-                    Collection<Long> filteredPartitionIds = pruner.prune();
-                    this.selectedPartitionNum = filteredPartitionIds.size();
-                    // 3. get partitions from cache
-                    String dbName = hmsTable.getDbName();
-                    String tblName = hmsTable.getName();
-                    String inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat();
-                    String basePath = metaClient.getBasePathV2().toString();
-                    Map<Long, String> partitionIdToNameMap = partitionValues.getPartitionIdToNameMap();
-                    Map<Long, List<String>> partitionValuesMap = partitionValues.getPartitionValuesMap();
-                    return filteredPartitionIds.stream().map(id -> {
-                        String path = basePath + "/" + partitionIdToNameMap.get(id);
-                        return new HivePartition(
-                                dbName, tblName, false, inputFormat, path, partitionValuesMap.get(id),
-                                Maps.newHashMap());
-                    }).collect(Collectors.toList());
-                } finally {
-                    partitionValues.readLock().unlock();
-                }
-            }
+            this.totalPartitionNum = selectedPartitions.totalPartitionNum;
+            Map<String, PartitionItem> prunedPartitions = selectedPartitions.selectedPartitions;
+            this.selectedPartitionNum = prunedPartitions.size();
+
+            String dbName = hmsTable.getDbName();
+            String tblName = hmsTable.getName();
+            String inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat();
+            String basePath = metaClient.getBasePathV2().toString();
+
+            List<HivePartition> hivePartitions = Lists.newArrayList();
+            prunedPartitions.forEach(
+                    (key, value) -> {
+                        String path = basePath + "/" + key;
+                        hivePartitions.add(new HivePartition(
+                                dbName, tblName, false, inputFormat, path,
+                                ((ListPartitionItem) value).getItems().get(0).getPartitionValuesAsStringList(),
+                                Maps.newHashMap()));
+                    }
+            );
+            return hivePartitions;
         }
         // unpartitioned table, create a dummy partition to save location and
         // inputformat,
@@ -420,7 +397,7 @@ public List<Split> getSplits() throws UserException {
         if (!partitionInit) {
             prunedPartitions = HiveMetaStoreClientHelper.ugiDoAs(
                     HiveMetaStoreClientHelper.getConfiguration(hmsTable),
-                    () -> getPrunedPartitions(hudiClient, snapshotTimestamp));
+                    () -> getPrunedPartitions(hudiClient));
             partitionInit = true;
         }
         List<Split> splits = Collections.synchronizedList(new ArrayList<>());
@@ -482,7 +459,7 @@ public boolean isBatchMode() {
             // Non partition table will get one dummy partition
             prunedPartitions = HiveMetaStoreClientHelper.ugiDoAs(
                     HiveMetaStoreClientHelper.getConfiguration(hmsTable),
-                    () -> getPrunedPartitions(hudiClient, snapshotTimestamp));
+                    () -> getPrunedPartitions(hudiClient));
             partitionInit = true;
         }
         int numPartitions = ConnectContext.get().getSessionVariable().getNumPartitionsInBatchMode();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 468a58bd6f6815..bea5eec432b2ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -653,6 +653,8 @@ public PlanFragment visitPhysicalHudiScan(PhysicalHudiScan fileScan, PlanTransla
         if (fileScan.getTableSnapshot().isPresent()) {
             ((FileQueryScanNode) scanNode).setQueryTableSnapshot(fileScan.getTableSnapshot().get());
         }
+        HudiScanNode hudiScanNode = (HudiScanNode) scanNode;
+        hudiScanNode.setSelectedPartitions(fileScan.getSelectedPartitions());
         return getPlanFragmentForPhysicalFileScan(fileScan, context, scanNode, table, tupleDescriptor);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
index 629690889432b3..51e68eb07631ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
@@ -84,7 +84,7 @@ protected LogicalHudiScan(RelationId id, ExternalTable table, List<String> quali
     public LogicalHudiScan(RelationId id, ExternalTable table, List<String> qualifier,
             Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
         this(id, table, qualifier, Optional.empty(), Optional.empty(),
-                SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot,
+                ((HMSExternalTable) table).initHudiSelectedPartitions(tableSnapshot), tableSample, tableSnapshot,
                 Optional.empty(), Optional.empty());
     }
 
diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_partition_prune.out b/regression-test/data/external_table_p2/hudi/test_hudi_partition_prune.out
new file mode 100644
index 00000000000000..fd3eafa0255722
--- /dev/null
+++ b/regression-test/data/external_table_p2/hudi/test_hudi_partition_prune.out
@@ -0,0 +1,357 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !time_travel_two_partition_1_3 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+
+-- !time_travel_two_partition_2_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !time_travel_two_partition_3_1 --
+4	David	US	2
+5	Eva	US	2
+
+-- !time_travel_two_partition_4_0 --
+
+-- !time_travel_two_partition_5_0 --
+
+-- !time_travel_two_partition_6_1 --
+1	Alice	US	1
+
+-- !one_partition_boolean --
+1	Alice	true
+2	Bob	true
+
+-- !one_partition_tinyint --
+1	Alice	1
+2	Bob	1
+
+-- !one_partition_smallint --
+1	Alice	10
+2	Bob	10
+
+-- !one_partition_int --
+1	Alice	100
+2	Bob	100
+
+-- !one_partition_bigint --
+1	Alice	1234567890
+2	Bob	1234567890
+
+-- !one_partition_string --
+1	Alice	RegionA
+2	Bob	RegionA
+
+-- !one_partition_date --
+1	Alice	2023-12-01
+2	Bob	2023-12-01
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !time_travel_two_partition_1_3 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+
+-- !time_travel_two_partition_2_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !time_travel_two_partition_3_1 --
+4	David	US	2
+5	Eva	US	2
+
+-- !time_travel_two_partition_4_0 --
+
+-- !time_travel_two_partition_5_0 --
+
+-- !time_travel_two_partition_6_1 --
+1	Alice	US	1
+
+-- !one_partition_boolean --
+1	Alice	true
+2	Bob	true
+
+-- !one_partition_tinyint --
+1	Alice	1
+2	Bob	1
+
+-- !one_partition_smallint --
+1	Alice	10
+2	Bob	10
+
+-- !one_partition_int --
+1	Alice	100
+2	Bob	100
+
+-- !one_partition_bigint --
+1	Alice	1234567890
+2	Bob	1234567890
+
+-- !one_partition_string --
+1	Alice	RegionA
+2	Bob	RegionA
+
+-- !one_partition_date --
+1	Alice	2023-12-01
+2	Bob	2023-12-01
+
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy
new file mode 100644
index 00000000000000..eea17b01711b44
--- /dev/null
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy
@@ -0,0 +1,333 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hudi_partition_prune", "p2,external,hudi,external_remote,external_remote_hudi") {
+    String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+        logger.info("disable hudi test")
+    }
+
+    String catalog_name = "test_hudi_partition_prune"
+    String props = context.config.otherConfigs.get("hudiEmrCatalog")
+    sql """drop catalog if exists ${catalog_name};"""
+    
+    for (String  use_hive_sync_partition : ['true','false']) {
+
+        sql """
+            create catalog if not exists ${catalog_name} properties (
+                ${props}
+                ,"use_hive_sync_partition"="${use_hive_sync_partition}"
+            );
+        """
+
+        sql """ switch ${catalog_name};"""
+        sql """ use regression_hudi;""" 
+        sql """ set enable_fallback_to_original_planner=false """
+
+
+
+        def one_partition_1_1 = """SELECT id,name,part1  FROM one_partition_tb WHERE part1 = 2024 ORDER BY id;"""
+        def one_partition_2_1 = """SELECT id,name,part1  FROM one_partition_tb WHERE part1 = 2025 ORDER BY id;"""
+        def one_partition_3_all = """SELECT id,name,part1  FROM one_partition_tb ORDER BY id;"""
+        def one_partition_4_all = """SELECT id,name,part1  FROM one_partition_tb WHERE id = 5 ORDER BY id;"""
+        def one_partition_5_1 = """SELECT id,name,part1  FROM one_partition_tb WHERE part1 = 2024 AND id >= 3 ORDER BY id;"""
+
+        def two_partition_1_1 = """SELECT id,name,part1,part2  FROM two_partition_tb WHERE part1 = 'US' AND part2 = 1 ORDER BY id;"""
+        def two_partition_2_1 = """SELECT id,name,part1,part2  FROM two_partition_tb WHERE part1 = 'EU' AND part2 = 2 ORDER BY id;"""
+        def two_partition_3_2 = """SELECT id,name,part1,part2  FROM two_partition_tb WHERE part1 = 'US' ORDER BY id;"""
+        def two_partition_4_all = """SELECT id,name,part1,part2  FROM two_partition_tb ORDER BY id;"""
+        def two_partition_5_1 = """SELECT id,name,part1,part2  FROM two_partition_tb WHERE part1 = 'US' AND part2 = 2 AND id > 5 ORDER BY id;"""
+        def two_partition_6_1 = """SELECT id,name,part1,part2  FROM two_partition_tb WHERE part1 = 'EU' AND part2 = 2 ORDER BY id;"""
+
+        def three_partition_1_1 = """SELECT id,name,part1,part2,part3  FROM three_partition_tb WHERE part1 = 'US' AND part2 = 2024 AND part3 = 'Q1' ORDER BY id;"""
+        def three_partition_2_1 = """SELECT id,name,part1,part2,part3  FROM three_partition_tb WHERE part1 = 'EU' AND part2 = 2025 AND part3 = 'Q2' ORDER BY id;"""
+        def three_partition_3_3 = """SELECT id,name,part1,part2,part3  FROM three_partition_tb WHERE part1 = 'AS' AND part2 = 2025 ORDER BY id;"""
+        def three_partition_4_2 = """SELECT id,name,part1,part2,part3  FROM three_partition_tb WHERE part1 = 'US' AND part3 = 'Q1' ORDER BY id;"""
+        def three_partition_5_all = """SELECT id,name,part1,part2,part3  FROM three_partition_tb ORDER BY id;"""
+        def three_partition_6_1 = """SELECT id,name,part1,part2,part3  FROM three_partition_tb WHERE part1 = 'EU' AND part2 = 2024 AND part3 = 'Q1' ORDER BY id;"""
+        def three_partition_7_7 = """SELECT id,name,part1,part2,part3  FROM three_partition_tb WHERE part2 = 2025 ORDER BY id;"""
+        def three_partition_8_2 = """SELECT id,name,part1,part2,part3  FROM three_partition_tb WHERE part1 = 'US' AND part3 = 'Q2' AND id BETWEEN 6 AND 10 ORDER BY id;"""
+
+        def one_partition_boolean = """SELECT id,name,part1  FROM boolean_partition_tb WHERE part1 = true ORDER BY id;"""
+        def one_partition_tinyint = """SELECT id,name,part1  FROM tinyint_partition_tb WHERE part1 = 1 ORDER BY id;"""
+        def one_partition_smallint = """SELECT id,name,part1  FROM smallint_partition_tb WHERE part1 = 10 ORDER BY id;"""
+        def one_partition_int = """SELECT id,name,part1  FROM int_partition_tb WHERE part1 = 100 ORDER BY id;"""
+        def one_partition_bigint = """SELECT id,name,part1  FROM bigint_partition_tb WHERE part1 = 1234567890 ORDER BY id;"""
+        def one_partition_string = """SELECT id,name,part1  FROM string_partition_tb WHERE part1 = 'RegionA' ORDER BY id;"""
+        def one_partition_date = """SELECT id,name,part1  FROM date_partition_tb WHERE part1 = '2023-12-01' ORDER BY id;"""
+        def one_partition_timestamp = """SELECT id,name,part1  FROM timestamp_partition_tb WHERE part1 = '2023-12-01 08:00:00' ORDER BY id;"""
+
+
+
+        qt_one_partition_1_1 one_partition_1_1
+        explain {
+            sql("${one_partition_1_1}")
+            contains "partition=1/2"
+        }
+
+        qt_one_partition_2_1 one_partition_2_1
+        explain {
+            sql("${one_partition_2_1}")
+            contains "partition=1/2"
+        }
+
+        qt_one_partition_3_all one_partition_3_all
+        explain {
+            sql("${one_partition_3_all}")
+            contains "partition=2/2"
+        }
+
+        qt_one_partition_4_all one_partition_4_all
+        explain {
+            sql("${one_partition_4_all}")
+            contains "partition=2/2"
+        }
+
+        qt_one_partition_5_1 one_partition_5_1
+        explain {
+            sql("${one_partition_5_1}")
+            contains "partition=1/2"
+        }
+
+
+        qt_two_partition_1_1 two_partition_1_1
+        explain {
+            sql("${two_partition_1_1}")
+            contains "partition=1/4"
+        }
+
+        qt_two_partition_2_1 two_partition_2_1
+        explain {
+            sql("${two_partition_2_1}")
+            contains "partition=1/4"
+        }
+
+        qt_two_partition_3_2 two_partition_3_2
+        explain {
+            sql("${two_partition_3_2}")
+            contains "partition=2/4"
+        }
+
+        qt_two_partition_4_all two_partition_4_all
+        explain {
+            sql("${two_partition_4_all}")
+            contains "partition=4/4"
+        }
+
+        qt_two_partition_5_1 two_partition_5_1
+        explain {
+            sql("${two_partition_5_1}")
+            contains "partition=1/4"
+        }
+
+        qt_two_partition_6_1 two_partition_6_1
+        explain {
+            sql("${two_partition_6_1}")
+            contains "partition=1/4"
+        }
+
+
+
+        qt_three_partition_1_1 three_partition_1_1
+        explain {
+            sql("${three_partition_1_1}")
+            contains "partition=1/10"
+        }
+
+        qt_three_partition_2_1 three_partition_2_1
+        explain {
+            sql("${three_partition_2_1}")
+            contains "partition=1/10"
+        }
+
+        qt_three_partition_3_3 three_partition_3_3
+        explain {
+            sql("${three_partition_3_3}")
+            contains "partition=3/10"
+        }
+
+        qt_three_partition_4_2 three_partition_4_2
+        explain {
+            sql("${three_partition_4_2}")
+            contains "partition=2/10"
+        }
+
+        qt_three_partition_5_all three_partition_5_all
+        explain {
+            sql("${three_partition_5_all}")
+            contains "partition=10/10"
+        }
+
+        qt_three_partition_6_1 three_partition_6_1
+        explain {
+            sql("${three_partition_6_1}")
+            contains "partition=1/10"
+        }
+
+        qt_three_partition_7_7 three_partition_7_7
+        explain {
+            sql("${three_partition_7_7}")
+            contains "partition=7/10"
+        }
+
+        qt_three_partition_8_2 three_partition_8_2
+        explain {
+            sql("${three_partition_8_2}")
+            contains "partition=2/10"
+        }
+
+
+        // 0 partitions
+        def one_partition_6_0 = """SELECT id,name,part1  FROM one_partition_tb WHERE part1 = 2023 ORDER BY id;"""
+        qt_one_partition_6_0 one_partition_6_0
+        explain {
+            sql("${one_partition_6_0}")
+            contains "partition=0/2"
+        }
+
+        def two_partition_7_0 = """SELECT id,name,part1  FROM two_partition_tb WHERE part1 = 'CN' AND part2 = 1 ORDER BY id;"""
+        qt_two_partition_7_0 two_partition_7_0
+        explain {
+            sql("${two_partition_7_0}")
+            contains "partition=0/4"
+        }
+
+        def two_partition_8_0 = """SELECT id,name,part1  FROM two_partition_tb WHERE part1 = 'US' AND part2 = 3 ORDER BY id;"""
+        qt_two_partition_8_0 two_partition_8_0
+        explain {
+            sql("${two_partition_8_0}")
+            contains "partition=0/4"
+        }
+
+        def three_partition_9_0 = """SELECT id,name,part1  FROM three_partition_tb WHERE part1 = 'US' AND part2 = 2023 AND part3 = 'Q1' ORDER BY id;"""
+        qt_three_partition_9_0 three_partition_9_0
+        explain {
+            sql("${three_partition_9_0}")
+            contains "partition=0/10"
+        }
+
+        def three_partition_10_0 = """SELECT id,name,part1  FROM three_partition_tb WHERE part1 = 'EU' AND part2 = 2024 AND part3 = 'Q4' ORDER BY id;"""
+        qt_three_partition_10_0 three_partition_10_0
+        explain {
+            sql("${three_partition_10_0}")
+            contains "partition=0/10"
+        }
+
+        def three_partition_11_0 = """SELECT id,name,part1  FROM three_partition_tb WHERE part1 = 'AS' AND part2 = 2025 AND part3 = 'Q4' ORDER BY id;"""
+        qt_three_partition_11_0 three_partition_11_0
+        explain {
+            sql("${three_partition_11_0}")
+            contains "partition=0/10"
+        }
+
+
+        //time travel 
+        def time_travel_two_partition_1_3 = "select id,name,part1,part2 from two_partition_tb  FOR TIME AS OF '20241202171226401' order by id;"
+        def time_travel_two_partition_2_2 = "select id,name,part1,part2 from two_partition_tb  FOR TIME AS OF '20241202171226401' where part1='US' order by id;"
+        def time_travel_two_partition_3_1 = "select id,name,part1,part2 from two_partition_tb  FOR TIME AS OF '20241202171226401' where part2=2 order by id;"
+        def time_travel_two_partition_4_0 = "select id,name,part1,part2 from two_partition_tb  FOR TIME AS OF '20241202171226401' where part2=10 order by id;"
+
+        qt_time_travel_two_partition_1_3 time_travel_two_partition_1_3
+        explain {
+            sql("${time_travel_two_partition_1_3}")
+            contains "partition=3/3"
+        }
+
+
+        qt_time_travel_two_partition_2_2 time_travel_two_partition_2_2
+        explain {
+            sql("${time_travel_two_partition_2_2}")
+            contains "partition=2/3"
+        }
+
+        qt_time_travel_two_partition_3_1 time_travel_two_partition_3_1
+        explain {
+            sql("${time_travel_two_partition_3_1}")
+            contains "partition=1/3"
+        }
+
+        qt_time_travel_two_partition_4_0 time_travel_two_partition_4_0
+        explain {
+            sql("${time_travel_two_partition_4_0}")
+            contains "partition=0/3"
+        }
+
+
+
+
+        def time_travel_two_partition_5_0 = "select id,name,part1,part2 from two_partition_tb  FOR TIME AS OF '20231126012025218' order by id;"
+        qt_time_travel_two_partition_5_0 time_travel_two_partition_5_0
+        explain {
+            sql("${time_travel_two_partition_5_0}")
+            contains "partition=0/0"
+        }
+
+        def time_travel_two_partition_6_1 = "select id,name,part1,part2 from two_partition_tb  FOR TIME AS OF '20241202171214902' order by id;"
+        qt_time_travel_two_partition_6_1 time_travel_two_partition_6_1
+        explain {
+            sql("${time_travel_two_partition_6_1}")
+            contains "partition=1/1"
+        }
+
+        // all types as partition
+        qt_one_partition_boolean one_partition_boolean
+        explain {
+            sql("${one_partition_boolean}")
+            contains "partition=1/2"
+        }
+        qt_one_partition_tinyint one_partition_tinyint
+        explain {
+            sql("${one_partition_tinyint}")
+            contains "partition=1/2"
+        }
+        qt_one_partition_smallint one_partition_smallint
+        explain {
+            sql("${one_partition_smallint}")
+            contains "partition=1/2"
+        }
+        qt_one_partition_int one_partition_int
+        explain {
+            sql("${one_partition_int}")
+            contains "partition=1/2"
+        }
+        qt_one_partition_bigint one_partition_bigint
+        explain {
+            sql("${one_partition_bigint}")
+            contains "partition=1/2"
+        }
+        qt_one_partition_string one_partition_string
+        explain {
+            sql("${one_partition_string}")
+            contains "partition=1/2"
+        }
+        qt_one_partition_date one_partition_date
+        explain {
+            sql("${one_partition_date}")
+            contains "partition=1/2"
+        }
+        // qt_one_partition_timestamp one_partition_timestamp
+        // explain {
+        //     sql("${one_partition_timestamp}")
+        //     contains "partition=1/2"
+        // }
+
+        sql """drop catalog if exists ${catalog_name};"""
+
+
+    }
+    
+}
\ No newline at end of file

From aec14e6a8ab90d0c46bde3c8c61c1a72f2e72691 Mon Sep 17 00:00:00 2001
From: Gavin Chou <gavin@selectdb.com>
Date: Fri, 6 Dec 2024 15:26:14 +0800
Subject: [PATCH 206/399] [opt](cloud) reset ms response message in case it is
 reused for retry (#45068)

---
 cloud/src/meta-service/meta_service.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h
index 7af96cbc14b8ee..5374cbea741fb0 100644
--- a/cloud/src/meta-service/meta_service.h
+++ b/cloud/src/meta-service/meta_service.h
@@ -729,6 +729,7 @@ class MetaServiceProxy final : public MetaService {
         int32_t retry_times = 0;
         uint64_t duration_ms = 0, retry_drift_ms = 0;
         while (true) {
+            resp->Clear(); // reset the response message in case it is reused for retry
             (impl_.get()->*method)(ctrl, req, resp, brpc::DoNothing());
             MetaServiceCode code = resp->status().code();
             if (code != MetaServiceCode::KV_TXN_STORE_GET_RETRYABLE &&

From c96699157f956a1ff17e27a5e18d4d20bcc043a8 Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Fri, 6 Dec 2024 15:59:13 +0800
Subject: [PATCH 207/399] [minor](runtime state) Removed unnecessary
 constructors from RuntimeState. (#45001)

### What problem does this PR solve?

Problem Summary:

Removed unnecessary constructors from RuntimeState
---
 .../partitioned_aggregation_sink_operator.cpp |  2 +-
 ...artitioned_aggregation_source_operator.cpp |  2 +-
 .../partitioned_hash_join_probe_operator.cpp  |  2 +-
 .../partitioned_hash_join_sink_operator.cpp   |  2 +-
 .../exec/spill_sort_sink_operator.cpp         |  2 +-
 .../exec/spill_sort_source_operator.cpp       |  2 +-
 be/src/pipeline/pipeline_fragment_context.cpp |  6 ++--
 be/src/runtime/runtime_state.cpp              | 31 -------------------
 be/src/runtime/runtime_state.h                |  7 +----
 9 files changed, 10 insertions(+), 46 deletions(-)

diff --git a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp
index ab0a43f4a635cf..5273960a5c1c29 100644
--- a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp
@@ -202,7 +202,7 @@ size_t PartitionedAggSinkOperatorX::revocable_mem_size(RuntimeState* state) cons
 
 Status PartitionedAggSinkLocalState::setup_in_memory_agg_op(RuntimeState* state) {
     _runtime_state = RuntimeState::create_unique(
-            nullptr, state->fragment_instance_id(), state->query_id(), state->fragment_id(),
+            state->fragment_instance_id(), state->query_id(), state->fragment_id(),
             state->query_options(), TQueryGlobals {}, state->exec_env(), state->get_query_ctx());
     _runtime_state->set_task_execution_context(state->get_task_execution_context().lock());
     _runtime_state->set_be_number(state->be_number());
diff --git a/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp b/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp
index 655a6e19725a9b..cdc6ef881d436d 100644
--- a/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp
@@ -166,7 +166,7 @@ Status PartitionedAggSourceOperatorX::get_block(RuntimeState* state, vectorized:
 
 Status PartitionedAggLocalState::setup_in_memory_agg_op(RuntimeState* state) {
     _runtime_state = RuntimeState::create_unique(
-            nullptr, state->fragment_instance_id(), state->query_id(), state->fragment_id(),
+            state->fragment_instance_id(), state->query_id(), state->fragment_id(),
             state->query_options(), TQueryGlobals {}, state->exec_env(), state->get_query_ctx());
     _runtime_state->set_task_execution_context(state->get_task_execution_context().lock());
     _runtime_state->set_be_number(state->be_number());
diff --git a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp
index 0e56acc1c574b2..20b25d54ff9f16 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp
@@ -606,7 +606,7 @@ Status PartitionedHashJoinProbeOperatorX::_setup_internal_operators(
     }
 
     local_state._runtime_state = RuntimeState::create_unique(
-            nullptr, state->fragment_instance_id(), state->query_id(), state->fragment_id(),
+            state->fragment_instance_id(), state->query_id(), state->fragment_id(),
             state->query_options(), TQueryGlobals {}, state->exec_env(), state->get_query_ctx());
 
     local_state._runtime_state->set_task_execution_context(
diff --git a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp
index d221eaeed0faba..878c3870946f1c 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp
@@ -438,7 +438,7 @@ Status PartitionedHashJoinSinkOperatorX::_setup_internal_operator(RuntimeState*
     auto& local_state = get_local_state(state);
 
     local_state._shared_state->inner_runtime_state = RuntimeState::create_unique(
-            nullptr, state->fragment_instance_id(), state->query_id(), state->fragment_id(),
+            state->fragment_instance_id(), state->query_id(), state->fragment_id(),
             state->query_options(), TQueryGlobals {}, state->exec_env(), state->get_query_ctx());
     local_state._shared_state->inner_runtime_state->set_task_execution_context(
             state->get_task_execution_context().lock());
diff --git a/be/src/pipeline/exec/spill_sort_sink_operator.cpp b/be/src/pipeline/exec/spill_sort_sink_operator.cpp
index 6e6689d4134deb..6071301c1d7bcc 100644
--- a/be/src/pipeline/exec/spill_sort_sink_operator.cpp
+++ b/be/src/pipeline/exec/spill_sort_sink_operator.cpp
@@ -80,7 +80,7 @@ Status SpillSortSinkLocalState::close(RuntimeState* state, Status execsink_statu
 
 Status SpillSortSinkLocalState::setup_in_memory_sort_op(RuntimeState* state) {
     _runtime_state = RuntimeState::create_unique(
-            nullptr, state->fragment_instance_id(), state->query_id(), state->fragment_id(),
+            state->fragment_instance_id(), state->query_id(), state->fragment_id(),
             state->query_options(), TQueryGlobals {}, state->exec_env(), state->get_query_ctx());
     _runtime_state->set_task_execution_context(state->get_task_execution_context().lock());
     _runtime_state->set_be_number(state->be_number());
diff --git a/be/src/pipeline/exec/spill_sort_source_operator.cpp b/be/src/pipeline/exec/spill_sort_source_operator.cpp
index e766cb27168de1..69ed816fa9142d 100644
--- a/be/src/pipeline/exec/spill_sort_source_operator.cpp
+++ b/be/src/pipeline/exec/spill_sort_source_operator.cpp
@@ -212,7 +212,7 @@ Status SpillSortLocalState::_create_intermediate_merger(
 }
 Status SpillSortLocalState::setup_in_memory_sort_op(RuntimeState* state) {
     _runtime_state = RuntimeState::create_unique(
-            nullptr, state->fragment_instance_id(), state->query_id(), state->fragment_id(),
+            state->fragment_instance_id(), state->query_id(), state->fragment_id(),
             state->query_options(), TQueryGlobals {}, state->exec_env(), state->get_query_ctx());
     _runtime_state->set_task_execution_context(state->get_task_execution_context().lock());
     _runtime_state->set_be_number(state->be_number());
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index e6f257f9da792a..8ceb63eb99324c 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -397,9 +397,9 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag
                         << print_id(_task_runtime_states[pip_idx][i]->fragment_instance_id()) << " "
                         << pipeline->debug_string();
                 _task_runtime_states[pip_idx][i] = RuntimeState::create_unique(
-                        this, local_params.fragment_instance_id, request.query_id,
-                        request.fragment_id, request.query_options, _query_ctx->query_globals,
-                        _exec_env, _query_ctx.get());
+                        local_params.fragment_instance_id, request.query_id, request.fragment_id,
+                        request.query_options, _query_ctx->query_globals, _exec_env,
+                        _query_ctx.get());
                 auto& task_runtime_state = _task_runtime_states[pip_idx][i];
                 _runtime_filter_states[i]->set_state(task_runtime_state.get());
                 {
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 579329c1082633..f3376d06858ec0 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -123,37 +123,6 @@ RuntimeState::RuntimeState(const TUniqueId& instance_id, const TUniqueId& query_
     DCHECK(_query_mem_tracker != nullptr && _query_mem_tracker->label() != "Orphan");
 }
 
-RuntimeState::RuntimeState(pipeline::PipelineFragmentContext*, const TUniqueId& instance_id,
-                           const TUniqueId& query_id, int32_t fragment_id,
-                           const TQueryOptions& query_options, const TQueryGlobals& query_globals,
-                           ExecEnv* exec_env, QueryContext* ctx)
-        : _profile("Fragment " + print_id(instance_id)),
-          _load_channel_profile("<unnamed>"),
-          _obj_pool(new ObjectPool()),
-          _unreported_error_idx(0),
-          _query_id(query_id),
-          _fragment_id(fragment_id),
-          _per_fragment_instance_idx(0),
-          _num_rows_load_total(0),
-          _num_rows_load_filtered(0),
-          _num_rows_load_unselected(0),
-          _num_rows_filtered_in_strict_mode_partial_update(0),
-          _num_print_error_rows(0),
-          _num_bytes_load_total(0),
-          _num_finished_scan_range(0),
-          _error_row_number(0),
-          _query_ctx(ctx) {
-    [[maybe_unused]] auto status = init(instance_id, query_options, query_globals, exec_env);
-    _query_mem_tracker = ctx->query_mem_tracker;
-#ifdef BE_TEST
-    if (_query_mem_tracker == nullptr) {
-        init_mem_trackers();
-    }
-#endif
-    DCHECK(_query_mem_tracker != nullptr && _query_mem_tracker->label() != "Orphan");
-    DCHECK(status.ok());
-}
-
 RuntimeState::RuntimeState(const TUniqueId& query_id, int32_t fragment_id,
                            const TQueryOptions& query_options, const TQueryGlobals& query_globals,
                            ExecEnv* exec_env, QueryContext* ctx)
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index ad63510e2af82c..a49567109a3b31 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -85,12 +85,7 @@ class RuntimeState {
                  const TQueryOptions& query_options, const TQueryGlobals& query_globals,
                  ExecEnv* exec_env, QueryContext* ctx);
 
-    // for only use in pipelineX
-    RuntimeState(pipeline::PipelineFragmentContext*, const TUniqueId& instance_id,
-                 const TUniqueId& query_id, int32 fragment_id, const TQueryOptions& query_options,
-                 const TQueryGlobals& query_globals, ExecEnv* exec_env, QueryContext* ctx);
-
-    // Used by pipelineX. This runtime state is only used for setup.
+    // Used by pipeline. This runtime state is only used for setup.
     RuntimeState(const TUniqueId& query_id, int32 fragment_id, const TQueryOptions& query_options,
                  const TQueryGlobals& query_globals, ExecEnv* exec_env, QueryContext* ctx);
 

From c9f1ff196b26cc302f12cca8c7bf2c1b38f8c758 Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Fri, 6 Dec 2024 15:59:32 +0800
Subject: [PATCH 208/399] [env](compile) enable the -Wshadow compiler option.
 (#45028)

### What problem does this PR solve?

In the past, Doris had enabled -Wshadow-field, which is a subset of
-Wshadow.
Therefore, we added -Wshadow in common/compile_check_begin.h and will
gradually enable it.
---
 be/src/common/compile_check_begin.h           |  1 +
 be/src/olap/base_tablet.cpp                   | 39 +++++++++----------
 ...aggregate_function_group_array_intersect.h |  5 +--
 .../aggregate_function_retention.h            |  6 +--
 .../aggregate_function_sequence_match.h       | 12 +++---
 be/src/vec/columns/column_object.cpp          |  8 ++--
 .../serde/data_type_jsonb_serde.cpp           | 12 +++---
 be/src/vec/functions/function_bitmap.cpp      |  4 +-
 be/src/vec/functions/function_conv.cpp        |  4 +-
 be/src/vec/functions/function_ip.h            | 10 ++---
 be/src/vec/functions/function_jsonb.cpp       |  7 ++--
 be/src/vec/sink/load_stream_stub.cpp          |  6 +--
 12 files changed, 54 insertions(+), 60 deletions(-)

diff --git a/be/src/common/compile_check_begin.h b/be/src/common/compile_check_begin.h
index 6da403f2894885..83da086688bc65 100644
--- a/be/src/common/compile_check_begin.h
+++ b/be/src/common/compile_check_begin.h
@@ -23,6 +23,7 @@
 #ifdef __clang__
 #pragma clang diagnostic push
 #pragma clang diagnostic error "-Wconversion"
+#pragma clang diagnostic error "-Wshadow"
 #pragma clang diagnostic ignored "-Wsign-conversion"
 #pragma clang diagnostic ignored "-Wfloat-conversion"
 #endif
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 9fc253d5a8d53e..845899e703f7c1 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -438,7 +438,6 @@ Status BaseTablet::lookup_row_data(const Slice& encoded_key, const RowLocation&
     StringRef value = string_column->get_data_at(0);
     values = value.to_string();
     if (write_to_cache) {
-        StringRef value = string_column->get_data_at(0);
         RowCache::instance()->insert({tablet_id(), encoded_key}, Slice {value.data, value.size});
     }
     return Status::OK();
@@ -476,12 +475,12 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest
         int num_segments = cast_set<int>(rs->num_segments());
         DCHECK_EQ(segments_key_bounds.size(), num_segments);
         std::vector<uint32_t> picked_segments;
-        for (int i = num_segments - 1; i >= 0; i--) {
-            if (key_without_seq.compare(segments_key_bounds[i].max_key()) > 0 ||
-                key_without_seq.compare(segments_key_bounds[i].min_key()) < 0) {
+        for (int j = num_segments - 1; j >= 0; j--) {
+            if (key_without_seq.compare(segments_key_bounds[j].max_key()) > 0 ||
+                key_without_seq.compare(segments_key_bounds[j].min_key()) < 0) {
                 continue;
             }
-            picked_segments.emplace_back(i);
+            picked_segments.emplace_back(j);
         }
         if (picked_segments.empty()) {
             continue;
@@ -778,11 +777,11 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
 
     if (config::enable_merge_on_write_correctness_check) {
         RowsetIdUnorderedSet rowsetids;
-        for (const auto& rowset : specified_rowsets) {
-            rowsetids.emplace(rowset->rowset_id());
+        for (const auto& specified_rowset : specified_rowsets) {
+            rowsetids.emplace(specified_rowset->rowset_id());
             VLOG_NOTICE << "[tabletID:" << tablet_id() << "]"
                         << "[add_sentinel_mark_to_delete_bitmap][end_version:" << end_version << "]"
-                        << "add:" << rowset->rowset_id();
+                        << "add:" << specified_rowset->rowset_id();
         }
         add_sentinel_mark_to_delete_bitmap(delete_bitmap.get(), rowsetids);
     }
@@ -892,11 +891,11 @@ Status BaseTablet::fetch_value_through_row_column(RowsetSharedPtr input_rowset,
     std::vector<std::string> default_values;
     default_values.resize(cids.size());
     for (int i = 0; i < cids.size(); ++i) {
-        const TabletColumn& column = tablet_schema.column(cids[i]);
+        const TabletColumn& tablet_column = tablet_schema.column(cids[i]);
         vectorized::DataTypePtr type =
-                vectorized::DataTypeFactory::instance().create_data_type(column);
-        col_uid_to_idx[column.unique_id()] = i;
-        default_values[i] = column.default_value();
+                vectorized::DataTypeFactory::instance().create_data_type(tablet_column);
+        col_uid_to_idx[tablet_column.unique_id()] = i;
+        default_values[i] = tablet_column.default_value();
         serdes[i] = type->get_serde();
     }
     vectorized::JsonbSerializeUtil::jsonb_to_block(serdes, *string_column, col_uid_to_idx, block,
@@ -1326,12 +1325,12 @@ Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap
                 required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator());
             }
         } else {
-            std::vector<RowsetSharedPtr> rowsets;
+            std::vector<RowsetSharedPtr> tablet_rowsets;
             {
                 std::shared_lock meta_rlock(_meta_lock);
-                rowsets = get_rowset_by_ids(&rowset_ids);
+                tablet_rowsets = get_rowset_by_ids(&rowset_ids);
             }
-            for (const auto& rowset : rowsets) {
+            for (const auto& rowset : tablet_rowsets) {
                 rapidjson::Value value;
                 std::string version_str = rowset->get_rowset_info_str();
                 value.SetString(version_str.c_str(),
@@ -1439,12 +1438,12 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf
                 txn_info->partial_update_info->max_version_in_flush_phase;
         DCHECK(max_version_in_flush_phase != -1);
         std::vector<RowsetSharedPtr> remained_rowsets;
-        for (const auto& rowset : specified_rowsets) {
-            if (rowset->end_version() <= max_version_in_flush_phase &&
-                rowset->produced_by_compaction()) {
-                rowsets_skip_alignment.emplace_back(rowset);
+        for (const auto& specified_rowset : specified_rowsets) {
+            if (specified_rowset->end_version() <= max_version_in_flush_phase &&
+                specified_rowset->produced_by_compaction()) {
+                rowsets_skip_alignment.emplace_back(specified_rowset);
             } else {
-                remained_rowsets.emplace_back(rowset);
+                remained_rowsets.emplace_back(specified_rowset);
             }
         }
         if (!rowsets_skip_alignment.empty()) {
diff --git a/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h b/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
index dfa9e7e37e5397..1bb6a7a170486c 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
@@ -99,7 +99,7 @@ struct AggregateFunctionGroupArrayIntersectData {
         value = std::make_unique<NullableNumericOrDateSetType>();
     }
 
-    void process_col_data(auto& column_data, size_t offset, size_t arr_size, bool& init, Set& set) {
+    void process_col_data(auto& column_data, size_t offset, size_t arr_size, Set& set) {
         const bool is_column_data_nullable = column_data.is_nullable();
 
         const ColumnNullable* col_null = nullptr;
@@ -175,7 +175,6 @@ class AggregateFunctionGroupArrayIntersect
     void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num,
              Arena*) const override {
         auto& data = this->data(place);
-        auto& init = data.init;
         auto& set = data.value;
 
         const bool col_is_nullable = (*columns[0]).is_nullable();
@@ -192,7 +191,7 @@ class AggregateFunctionGroupArrayIntersect
         const auto arr_size = offsets[row_num] - offset;
         const auto& column_data = column.get_data();
 
-        data.process_col_data(column_data, offset, arr_size, init, set);
+        data.process_col_data(column_data, offset, arr_size, set);
     }
 
     void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_retention.h b/be/src/vec/aggregate_functions/aggregate_function_retention.h
index 3318ac96ef36cf..95b2c8f9bb2e7a 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_retention.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_retention.h
@@ -94,18 +94,18 @@ struct RetentionState {
         }
     }
 
-    void insert_result_into(IColumn& to, size_t events_size, const uint8_t* events) const {
+    void insert_result_into(IColumn& to, size_t events_size, const uint8_t* arg_events) const {
         auto& data_to = assert_cast<ColumnUInt8&>(to).get_data();
 
         ColumnArray::Offset64 current_offset = data_to.size();
         data_to.resize(current_offset + events_size);
 
-        bool first_flag = events[0];
+        bool first_flag = arg_events[0];
         data_to[current_offset] = first_flag;
         ++current_offset;
 
         for (size_t i = 1; i < events_size; ++i) {
-            data_to[current_offset] = (first_flag && events[i]);
+            data_to[current_offset] = (first_flag && arg_events[i]);
             ++current_offset;
         }
     }
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h
index 827e506a3b0ed9..8fd0a1f5525cd1 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h
@@ -89,10 +89,10 @@ struct AggregateFunctionSequenceMatchData final {
 
     size_t get_arg_count() const { return arg_count; }
 
-    void init(const std::string pattern, size_t arg_count) {
+    void init(const std::string pattern_, size_t arg_count_) {
         if (!init_flag) {
-            this->pattern = pattern;
-            this->arg_count = arg_count;
+            this->pattern = pattern_;
+            this->arg_count = arg_count_;
             parse_pattern();
             init_flag = true;
         }
@@ -629,8 +629,7 @@ class AggregateFunctionSequenceBase
     void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs,
                Arena*) const override {
         const std::string pattern = this->data(rhs).get_pattern();
-        size_t arg_count = this->data(rhs).get_arg_count();
-        this->data(place).init(pattern, arg_count);
+        this->data(place).init(pattern, this->data(rhs).get_arg_count());
         this->data(place).merge(this->data(rhs));
     }
 
@@ -642,8 +641,7 @@ class AggregateFunctionSequenceBase
                      Arena*) const override {
         this->data(place).read(buf);
         const std::string pattern = this->data(place).get_pattern();
-        size_t arg_count = this->data(place).get_arg_count();
-        this->data(place).init(pattern, arg_count);
+        this->data(place).init(pattern, this->data(place).get_arg_count());
     }
 
 private:
diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
index 3e8d3722305e8d..d67a70d2f630f0 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -645,8 +645,8 @@ void ColumnObject::resize(size_t n) {
     num_rows = n;
 }
 
-bool ColumnObject::Subcolumn::check_if_sparse_column(size_t num_rows) {
-    if (num_rows < config::variant_threshold_rows_to_estimate_sparse_column) {
+bool ColumnObject::Subcolumn::check_if_sparse_column(size_t arg_num_rows) {
+    if (arg_num_rows < config::variant_threshold_rows_to_estimate_sparse_column) {
         return false;
     }
     std::vector<double> defaults_ratio;
@@ -1603,7 +1603,7 @@ Status ColumnObject::merge_sparse_to_root_column() {
     return Status::OK();
 }
 
-void ColumnObject::unnest(Subcolumns::NodePtr& entry, Subcolumns& subcolumns) const {
+void ColumnObject::unnest(Subcolumns::NodePtr& entry, Subcolumns& arg_subcolumns) const {
     entry->data.finalize();
     auto nested_column = entry->data.get_finalized_column_ptr()->assume_mutable();
     auto* nested_column_nullable = assert_cast<ColumnNullable*>(nested_column.get());
@@ -1634,7 +1634,7 @@ void ColumnObject::unnest(Subcolumns::NodePtr& entry, Subcolumns& subcolumns) co
         auto type = make_nullable(
                 std::make_shared<DataTypeArray>(nested_entry->data.least_common_type.get()));
         Subcolumn subcolumn(nullable_subnested_column->assume_mutable(), type, is_nullable);
-        subcolumns.add(path_builder.build(), subcolumn);
+        arg_subcolumns.add(path_builder.build(), subcolumn);
     }
 }
 
diff --git a/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp b/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
index adc041f511198e..e597cdba224376 100644
--- a/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
@@ -212,9 +212,9 @@ void convert_jsonb_to_rapidjson(const JsonbValue& val, rapidjson::Value& target,
         }
         target.Reserve(array.numElem(), allocator);
         for (auto it = array.begin(); it != array.end(); ++it) {
-            rapidjson::Value val;
-            convert_jsonb_to_rapidjson(*static_cast<const JsonbValue*>(it), val, allocator);
-            target.PushBack(val, allocator);
+            rapidjson::Value array_val;
+            convert_jsonb_to_rapidjson(*static_cast<const JsonbValue*>(it), array_val, allocator);
+            target.PushBack(array_val, allocator);
         }
         break;
     }
@@ -222,9 +222,9 @@ void convert_jsonb_to_rapidjson(const JsonbValue& val, rapidjson::Value& target,
         target.SetObject();
         const ObjectVal& obj = static_cast<const ObjectVal&>(val);
         for (auto it = obj.begin(); it != obj.end(); ++it) {
-            rapidjson::Value val;
-            convert_jsonb_to_rapidjson(*it->value(), val, allocator);
-            target.AddMember(rapidjson::GenericStringRef(it->getKeyStr(), it->klen()), val,
+            rapidjson::Value obj_val;
+            convert_jsonb_to_rapidjson(*it->value(), obj_val, allocator);
+            target.AddMember(rapidjson::GenericStringRef(it->getKeyStr(), it->klen()), obj_val,
                              allocator);
         }
         break;
diff --git a/be/src/vec/functions/function_bitmap.cpp b/be/src/vec/functions/function_bitmap.cpp
index d5f2075dffb6d2..92a5dba7b7a4d4 100644
--- a/be/src/vec/functions/function_bitmap.cpp
+++ b/be/src/vec/functions/function_bitmap.cpp
@@ -685,8 +685,8 @@ ColumnPtr handle_bitmap_op_count_null_value(ColumnPtr& src, const Block& block,
             continue;
         }
 
-        if (auto* nullable = assert_cast<const ColumnNullable*>(elem.column.get())) {
-            const ColumnPtr& null_map_column = nullable->get_null_map_column_ptr();
+        if (const auto* nullable_column = assert_cast<const ColumnNullable*>(elem.column.get())) {
+            const ColumnPtr& null_map_column = nullable_column->get_null_map_column_ptr();
             const NullMap& src_null_map =
                     assert_cast<const ColumnUInt8&>(*null_map_column).get_data();
 
diff --git a/be/src/vec/functions/function_conv.cpp b/be/src/vec/functions/function_conv.cpp
index 085d982a1cbf1e..38932530c5a2b4 100644
--- a/be/src/vec/functions/function_conv.cpp
+++ b/be/src/vec/functions/function_conv.cpp
@@ -223,8 +223,8 @@ struct ConvStringImpl {
         if (!MathFunctions::handle_parse_result(dst_base, &decimal_num, parse_res)) {
             result_column->insert_data("0", 1);
         } else {
-            StringRef str = MathFunctions::decimal_to_base(context, decimal_num, dst_base);
-            result_column->insert_data(reinterpret_cast<const char*>(str.data), str.size);
+            StringRef str_base = MathFunctions::decimal_to_base(context, decimal_num, dst_base);
+            result_column->insert_data(reinterpret_cast<const char*>(str_base.data), str_base.size);
         }
     }
 };
diff --git a/be/src/vec/functions/function_ip.h b/be/src/vec/functions/function_ip.h
index 928ce4369726b2..67edad5015aeaf 100644
--- a/be/src/vec/functions/function_ip.h
+++ b/be/src/vec/functions/function_ip.h
@@ -458,10 +458,10 @@ ColumnPtr convert_to_ipv6(const StringColumnType& string_column,
                 std::reverse(res_value, res_value + IPV6_BINARY_LENGTH);
             }
             if constexpr (std::is_same_v<ToColumn, ColumnString>) {
-                auto* column_string = assert_cast<ColumnString*>(col_res.get());
+                auto* column_string_res = assert_cast<ColumnString*>(col_res.get());
                 std::copy(res_value, res_value + IPV6_BINARY_LENGTH,
-                          column_string->get_chars().begin() + i * IPV6_BINARY_LENGTH);
-                column_string->get_offsets().push_back((i + 1) * IPV6_BINARY_LENGTH);
+                          column_string_res->get_chars().begin() + i * IPV6_BINARY_LENGTH);
+                column_string_res->get_offsets().push_back((i + 1) * IPV6_BINARY_LENGTH);
             } else {
                 col_res->insert_data(reinterpret_cast<const char*>(res_value), IPV6_BINARY_LENGTH);
             }
@@ -471,8 +471,8 @@ ColumnPtr convert_to_ipv6(const StringColumnType& string_column,
             }
             std::fill_n(&vec_res[out_offset], offset_inc, 0);
             if constexpr (std::is_same_v<ToColumn, ColumnString>) {
-                auto* column_string = assert_cast<ColumnString*>(col_res.get());
-                column_string->get_offsets().push_back((i + 1) * IPV6_BINARY_LENGTH);
+                auto* column_string_res = assert_cast<ColumnString*>(col_res.get());
+                column_string_res->get_offsets().push_back((i + 1) * IPV6_BINARY_LENGTH);
             }
             if constexpr (exception_mode == IPConvertExceptionMode::Null) {
                 (*vec_null_map_to)[i] = true;
diff --git a/be/src/vec/functions/function_jsonb.cpp b/be/src/vec/functions/function_jsonb.cpp
index 0e78eb894b20c5..463508169aadc6 100644
--- a/be/src/vec/functions/function_jsonb.cpp
+++ b/be/src/vec/functions/function_jsonb.cpp
@@ -1638,17 +1638,16 @@ class FunctionJsonSearch : public IFunction {
                       LikeState* state, JsonbPath* cur_path,
                       std::unordered_set<std::string>* matches) const {
         if (element.isString()) {
-            const std::string_view str = element.getString();
+            const std::string_view element_str = element.getString();
             unsigned char res;
-            RETURN_IF_ERROR(matched(str, state, &res));
+            RETURN_IF_ERROR(matched(element_str, state, &res));
             if (res) {
                 std::string str;
                 auto valid = cur_path->to_string(&str);
                 if (!valid) {
                     return false;
                 }
-                auto res = matches->insert(str);
-                return res.second;
+                return matches->insert(str).second;
             } else {
                 return false;
             }
diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index 979daf6a85e682..59719765ab3964 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -65,13 +65,11 @@ int LoadStreamReplyHandler::on_received_messages(brpc::StreamId id, butil::IOBuf
         if (response.failed_tablets_size() > 0) {
             ss << ", failed tablet ids:";
             for (auto pb : response.failed_tablets()) {
-                Status st = Status::create(pb.status());
-                ss << " " << pb.id() << ":" << st;
+                ss << " " << pb.id() << ":" << Status::create(pb.status());
             }
             std::lock_guard<bthread::Mutex> lock(stub->_failed_tablets_mutex);
             for (auto pb : response.failed_tablets()) {
-                Status st = Status::create(pb.status());
-                stub->_failed_tablets.emplace(pb.id(), st);
+                stub->_failed_tablets.emplace(pb.id(), Status::create(pb.status()));
             }
         }
         if (response.tablet_schemas_size() > 0) {

From 1f45e7fc838cfced4009dbb4d4c4140932ca58e4 Mon Sep 17 00:00:00 2001
From: airborne12 <jiangkai@selectdb.com>
Date: Fri, 6 Dec 2024 16:13:18 +0800
Subject: [PATCH 209/399] [feat](test) add fe ut for bf index create table
 (#44973)

Problem Summary:
1. Fix wrong gram_size 256 for ngram bf index DDL.
2. Enhance bloom_filter_columns empty properties prcocessing in bf index
DDL.
3. Add FE UT for ngram and bf index DDL.
---
 .../org/apache/doris/analysis/IndexDef.java   |  41 +-
 .../doris/common/util/PropertyAnalyzer.java   |   3 +
 .../CreateTableWithBloomFilterIndexTest.java  | 905 ++++++++++++++++++
 .../test_ngram_bloomfilter_index.groovy       |   6 +-
 4 files changed, 940 insertions(+), 15 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index 7f406d2a7e1365..f51e63e4fbec0f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -43,12 +43,17 @@ public class IndexDef {
     private boolean isBuildDeferred = false;
     private PartitionNames partitionNames;
     private List<Integer> columnUniqueIds = Lists.newArrayList();
+    private static final int MIN_NGRAM_SIZE = 1;
+    private static final int MAX_NGRAM_SIZE = 255;
+    private static final int MIN_BF_SIZE = 64;
+    private static final int MAX_BF_SIZE = 65535;
 
     public static final String NGRAM_SIZE_KEY = "gram_size";
     public static final String NGRAM_BF_SIZE_KEY = "bf_size";
     public static final String DEFAULT_NGRAM_SIZE = "2";
     public static final String DEFAULT_NGRAM_BF_SIZE = "256";
 
+
     public IndexDef(String indexName, boolean ifNotExists, List<String> columns, IndexType indexType,
                     Map<String, String> properties, String comment) {
         this.indexName = indexName;
@@ -238,8 +243,8 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe
                     throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table"
                         + " or key columns of AGG_KEYS table. invalid index: " + indexName);
                 } else if (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite
-                               && indexType == IndexType.INVERTED && properties != null
-                               && properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
+                        && indexType == IndexType.INVERTED && properties != null
+                        && properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
                     throw new AnalysisException("INVERTED index with parser can NOT be used in value columns of"
                         + " UNIQUE_KEYS table with merge_on_write disable. invalid index: " + indexName);
                 }
@@ -256,21 +261,29 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe
                 if (properties.size() != 2) {
                     throw new AnalysisException("ngram_bf index should have gram_size and bf_size properties");
                 }
-                try {
-                    int ngramSize = Integer.parseInt(properties.get(NGRAM_SIZE_KEY));
-                    int bfSize = Integer.parseInt(properties.get(NGRAM_BF_SIZE_KEY));
-                    if (ngramSize > 256 || ngramSize < 1) {
-                        throw new AnalysisException("gram_size should be integer and less than 256");
-                    }
-                    if (bfSize > 65535 || bfSize < 64) {
-                        throw new AnalysisException("bf_size should be integer and between 64 and 65535");
-                    }
-                } catch (NumberFormatException e) {
-                    throw new AnalysisException("invalid ngram properties:" + e.getMessage(), e);
-                }
+
+                parseAndValidateProperty(properties, NGRAM_SIZE_KEY, MIN_NGRAM_SIZE, MAX_NGRAM_SIZE);
+                parseAndValidateProperty(properties, NGRAM_BF_SIZE_KEY, MIN_BF_SIZE, MAX_BF_SIZE);
             }
         } else {
             throw new AnalysisException("Unsupported index type: " + indexType);
         }
     }
+
+    private void parseAndValidateProperty(Map<String, String> properties, String key, int minValue, int maxValue)
+            throws AnalysisException {
+        String valueStr = properties.get(key);
+        if (valueStr == null) {
+            throw new AnalysisException("Property '" + key + "' is missing.");
+        }
+        try {
+            int value = Integer.parseInt(valueStr);
+            if (value < minValue || value > maxValue) {
+                throw new AnalysisException("'" + key + "' should be an integer between "
+                                                + minValue + " and " + maxValue + ".");
+            }
+        } catch (NumberFormatException e) {
+            throw new AnalysisException("Invalid value for '" + key + "': " + valueStr, e);
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 915b5d48f013b9..536ca5633bf30a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -637,6 +637,9 @@ public static Set<String> analyzeBloomFilterColumns(Map<String, String> properti
             }
 
             String[] bfColumnArr = bfColumnsStr.split(COMMA_SEPARATOR);
+            if (bfColumnArr.length == 0) {
+                return bfColumns;
+            }
             Set<String> bfColumnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
             for (String bfColumn : bfColumnArr) {
                 bfColumn = bfColumn.trim();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java
new file mode 100644
index 00000000000000..54787ee8eefe92
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java
@@ -0,0 +1,905 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ExceptionChecker;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+public class CreateTableWithBloomFilterIndexTest extends TestWithFeService {
+    private static String runningDir = "fe/mocked/CreateTableWithBloomFilterIndexTest/"
+            + UUID.randomUUID().toString() + "/";
+
+    @Override
+    protected void runBeforeAll() throws Exception {
+        createDatabase("test");
+    }
+
+    @Test
+    public void testCreateTableWithTinyIntBloomFilterIndex() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "TINYINT is not supported in bloom filter index. invalid column: k1",
+                () -> createTable("CREATE TABLE test.tbl_tinyint_bf (\n"
+                        + "k1 TINYINT, \n"
+                        + "v1 INT\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k1\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithSupportedIntBloomFilterIndex() throws Exception {
+        // smallint
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_smallint_bf (\n"
+                + "k1 SMALLINT, \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+
+        // int
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_int_bf (\n"
+                + "k1 INT, \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+
+        // bigint
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_bigint_bf (\n"
+                + "k1 BIGINT, \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+
+        // largeint
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_largeint_bf (\n"
+                + "k1 LARGEINT, \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithFloatBloomFilterIndex() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "FLOAT is not supported in bloom filter index. invalid column: k2",
+                () -> createTable("CREATE TABLE test.tbl_float_bf (\n"
+                        + "k1 INT, \n"
+                        + "k2 FLOAT, \n"
+                        + "v1 INT\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k2\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithDoubleBloomFilterIndex() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "DOUBLE is not supported in bloom filter index. invalid column: k2",
+                () -> createTable("CREATE TABLE test.tbl_double_bf (\n"
+                        + "k1 INT, \n"
+                        + "k2 DOUBLE, \n"
+                        + "v1 INT\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k2\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithDecimalBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_decimal_bf (\n"
+                + "k1 DECIMAL(10,2), \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithCharBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_char_bf (\n"
+                + "k1 CHAR(20), \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithVarcharBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_varchar_bf (\n"
+                + "k1 VARCHAR(20), \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithTextBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_text_bf (\n"
+                + "k1 INT, \n"
+                + "k2 TEXT, \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k2\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithDecimalV3BloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_decimalv3_bf (\n"
+                + "k1 DECIMALV3(10,2), \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithIPv4BloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_ipv4_bf (\n"
+                + "k1 IPV4, \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithIPv6BloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_ipv6_bf (\n"
+                + "k1 IPV6, \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithDateBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_date_bf (\n"
+                + "k1 DATE, \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithDateTimeBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_datetime_bf (\n"
+                + "k1 DATETIME, \n"
+                + "v1 INT\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithCharNgramBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_char_ngram_bf (\n"
+                + "k1 CHAR(20), \n"
+                + "v1 INT,\n"
+                + "INDEX idx_k1_ngram (k1) USING NGRAM_BF PROPERTIES(\"gram_size\"=\"3\", \"bf_size\"=\"1024\")\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithVarcharNgramBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_varchar_ngram_bf (\n"
+                + "k1 VARCHAR(50), \n"
+                + "v1 INT,\n"
+                + "INDEX idx_k1_ngram (k1) USING NGRAM_BF PROPERTIES(\"gram_size\"=\"3\", \"bf_size\"=\"1024\")\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithStringNgramBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_string_ngram_bf (\n"
+                + "k1 INT, \n"
+                + "k2 STRING, \n"
+                + "v1 INT,\n"
+                + "INDEX idx_k2_ngram (k2) USING NGRAM_BF PROPERTIES(\"gram_size\"=\"3\", \"bf_size\"=\"1024\")\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithArrayNumericBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                        "ARRAY is not supported in bloom filter index. invalid column: k1",
+                        () -> createTable("CREATE TABLE test.tbl_array_numeric_bf (\n"
+                + "v1 INT,\n"
+                + "k1 ARRAY<INT>\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(v1)\n"
+                + "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithArrayDateBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                        "ARRAY is not supported in bloom filter index. invalid column: k1",
+                        () -> createTable("CREATE TABLE test.tbl_array_date_bf (\n"
+                + "v1 INT,\n"
+                + "k1 ARRAY<DATE>\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(v1)\n"
+                + "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithArrayStringNgramBloomFilterIndex() {
+        ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+                " ARRAY is not supported in ngram_bf index. invalid column: k1",
+                () -> createTable("CREATE TABLE test.tbl_array_string_ngram_bf (\n"
+                        + "v1 INT,\n"
+                        + "k1 ARRAY<STRING>,\n"
+                        + "INDEX idx_k1_ngram (k1) USING NGRAM_BF PROPERTIES(\"gram_size\"=\"3\", \"bf_size\"=\"1024\")\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(v1)\n"
+                        + "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithMapBloomFilterIndex() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "MAP is not supported in bloom filter index. invalid column: k1",
+                () -> createTable("CREATE TABLE test.tbl_map_bf (\n"
+                        + "v1 INT,\n"
+                        + "k1 MAP<INT, STRING>\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(v1)\n"
+                        + "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k1\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithStructBloomFilterIndex() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "STRUCT is not supported in bloom filter index. invalid column: k1",
+                () -> createTable("CREATE TABLE test.tbl_struct_bf (\n"
+                        + "v1 INT,\n"
+                        + "k1 STRUCT<f1:INT, f2:STRING>\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(v1)\n"
+                        + "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k1\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithJsonBloomFilterIndex() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                " JSON is not supported in bloom filter index. invalid column: k1",
+                () -> createTable("CREATE TABLE test.tbl_json_bf (\n"
+                        + "v1 INT,\n"
+                        + "k1 JSON\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(v1)\n"
+                        + "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k1\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testCreateTableWithHllBloomFilterIndex() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                " HLL is not supported in bloom filter index. invalid column: k1",
+                () -> createTable("CREATE TABLE test.tbl_hll_bf (\n"
+                        + "v1 INT,\n"
+                        + "k1 HLL\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(v1)\n"
+                        + "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k1\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testCreateMowTableWithBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_mow_bf (\n"
+                + "k1 INT, \n"
+                + "v1 VARCHAR(20)\n"
+                + ") ENGINE=OLAP\n"
+                + "UNIQUE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                + "PROPERTIES (\n"
+                + "\"replication_num\" = \"1\",\n"
+                + "\"bloom_filter_columns\" = \"v1\",\n"
+                + "\"enable_unique_key_merge_on_write\" = \"true\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateDuplicateTableWithBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_duplicate_bf (\n"
+                + "k1 INT, \n"
+                + "v1 VARCHAR(20)\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"v1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateMorTableWithBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_mor_bf (\n"
+                + "k1 INT, \n"
+                + "v1 VARCHAR(20)\n"
+                + ") ENGINE=OLAP\n"
+                + "UNIQUE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                + "PROPERTIES (\n"
+                + "\"replication_num\" = \"1\",\n"
+                + "\"bloom_filter_columns\" = \"v1\",\n"
+                + "\"enable_unique_key_merge_on_write\" = \"false\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testCreateAggTableWithBloomFilterIndex() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_agg_bf (\n"
+                + "k1 INT, \n"
+                + "v1 INT SUM\n"
+                + ") ENGINE=OLAP\n"
+                + "AGGREGATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsValidCharacters() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_bf_valid_chars (\n"
+                + "k1 INT, \n"
+                + "v1 VARCHAR(20)\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1,v1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsInvalidCharacters() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter column does not exist in table. invalid column: k1;v1",
+                () -> createTable("CREATE TABLE test.tbl_bf_invalid_chars (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k1;v1\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterFppValidInput() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_bf_fpp_valid (\n"
+                + "k1 INT, \n"
+                + "v1 VARCHAR(20)\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"v1\",\n"
+                + "\"bloom_filter_fpp\" = \"0.05\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testBloomFilterFppInvalidInput() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter fpp should in [1.0E-4, 0.05]",
+                () -> createTable("CREATE TABLE test.tbl_bf_fpp_invalid (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"v1\",\n"
+                        + "\"bloom_filter_fpp\" = \"-0.05\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testNgramBloomFilterGramSizeValidInput() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_ngram_gramsize_valid (\n"
+                + "k1 INT, \n"
+                + "k2 STRING, \n"
+                + "v1 INT,\n"
+                + "INDEX idx_k2_ngram (k2) USING NGRAM_BF PROPERTIES(\"gram_size\"=\"4\")\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                + "PROPERTIES (\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testNgramBloomFilterGramSizeInvalidInput() {
+        ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+                "'gram_size' should be an integer between 1 and 255",
+                () -> createTable("CREATE TABLE test.tbl_ngram_gramsize_invalid (\n"
+                        + "k1 INT, \n"
+                        + "k2 STRING, \n"
+                        + "v1 INT,\n"
+                        + "INDEX idx_k2_ngram (k2) USING NGRAM_BF PROPERTIES(\"gram_size\"=\"-1\")\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                        + "PROPERTIES (\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testNgramBloomFilterGramSizeInvalidInput256() {
+        ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+                "'gram_size' should be an integer between 1 and 255",
+                () -> createTable("CREATE TABLE test.tbl_ngram_gram_size_invalid (\n"
+                        + "k1 INT, \n"
+                        + "k2 STRING, \n"
+                        + "v1 INT,\n"
+                        + "INDEX idx_k2_ngram (k2) USING NGRAM_BF PROPERTIES(\"gram_size\"=\"256\")\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                        + "PROPERTIES (\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testNgramBloomFilterBfSizeValidInput() throws Exception {
+        ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+                "'bf_size' should be an integer between 64 and 65535",
+                () -> createTable("CREATE TABLE test.tbl_ngram_bfsize_valid (\n"
+                        + "k1 INT, \n"
+                        + "k2 STRING, \n"
+                        + "v1 INT,\n"
+                        + "INDEX idx_k2_ngram (k2) USING NGRAM_BF PROPERTIES(\"bf_size\"=\"256000000\")\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                        + "PROPERTIES (\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testNgramBloomFilterBfSizeInvalidInput() {
+        ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+                "'bf_size' should be an integer between 64 and 65535",
+                () -> createTable("CREATE TABLE test.tbl_ngram_bfsize_invalid (\n"
+                        + "k1 INT, \n"
+                        + "k2 STRING, \n"
+                        + "v1 INT,\n"
+                        + "INDEX idx_k2_ngram (k2) USING NGRAM_BF PROPERTIES(\"bf_size\"=\"-256000000\")\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                        + "PROPERTIES (\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testNgramBloomFilterBfSizeInvalidInput65536() {
+        ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+                "'bf_size' should be an integer between 64 and 65535",
+                () -> createTable("CREATE TABLE test.tbl_ngram_bf_size_invalid (\n"
+                        + "k1 INT, \n"
+                        + "k2 STRING, \n"
+                        + "v1 INT,\n"
+                        + "INDEX idx_k2_ngram (k2) USING NGRAM_BF PROPERTIES(\"bf_size\"=\"65536\")\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                        + "PROPERTIES (\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsDuplicated() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Reduplicated bloom filter column: k1",
+                () -> createTable("CREATE TABLE test.tbl_bf_duplicated_columns (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k1,k1\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnDoesNotExist() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter column does not exist in table. invalid column: k3",
+                () -> createTable("CREATE TABLE test.tbl_bf_column_not_exist (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k3\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnInvalidType() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "BOOLEAN is not supported in bloom filter index. invalid column: k2",
+                () -> createTable("CREATE TABLE test.tbl_bf_invalid_type (\n"
+                        + "k1 INT, \n"
+                        + "k2 BOOLEAN,\n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k2\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnNonKeyInAggKeys() throws Exception {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                        "Bloom filter index should only be used in columns of UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table. invalid column: v1",
+                        () -> createTable("CREATE TABLE test.tbl_bf_nonkey_in_agg (\n"
+                + "k1 INT, \n"
+                + "v1 INT SUM\n"
+                + ") ENGINE=OLAP\n"
+                + "AGGREGATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"v1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testBloomFilterFppNotDouble() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter fpp is not Double",
+                () -> createTable("CREATE TABLE test.tbl_bf_fpp_not_double (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"v1\",\n"
+                        + "\"bloom_filter_fpp\" = \"abc\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterFppOutOfRange() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter fpp should in [1.0E-4, 0.05]",
+                () -> createTable("CREATE TABLE test.tbl_bf_fpp_out_of_range (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"v1\",\n"
+                        + "\"bloom_filter_fpp\" = \"0.1\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterFppBelowMin() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter fpp should in [1.0E-4, 0.05]",
+                () -> createTable("CREATE TABLE test.tbl_bf_fpp_below_min (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"v1\",\n"
+                        + "\"bloom_filter_fpp\" = \"1e-5\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsEmptyString() throws Exception {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                        "Unknown properties: {bloom_filter_columns=}",
+                        () -> createTable("CREATE TABLE test.tbl_bf_empty_columns (\n"
+                + "k1 INT, \n"
+                + "v1 VARCHAR(20)\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsOnlyCommas() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Unknown properties: {bloom_filter_columns=,,,}",
+                () -> createTable("CREATE TABLE test.tbl_bf_only_commas (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \",,,\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsNonExistingColumns() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter column does not exist in table. invalid column: k3",
+                () -> createTable("CREATE TABLE test.tbl_bf_non_existing_columns (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20),\n"
+                        + "k2 INT\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k2,k3\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsWithSpecialCharacters() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter column does not exist in table. invalid column: k1@",
+                () -> createTable("CREATE TABLE test.tbl_bf_special_chars (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20),\n"
+                        + "k2 INT\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"k1@,v1#\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsWithDifferentCase() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_bf_different_case (\n"
+                + "k1 INT, \n"
+                + "V1 VARCHAR(20)\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(K1)\n"
+                + "DISTRIBUTED BY HASH(K1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"k1,v1\",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsWithSpaces() throws Exception {
+        ExceptionChecker.expectThrowsNoException(() -> createTable("CREATE TABLE test.tbl_bf_columns_with_spaces (\n"
+                + "k1 INT, \n"
+                + "v1 VARCHAR(20)\n"
+                + ") ENGINE=OLAP\n"
+                + "DUPLICATE KEY(k1)\n"
+                + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                + "PROPERTIES (\n"
+                + "\"bloom_filter_columns\" = \"  k1  ,  v1  \",\n"
+                + "\"replication_num\" = \"1\"\n"
+                + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsWithLongColumnName() throws Exception {
+        StringBuilder sb = new StringBuilder("k");
+        for (int i = 0; i < 1000; i++) {
+            sb.append('1');
+        }
+        String longColumnName = sb.toString();
+
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter column does not exist in table. invalid column: " + longColumnName,
+                () -> createTable("CREATE TABLE test.tbl_bf_long_column_name (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20),\n"
+                        + "k2 INT\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"" + longColumnName + "\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsWithUnicodeCharacters() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter column does not exist in table. invalid column: 名字",
+                () -> createTable("CREATE TABLE test.tbl_bf_unicode_columns (\n"
+                        + "k1 INT, \n"
+                        + "name VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \"名字\",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+
+    @Test
+    public void testBloomFilterColumnsWithNullOrWhitespace() {
+        ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+                "Bloom filter column does not exist in table. invalid column: ",
+                () -> createTable("CREATE TABLE test.tbl_bf_null_or_whitespace (\n"
+                        + "k1 INT, \n"
+                        + "v1 VARCHAR(20)\n"
+                        + ") ENGINE=OLAP\n"
+                        + "DUPLICATE KEY(k1)\n"
+                        + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
+                        + "PROPERTIES (\n"
+                        + "\"bloom_filter_columns\" = \" , \",\n"
+                        + "\"replication_num\" = \"1\"\n"
+                        + ");"));
+    }
+}
diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy
index e2ab9b9c117f1c..cce6ed9fd9d014 100644
--- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy
+++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy
@@ -104,6 +104,10 @@ suite("test_ngram_bloomfilter_index") {
         """
     test {
         sql """ALTER TABLE  ${tableName3} ADD INDEX idx_http_url(http_url) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="65536") COMMENT 'http_url ngram_bf index'"""
-        exception "bf_size should be integer and between 64 and 65535"
+        exception "'bf_size' should be an integer between 64 and 65535"
+    }
+    test {
+        sql """ALTER TABLE  ${tableName3} ADD INDEX idx_http_url(http_url) USING NGRAM_BF PROPERTIES("gram_size"="256", "bf_size"="65535") COMMENT 'http_url ngram_bf index'"""
+        exception "'gram_size' should be an integer between 1 and 255"
     }
 }

From 2da3c8b8dddc23b90050f0f3a22fd465888496eb Mon Sep 17 00:00:00 2001
From: zhangstar333 <zhangsida@selectdb.com>
Date: Fri, 6 Dec 2024 17:07:22 +0800
Subject: [PATCH 210/399] [improve](udf) support handle const column in
 java-udf (#44926)

### What problem does this PR solve?

Problem Summary:
support handle const column in java-udf
the before and after execute time
```

mysql [ssb]>select count(test_query_qa.java_udf_test3(c_custkey,"asd")) from test_udf;
+-----------------------------------------+
| count(java_udf_test3(c_custkey, 'asd')) |
+-----------------------------------------+
|                                36000000 |
+-----------------------------------------+
1 row in set (0.68 sec)

------------

mysql [ssb]>select count(test_query_qa.java_udf_test3(c_custkey,"asd")) from test_udf;
+-----------------------------------------+
| count(java_udf_test3(c_custkey, 'asd')) |
+-----------------------------------------+
|                                36000000 |
+-----------------------------------------+
1 row in set (0.52 sec)


```

### Release note
support handle const column in java-udf
---
 be/src/vec/exec/jni_connector.cpp             |  76 +++----
 be/src/vec/exec/jni_connector.h               |   8 +-
 .../doris/common/jni/vec/ColumnType.java      |  20 +-
 .../doris/common/jni/vec/VectorColumn.java    |  41 +++-
 .../doris/common/jni/vec/VectorTable.java     |  19 ++
 .../org/apache/doris/udf/UdfExecutor.java     |   3 +-
 .../javaudf_p0/test_javaudf_const_test.out    |  13 ++
 .../org/apache/doris/udf/UDFConstTest.java    |  99 ++++++++++
 .../javaudf_p0/test_javaudf_const_test.groovy | 185 ++++++++++++++++++
 9 files changed, 409 insertions(+), 55 deletions(-)
 create mode 100644 regression-test/data/javaudf_p0/test_javaudf_const_test.out
 create mode 100644 regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDFConstTest.java
 create mode 100644 regression-test/suites/javaudf_p0/test_javaudf_const_test.groovy

diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp
index a7b0d5144ee623..4c977b69ad6a42 100644
--- a/be/src/vec/exec/jni_connector.cpp
+++ b/be/src/vec/exec/jni_connector.cpp
@@ -599,68 +599,79 @@ std::string JniConnector::get_jni_type(const TypeDescriptor& desc) {
     }
 }
 
-Status JniConnector::_fill_column_meta(ColumnPtr& doris_column, DataTypePtr& data_type,
+Status JniConnector::_fill_column_meta(const ColumnPtr& doris_column, const DataTypePtr& data_type,
                                        std::vector<long>& meta_data) {
     TypeIndex logical_type = remove_nullable(data_type)->get_type_id();
+    const IColumn* column = nullptr;
+    // insert const flag
+    if (is_column_const(*doris_column)) {
+        meta_data.emplace_back((long)1);
+        const auto& const_column = assert_cast<const ColumnConst&>(*doris_column);
+        column = &(const_column.get_data_column());
+    } else {
+        meta_data.emplace_back((long)0);
+        column = &(*doris_column);
+    }
+
     // insert null map address
-    MutableColumnPtr data_column;
-    if (doris_column->is_nullable()) {
-        auto* nullable_column =
-                reinterpret_cast<vectorized::ColumnNullable*>(doris_column->assume_mutable().get());
-        data_column = nullable_column->get_nested_column_ptr();
-        NullMap& null_map = nullable_column->get_null_map_data();
+    const IColumn* data_column = nullptr;
+    if (column->is_nullable()) {
+        const auto& nullable_column = assert_cast<const vectorized::ColumnNullable&>(*column);
+        data_column = &(nullable_column.get_nested_column());
+        const auto& null_map = nullable_column.get_null_map_data();
         meta_data.emplace_back((long)null_map.data());
     } else {
         meta_data.emplace_back(0);
-        data_column = doris_column->assume_mutable();
+        data_column = column;
     }
     switch (logical_type) {
-#define DISPATCH(TYPE_INDEX, COLUMN_TYPE, CPP_TYPE)                                         \
-    case TYPE_INDEX: {                                                                      \
-        meta_data.emplace_back(_get_fixed_length_column_address<COLUMN_TYPE>(data_column)); \
-        break;                                                                              \
+#define DISPATCH(TYPE_INDEX, COLUMN_TYPE, CPP_TYPE)                                          \
+    case TYPE_INDEX: {                                                                       \
+        meta_data.emplace_back(_get_fixed_length_column_address<COLUMN_TYPE>(*data_column)); \
+        break;                                                                               \
     }
         FOR_FIXED_LENGTH_TYPES(DISPATCH)
 #undef DISPATCH
     case TypeIndex::String:
         [[fallthrough]];
     case TypeIndex::FixedString: {
-        auto& string_column = static_cast<ColumnString&>(*data_column);
+        const auto& string_column = assert_cast<const ColumnString&>(*data_column);
         // inert offsets
         meta_data.emplace_back((long)string_column.get_offsets().data());
         meta_data.emplace_back((long)string_column.get_chars().data());
         break;
     }
     case TypeIndex::Array: {
-        ColumnPtr& element_column = static_cast<ColumnArray&>(*data_column).get_data_ptr();
-        meta_data.emplace_back((long)static_cast<ColumnArray&>(*data_column).get_offsets().data());
-        DataTypePtr& element_type = const_cast<DataTypePtr&>(
-                (reinterpret_cast<const DataTypeArray*>(remove_nullable(data_type).get()))
+        const auto& element_column = assert_cast<const ColumnArray&>(*data_column).get_data_ptr();
+        meta_data.emplace_back(
+                (long)assert_cast<const ColumnArray&>(*data_column).get_offsets().data());
+        const auto& element_type = assert_cast<const DataTypePtr&>(
+                (assert_cast<const DataTypeArray*>(remove_nullable(data_type).get()))
                         ->get_nested_type());
         RETURN_IF_ERROR(_fill_column_meta(element_column, element_type, meta_data));
         break;
     }
     case TypeIndex::Struct: {
-        auto& doris_struct = static_cast<ColumnStruct&>(*data_column);
-        const DataTypeStruct* doris_struct_type =
-                reinterpret_cast<const DataTypeStruct*>(remove_nullable(data_type).get());
+        const auto& doris_struct = assert_cast<const ColumnStruct&>(*data_column);
+        const auto* doris_struct_type =
+                assert_cast<const DataTypeStruct*>(remove_nullable(data_type).get());
         for (int i = 0; i < doris_struct.tuple_size(); ++i) {
-            ColumnPtr& struct_field = doris_struct.get_column_ptr(i);
-            DataTypePtr& field_type = const_cast<DataTypePtr&>(doris_struct_type->get_element(i));
+            const auto& struct_field = doris_struct.get_column_ptr(i);
+            const auto& field_type =
+                    assert_cast<const DataTypePtr&>(doris_struct_type->get_element(i));
             RETURN_IF_ERROR(_fill_column_meta(struct_field, field_type, meta_data));
         }
         break;
     }
     case TypeIndex::Map: {
-        auto& map = static_cast<ColumnMap&>(*data_column);
-        DataTypePtr& key_type = const_cast<DataTypePtr&>(
-                reinterpret_cast<const DataTypeMap*>(remove_nullable(data_type).get())
-                        ->get_key_type());
-        DataTypePtr& value_type = const_cast<DataTypePtr&>(
-                reinterpret_cast<const DataTypeMap*>(remove_nullable(data_type).get())
+        const auto& map = assert_cast<const ColumnMap&>(*data_column);
+        const auto& key_type = assert_cast<const DataTypePtr&>(
+                assert_cast<const DataTypeMap*>(remove_nullable(data_type).get())->get_key_type());
+        const auto& value_type = assert_cast<const DataTypePtr&>(
+                assert_cast<const DataTypeMap*>(remove_nullable(data_type).get())
                         ->get_value_type());
-        ColumnPtr& key_column = map.get_keys_ptr();
-        ColumnPtr& value_column = map.get_values_ptr();
+        const auto& key_column = map.get_keys_ptr();
+        const auto& value_column = map.get_values_ptr();
         meta_data.emplace_back((long)map.get_offsets().data());
         RETURN_IF_ERROR(_fill_column_meta(key_column, key_type, meta_data));
         RETURN_IF_ERROR(_fill_column_meta(value_column, value_type, meta_data));
@@ -686,11 +697,6 @@ Status JniConnector::to_java_table(Block* block, size_t num_rows, const ColumnNu
     // insert number of rows
     meta_data.emplace_back(num_rows);
     for (size_t i : arguments) {
-        if (is_column_const(*(block->get_by_position(i).column))) {
-            auto doris_column = block->get_by_position(i).column->convert_to_full_column_if_const();
-            bool is_nullable = block->get_by_position(i).type->is_nullable();
-            block->replace_by_position(i, is_nullable ? make_nullable(doris_column) : doris_column);
-        }
         auto& column_with_type_and_name = block->get_by_position(i);
         RETURN_IF_ERROR(_fill_column_meta(column_with_type_and_name.column,
                                           column_with_type_and_name.type, meta_data));
diff --git a/be/src/vec/exec/jni_connector.h b/be/src/vec/exec/jni_connector.h
index 52a3fb2e7782ca..e06654dcfe3ca8 100644
--- a/be/src/vec/exec/jni_connector.h
+++ b/be/src/vec/exec/jni_connector.h
@@ -336,13 +336,13 @@ class JniConnector : public ProfileCollector {
     static Status _fill_struct_column(TableMetaAddress& address, MutableColumnPtr& doris_column,
                                       DataTypePtr& data_type, size_t num_rows);
 
-    static Status _fill_column_meta(ColumnPtr& doris_column, DataTypePtr& data_type,
+    static Status _fill_column_meta(const ColumnPtr& doris_column, const DataTypePtr& data_type,
                                     std::vector<long>& meta_data);
 
     template <typename COLUMN_TYPE, typename CPP_TYPE>
     static Status _fill_fixed_length_column(MutableColumnPtr& doris_column, CPP_TYPE* ptr,
                                             size_t num_rows) {
-        auto& column_data = static_cast<COLUMN_TYPE&>(*doris_column).get_data();
+        auto& column_data = assert_cast<COLUMN_TYPE&>(*doris_column).get_data();
         size_t origin_size = column_data.size();
         column_data.resize(origin_size + num_rows);
         memcpy(column_data.data() + origin_size, ptr, sizeof(CPP_TYPE) * num_rows);
@@ -350,8 +350,8 @@ class JniConnector : public ProfileCollector {
     }
 
     template <typename COLUMN_TYPE>
-    static long _get_fixed_length_column_address(MutableColumnPtr& doris_column) {
-        return (long)static_cast<COLUMN_TYPE&>(*doris_column).get_data().data();
+    static long _get_fixed_length_column_address(const IColumn& doris_column) {
+        return (long)assert_cast<const COLUMN_TYPE&>(doris_column).get_data().data();
     }
 
     void _generate_predicates(
diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
index 1a919d846310d0..37a58075978c18 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
@@ -207,16 +207,16 @@ public int getScale() {
     public int metaSize() {
         switch (type) {
             case UNSUPPORTED:
-                // set nullMap address as 0.
-                return 1;
+                // const flag / set nullMap address as 0.
+                return 2;
             case ARRAY:
             case MAP:
             case STRUCT:
-                // array & map : [nullMap | offsets | ... ]
-                // struct : [nullMap | ... ]
-                int size = 2;
+                // array & map : [const | nullMap | offsets | ... ]
+                // struct : [const | nullMap | ... ]
+                int size = 3;
                 if (type == Type.STRUCT) {
-                    size = 1;
+                    size = 2;
                 }
                 for (ColumnType c : childTypes) {
                     size += c.metaSize();
@@ -226,11 +226,11 @@ public int metaSize() {
             case BINARY:
             case CHAR:
             case VARCHAR:
-                // [nullMap | offsets | data ]
-                return 3;
+                // [const | nullMap | offsets | data ]
+                return 4;
             default:
-                // [nullMap | data]
-                return 2;
+                // [const | nullMap | data]
+                return 3;
         }
     }
 
diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
index 1542174cc1c587..839027b03b61c5 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
@@ -24,6 +24,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
+import org.apache.log4j.Logger;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -41,10 +42,14 @@
  * see <a href="https://github.com/apache/spark/blob/master/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java">WritableColumnVector</a>
  */
 public class VectorColumn {
+    public static final Logger LOG = Logger.getLogger(VectorColumn.class);
     // String is stored as array<byte>
     // The default string length to initialize the capacity.
     private static final int DEFAULT_STRING_LENGTH = 4;
 
+    //add a new flag for const column
+    private boolean isConst = false;
+
     // NullMap column address
     private long nullMap;
     private boolean[] nulls = null;
@@ -75,6 +80,7 @@ public class VectorColumn {
 
     // Create writable column
     private VectorColumn(ColumnType columnType, int capacity) {
+        this.isConst = false;
         this.columnType = columnType;
         this.capacity = 0;
         this.nullMap = 0;
@@ -106,6 +112,7 @@ private VectorColumn(ColumnType columnType, int capacity) {
 
     // restore the child of string column & restore meta column
     private VectorColumn(long address, int capacity, ColumnType columnType) {
+        this.isConst = false;
         this.columnType = columnType;
         this.capacity = capacity;
         this.nullMap = 0;
@@ -122,7 +129,7 @@ private VectorColumn(long address, int capacity, ColumnType columnType) {
         }
     }
 
-    // Create readable column
+    // Create readable column, and maybe pass by const column in Readable column
     private VectorColumn(ColumnType columnType, int numRows, long columnMetaAddress) {
         if (columnType.isUnsupported()) {
             throw new RuntimeException("Unsupported type for column: " + columnType.getName());
@@ -130,14 +137,27 @@ private VectorColumn(ColumnType columnType, int numRows, long columnMetaAddress)
         long address = columnMetaAddress;
         this.capacity = numRows;
         this.columnType = columnType;
+        Long constFlag = OffHeap.getLong(null, address);
+        address += 8;
+        if (constFlag != 0) {
+            this.isConst = true;
+        }
+        // record the real rows even if it's const column
+        // as only one rows in const column
+        int realRows = this.isConst ? 1 : numRows;
         this.nullMap = OffHeap.getLong(null, address);
         address += 8;
         this.numNulls = 0;
         if (this.nullMap != 0) {
-            nulls = OffHeap.getBoolean(null, nullMap, numRows);
+            nulls = OffHeap.getBoolean(null, nullMap, realRows);
             for (boolean isNull : nulls) {
                 if (isNull) {
-                    this.numNulls++;
+                    if (this.isConst) {
+                        // all of const is null value
+                        this.numNulls += numRows;
+                    } else {
+                        this.numNulls++;
+                    }
                 }
             }
         }
@@ -149,7 +169,7 @@ private VectorColumn(ColumnType columnType, int numRows, long columnMetaAddress)
             if (!columnType.isStruct()) {
                 this.offsets = OffHeap.getLong(null, address);
                 address += 8;
-                childRows = getArrayEndOffset(numRows - 1);
+                childRows = getArrayEndOffset(realRows - 1);
             }
             this.data = 0;
             List<ColumnType> children = columnType.getChildTypes();
@@ -162,7 +182,7 @@ private VectorColumn(ColumnType columnType, int numRows, long columnMetaAddress)
             this.offsets = OffHeap.getLong(null, address);
             address += 8;
             this.data = 0;
-            int length = OffHeap.getInt(null, this.offsets + (numRows - 1) * 4L);
+            int length = OffHeap.getInt(null, this.offsets + (realRows - 1) * 4L);
             childColumns = new VectorColumn[1];
             childColumns[0] = new VectorColumn(OffHeap.getLong(null, address), length,
                     new ColumnType("#stringBytes", Type.BYTE));
@@ -221,6 +241,10 @@ public ColumnType getColumnType() {
         return columnType;
     }
 
+    public boolean isConst() {
+        return isConst;
+    }
+
     /**
      * Release columns and meta information
      */
@@ -248,6 +272,7 @@ public void close() {
         capacity = 0;
         numNulls = 0;
         appendIndex = 0;
+        isConst = false;
     }
 
     private void throwReserveException(int requiredCapacity, Throwable cause) {
@@ -1450,6 +1475,11 @@ public void appendObjectColumn(Object[] batch, boolean isNullable) {
     }
 
     public Object[] getObjectColumn(int start, int end) {
+        // for const column only one row in column
+        if (isConst()) {
+            start = 0;
+            end = 1;
+        }
         switch (columnType.getType()) {
             case BOOLEAN:
                 return getBooleanColumn(start, end);
@@ -1583,6 +1613,7 @@ public void appendValue(ColumnValue o) {
 
     // for test only.
     public void dump(StringBuilder sb, int i) {
+        i = isConst() ? 0 : i;
         if (isNullAt(i)) {
             sb.append("NULL");
             return;
diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java
index a8849d1fa2044d..c94c95b3e8be31 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java
@@ -21,6 +21,8 @@
 import org.apache.doris.common.jni.utils.OffHeap;
 import org.apache.doris.common.jni.vec.ColumnType.Type;
 
+import org.apache.log4j.Logger;
+
 import java.util.Collections;
 import java.util.Map;
 
@@ -28,6 +30,7 @@
  * Store a batch of data as vector table.
  */
 public class VectorTable {
+    public static final Logger LOG = Logger.getLogger(VectorTable.class);
     private final VectorColumn[] columns;
     private final ColumnType[] columnTypes;
     private final String[] fields;
@@ -199,6 +202,10 @@ public int getNumColumns() {
         return columns.length;
     }
 
+    public boolean isConstColumn(int idx) {
+        return columns[idx].isConst();
+    }
+
     public long getMetaAddress() {
         if (!onlyReadable) {
             meta.reset();
@@ -229,6 +236,18 @@ public void close() {
     // for test only.
     public String dump(int rowLimit) {
         StringBuilder sb = new StringBuilder();
+        for (int col = 0; col < columns.length; col++) {
+            ColumnType.Type typeValue = columns[col].getColumnPrimitiveType();
+            sb.append(typeValue.name());
+            sb.append("(rows: " + columns[col].numRows());
+            sb.append(")(const: ");
+            sb.append(columns[col].isConst() ? "true) " : "false) ");
+            if (col != 0) {
+                sb.append(",    ");
+            }
+        }
+        sb.append("\n");
+
         for (int i = 0; i < rowLimit && i < getNumRows(); i++) {
             for (int j = 0; j < columns.length; j++) {
                 if (j != 0) {
diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
index 685e20de843869..1b5bff1e7c11d1 100644
--- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
+++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
@@ -116,7 +116,8 @@ public long evaluate(Map<String, String> inputParams, Map<String, String> output
             Object[] parameters = new Object[numColumns];
             for (int i = 0; i < numRows; ++i) {
                 for (int j = 0; j < numColumns; ++j) {
-                    parameters[j] = inputs[j][i];
+                    int row = inputTable.isConstColumn(j) ? 0 : i;
+                    parameters[j] = inputs[j][row];
                 }
                 result[i] = methodAccess.invoke(udf, evaluateIndex, parameters);
             }
diff --git a/regression-test/data/javaudf_p0/test_javaudf_const_test.out b/regression-test/data/javaudf_p0/test_javaudf_const_test.out
new file mode 100644
index 00000000000000..984cd976b3ec40
--- /dev/null
+++ b/regression-test/data/javaudf_p0/test_javaudf_const_test.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !java_udf_all_types --
+1	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+2	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+3	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+4	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+5	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+6	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+7	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+8	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+9	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+10	true	2	3	4.4000	5.5	6.6	2020-02-02	2022-02-03T10:10:10	asd	["a", "b"]	{"s_id":2, "s_name":"sa2"}	{"aa":"bb"}
+
diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDFConstTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDFConstTest.java
new file mode 100644
index 00000000000000..86370356998fed
--- /dev/null
+++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDFConstTest.java
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.udf;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+public class UDFConstTest {
+    public static class ConstBoolean {
+        public Boolean evaluate(Integer i, Boolean value) {
+            return value;
+        }
+    }
+
+    public static class ConstInt {
+        public Integer evaluate(Integer i, Integer value) {
+            return value;
+        }
+    }
+
+    public static class ConstFloat {
+        public Float evaluate(Integer i, Float value) {
+            return value;
+        }
+    }
+
+    public static class ConstDouble {
+        public Double evaluate(Integer i, Double value) {
+            return value;
+        }
+    }
+
+    public static class ConstLargeInt {
+        public BigInteger evaluate(Integer i, BigInteger value) {
+            return value;
+        }
+    }
+
+    public static class ConstDecimal {
+        public BigDecimal evaluate(Integer i, BigDecimal value) {
+            return value;
+        }
+    }
+
+    public static class ConstDate {
+        public LocalDate evaluate(Integer i, LocalDate value) {
+            return value;
+        }
+    }
+
+    public static class ConstDateTime {
+        public LocalDateTime evaluate(Integer i, LocalDateTime value) {
+            return value;
+        }
+    }
+
+    public static class ConstString {
+        public String evaluate(Integer i, String value) {
+            return value;
+        }
+    }
+
+    public static class ConstArray {
+        public ArrayList<String> evaluate(Integer i, ArrayList<String> value) {
+            return value;
+        }
+    }
+
+    public static class ConstMap {
+        public HashMap<String, String> evaluate(Integer i, HashMap<String, String> value) {
+            return value;
+        }
+    }
+
+    public static class ConstStruct {
+        public ArrayList<Object> evaluate(Integer i, ArrayList<Object> value) {
+            return value;
+        }
+    }
+}
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_const_test.groovy b/regression-test/suites/javaudf_p0/test_javaudf_const_test.groovy
new file mode 100644
index 00000000000000..31323bb46ed100
--- /dev/null
+++ b/regression-test/suites/javaudf_p0/test_javaudf_const_test.groovy
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("test_javaudf_const_test") {
+    def tableName = "test_javaudf_const_test"
+    def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar"""
+    // scp_udf_file_to_all_be(jarPath)
+
+    log.info("Jar path: ${jarPath}".toString())
+    try {
+        sql """ DROP TABLE IF EXISTS ${tableName} """
+        sql """
+        CREATE TABLE IF NOT EXISTS test_javaudf_const_test (
+            int_col int,
+            boolean_col boolean,
+            tinyint_col tinyint,
+            smallint_col smallint,
+            bigint_col bigint,
+            largeint_col largeint,
+            decimal_col decimal(15, 4),
+            float_col float,
+            double_col double,
+            date_col date,
+            datetime_col datetime(6),
+            string_col string,
+            array_col array<string>,
+            struct_col STRUCT<s_id:int, s_name:string>,
+            map_col map<string, string>
+            )
+            DISTRIBUTED BY HASH(int_col) PROPERTIES("replication_num" = "1");
+        """
+        StringBuilder sb = new StringBuilder()
+        int i = 1
+        for (; i < 10; i++) {
+            sb.append("""
+                (${i},${i%2},${i},${i}*2,${i}*3,${i}*4,${3.33/i},${(7.77/i).round(3)},${(3.1415/i).round(5)},"2023-10-${i+17}","2023-10-${i+10} 10:1${i}:11.234","row${i}",array(null, "nested${i}"),struct(${i}, "sa${i}"), {"k${i}":null,"k${i+1}":${i}}),
+            """)
+            log.info("${sb.toString()}");
+        }
+        sb.append("""
+                (${i},${i%2},null,${i}*2,${i}*3,${i}*4,null,${(7.77/i).round(3)},${(3.1415/i).round(5)},null,"2023-10-${i+10} 10:${i}:11.234",null,array(null, "nested${i}"),struct(${i}, "saaaaaa"),{"k${i}":null,"k${i+1}":${i}})
+            """)
+        sql """ INSERT INTO ${tableName} VALUES
+             ${sb.toString()}
+            """
+        File path = new File(jarPath)
+        if (!path.exists()) {
+            throw new IllegalStateException("""${jarPath} doesn't exist! """)
+        }
+
+        sql """DROP FUNCTION IF EXISTS const_boolean(int,boolean);"""
+        sql """CREATE FUNCTION const_boolean(int,boolean) RETURNS boolean PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstBoolean",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_int(int,int);"""
+        sql """CREATE FUNCTION const_int(int,int) RETURNS int PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstInt",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_largeint(int,largeint);"""
+        sql """CREATE FUNCTION const_largeint(int,largeint) RETURNS largeint PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstLargeInt",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_decimal(int,decimal(15, 4));"""
+        sql """CREATE FUNCTION const_decimal(int,decimal(15, 4)) RETURNS decimal(15, 4) PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstDecimal",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_float(int,float);"""
+        sql """CREATE FUNCTION const_float(int,float) RETURNS float PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstFloat",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_double(int,double);"""
+        sql """CREATE FUNCTION const_double(int,double) RETURNS double PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstDouble",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_date(int,date);"""
+        sql """CREATE FUNCTION const_date(int,date) RETURNS date PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstDate",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_datetime(int,datetime(6));"""
+        sql """CREATE FUNCTION const_datetime(int,datetime(6)) RETURNS datetime(6) PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstDateTime",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_string(int,string);"""
+        sql """CREATE FUNCTION const_string(int,string) RETURNS string PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstString",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_array(int,array<string>);"""
+        sql """CREATE FUNCTION const_array(int,array<string>) RETURNS array<string> PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstArray",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_struct(int,STRUCT<s_id:int, s_name:string>);"""
+        sql """CREATE FUNCTION const_struct(int,STRUCT<s_id:int, s_name:string>) RETURNS STRUCT<s_id:int, s_name:string> PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstStruct",
+            "type"="JAVA_UDF"
+        );"""
+
+        sql """DROP FUNCTION IF EXISTS const_map(int,map<string,string>);"""
+        sql """CREATE FUNCTION const_map(int,map<string,string>) RETURNS map<string,string> PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.UDFConstTest\$ConstMap",
+            "type"="JAVA_UDF"
+        );"""
+
+        qt_java_udf_all_types """select
+            int_col,
+            const_boolean(int_col,true),
+            const_int(int_col,2),
+            const_largeint(int_col,3),
+            const_decimal(int_col,4.4),
+            const_float(int_col,5.5),
+            const_double(int_col,6.6),
+            const_date(int_col,'2020-02-02'),
+            const_datetime(int_col,'2022-02-03 10:10:10'),
+            const_string(int_col,'asd'),
+            const_array(int_col,['a','b']),
+            const_struct(int_col,struct(2, 'sa2')),
+            const_map(int_col,{"aa":"bb"})
+            from ${tableName} order by int_col;"""
+    } finally {
+        try_sql """DROP FUNCTION IF EXISTS const_boolean(int,boolean);"""
+        try_sql """DROP FUNCTION IF EXISTS const_int(int,int);"""
+        try_sql """DROP FUNCTION IF EXISTS const_largeint(int,largeint);"""
+        try_sql """DROP FUNCTION IF EXISTS const_decimal(int,decimal(15, 4));"""
+        try_sql """DROP FUNCTION IF EXISTS const_float(int,float);"""
+        try_sql """DROP FUNCTION IF EXISTS const_double(int,double);"""
+        try_sql """DROP FUNCTION IF EXISTS const_date(int,date);"""
+        try_sql """DROP FUNCTION IF EXISTS const_datetime(int,datetime(6));"""
+        try_sql """DROP FUNCTION IF EXISTS const_string(int,string);"""
+        try_sql """DROP FUNCTION IF EXISTS const_array(int,array<string>);"""
+        try_sql """DROP FUNCTION IF EXISTS const_struct(int,STRUCT<s_id:int, s_name:string>);"""
+        try_sql """DROP FUNCTION IF EXISTS const_map(int,map<string,string>);"""
+        try_sql("""DROP TABLE IF EXISTS ${tableName};""")
+    }
+}

From f3aff7c0bb1f1cc9de95db1543aeeba5ae89d4cd Mon Sep 17 00:00:00 2001
From: huanghaibin <huanghaibin@selectdb.com>
Date: Fri, 6 Dec 2024 19:08:33 +0800
Subject: [PATCH 211/399] [chore](check) Add code check for some mow code
 (#44439)

Related PR: #44410

Problem Summary:
Add code check for some mow code
---
 be/src/cloud/cloud_cumulative_compaction.cpp       |  8 ++++++--
 be/src/cloud/cloud_cumulative_compaction.h         |  2 ++
 .../cloud/cloud_cumulative_compaction_policy.cpp   | 12 +++++++-----
 be/src/cloud/cloud_cumulative_compaction_policy.h  |  8 +++++---
 be/src/cloud/cloud_delete_bitmap_action.cpp        |  2 ++
 be/src/cloud/cloud_delete_bitmap_action.h          |  2 ++
 .../cloud/cloud_engine_calc_delete_bitmap_task.cpp |  2 ++
 be/src/cloud/cloud_meta_mgr.cpp                    |  4 +++-
 be/src/cloud/cloud_meta_mgr.h                      |  2 ++
 be/src/cloud/cloud_storage_engine.cpp              | 14 +++++++++-----
 be/src/cloud/cloud_tablet.cpp                      | 11 +++++++----
 be/src/cloud/cloud_tablet.h                        |  2 +-
 be/src/cloud/config.cpp                            |  2 ++
 be/src/cloud/config.h                              |  2 ++
 be/src/common/compile_check_begin.h                |  2 +-
 be/src/common/compile_check_end.h                  |  2 +-
 be/src/olap/base_tablet.cpp                        |  2 +-
 be/src/olap/base_tablet.h                          |  2 +-
 be/src/olap/rowset/beta_rowset_writer.cpp          | 14 ++++++++------
 be/src/olap/rowset/beta_rowset_writer.h            |  4 ++--
 be/src/olap/tablet_meta.cpp                        |  9 ++++++---
 be/src/olap/tablet_meta.h                          |  8 +++++---
 be/src/service/http_service.cpp                    |  2 ++
 be/src/util/hash_util.hpp                          |  2 +-
 24 files changed, 80 insertions(+), 40 deletions(-)

diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp
index 4e25cd74209e6e..f63054563aa18a 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -33,6 +33,7 @@
 #include "util/uuid_generator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 using namespace ErrorCode;
 
 bvar::Adder<uint64_t> cumu_output_size("cumu_compaction", "output_size");
@@ -488,8 +489,10 @@ Status CloudCumulativeCompaction::pick_rowsets_to_compact() {
     }
 
     int64_t max_score = config::cumulative_compaction_max_deltas;
-    auto process_memory_usage = doris::GlobalMemoryArbitrator::process_memory_usage();
-    bool memory_usage_high = process_memory_usage > MemInfo::soft_mem_limit() * 0.8;
+    double process_memory_usage =
+            cast_set<double>(doris::GlobalMemoryArbitrator::process_memory_usage());
+    bool memory_usage_high =
+            process_memory_usage > cast_set<double>(MemInfo::soft_mem_limit()) * 0.8;
     if (cloud_tablet()->last_compaction_status.is<ErrorCode::MEM_LIMIT_EXCEEDED>() ||
         memory_usage_high) {
         max_score = std::max(config::cumulative_compaction_max_deltas /
@@ -619,4 +622,5 @@ void CloudCumulativeCompaction::do_lease() {
     }
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/cloud/cloud_cumulative_compaction.h b/be/src/cloud/cloud_cumulative_compaction.h
index 1159dcb59ceef1..87fc0b62c9c389 100644
--- a/be/src/cloud/cloud_cumulative_compaction.h
+++ b/be/src/cloud/cloud_cumulative_compaction.h
@@ -24,6 +24,7 @@
 #include "olap/compaction.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 class CloudCumulativeCompaction : public CloudCompactionMixin {
 public:
@@ -60,4 +61,5 @@ class CloudCumulativeCompaction : public CloudCompactionMixin {
     Version _last_delete_version {-1, -1};
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
index 5a9879387b2327..92a47fcc69f8d7 100644
--- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
@@ -31,6 +31,7 @@
 #include "olap/tablet_meta.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 CloudSizeBasedCumulativeCompactionPolicy::CloudSizeBasedCumulativeCompactionPolicy(
         int64_t promotion_size, double promotion_ratio, int64_t promotion_min_size,
@@ -48,7 +49,7 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::_level_size(const int64_t size
     return (int64_t)1 << (sizeof(size) * 8 - 1 - __builtin_clzl(size));
 }
 
-int32_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets(
+int64_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets(
         CloudTablet* tablet, const std::vector<RowsetSharedPtr>& candidate_rowsets,
         const int64_t max_compaction_score, const int64_t min_compaction_score,
         std::vector<RowsetSharedPtr>* input_rowsets, Version* last_delete_version,
@@ -114,8 +115,8 @@ int32_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets(
     size_t new_compaction_score = *compaction_score;
     while (rs_begin != input_rowsets->end()) {
         auto& rs_meta = (*rs_begin)->rowset_meta();
-        int current_level = _level_size(rs_meta->total_disk_size());
-        int remain_level = _level_size(total_size - rs_meta->total_disk_size());
+        int64_t current_level = _level_size(rs_meta->total_disk_size());
+        int64_t remain_level = _level_size(total_size - rs_meta->total_disk_size());
         // if current level less then remain level, input rowsets contain current rowset
         // and process return; otherwise, input rowsets do not contain current rowset.
         if (current_level <= remain_level) {
@@ -185,7 +186,7 @@ int32_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets(
 }
 
 int64_t CloudSizeBasedCumulativeCompactionPolicy::cloud_promotion_size(CloudTablet* t) const {
-    int64_t promotion_size = int64_t(t->base_size() * _promotion_ratio);
+    int64_t promotion_size = int64_t(cast_set<double>(t->base_size()) * _promotion_ratio);
     // promotion_size is between _size_based_promotion_size and _size_based_promotion_min_size
     return promotion_size > _promotion_size       ? _promotion_size
            : promotion_size < _promotion_min_size ? _promotion_min_size
@@ -215,7 +216,7 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point(
                    : last_cumulative_point;
 }
 
-int32_t CloudTimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
+int64_t CloudTimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
         CloudTablet* tablet, const std::vector<RowsetSharedPtr>& candidate_rowsets,
         const int64_t max_compaction_score, const int64_t min_compaction_score,
         std::vector<RowsetSharedPtr>* input_rowsets, Version* last_delete_version,
@@ -377,4 +378,5 @@ int64_t CloudTimeSeriesCumulativeCompactionPolicy::new_cumulative_point(
     return output_rowset->end_version() + 1;
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.h b/be/src/cloud/cloud_cumulative_compaction_policy.h
index c142a8a6d3dffe..9373728547241b 100644
--- a/be/src/cloud/cloud_cumulative_compaction_policy.h
+++ b/be/src/cloud/cloud_cumulative_compaction_policy.h
@@ -30,6 +30,7 @@
 #include "olap/rowset/rowset_meta.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 class Tablet;
 struct Version;
@@ -44,7 +45,7 @@ class CloudCumulativeCompactionPolicy {
 
     virtual int64_t new_compaction_level(const std::vector<RowsetSharedPtr>& input_rowsets) = 0;
 
-    virtual int32_t pick_input_rowsets(CloudTablet* tablet,
+    virtual int64_t pick_input_rowsets(CloudTablet* tablet,
                                        const std::vector<RowsetSharedPtr>& candidate_rowsets,
                                        const int64_t max_compaction_score,
                                        const int64_t min_compaction_score,
@@ -71,7 +72,7 @@ class CloudSizeBasedCumulativeCompactionPolicy : public CloudCumulativeCompactio
         return 0;
     }
 
-    int32_t pick_input_rowsets(CloudTablet* tablet,
+    int64_t pick_input_rowsets(CloudTablet* tablet,
                                const std::vector<RowsetSharedPtr>& candidate_rowsets,
                                const int64_t max_compaction_score,
                                const int64_t min_compaction_score,
@@ -106,7 +107,7 @@ class CloudTimeSeriesCumulativeCompactionPolicy : public CloudCumulativeCompacti
 
     int64_t new_compaction_level(const std::vector<RowsetSharedPtr>& input_rowsets) override;
 
-    int32_t pick_input_rowsets(CloudTablet* tablet,
+    int64_t pick_input_rowsets(CloudTablet* tablet,
                                const std::vector<RowsetSharedPtr>& candidate_rowsets,
                                const int64_t max_compaction_score,
                                const int64_t min_compaction_score,
@@ -115,4 +116,5 @@ class CloudTimeSeriesCumulativeCompactionPolicy : public CloudCumulativeCompacti
                                bool allow_delete = false) override;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp b/be/src/cloud/cloud_delete_bitmap_action.cpp
index 86cc535e1bc88e..3d834bfe7b373c 100644
--- a/be/src/cloud/cloud_delete_bitmap_action.cpp
+++ b/be/src/cloud/cloud_delete_bitmap_action.cpp
@@ -50,6 +50,7 @@
 #include "util/stopwatch.hpp"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 using namespace ErrorCode;
 
 namespace {
@@ -177,4 +178,5 @@ void CloudDeleteBitmapAction::handle(HttpRequest* req) {
     }
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/cloud/cloud_delete_bitmap_action.h b/be/src/cloud/cloud_delete_bitmap_action.h
index 35739a7373efc8..ce507ee9991757 100644
--- a/be/src/cloud/cloud_delete_bitmap_action.h
+++ b/be/src/cloud/cloud_delete_bitmap_action.h
@@ -27,6 +27,7 @@
 #include "olap/tablet.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class HttpRequest;
 
 class ExecEnv;
@@ -52,4 +53,5 @@ class CloudDeleteBitmapAction : public HttpHandlerWithAuth {
     CloudStorageEngine& _engine;
     DeleteBitmapActionType _delete_bitmap_action_type;
 };
+#include "common/compile_check_end.h"
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
index 336117d1012d4d..fbf4b9cf303570 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
@@ -34,6 +34,7 @@
 #include "runtime/memory/mem_tracker_limiter.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
         CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
@@ -325,4 +326,5 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset(
     return status;
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 8e21498b0d873d..477cc802910869 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -64,6 +64,7 @@
 #include "util/thrift_rpc_helper.h"
 
 namespace doris::cloud {
+#include "common/compile_check_begin.h"
 using namespace ErrorCode;
 
 Status bthread_fork_join(const std::vector<std::function<Status()>>& tasks, int concurrency) {
@@ -717,7 +718,7 @@ Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_
                 "rowset_ids.size={},segment_ids.size={},vers.size={},delete_bitmaps.size={}",
                 rowset_ids.size(), segment_ids.size(), vers.size(), delete_bitmaps.size());
     }
-    for (size_t i = 0; i < rowset_ids.size(); i++) {
+    for (int i = 0; i < rowset_ids.size(); i++) {
         RowsetId rst_id;
         rst_id.init(rowset_ids[i]);
         delete_bitmap->merge(
@@ -1286,4 +1287,5 @@ int64_t CloudMetaMgr::get_inverted_index_file_szie(const RowsetMeta& rs_meta) {
     return total_inverted_index_size;
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::cloud
diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h
index c49b036ad90c15..913ef59489a1b3 100644
--- a/be/src/cloud/cloud_meta_mgr.h
+++ b/be/src/cloud/cloud_meta_mgr.h
@@ -27,6 +27,7 @@
 #include "util/s3_util.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 class DeleteBitmap;
 class StreamLoadContext;
@@ -124,4 +125,5 @@ class CloudMetaMgr {
 };
 
 } // namespace cloud
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp
index dc6abbac31ba1b..2944ccf21f276b 100644
--- a/be/src/cloud/cloud_storage_engine.cpp
+++ b/be/src/cloud/cloud_storage_engine.cpp
@@ -52,6 +52,7 @@
 #include "util/parse_util.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 using namespace std::literals;
 
@@ -166,7 +167,8 @@ Status CloudStorageEngine::open() {
 
     _memtable_flush_executor = std::make_unique<MemTableFlushExecutor>();
     // Use file cache disks number
-    _memtable_flush_executor->init(io::FileCacheFactory::instance()->get_cache_instance_size());
+    _memtable_flush_executor->init(
+            cast_set<int32_t>(io::FileCacheFactory::instance()->get_cache_instance_size()));
 
     _calc_delete_bitmap_executor = std::make_unique<CalcDeleteBitmapExecutor>();
     _calc_delete_bitmap_executor->init();
@@ -321,7 +323,7 @@ void CloudStorageEngine::_check_file_cache_ttl_block_valid() {
         for (const auto& rowset : rowsets) {
             int64_t ttl_seconds = tablet->tablet_meta()->ttl_seconds();
             if (rowset->newest_write_timestamp() + ttl_seconds <= UnixSeconds()) continue;
-            for (int64_t seg_id = 0; seg_id < rowset->num_segments(); seg_id++) {
+            for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); seg_id++) {
                 auto hash = Segment::file_cache_key(rowset->rowset_id().to_string(), seg_id);
                 auto* file_cache = io::FileCacheFactory::instance()->get_by_path(hash);
                 file_cache->update_ttl_atime(hash);
@@ -544,7 +546,8 @@ std::vector<CloudTabletSPtr> CloudStorageEngine::_generate_cloud_compaction_task
     int num_cumu =
             std::accumulate(submitted_cumu_compactions.begin(), submitted_cumu_compactions.end(), 0,
                             [](int a, auto& b) { return a + b.second.size(); });
-    int num_base = submitted_base_compactions.size() + submitted_full_compactions.size();
+    int num_base =
+            cast_set<int>(submitted_base_compactions.size() + submitted_full_compactions.size());
     int n = thread_per_disk - num_cumu - num_base;
     if (compaction_type == CompactionType::BASE_COMPACTION) {
         // We need to reserve at least one thread for cumulative compaction,
@@ -822,7 +825,7 @@ Status CloudStorageEngine::get_compaction_status_json(std::string* result) {
     // cumu
     std::string_view cumu = "CumulativeCompaction";
     rapidjson::Value cumu_key;
-    cumu_key.SetString(cumu.data(), cumu.length(), root.GetAllocator());
+    cumu_key.SetString(cumu.data(), cast_set<uint32_t>(cumu.length()), root.GetAllocator());
     rapidjson::Document cumu_arr;
     cumu_arr.SetArray();
     for (auto& [tablet_id, v] : _submitted_cumu_compactions) {
@@ -834,7 +837,7 @@ Status CloudStorageEngine::get_compaction_status_json(std::string* result) {
     // base
     std::string_view base = "BaseCompaction";
     rapidjson::Value base_key;
-    base_key.SetString(base.data(), base.length(), root.GetAllocator());
+    base_key.SetString(base.data(), cast_set<uint32_t>(base.length()), root.GetAllocator());
     rapidjson::Document base_arr;
     base_arr.SetArray();
     for (auto& [tablet_id, _] : _submitted_base_compactions) {
@@ -857,4 +860,5 @@ std::shared_ptr<CloudCumulativeCompactionPolicy> CloudStorageEngine::cumu_compac
     return _cumulative_compaction_policies.at(compaction_policy);
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 86cf23e39cc335..d545f827bc4e21 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -50,6 +50,7 @@
 #include "vec/common/schema_util.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 using namespace ErrorCode;
 
 static constexpr int COMPACTION_DELETE_BITMAP_LOCK_ID = -1;
@@ -380,7 +381,7 @@ void CloudTablet::delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete,
     _tablet_meta->modify_rs_metas({}, rs_metas, false);
 }
 
-int CloudTablet::delete_expired_stale_rowsets() {
+uint64_t CloudTablet::delete_expired_stale_rowsets() {
     std::vector<RowsetSharedPtr> expired_rowsets;
     int64_t expired_stale_sweep_endtime =
             ::time(nullptr) - config::tablet_rowset_stale_sweep_time_sec;
@@ -539,7 +540,7 @@ Result<std::unique_ptr<RowsetWriter>> CloudTablet::create_transient_rowset_write
 
     return RowsetFactory::create_rowset_writer(_engine, context, false)
             .transform([&](auto&& writer) {
-                writer->set_segment_start_id(rowset.num_segments());
+                writer->set_segment_start_id(cast_set<int32_t>(rowset.num_segments()));
                 return writer;
             });
 }
@@ -617,7 +618,8 @@ void CloudTablet::get_compaction_status(std::string* json_result) {
         }
         rapidjson::Value value;
         std::string version_str = rowset->get_rowset_info_str();
-        value.SetString(version_str.c_str(), version_str.length(), versions_arr.GetAllocator());
+        value.SetString(version_str.c_str(), cast_set<uint32_t>(version_str.length()),
+                        versions_arr.GetAllocator());
         versions_arr.PushBack(value, versions_arr.GetAllocator());
         last_version = ver.second;
     }
@@ -630,7 +632,7 @@ void CloudTablet::get_compaction_status(std::string* json_result) {
     for (auto& rowset : stale_rowsets) {
         rapidjson::Value value;
         std::string version_str = rowset->get_rowset_info_str();
-        value.SetString(version_str.c_str(), version_str.length(),
+        value.SetString(version_str.c_str(), cast_set<uint32_t>(version_str.length()),
                         stale_versions_arr.GetAllocator());
         stale_versions_arr.PushBack(value, stale_versions_arr.GetAllocator());
     }
@@ -925,4 +927,5 @@ void CloudTablet::build_tablet_report_info(TTabletInfo* tablet_info) {
     // but it may be used in the future.
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index 80038e569ba2fc..fc0d64a493d316 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -92,7 +92,7 @@ class CloudTablet final : public BaseTablet {
     void clear_cache() override;
 
     // Return number of deleted stale rowsets
-    int delete_expired_stale_rowsets();
+    uint64_t delete_expired_stale_rowsets();
 
     bool has_stale_rowsets() const { return !_stale_rs_version_map.empty(); }
 
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index e95c295ae1daa5..f90bf536f63018 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -20,6 +20,7 @@
 #include "common/status.h"
 
 namespace doris::config {
+#include "common/compile_check_begin.h"
 
 DEFINE_String(deploy_mode, "");
 DEFINE_mString(cloud_unique_id, "");
@@ -76,4 +77,5 @@ DEFINE_mInt32(tablet_txn_info_min_expired_seconds, "120");
 DEFINE_mBool(enable_use_cloud_unique_id_from_fe, "true");
 
 DEFINE_mBool(enable_cloud_tablet_report, "true");
+#include "common/compile_check_end.h"
 } // namespace doris::config
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index b345e6355921a4..a8a7c0c48ec91f 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -20,6 +20,7 @@
 #include "common/config.h"
 
 namespace doris::config {
+#include "common/compile_check_begin.h"
 
 DECLARE_String(deploy_mode);
 // deprecated do not configure directly
@@ -110,4 +111,5 @@ DECLARE_mBool(enable_use_cloud_unique_id_from_fe);
 
 DECLARE_Bool(enable_cloud_tablet_report);
 
+#include "common/compile_check_end.h"
 } // namespace doris::config
diff --git a/be/src/common/compile_check_begin.h b/be/src/common/compile_check_begin.h
index 83da086688bc65..4d860d39d1cf72 100644
--- a/be/src/common/compile_check_begin.h
+++ b/be/src/common/compile_check_begin.h
@@ -28,4 +28,4 @@
 #pragma clang diagnostic ignored "-Wfloat-conversion"
 #endif
 
-//#include "common/compile_check_begin.h"
\ No newline at end of file
+//#include "common/compile_check_begin.h"
diff --git a/be/src/common/compile_check_end.h b/be/src/common/compile_check_end.h
index 0897965dc74a3d..40df41b6bdfc6c 100644
--- a/be/src/common/compile_check_end.h
+++ b/be/src/common/compile_check_end.h
@@ -20,4 +20,4 @@
 #endif
 #undef COMPILE_CHECK
 
-// #include "common/compile_check_end.h"
\ No newline at end of file
+// #include "common/compile_check_end.h"
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 845899e703f7c1..82dc122e19f5ef 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -1757,7 +1757,7 @@ std::vector<RowsetSharedPtr> BaseTablet::get_snapshot_rowset(bool include_stale_
 
 void BaseTablet::calc_consecutive_empty_rowsets(
         std::vector<RowsetSharedPtr>* empty_rowsets,
-        const std::vector<RowsetSharedPtr>& candidate_rowsets, int limit) {
+        const std::vector<RowsetSharedPtr>& candidate_rowsets, int64_t limit) {
     int len = cast_set<int>(candidate_rowsets.size());
     for (int i = 0; i < len - 1; ++i) {
         auto rowset = candidate_rowsets[i];
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index f961f4c49eedd6..bd46cdbbe14854 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -276,7 +276,7 @@ class BaseTablet {
     // Find the first consecutive empty rowsets. output->size() >= limit
     void calc_consecutive_empty_rowsets(std::vector<RowsetSharedPtr>* empty_rowsets,
                                         const std::vector<RowsetSharedPtr>& candidate_rowsets,
-                                        int limit);
+                                        int64_t limit);
 
     // Return the merged schema of all rowsets
     virtual TabletSchemaSPtr merged_tablet_schema() const { return _max_version_schema; }
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
index 475a04df4911fd..ab5bc48db80b00 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -60,6 +60,7 @@
 #include "vec/data_types/data_type_factory.hpp"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 using namespace ErrorCode;
 
 namespace {
@@ -475,15 +476,15 @@ Status BetaRowsetWriter::_rename_compacted_segments(int64_t begin, int64_t end)
     return Status::OK();
 }
 
-void BetaRowsetWriter::_clear_statistics_for_deleting_segments_unsafe(uint64_t begin,
-                                                                      uint64_t end) {
+void BetaRowsetWriter::_clear_statistics_for_deleting_segments_unsafe(uint32_t begin,
+                                                                      uint32_t end) {
     VLOG_DEBUG << "_segid_statistics_map clear record segid range from:" << begin << " to:" << end;
-    for (int i = begin; i <= end; ++i) {
+    for (uint32_t i = begin; i <= end; ++i) {
         _segid_statistics_map.erase(i);
     }
 }
 
-Status BetaRowsetWriter::_rename_compacted_segment_plain(uint64_t seg_id) {
+Status BetaRowsetWriter::_rename_compacted_segment_plain(uint32_t seg_id) {
     if (seg_id == _num_segcompacted) {
         ++_num_segcompacted;
         return Status::OK();
@@ -653,7 +654,7 @@ Status BaseBetaRowsetWriter::add_rowset(RowsetSharedPtr rowset) {
     _num_rows_written += rowset->num_rows();
     _total_data_size += rowset->rowset_meta()->data_disk_size();
     _total_index_size += rowset->rowset_meta()->index_disk_size();
-    _num_segment += rowset->num_segments();
+    _num_segment += cast_set<int32_t>(rowset->num_segments());
     // append key_bounds to current rowset
     RETURN_IF_ERROR(rowset->get_segments_key_bounds(&_segments_encoded_key_bounds));
 
@@ -1043,7 +1044,7 @@ Status BaseBetaRowsetWriter::add_segment(uint32_t segment_id, const SegmentStati
         if (segment_id >= _segment_num_rows.size()) {
             _segment_num_rows.resize(segment_id + 1);
         }
-        _segment_num_rows[segid_offset] = segstat.row_num;
+        _segment_num_rows[segid_offset] = cast_set<uint32_t>(segstat.row_num);
     }
     VLOG_DEBUG << "_segid_statistics_map add new record. segment_id:" << segment_id
                << " row_num:" << segstat.row_num << " data_size:" << segstat.data_size
@@ -1111,4 +1112,5 @@ Status BetaRowsetWriter::flush_segment_writer_for_segcompaction(
     return Status::OK();
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h
index d96301af22630d..a69d1063a55086 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -298,9 +298,9 @@ class BetaRowsetWriter : public BaseBetaRowsetWriter {
     Status _load_noncompacted_segment(segment_v2::SegmentSharedPtr& segment, int32_t segment_id);
     Status _find_longest_consecutive_small_segment(SegCompactionCandidatesSharedPtr& segments);
     Status _rename_compacted_segments(int64_t begin, int64_t end);
-    Status _rename_compacted_segment_plain(uint64_t seg_id);
+    Status _rename_compacted_segment_plain(uint32_t seg_id);
     Status _rename_compacted_indices(int64_t begin, int64_t end, uint64_t seg_id);
-    void _clear_statistics_for_deleting_segments_unsafe(uint64_t begin, uint64_t end);
+    void _clear_statistics_for_deleting_segments_unsafe(uint32_t begin, uint32_t end);
 
     StorageEngine& _engine;
 
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 9f905918a66e9a..3247f34656fb5d 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -57,6 +57,7 @@ using std::unordered_map;
 using std::vector;
 
 namespace doris {
+#include "common/compile_check_begin.h"
 using namespace ErrorCode;
 
 TabletMetaSharedPtr TabletMeta::create(
@@ -106,7 +107,7 @@ TabletMeta::TabletMeta()
           _delete_bitmap(new DeleteBitmap(_tablet_id)) {}
 
 TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id,
-                       int64_t replica_id, int32_t schema_hash, uint64_t shard_id,
+                       int64_t replica_id, int32_t schema_hash, int32_t shard_id,
                        const TTabletSchema& tablet_schema, uint32_t next_unique_id,
                        const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
                        TabletUid tablet_uid, TTabletType::type tabletType,
@@ -574,7 +575,8 @@ void TabletMeta::serialize(string* meta_binary) {
 
 Status TabletMeta::deserialize(std::string_view meta_binary) {
     TabletMetaPB tablet_meta_pb;
-    bool parsed = tablet_meta_pb.ParseFromArray(meta_binary.data(), meta_binary.size());
+    bool parsed = tablet_meta_pb.ParseFromArray(meta_binary.data(),
+                                                static_cast<int32_t>(meta_binary.size()));
     if (!parsed) {
         return Status::Error<INIT_FAILED>("parse tablet meta failed");
     }
@@ -667,7 +669,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) {
         int seg_maps_size = tablet_meta_pb.delete_bitmap().segment_delete_bitmaps_size();
         CHECK(rst_ids_size == seg_ids_size && seg_ids_size == seg_maps_size &&
               seg_maps_size == versions_size);
-        for (size_t i = 0; i < rst_ids_size; ++i) {
+        for (int i = 0; i < rst_ids_size; ++i) {
             RowsetId rst_id;
             rst_id.init(tablet_meta_pb.delete_bitmap().rowset_ids(i));
             auto seg_id = tablet_meta_pb.delete_bitmap().segment_ids(i);
@@ -1309,4 +1311,5 @@ std::string tablet_state_name(TabletState state) {
     }
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index a8c82a4abd2921..25f6bcd569be43 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -51,6 +51,7 @@
 #include "util/uid_util.h"
 
 namespace json2pb {
+#include "common/compile_check_begin.h"
 struct Pb2JsonOptions;
 } // namespace json2pb
 
@@ -100,7 +101,7 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
 
     TabletMeta();
     TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int64_t replica_id,
-               int32_t schema_hash, uint64_t shard_id, const TTabletSchema& tablet_schema,
+               int32_t schema_hash, int32_t shard_id, const TTabletSchema& tablet_schema,
                uint32_t next_unique_id,
                const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id,
                TabletUid tablet_uid, TTabletType::type tabletType,
@@ -152,7 +153,7 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
     int64_t replica_id() const;
     void set_replica_id(int64_t replica_id) { _replica_id = replica_id; }
     int32_t schema_hash() const;
-    int16_t shard_id() const;
+    int32_t shard_id() const;
     void set_shard_id(int32_t shard_id);
     int64_t creation_time() const;
     void set_creation_time(int64_t creation_time);
@@ -610,7 +611,7 @@ inline int32_t TabletMeta::schema_hash() const {
     return _schema_hash;
 }
 
-inline int16_t TabletMeta::shard_id() const {
+inline int32_t TabletMeta::shard_id() const {
     return _shard_id;
 }
 
@@ -774,4 +775,5 @@ std::string tablet_state_name(TabletState state);
 bool operator==(const TabletMeta& a, const TabletMeta& b);
 bool operator!=(const TabletMeta& a, const TabletMeta& b);
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp
index 57600d1f56aae9..912f9f5ff403e7 100644
--- a/be/src/service/http_service.cpp
+++ b/be/src/service/http_service.cpp
@@ -80,6 +80,7 @@
 #include "util/doris_metrics.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace {
 std::shared_ptr<bufferevent_rate_limit_group> get_rate_limit_group(event_base* event_base) {
     auto rate_limit = config::download_binlog_rate_limit_kbs;
@@ -473,4 +474,5 @@ int HttpService::get_real_port() const {
     return _ev_http_server->get_real_port();
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/util/hash_util.hpp b/be/src/util/hash_util.hpp
index bed8569afaca32..fbf10b75ae02c0 100644
--- a/be/src/util/hash_util.hpp
+++ b/be/src/util/hash_util.hpp
@@ -297,7 +297,7 @@ class HashUtil {
 #endif
     }
 
-    static uint64_t hash64(const void* data, uint32_t bytes, uint64_t seed) {
+    static uint64_t hash64(const void* data, uint64_t bytes, uint64_t seed) {
 #ifdef _SSE4_2_
         if (LIKELY(CpuInfo::is_supported(CpuInfo::SSE4_2))) {
             return crc_hash64(data, bytes, seed);

From 20ffcde8819020becb504340da46849c13f4d46e Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Fri, 6 Dec 2024 23:07:47 +0800
Subject: [PATCH 212/399] [Bug](runtime-filter) avoid missing eos make rf
 process meet error (#45105)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What problem does this PR solve?
a instance may closed early coz sink is finished, then this instance
will missing eos.
shared hash table join build sink will copy rf from other instance when
sink meet eos, then use those rf on close() method.
for avoid rf process meet error, this pr move copy rf logic to close()
method.


![图片](https://github.com/user-attachments/assets/f80aee0b-4b23-4095-b49b-68f6e2e56f25)
---
 be/src/pipeline/dependency.cpp               |  9 ++++-----
 be/src/pipeline/dependency.h                 | 19 ++-----------------
 be/src/pipeline/exec/hashjoin_build_sink.cpp |  6 +++---
 3 files changed, 9 insertions(+), 25 deletions(-)

diff --git a/be/src/pipeline/dependency.cpp b/be/src/pipeline/dependency.cpp
index 983429f15e2fd5..5fef018423df25 100644
--- a/be/src/pipeline/dependency.cpp
+++ b/be/src/pipeline/dependency.cpp
@@ -92,11 +92,10 @@ std::string Dependency::debug_string(int indentation_level) {
 
 std::string CountedFinishDependency::debug_string(int indentation_level) {
     fmt::memory_buffer debug_string_buffer;
-    fmt::format_to(
-            debug_string_buffer,
-            "{}{}: id={}, block_task={}, ready={}, _always_ready={}, count={}, _stack_set_ready={}",
-            std::string(indentation_level * 2, ' '), _name, _node_id, _blocked_task.size(), _ready,
-            _always_ready, _counter, _stack_set_ready);
+    fmt::format_to(debug_string_buffer,
+                   "{}{}: id={}, block_task={}, ready={}, _always_ready={}, count={}",
+                   std::string(indentation_level * 2, ' '), _name, _node_id, _blocked_task.size(),
+                   _ready, _always_ready, _counter);
     return fmt::to_string(debug_string_buffer);
 }
 
diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h
index 71ef2d8b8fd85f..ad018c8b4f8f3d 100644
--- a/be/src/pipeline/dependency.h
+++ b/be/src/pipeline/dependency.h
@@ -173,26 +173,11 @@ struct FakeSharedState final : public BasicSharedState {
     ENABLE_FACTORY_CREATOR(FakeSharedState)
 };
 
-class DependencyWithStack : public Dependency {
-public:
-    using SharedState = FakeSharedState;
-    DependencyWithStack(int id, int node_id, std::string name, bool ready = false)
-            : Dependency(id, node_id, name, ready) {}
-
-    void set_ready() override {
-        _stack_set_ready = get_stack_trace();
-        Dependency::set_ready();
-    }
-
-protected:
-    std::string _stack_set_ready;
-};
-
-class CountedFinishDependency final : public DependencyWithStack {
+class CountedFinishDependency final : public Dependency {
 public:
     using SharedState = FakeSharedState;
     CountedFinishDependency(int id, int node_id, std::string name)
-            : DependencyWithStack(id, node_id, name, true) {}
+            : Dependency(id, node_id, name, true) {}
 
     void add() {
         std::unique_lock<std::mutex> l(_mtx);
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index ccd8757aba7f28..b2a79a941f79e7 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -170,6 +170,9 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
                    (p._shared_hash_table_context &&
                     !p._shared_hash_table_context->complete_build_stage)) {
             throw Exception(ErrorCode::INTERNAL_ERROR, "build_sink::close meet error state");
+        } else {
+            RETURN_IF_ERROR(
+                    _runtime_filter_slots->copy_from_shared_context(p._shared_hash_table_context));
         }
 
         SCOPED_TIMER(_publish_runtime_filter_timer);
@@ -553,9 +556,6 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
             return _shared_hash_table_context->status;
         }
 
-        RETURN_IF_ERROR(local_state._runtime_filter_slots->copy_from_shared_context(
-                _shared_hash_table_context));
-
         local_state.profile()->add_info_string(
                 "SharedHashTableFrom",
                 print_id(

From 140471ffacd4f9cc6732755c9d56bd7f4842261d Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Fri, 6 Dec 2024 23:30:10 +0800
Subject: [PATCH 213/399] [fix](compaction) tablet is not removed in compaction
 queue if dropped (#45085)

Problem Summary:
it will cause the data dir can not be deleted because the table shared
ptr use count is 2
---
 be/src/agent/task_worker_pool.cpp   |  4 +++-
 be/src/olap/olap_server.cpp         |  3 ++-
 be/src/olap/tablet_manager.cpp      | 20 ++++++++++----------
 be/src/olap/tablet_meta_manager.cpp |  3 +--
 4 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index c0f16d304a2b72..a8ab93de455c3b 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1630,11 +1630,13 @@ void drop_tablet_callback(StorageEngine& engine, const TAgentTaskRequest& req) {
                 dropped_tablet->tablet_uid());
         LOG_INFO("successfully drop tablet")
                 .tag("signature", req.signature)
-                .tag("tablet_id", drop_tablet_req.tablet_id);
+                .tag("tablet_id", drop_tablet_req.tablet_id)
+                .tag("replica_id", drop_tablet_req.replica_id);
     } else {
         LOG_WARNING("failed to drop tablet")
                 .tag("signature", req.signature)
                 .tag("tablet_id", drop_tablet_req.tablet_id)
+                .tag("replica_id", drop_tablet_req.replica_id)
                 .error(status);
     }
 
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 736bdaa99304d3..90d0883984e78b 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1071,7 +1071,8 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet,
             if (!tablet->can_do_compaction(tablet->data_dir()->path_hash(), compaction_type)) {
                 LOG(INFO) << "Tablet state has been changed, no need to begin this compaction "
                              "task, tablet_id="
-                          << tablet->tablet_id() << "tablet_state=" << tablet->tablet_state();
+                          << tablet->tablet_id() << ", tablet_state=" << tablet->tablet_state();
+                _pop_tablet_from_submitted_compaction(tablet, compaction_type);
                 return;
             }
             tablet->compaction_stage = CompactionStage::EXECUTING;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 59f0e1893d84d2..33fee7ca350900 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1170,14 +1170,14 @@ bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) {
             if (tablet_in_not_shutdown->tablet_path() != tablet->tablet_path()) {
                 LOG(INFO) << "tablet path not eq shutdown tablet path, move it to trash, tablet_id="
                           << tablet_in_not_shutdown->tablet_id()
-                          << " mem manager tablet path=" << tablet_in_not_shutdown->tablet_path()
-                          << " shutdown tablet path=" << tablet->tablet_path();
+                          << ", mem manager tablet path=" << tablet_in_not_shutdown->tablet_path()
+                          << ", shutdown tablet path=" << tablet->tablet_path();
                 return tablet->data_dir()->move_to_trash(tablet->tablet_path());
             } else {
                 LOG(INFO) << "tablet path eq shutdown tablet path, not move to trash, tablet_id="
                           << tablet_in_not_shutdown->tablet_id()
-                          << " mem manager tablet path=" << tablet_in_not_shutdown->tablet_path()
-                          << " shutdown tablet path=" << tablet->tablet_path();
+                          << ", mem manager tablet path=" << tablet_in_not_shutdown->tablet_path()
+                          << ", shutdown tablet path=" << tablet->tablet_path();
                 return true;
             }
         }
@@ -1282,7 +1282,7 @@ Status TabletManager::register_transition_tablet(int64_t tablet_id, std::string
         // not found
         shard.tablets_under_transition[tablet_id] = std::make_tuple(reason, thread_id, 1);
         LOG(INFO) << "add tablet_id= " << tablet_id << " to map, reason=" << reason
-                  << " lock times=1 thread_id_in_map=" << thread_id;
+                  << ", lock times=1, thread_id_in_map=" << thread_id;
         return Status::OK();
     } else {
         // found
@@ -1290,15 +1290,15 @@ Status TabletManager::register_transition_tablet(int64_t tablet_id, std::string
         if (thread_id != thread_id_in_map) {
             // other thread, failed
             LOG(INFO) << "tablet_id = " << tablet_id << " is doing " << r
-                      << " thread_id_in_map=" << thread_id_in_map << " , add reason=" << reason
-                      << " thread_id=" << thread_id;
+                      << ", thread_id_in_map=" << thread_id_in_map << " , add reason=" << reason
+                      << ", thread_id=" << thread_id;
             return Status::InternalError<false>("{} failed try later, tablet_id={}", reason,
                                                 tablet_id);
         }
         // add lock times
         ++lock_times;
         LOG(INFO) << "add tablet_id= " << tablet_id << " to map, reason=" << reason
-                  << " lock times=" << lock_times << " thread_id_in_map=" << thread_id_in_map;
+                  << ", lock times=" << lock_times << ", thread_id_in_map=" << thread_id_in_map;
         return Status::OK();
     }
 }
@@ -1322,10 +1322,10 @@ void TabletManager::unregister_transition_tablet(int64_t tablet_id, std::string
         --lock_times;
         if (lock_times != 0) {
             LOG(INFO) << "erase tablet_id= " << tablet_id << " from map, reason=" << reason
-                      << " left=" << lock_times << " thread_id_in_map=" << thread_id_in_map;
+                      << ", left=" << lock_times << ", thread_id_in_map=" << thread_id_in_map;
         } else {
             LOG(INFO) << "erase tablet_id= " << tablet_id << " from map, reason=" << reason
-                      << " thread_id_in_map=" << thread_id_in_map;
+                      << ", thread_id_in_map=" << thread_id_in_map;
             shard.tablets_under_transition.erase(tablet_id);
         }
     }
diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp
index 6f27dd4db4e672..7c08d7856200f9 100644
--- a/be/src/olap/tablet_meta_manager.cpp
+++ b/be/src/olap/tablet_meta_manager.cpp
@@ -291,8 +291,7 @@ Status TabletMetaManager::remove_old_version_delete_bitmap(DataDir* store, TTabl
         return true;
     };
     LOG(INFO) << "remove old version delete bitmap, tablet_id: " << tablet_id
-              << " version: " << version << " removed keys size: " << remove_keys.size();
-    ;
+              << " version: " << version << ", removed keys size: " << remove_keys.size();
     RETURN_IF_ERROR(meta->iterate(META_COLUMN_FAMILY_INDEX, begin_key, get_remove_keys_func));
     return meta->remove(META_COLUMN_FAMILY_INDEX, remove_keys);
 }

From 0671f57221378144e7369490a28c2292b940f90a Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Sat, 7 Dec 2024 06:18:45 +0800
Subject: [PATCH 214/399] [feat](catalog)Replace HadoopUGI with
 HadoopKerberosAuthenticator to Support Kerberos Ticket Auto-Renewal (#44916)

### Background
The current implementation uses the HadoopUGI method, which invokes the
ugiDoAs function for each operation to log in and execute actions based
on the configuration. However, this approach has the following issues:

- Lack of Auto-Renewal: If the Kerberos TGT (Ticket Granting Ticket)
expires, manual re-login is required as there is no support for
automatic ticket renewal.
- Redundant Login Overhead: Each operation requires reinitializing or
checking UserGroupInformation, potentially causing performance
bottlenecks.
- Complex Management: The HadoopUGI design does not unify the lifecycle
management of UGI instances, leading to duplicated logic across the
codebase.
### Objective

- Auto-Renewal: Automatically renew Kerberos credentials when the TGT is
expired or near expiry.
- UGI Caching: Maintain reusable UserGroupInformation instances during
their lifecycle to avoid repetitive logins.
- Unified Management: Simplify the management of UGI instances and
Kerberos credentials.
---
 .../org/apache/doris/hudi/HudiJniScanner.java |  16 +--
 .../java/org/apache/doris/hudi/Utils.java     |  12 +-
 .../security/authentication/HadoopUGI.java    | 103 ------------------
 .../hive/HiveMetaStoreClientHelper.java       |  20 ++--
 .../paimon/PaimonExternalCatalog.java         |  88 +++++++++------
 5 files changed, 84 insertions(+), 155 deletions(-)
 delete mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java

diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
index a284c7adcdd105..bc082e56732476 100644
--- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
+++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
@@ -17,11 +17,10 @@
 
 package org.apache.doris.hudi;
 
-
 import org.apache.doris.common.jni.JniScanner;
 import org.apache.doris.common.jni.vec.ColumnType;
 import org.apache.doris.common.security.authentication.AuthenticationConfig;
-import org.apache.doris.common.security.authentication.HadoopUGI;
+import org.apache.doris.common.security.authentication.HadoopAuthenticator;
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.avro.generic.GenericDatumReader;
@@ -160,14 +159,15 @@ public void open() throws IOException {
             cleanResolverLock.readLock().lock();
             try {
                 lastUpdateTime.set(System.currentTimeMillis());
+                AuthenticationConfig authenticationConfig = AuthenticationConfig.getKerberosConfig(split.hadoopConf());
+                HadoopAuthenticator hadoopAuthenticator = HadoopAuthenticator
+                        .getHadoopAuthenticator(authenticationConfig);
                 if (split.incrementalRead()) {
-                    recordIterator = HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(
-                                    split.hadoopConf()),
-                            () -> new MORIncrementalSplitReader(split).buildScanIterator(new Filter[0]));
+                    recordIterator = hadoopAuthenticator.doAs(() -> new MORIncrementalSplitReader(split)
+                            .buildScanIterator(new Filter[0]));
                 } else {
-                    recordIterator = HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(
-                                    split.hadoopConf()),
-                            () -> new MORSnapshotSplitReader(split).buildScanIterator(new Filter[0]));
+                    recordIterator = hadoopAuthenticator.doAs(() -> new MORSnapshotSplitReader(split)
+                            .buildScanIterator(new Filter[0]));
                 }
                 if (AVRO_RESOLVER_CACHE != null && AVRO_RESOLVER_CACHE.get() != null) {
                     cachedResolvers.computeIfAbsent(Thread.currentThread().getId(),
diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
index 3e07c8917905a3..c0fbec633e897c 100644
--- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
+++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
@@ -18,7 +18,7 @@
 package org.apache.doris.hudi;
 
 import org.apache.doris.common.security.authentication.AuthenticationConfig;
-import org.apache.doris.common.security.authentication.HadoopUGI;
+import org.apache.doris.common.security.authentication.HadoopAuthenticator;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -77,7 +77,13 @@ public static void killProcess(long pid) {
 
     public static HoodieTableMetaClient getMetaClient(Configuration conf, String basePath) {
         HadoopStorageConfiguration hadoopStorageConfiguration = new HadoopStorageConfiguration(conf);
-        return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf), () -> HoodieTableMetaClient.builder()
-                .setConf(hadoopStorageConfiguration).setBasePath(basePath).build());
+        AuthenticationConfig authenticationConfig = AuthenticationConfig.getKerberosConfig(conf);
+        HadoopAuthenticator hadoopAuthenticator = HadoopAuthenticator.getHadoopAuthenticator(authenticationConfig);
+        try {
+            return hadoopAuthenticator.doAs(() -> HoodieTableMetaClient.builder()
+                    .setConf(hadoopStorageConfiguration).setBasePath(basePath).build());
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to get HoodieTableMetaClient", e);
+        }
     }
 }
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java
deleted file mode 100644
index 2f73440ecfa368..00000000000000
--- a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java
+++ /dev/null
@@ -1,103 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.common.security.authentication;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-
-@Deprecated
-public class HadoopUGI {
-    private static final Logger LOG = LogManager.getLogger(HadoopUGI.class);
-
-    /**
-     * login and return hadoop ugi
-     * @param config auth config
-     * @return ugi
-     */
-    private static UserGroupInformation loginWithUGI(AuthenticationConfig config) {
-        if (config == null || !config.isValid()) {
-            return null;
-        }
-        if (config instanceof KerberosAuthenticationConfig) {
-            try {
-                // TODO: remove after iceberg and hudi kerberos test case pass
-                try {
-                    //  login hadoop with keytab and try checking TGT
-                    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-                    LOG.debug("Current login user: {}", ugi.getUserName());
-                    String principal = ((KerberosAuthenticationConfig) config).getKerberosPrincipal();
-                    if (ugi.hasKerberosCredentials() && StringUtils.equals(ugi.getUserName(), principal)) {
-                        // if the current user is logged by kerberos and is the same user
-                        // just use checkTGTAndReloginFromKeytab because this method will only relogin
-                        // when the TGT is expired or is close to expiry
-                        ugi.checkTGTAndReloginFromKeytab();
-                        return ugi;
-                    }
-                } catch (IOException e) {
-                    LOG.warn("A SecurityException occurs with kerberos, do login immediately.", e);
-                }
-                return new HadoopKerberosAuthenticator((KerberosAuthenticationConfig) config).getUGI();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        } else {
-            String hadoopUserName = ((SimpleAuthenticationConfig) config).getUsername();
-            if (hadoopUserName == null) {
-                hadoopUserName = "hadoop";
-                ((SimpleAuthenticationConfig) config).setUsername(hadoopUserName);
-                LOG.debug(AuthenticationConfig.HADOOP_USER_NAME + " is unset, use default user: hadoop");
-            }
-
-            UserGroupInformation ugi;
-            try {
-                ugi = UserGroupInformation.getLoginUser();
-                if (ugi.getUserName().equals(hadoopUserName)) {
-                    return ugi;
-                }
-            } catch (IOException e) {
-                LOG.warn("A SecurityException occurs with simple, do login immediately.", e);
-            }
-
-            ugi = UserGroupInformation.createRemoteUser(hadoopUserName);
-            UserGroupInformation.setLoginUser(ugi);
-            LOG.debug("Login by proxy user, hadoop.username: {}", hadoopUserName);
-            return ugi;
-        }
-    }
-
-    public static <T> T ugiDoAs(AuthenticationConfig authConf, PrivilegedExceptionAction<T> action) {
-        UserGroupInformation ugi = HadoopUGI.loginWithUGI(authConf);
-        try {
-            if (ugi != null) {
-                if (authConf instanceof KerberosAuthenticationConfig) {
-                    ugi.checkTGTAndReloginFromKeytab();
-                }
-                return ugi.doAs(action);
-            } else {
-                return action.run();
-            }
-        } catch (Exception e) {
-            throw new RuntimeException(e.getMessage(), e);
-        }
-    }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index 97f86612a495da..884cfbee45ba9f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -40,7 +40,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.security.authentication.AuthenticationConfig;
-import org.apache.doris.common.security.authentication.HadoopUGI;
+import org.apache.doris.common.security.authentication.HadoopAuthenticator;
 import org.apache.doris.datasource.ExternalCatalog;
 import org.apache.doris.fs.remote.dfs.DFSFileSystem;
 import org.apache.doris.thrift.TExprOpcode;
@@ -68,6 +68,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
@@ -823,19 +824,22 @@ public static <T> T ugiDoAs(long catalogId, PrivilegedExceptionAction<T> action)
 
     public static <T> T ugiDoAs(Configuration conf, PrivilegedExceptionAction<T> action) {
         // if hive config is not ready, then use hadoop kerberos to login
-        AuthenticationConfig krbConfig = AuthenticationConfig.getKerberosConfig(conf,
-                AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL,
-                AuthenticationConfig.HADOOP_KERBEROS_KEYTAB);
-        return HadoopUGI.ugiDoAs(krbConfig, action);
+        AuthenticationConfig authenticationConfig = AuthenticationConfig.getKerberosConfig(conf);
+        HadoopAuthenticator hadoopAuthenticator = HadoopAuthenticator.getHadoopAuthenticator(authenticationConfig);
+        try {
+            return hadoopAuthenticator.doAs(action);
+        } catch (IOException e) {
+            LOG.warn("HiveMetaStoreClientHelper ugiDoAs failed.", e);
+            throw new RuntimeException(e);
+        }
     }
 
     public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
         String hudiBasePath = table.getRemoteTable().getSd().getLocation();
         Configuration conf = getConfiguration(table);
         HadoopStorageConfiguration hadoopStorageConfiguration = new HadoopStorageConfiguration(conf);
-        return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf),
-                () -> HoodieTableMetaClient.builder().setConf(hadoopStorageConfiguration).setBasePath(hudiBasePath)
-                        .build());
+        return ugiDoAs(conf, () -> HoodieTableMetaClient.builder().setConf(hadoopStorageConfiguration)
+                .setBasePath(hudiBasePath).build());
     }
 
     public static Configuration getConfiguration(HMSExternalTable table) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
index 5a9e6feb5ad5b8..eb25336ab0b033 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
@@ -19,7 +19,7 @@
 
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.security.authentication.AuthenticationConfig;
-import org.apache.doris.common.security.authentication.HadoopUGI;
+import org.apache.doris.common.security.authentication.HadoopAuthenticator;
 import org.apache.doris.datasource.CatalogProperty;
 import org.apache.doris.datasource.ExternalCatalog;
 import org.apache.doris.datasource.InitCatalogLog;
@@ -40,6 +40,7 @@
 import org.apache.paimon.catalog.Identifier;
 import org.apache.paimon.options.Options;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -53,6 +54,7 @@ public abstract class PaimonExternalCatalog extends ExternalCatalog {
     protected String catalogType;
     protected Catalog catalog;
     protected AuthenticationConfig authConf;
+    protected HadoopAuthenticator hadoopAuthenticator;
 
     private static final List<String> REQUIRED_PROPERTIES = ImmutableList.of(
             PaimonProperties.WAREHOUSE
@@ -71,9 +73,8 @@ protected void initLocalObjectsImpl() {
         for (Map.Entry<String, String> propEntry : this.catalogProperty.getHadoopProperties().entrySet()) {
             conf.set(propEntry.getKey(), propEntry.getValue());
         }
-        authConf = AuthenticationConfig.getKerberosConfig(conf,
-                AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL,
-                AuthenticationConfig.HADOOP_KERBEROS_KEYTAB);
+        authConf = AuthenticationConfig.getKerberosConfig(conf);
+        hadoopAuthenticator = HadoopAuthenticator.getHadoopAuthenticator(authConf);
     }
 
     public String getCatalogType() {
@@ -82,40 +83,57 @@ public String getCatalogType() {
     }
 
     protected List<String> listDatabaseNames() {
-        return HadoopUGI.ugiDoAs(authConf, () -> new ArrayList<>(catalog.listDatabases()));
+        try {
+            return hadoopAuthenticator.doAs(() -> new ArrayList<>(catalog.listDatabases()));
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to list databases names, catalog name: " + getName(), e);
+        }
     }
 
     @Override
     public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
         makeSureInitialized();
-        return HadoopUGI.ugiDoAs(authConf, () -> catalog.tableExists(Identifier.create(dbName, tblName)));
+        try {
+            return hadoopAuthenticator.doAs(() -> catalog.tableExists(Identifier.create(dbName, tblName)));
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to check table existence, catalog name: " + getName(), e);
+        }
     }
 
     @Override
     public List<String> listTableNames(SessionContext ctx, String dbName) {
         makeSureInitialized();
-        return HadoopUGI.ugiDoAs(authConf, () -> {
-            List<String> tableNames = null;
-            try {
-                tableNames = catalog.listTables(dbName);
-            } catch (Catalog.DatabaseNotExistException e) {
-                LOG.warn("DatabaseNotExistException", e);
-            }
-            return tableNames;
-        });
+        try {
+            return hadoopAuthenticator.doAs(() -> {
+                List<String> tableNames = null;
+                try {
+                    tableNames = catalog.listTables(dbName);
+                } catch (Catalog.DatabaseNotExistException e) {
+                    LOG.warn("DatabaseNotExistException", e);
+                }
+                return tableNames;
+            });
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to list table names, catalog name: " + getName(), e);
+        }
     }
 
     public org.apache.paimon.table.Table getPaimonTable(String dbName, String tblName) {
         makeSureInitialized();
-        return HadoopUGI.ugiDoAs(authConf, () -> {
-            org.apache.paimon.table.Table table = null;
-            try {
-                table = catalog.getTable(Identifier.create(dbName, tblName));
-            } catch (Catalog.TableNotExistException e) {
-                LOG.warn("TableNotExistException", e);
-            }
-            return table;
-        });
+        try {
+            return hadoopAuthenticator.doAs(() -> {
+                org.apache.paimon.table.Table table = null;
+                try {
+                    table = catalog.getTable(Identifier.create(dbName, tblName));
+                } catch (Catalog.TableNotExistException e) {
+                    LOG.warn("TableNotExistException", e);
+                }
+                return table;
+            });
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to get Paimon table, catalog name: " + getName() + ", db: "
+                    + dbName + ", table: " + tblName, e);
+        }
     }
 
     protected String getPaimonCatalogType(String catalogType) {
@@ -127,15 +145,19 @@ protected String getPaimonCatalogType(String catalogType) {
     }
 
     protected Catalog createCatalog() {
-        return HadoopUGI.ugiDoAs(authConf, () -> {
-            Options options = new Options();
-            Map<String, String> paimonOptionsMap = getPaimonOptionsMap();
-            for (Map.Entry<String, String> kv : paimonOptionsMap.entrySet()) {
-                options.set(kv.getKey(), kv.getValue());
-            }
-            CatalogContext context = CatalogContext.create(options, getConfiguration());
-            return createCatalogImpl(context);
-        });
+        try {
+            return hadoopAuthenticator.doAs(() -> {
+                Options options = new Options();
+                Map<String, String> paimonOptionsMap = getPaimonOptionsMap();
+                for (Map.Entry<String, String> kv : paimonOptionsMap.entrySet()) {
+                    options.set(kv.getKey(), kv.getValue());
+                }
+                CatalogContext context = CatalogContext.create(options, getConfiguration());
+                return createCatalogImpl(context);
+            });
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to create catalog, catalog name: " + getName(), e);
+        }
     }
 
     protected Catalog createCatalogImpl(CatalogContext context) {

From 9335d2eec5a137888445b22ae459a80662e456ae Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Sat, 7 Dec 2024 06:35:46 +0800
Subject: [PATCH 215/399] [fix](case)fix manager show proc case unstable.
 (#45040)

### What problem does this PR solve?
Problem Summary:
fix manager show proc case unstable.
Replace   pipeline-unfriendly `select sleep(xxx)`  in case.
---
 .../manager/test_manager_interface_1.groovy   | 122 ---------------
 .../manager/test_manager_interface_4.groovy   | 146 ++++++++++++++++++
 2 files changed, 146 insertions(+), 122 deletions(-)
 create mode 100644 regression-test/suites/manager/test_manager_interface_4.groovy

diff --git a/regression-test/suites/manager/test_manager_interface_1.groovy b/regression-test/suites/manager/test_manager_interface_1.groovy
index 249ce252ffbf86..02f6c210f1fd09 100644
--- a/regression-test/suites/manager/test_manager_interface_1.groovy
+++ b/regression-test/suites/manager/test_manager_interface_1.groovy
@@ -496,128 +496,6 @@ DISTRIBUTED BY HASH(`k1`) BUCKETS 1"""))
     test_table_index()
 
 
-
-
-
-// show proc '/current_query_stmts'
-// show proc '/current_queries'
-// show processlist
-// kill query $query_id
-// SHOW PROC '/cluster_health/tablet_health' 
-    def test_proc = {
-
-        def futures = []
-        
-
-        futures.add( thread {
-
-            try{
-                sql """ select sleep(4.809); """
-            }catch(Exception e){
-                
-            }
-        })
-        futures.add( thread {
-            sleep(1000);
-            List<List<Object>> result = sql """ show proc '/current_query_stmts' """ 
-            logger.info("result = ${result}" )
-            def x = 0
-            def queryid = ""
-            logger.info("result = ${result}")
-
-            for( int i = 0;i<result.size();i++) {
-                if (result[i][7] != null && result[i][7].contains("sleep(4.809)") )//Statement
-                {
-                    x = 1
-                    queryid = result[i][0]
-                    logger.info("query ID = ${queryid}")
-                    assertTrue(result[i][0]!= null) //QueryId
-                    assertTrue(result[i][1]!= null) //ConnectionId
-                    assertTrue(result[i][2]!= null)//Catalog
-                    assertTrue(result[i][3]!= null)//Database
-                    assertTrue(result[i][4]!= null)//User
-                    assertTrue(result[i][5]!= null)//ExecTime
-                    assertTrue(result[i][5].isNumber())//ExecTime
-                    assertTrue(result[i][6]!= null)//SqlHash
-                }
-            }
-            assertTrue(x == 1)
-            
-            x = 0 
-            result = sql """  show proc '/current_queries' """
-            logger.info("result = ${result}")
-            for( int i = 0;i<result.size();i++) {
-                if (result[i][0] == queryid )//QueryId
-                {
-                    x = 1
-                    assertTrue(result[i][5]!= null)//ScanBytes
-                    assertTrue(result[i][6]!= null)//ProcessBytes
-                }
-            }
-            assertTrue(x == 1)
-
-            result = sql """  show processlist  """
-            logger.info("result = ${result}")
-            for( int i =0 ;i < result.size();i++ ){
-                assertTrue( result[i][2].toLowerCase() != "null"  )//User
-                assertTrue( result[i][3].toLowerCase() != "null"  )//Host
-                assertTrue( result[i][5].toLowerCase() != "null"  )//Catalog
-                assertTrue( result[i][6].toLowerCase() != "null"  )//Db
-                assertTrue( result[i][10].toLowerCase() != "null"  )//QueryId
-                if (result[i][10] == queryid) {
-                    x = 1
-                }
-            }
-
-            assertTrue(x == 1)
-            sql """ kill query "${queryid}" """
-            
-            x =  0
-            sleep(5000)
-
-            result = sql """  show proc '/current_queries' """
-            logger.info("result = ${result}")
-            for( int i = 0;i<result.size();i++) {
-                if (result[i][0] == queryid )//QueryId
-                {
-                    x = 1
-                }
-            }
-            assertTrue(x == 0)
-        })
-        futures.each { it.get() }
-
-
-        def tablet_num  = 0;
-        def healthy_num = 0;
-        def total_tablet_num  = 0;
-        def total_healthy_num = 0;
-        result = sql """  SHOW PROC '/cluster_health/tablet_health' """ 
-        logger.info("result = ${result}" )
-
-        for( int i =0 ;i < result.size();i++ ){
-            assertTrue(result[i][0].toLowerCase() != null ) // DbId
-            if (result[i][0].toLowerCase() == "total") {
-                total_tablet_num = result[i][2].toBigInteger();
-                total_healthy_num = result[i][3].toBigInteger();
-            }else {
-                tablet_num += result[i][2].toBigInteger();
-                healthy_num += result[i][3].toBigInteger();
-                
-            }
-            // assertTrue(result[i][2]()) // TabletNum
-            // assertTrue(result[i][3]()) // HealthyNum
-        }
-        assertTrue(total_healthy_num == healthy_num )
-        assertTrue(total_healthy_num == healthy_num )
-    
-
-
-    }
-    test_proc();
-
-
-
 //select a.*, b.*, c.NAME as WORKLOAD_GROUP_NAME from information_schema.active_queries a left join information_schema.backend_active_tasks b on a.QUERY_ID = b.QUERY_ID left join information_schema.workload_groups c on a.WORKLOAD_GROUP_ID = c.ID
     def  test_active_query = {
 
diff --git a/regression-test/suites/manager/test_manager_interface_4.groovy b/regression-test/suites/manager/test_manager_interface_4.groovy
new file mode 100644
index 00000000000000..355d96e1e0310a
--- /dev/null
+++ b/regression-test/suites/manager/test_manager_interface_4.groovy
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+import org.apache.doris.regression.util.NodeType
+
+import java.time.LocalDateTime
+import java.time.Duration
+import java.time.format.DateTimeFormatter
+
+
+
+suite('test_manager_interface_4',"p0") {
+
+
+// show proc '/current_query_stmts'
+// show proc '/current_queries'
+// show processlist
+// kill query $query_id
+// SHOW PROC '/cluster_health/tablet_health' 
+    def test_proc = {
+
+        def futures = []
+        
+
+        sql """set parallel_pipeline_task_num=1;"""
+        futures.add( thread {
+
+            try{
+                sql """ select *  from numbers("number" = "9910") as  a  join numbers('number'="18880094567") as b on a.number = b.number; """
+            }catch(Exception e){
+                
+            }
+        })
+
+        futures.add( thread {
+            sleep(500);
+            List<List<Object>> result = sql """ show proc '/current_query_stmts' """ 
+            logger.info("result = ${result}" )
+            def x = 0
+            def queryid = ""
+            logger.info("result = ${result}")
+
+            for( int i = 0;i<result.size();i++) {
+                if (result[i][7] != null && result[i][7].contains("18880094567") )//Statement
+                {
+                    x = 1
+                    queryid = result[i][0]
+                    logger.info("query ID = ${queryid}")
+                    assertTrue(result[i][0]!= null) //QueryId
+                    assertTrue(result[i][1]!= null) //ConnectionId
+                    assertTrue(result[i][2]!= null)//Catalog
+                    assertTrue(result[i][3]!= null)//Database
+                    assertTrue(result[i][4]!= null)//User
+                    assertTrue(result[i][5]!= null)//ExecTime
+                    assertTrue(result[i][5].isNumber())//ExecTime
+                    assertTrue(result[i][6]!= null)//SqlHash
+                }
+            }
+            assertTrue(x == 1)
+            
+            x = 0 
+            result = sql """  show proc '/current_queries' """
+            logger.info("result = ${result}")
+            for( int i = 0;i<result.size();i++) {
+                if (result[i][0] == queryid )//QueryId
+                {
+                    x = 1
+                    assertTrue(result[i][5]!= null)//ScanBytes
+                    assertTrue(result[i][6]!= null)//ProcessBytes
+                }
+            }
+            assertTrue(x == 1)
+
+            result = sql """  show processlist  """
+            logger.info("result = ${result}")
+            for( int i =0 ;i < result.size();i++ ){
+                assertTrue( result[i][2].toLowerCase() != "null"  )//User
+                assertTrue( result[i][3].toLowerCase() != "null"  )//Host
+                assertTrue( result[i][5].toLowerCase() != "null"  )//Catalog
+                assertTrue( result[i][6].toLowerCase() != "null"  )//Db
+                assertTrue( result[i][10].toLowerCase() != "null"  )//QueryId
+                if (result[i][10] == queryid) {
+                    x = 1
+                }
+            }
+
+            assertTrue(x == 1)
+            sql """ kill query "${queryid}" """
+            
+            x =  0
+            sleep(3000)
+
+            result = sql """  show proc '/current_queries' """
+            logger.info("result = ${result}")
+            for( int i = 0;i<result.size();i++) {
+                if (result[i][0] == queryid )//QueryId
+                {
+                    x = 1
+                }
+            }
+            assertTrue(x == 0)
+        })
+        futures.each { it.get() }
+
+
+        def tablet_num  = 0;
+        def healthy_num = 0;
+        def total_tablet_num  = 0;
+        def total_healthy_num = 0;
+        result = sql """  SHOW PROC '/cluster_health/tablet_health' """ 
+        logger.info("result = ${result}" )
+
+        for( int i =0 ;i < result.size();i++ ){
+            assertTrue(result[i][0].toLowerCase() != null ) // DbId
+            if (result[i][0].toLowerCase() == "total") {
+                total_tablet_num = result[i][2].toBigInteger();
+                total_healthy_num = result[i][3].toBigInteger();
+            }else {
+                tablet_num += result[i][2].toBigInteger();
+                healthy_num += result[i][3].toBigInteger();
+                
+            }
+            // assertTrue(result[i][2]()) // TabletNum
+            // assertTrue(result[i][3]()) // HealthyNum
+        }
+        assertTrue(total_healthy_num == healthy_num )
+        assertTrue(total_healthy_num == healthy_num )
+
+    }
+    test_proc();
+}
\ No newline at end of file

From 33ee349cdb6e2d765801b73fcfdbc0e60bcb7df5 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Sat, 7 Dec 2024 06:41:54 +0800
Subject: [PATCH 216/399] [chore](job-test)Add comments to the job's test cases
 (#44627)

### What problem does this PR solve?

Add comments to the job's test cases
---
 .../suites/job_p0/test_base_insert_job.groovy | 42 +++++++++++++------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy
index 8a0bb34ca43fd5..fc97e367483b86 100644
--- a/regression-test/suites/job_p0/test_base_insert_job.groovy
+++ b/regression-test/suites/job_p0/test_base_insert_job.groovy
@@ -76,6 +76,7 @@ suite("test_base_insert_job") {
         insert into ${tableName} values
         ('2023-03-18', 1, 1)
         """
+    // create recurring job
     sql """
        CREATE JOB ${jobName}  ON SCHEDULE every 1 second   comment 'test' DO INSERT INTO ${tableName} (`timestamp`, `type`, `user_id`)
         WITH
@@ -97,23 +98,25 @@ suite("test_base_insert_job") {
     """
     Awaitility.await().atMost(30, SECONDS).until(
             {
-                def onceJob = sql """ select SucceedTaskCount from jobs("type"="insert") where Name like '%${jobName}%' and ExecuteType='RECURRING' """
-                println(onceJob)
-                onceJob.size() == 1 && '1' <= onceJob.get(0).get(0)
+                def jobSuccendCount = sql """ select SucceedTaskCount from jobs("type"="insert") where Name like '%${jobName}%' and ExecuteType='RECURRING' """
+                // check job status and succeed task count larger than 1
+                jobSuccendCount.size() == 1 && '1' <= jobSuccendCount.get(0).get(0)
 
             }
     )
     sql """
         PAUSE JOB where jobname =  '${jobName}'
     """
+    def pausedJobStatus = sql """
+        select status from jobs("type"="insert") where Name='${jobName}'
+    """
+    assert pausedJobStatus.get(0).get(0) == "PAUSED"
     def tblDatas = sql """select * from ${tableName}"""
-    println tblDatas
     assert tblDatas.size() >= 2 //at least 2 records
-    def pauseJobId = sql """select id from jobs("type"="insert") where Name='${jobName}'"""
-    def taskStatus = sql """select status from tasks("type"="insert") where jobid= '${pauseJobId.get(0).get(0)}'"""
-    println taskStatus
+
+    def taskStatus = sql """select status from tasks("type"="insert") where JobName ='${jobName}'"""
     for (int i = 0; i < taskStatus.size(); i++) {
-        assert taskStatus.get(i).get(0) != "FAILED" || taskStatus.get(i).get(0) != "STOPPED" || taskStatus.get(i).get(0) != "STOPPED"
+        assert taskStatus.get(i).get(0) =="CANCELLED" || taskStatus.get(i).get(0) =="FINISHED"
     }
     sql """
        CREATE JOB ${jobMixedName}  ON SCHEDULE every 1 second  DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
@@ -122,6 +125,7 @@ suite("test_base_insert_job") {
     println mixedNameJobs
     assert mixedNameJobs.size() == 1 && mixedNameJobs.get(0).get(0) == jobMixedName
     assert mixedNameJobs.get(0).get(1) == ''
+    // clean up job and table
     sql """
         DROP JOB IF EXISTS where jobname =  '${jobName}'
     """
@@ -145,23 +149,25 @@ suite("test_base_insert_job") {
         """
     def dataCount = sql """select count(*) from ${tableName}"""
     assert dataCount.get(0).get(0) == 0
+    // create one time job
     sql """
           CREATE JOB ${jobName}  ON SCHEDULE at current_timestamp   comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName}  values  ('2023-07-19', 2, 1001);
      """
-
+    // wait job finished
     Awaitility.await("create-one-time-job-test").atMost(30, SECONDS).until(
             {
                 def onceJob = sql """ select SucceedTaskCount from jobs("type"="insert") where Name like '%${jobName}%' and ExecuteType='ONE_TIME' """
                 onceJob.size() == 1 && '1' == onceJob.get(0).get(0)
             }
     )
-    def onceJob = sql """ select SucceedTaskCount from jobs("type"="insert") where Name like '%${jobName}%' and ExecuteType='ONE_TIME' """
+    def onceJob = sql """ select SucceedTaskCount  from jobs("type"="insert") where Name like '%${jobName}%' and ExecuteType='ONE_TIME' """
     assert onceJob.size() == 1
     //check succeed task count
     assert '1' == onceJob.get(0).get(0)
     def datas = sql """select status,taskid from tasks("type"="insert") where jobName= '${jobName}'"""
-    println datas
+    // table should have one record after job finished
     assert datas.size() == 1
+    // one time job only has one task. when job finished, task status should be FINISHED
     assert datas.get(0).get(0) == "FINISHED"
     // check table data
     def dataCount1 = sql """select count(1) from ${tableName} where user_id=1001"""
@@ -175,19 +181,20 @@ suite("test_base_insert_job") {
     sql """
         DROP JOB IF EXISTS where jobname =  'press'
     """
-
+    // create job with start time is current time and interval is 10 hours
     sql """
           CREATE JOB press  ON SCHEDULE every 10 hour starts CURRENT_TIMESTAMP  comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName}  values  ('2023-07-19', 99, 99);
      """
     Awaitility.await("create-immediately-job-test").atMost(60, SECONDS).until({
         def pressJob = sql """ select SucceedTaskCount from jobs("type"="insert") where name='press'"""
-        println pressJob
+        // check job status and succeed task count is 1
         pressJob.size() == 1 && '1' == onceJob.get(0).get(0)
     })
 
     sql """
         DROP JOB IF EXISTS where jobname =  'past_start_time'
     """
+    // create job with start time is past time, job should be running
     sql """
           CREATE JOB past_start_time  ON SCHEDULE every 10 hour starts '2023-11-13 14:18:07'  comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName}  values  ('2023-07-19', 99, 99);
      """
@@ -214,6 +221,10 @@ suite("test_base_insert_job") {
     sql """
         PAUSE JOB where jobname =  '${jobName}'
     """
+    pausedJobStatus = sql """
+        select status from jobs("type"="insert") where Name='${jobName}'
+    """
+    assert pausedJobStatus.get(0).get(0) == "PAUSED"
     def tasks = sql """ select status from tasks("type"="insert") where JobName= '${jobName}'  """
     sql """
         RESUME JOB where jobname =  '${jobName}'
@@ -226,6 +237,11 @@ suite("test_base_insert_job") {
         //resume tasks size should be greater than before pause
         afterResumeTasks.size() > tasks.size()
     })
+    // check resume job status
+    def afterResumeJobStatus = sql """
+        select status from jobs("type"="insert") where Name='${jobName}'
+    """
+    assert afterResumeJobStatus.get(0).get(0) == "RUNNING"
 
     // assert same job name
     try {

From ab1ade0aeae420b9b66f2aa64b635142cacd8150 Mon Sep 17 00:00:00 2001
From: sparrow <38098988+biohazard4321@users.noreply.github.com>
Date: Sat, 7 Dec 2024 09:05:49 +0800
Subject: [PATCH 217/399] [bugfix](hive)Fix split assignment leak. releated to
 issue (#40683). (#44845)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Also fix stop split too early in select join statement when
enable_profile is true. Releated to issue (#40683).

Case appearing condition: use batchmode and set enable_profile=true，hive
table has enough split （like more than 10）

### What problem does this PR solve?
Under enable_profile is true, in query with join like "select
dt.d_year,dt.d_moy from date_dim dt ,store_sales where dt.d_date_sk =
store_sales.ss_sold_date_sk group by dt.d_year, dt.d_moy order by
dt.d_year, dt.d_moy limit 100",
the old fix in commit 83797e36aeb stops SplitAssignment in
FileScanNode.getNodeExplainString, that would be called in normal query
(in StmtExecutor.updateProfile)，then BE would get empty split and make
wrong result。
So i changed the place where SplitAssignment need stop in explain only
statement.
---
 .../main/java/org/apache/doris/datasource/FileScanNode.java   | 1 -
 .../doris/nereids/trees/plans/commands/ExplainCommand.java    | 4 ++++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index 9845c0070fc430..29fdb2b09acfcd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -126,7 +126,6 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
         output.append(prefix);
         if (isBatchMode()) {
             output.append("(approximate)");
-            splitAssignment.stop();
         }
         output.append("inputSplitNum=").append(selectedSplitNum).append(", totalFileSize=")
             .append(totalFileSize).append(", scanRanges=").append(scanRangeLocations.size()).append("\n");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
index d4fce906693a68..e3f2f1d732ae5a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
@@ -27,6 +27,7 @@
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.planner.ScanNode;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.StmtExecutor;
 
@@ -95,6 +96,9 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
         } else {
             executor.handleExplainStmt(planner.getExplainString(explainOptions), true);
         }
+        for (ScanNode scanNode : planner.getScanNodes()) {
+            scanNode.stop();
+        }
     }
 
     @Override

From 5f7a4a89ac8a945fdb038390eebaf38ff8c22667 Mon Sep 17 00:00:00 2001
From: huanghaibin <huanghaibin@selectdb.com>
Date: Sat, 7 Dec 2024 10:56:34 +0800
Subject: [PATCH 218/399] [compile](fix) mow code compile error (#45134)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #44439
---
 be/src/cloud/cloud_meta_mgr.cpp       |  4 ++--
 be/src/cloud/cloud_storage_engine.cpp | 20 ++++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 477cc802910869..835e74ca7d5687 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -758,10 +758,10 @@ Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta,
     Status st = retry_rpc("prepare rowset", req, &resp, &MetaService_Stub::prepare_rowset);
     if (!st.ok() && resp.status().code() == MetaServiceCode::ALREADY_EXISTED) {
         if (existed_rs_meta != nullptr && resp.has_existed_rowset_meta()) {
-            RowsetMetaPB doris_rs_meta =
+            RowsetMetaPB doris_rs_meta_tmp =
                     cloud_rowset_meta_to_doris(std::move(*resp.mutable_existed_rowset_meta()));
             *existed_rs_meta = std::make_shared<RowsetMeta>();
-            (*existed_rs_meta)->init_from_pb(doris_rs_meta);
+            (*existed_rs_meta)->init_from_pb(doris_rs_meta_tmp);
         }
         return Status::AlreadyExist("failed to prepare rowset: {}", resp.status().msg());
     }
diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp
index 2944ccf21f276b..b66a9cfbdb2245 100644
--- a/be/src/cloud/cloud_storage_engine.cpp
+++ b/be/src/cloud/cloud_storage_engine.cpp
@@ -352,11 +352,11 @@ void CloudStorageEngine::sync_storage_vault() {
 
     for (auto& [id, vault_info, path_format] : vault_infos) {
         auto fs = get_filesystem(id);
-        auto st = (fs == nullptr)
-                          ? std::visit(VaultCreateFSVisitor {id, path_format}, vault_info)
-                          : std::visit(RefreshFSVaultVisitor {id, std::move(fs), path_format},
-                                       vault_info);
-        if (!st.ok()) [[unlikely]] {
+        auto status = (fs == nullptr)
+                              ? std::visit(VaultCreateFSVisitor {id, path_format}, vault_info)
+                              : std::visit(RefreshFSVaultVisitor {id, std::move(fs), path_format},
+                                           vault_info);
+        if (!status.ok()) [[unlikely]] {
             LOG(WARNING) << vault_process_error(id, vault_info, std::move(st));
         }
     }
@@ -506,13 +506,13 @@ void CloudStorageEngine::_compaction_tasks_producer_callback() {
             /// If it is not cleaned up, the reference count of the tablet will always be greater than 1,
             /// thus cannot be collected by the garbage collector. (TabletManager::start_trash_sweep)
             for (const auto& tablet : tablets_compaction) {
-                Status st = submit_compaction_task(tablet, compaction_type);
-                if (st.ok()) continue;
-                if ((!st.is<ErrorCode::BE_NO_SUITABLE_VERSION>() &&
-                     !st.is<ErrorCode::CUMULATIVE_NO_SUITABLE_VERSION>()) ||
+                Status status = submit_compaction_task(tablet, compaction_type);
+                if (status.ok()) continue;
+                if ((!status.is<ErrorCode::BE_NO_SUITABLE_VERSION>() &&
+                     !status.is<ErrorCode::CUMULATIVE_NO_SUITABLE_VERSION>()) ||
                     VLOG_DEBUG_IS_ON) {
                     LOG(WARNING) << "failed to submit compaction task for tablet: "
-                                 << tablet->tablet_id() << ", err: " << st;
+                                 << tablet->tablet_id() << ", err: " << status;
                 }
             }
             interval = config::generate_compaction_tasks_interval_ms;

From fe63640cd014fc7381dfcf4318a22a4684c954e3 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <chenkaijie@selectdb.com>
Date: Sat, 7 Dec 2024 21:57:50 +0800
Subject: [PATCH 219/399] [fix](move-memtable) tolerate non-open streams in
 close wait (#44680)

Related PR: #44344
`VTabletWriterV2::_select_streams()` is already checking if there is
enough downstream BE to meet the replication requirements.
`VTabletWriterV2::close()` should tolerate those non-open streams on
close wait.

Debug point `VTabletWriterV2._open_streams.skip_two_backends` is added
along with `VTabletWriterV2._open_streams.skip_one_backend` to check
this behavior.
---
 be/src/vec/sink/load_stream_stub.cpp             | 16 ++++++----------
 be/src/vec/sink/writer/vtablet_writer_v2.cpp     | 12 +++++++++---
 .../test_multi_replica_fault_injection.groovy    | 13 ++++++++-----
 3 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index 59719765ab3964..d5b04c636d4d6c 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -219,11 +219,7 @@ Status LoadStreamStub::append_data(int64_t partition_id, int64_t index_id, int64
         add_failed_tablet(tablet_id, _status);
         return _status;
     }
-    DBUG_EXECUTE_IF("LoadStreamStub.only_send_segment_0", {
-        if (segment_id != 0) {
-            return Status::OK();
-        }
-    });
+    DBUG_EXECUTE_IF("LoadStreamStub.skip_send_segment", { return Status::OK(); });
     PStreamHeader header;
     header.set_src_id(_src_id);
     *header.mutable_load_id() = _load_id;
@@ -246,11 +242,7 @@ Status LoadStreamStub::add_segment(int64_t partition_id, int64_t index_id, int64
         add_failed_tablet(tablet_id, _status);
         return _status;
     }
-    DBUG_EXECUTE_IF("LoadStreamStub.only_send_segment_0", {
-        if (segment_id != 0) {
-            return Status::OK();
-        }
-    });
+    DBUG_EXECUTE_IF("LoadStreamStub.skip_send_segment", { return Status::OK(); });
     PStreamHeader header;
     header.set_src_id(_src_id);
     *header.mutable_load_id() = _load_id;
@@ -340,6 +332,10 @@ Status LoadStreamStub::wait_for_schema(int64_t partition_id, int64_t index_id, i
 
 Status LoadStreamStub::close_wait(RuntimeState* state, int64_t timeout_ms) {
     DBUG_EXECUTE_IF("LoadStreamStub::close_wait.long_wait", DBUG_BLOCK);
+    if (!_is_open.load()) {
+        // we don't need to close wait on non-open streams
+        return Status::OK();
+    }
     if (!_is_closing.load()) {
         return _status;
     }
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index 3dc58be3bcde88..cd196a8f2b3df2 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -269,14 +269,20 @@ Status VTabletWriterV2::open(RuntimeState* state, RuntimeProfile* profile) {
 }
 
 Status VTabletWriterV2::_open_streams() {
-    bool fault_injection_skip_be = true;
+    int fault_injection_skip_be = 0;
     bool any_backend = false;
     bool any_success = false;
     for (auto& [dst_id, _] : _tablets_for_node) {
         auto streams = _load_stream_map->get_or_create(dst_id);
         DBUG_EXECUTE_IF("VTabletWriterV2._open_streams.skip_one_backend", {
-            if (fault_injection_skip_be) {
-                fault_injection_skip_be = false;
+            if (fault_injection_skip_be < 1) {
+                fault_injection_skip_be++;
+                continue;
+            }
+        });
+        DBUG_EXECUTE_IF("VTabletWriterV2._open_streams.skip_two_backends", {
+            if (fault_injection_skip_be < 2) {
+                fault_injection_skip_be++;
                 continue;
             }
         });
diff --git a/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
index 2f6afd5ca6925b..d09983d52d0dc3 100644
--- a/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
@@ -75,14 +75,15 @@ suite("test_multi_replica_fault_injection", "nonConcurrent") {
             file "baseall.txt"
         }
 
-        def load_with_injection = { injection, error_msg->
+        def load_with_injection = { injection, error_msg, success=false->
             try {
                 sql "truncate table test"
                 GetDebugPoint().enableDebugPointForAllBEs(injection)
                 sql "insert into test select * from baseall where k1 <= 3"
+                assertTrue(success, String.format("Expected Exception '%s', actual success", error_msg))
             } catch(Exception e) {
                 logger.info(e.getMessage())
-                assertTrue(e.getMessage().contains(error_msg))
+                assertTrue(e.getMessage().contains(error_msg), e.toString())
             } finally {
                 GetDebugPoint().disableDebugPointForAllBEs(injection)
             }
@@ -90,15 +91,17 @@ suite("test_multi_replica_fault_injection", "nonConcurrent") {
 
         // StreamSinkFileWriter appendv write segment failed one replica
         // success
-        load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", "sucess")
+        load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", "sucess", true)
         // StreamSinkFileWriter appendv write segment failed two replica
         load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", "add segment failed")
         // StreamSinkFileWriter appendv write segment failed all replica
         load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", "failed to send segment data to any replicas")
         // test segment num check when LoadStreamStub missed tail segments
-        load_with_injection("LoadStreamStub.only_send_segment_0", "segment num mismatch")
+        load_with_injection("LoadStreamStub.skip_send_segment", "segment num mismatch")
         // test one backend open failure
-        load_with_injection("VTabletWriterV2._open_streams.skip_one_backend", "success")
+        load_with_injection("VTabletWriterV2._open_streams.skip_one_backend", "success", true)
+        // test two backend open failure
+        load_with_injection("VTabletWriterV2._open_streams.skip_two_backends", "not enough streams 1/3")
         sql """ set enable_memtable_on_sink_node=false """
     }
 }

From 5d3bc2b051ffb134c8ad5054d1fe5ced5aef5527 Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Sun, 8 Dec 2024 09:56:29 +0800
Subject: [PATCH 220/399] [fix](regression)Add log to investigate
 nereids_p0/stats/partition_key_minmax case failure. (#45121)

### What problem does this PR solve?
Add log to investigate nereids_p0/stats/partition_key_minmax case
failure.

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None
---
 .../suites/nereids_p0/stats/partition_key_minmax.groovy      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/regression-test/suites/nereids_p0/stats/partition_key_minmax.groovy b/regression-test/suites/nereids_p0/stats/partition_key_minmax.groovy
index 3a58d1b6f23ed2..e7d733611e79d1 100644
--- a/regression-test/suites/nereids_p0/stats/partition_key_minmax.groovy
+++ b/regression-test/suites/nereids_p0/stats/partition_key_minmax.groovy
@@ -32,6 +32,8 @@ suite("partition_key_minmax") {
 
         analyze table rangetable with sync;
     """
+    def columnStats = sql """show column cached stats rangetable"""
+    logger.info("rangetable cached stats: " + columnStats)
     explain {
         sql """memo plan
             select * from rangetable where a < 250;
@@ -56,6 +58,9 @@ suite("partition_key_minmax") {
     analyze table listtable with sync;
     """
 
+    columnStats = sql """show column cached stats listtable"""
+    logger.info("listtable cached stats: " + columnStats)
+
     explain {
         sql """
          memo plan select * from listtable where id >=3;

From 9f31a39d1aa4551d081d89124658b24ea54b6f63 Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Mon, 9 Dec 2024 02:30:36 +0800
Subject: [PATCH 221/399] [enhance](auth)When authorization includes create,
 not check if resources exist (#45125)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #39597

Problem Summary:
When authorization includes create, not check if resources exist
---
 .../main/java/org/apache/doris/mysql/privilege/Auth.java  | 8 ++++++--
 .../suites/auth_p0/test_grant_nonexist_table.groovy       | 4 +++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
index 8a13c6dd125548..a1fb57d01cafa4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
@@ -686,7 +686,7 @@ private void grantInternal(UserIdentity userIdent, String role, TablePattern tbl
         writeLock();
         try {
             if (!isReplay) {
-                checkTablePatternExist(tblPattern);
+                checkTablePatternExist(tblPattern, privs);
             }
             if (role == null) {
                 if (!doesUserExist(userIdent)) {
@@ -706,8 +706,12 @@ private void grantInternal(UserIdentity userIdent, String role, TablePattern tbl
         }
     }
 
-    private void checkTablePatternExist(TablePattern tablePattern) throws DdlException {
+    private void checkTablePatternExist(TablePattern tablePattern, PrivBitSet privs) throws DdlException {
         Objects.requireNonNull(tablePattern, "tablePattern can not be null");
+        Objects.requireNonNull(privs, "privs can not be null");
+        if (privs.containsPrivs(Privilege.CREATE_PRIV)) {
+            return;
+        }
         PrivLevel privLevel = tablePattern.getPrivLevel();
         if (privLevel == PrivLevel.GLOBAL) {
             return;
diff --git a/regression-test/suites/auth_p0/test_grant_nonexist_table.groovy b/regression-test/suites/auth_p0/test_grant_nonexist_table.groovy
index 36e75707be7252..74d211e5010201 100644
--- a/regression-test/suites/auth_p0/test_grant_nonexist_table.groovy
+++ b/regression-test/suites/auth_p0/test_grant_nonexist_table.groovy
@@ -39,7 +39,9 @@ suite("test_grant_nonexist_table","p0,auth") {
             sql """grant select_priv on internal.${dbName}.non_exist_table to ${user}"""
             exception "table"
         }
-
+    // contain create_triv should not check name, Same behavior as MySQL
+    sql """grant create_priv on internal.${dbName}.non_exist_table to ${user}"""
+    sql """grant create_priv,select_priv on internal.${dbName}.non_exist_table to ${user}"""
 
     try_sql("DROP USER ${user}")
 }

From 4cf908c43c60abc6652c3bd180ae9397d236718d Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Mon, 9 Dec 2024 08:26:06 +0800
Subject: [PATCH 222/399] [enchement](mc)Optimize reading of maxcompute
 partition tables. (#45148)

### What problem does this PR solve?
Problem Summary:
Optimize reading of maxcompute partition tables:
1. Introduce batch mode to generate splits for Maxcompute partition
tables to optimize scenarios with a large number of partitions. Control
it through the variable `num_partitions_in_batch_mode`.
2. Introduce catalog parameter `mc.split_cross_partition`. The parameter
is true, which is more friendly to reading partition tables, and false,
which is more friendly to debug.
3. Add `-Darrow.enable_null_check_for_get=false` to be jvm to improve
the efficiency of mc arrow data conversion.
---
 conf/be.conf                                  |    2 +-
 .../maxcompute/MaxComputeExternalCatalog.java |    8 +-
 .../maxcompute/source/MaxComputeScanNode.java |  276 ++--
 .../property/constants/MCProperties.java      |    7 +
 .../test_max_compute_partition_prune.out      | 1364 +++++++++++++++++
 .../test_max_compute_partition_prune.groovy   |  356 ++---
 6 files changed, 1747 insertions(+), 266 deletions(-)

diff --git a/conf/be.conf b/conf/be.conf
index ae9524313d860a..ddbc385522e848 100644
--- a/conf/be.conf
+++ b/conf/be.conf
@@ -24,7 +24,7 @@ LOG_DIR="${DORIS_HOME}/log/"
 JAVA_OPTS="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xloggc:$LOG_DIR/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
 
 # For jdk 17, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djol.skipHotspotSAAttach=true -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED"
+JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djol.skipHotspotSAAttach=true -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED -Darrow.enable_null_check_for_get=false"
 
 # Set your own JAVA_HOME
 # JAVA_HOME=/path/to/jdk/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
index 06c1e55dcf6f4b..27e4976aedd85e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
@@ -164,22 +164,24 @@ protected void initLocalObjectsImpl() {
         defaultProject = props.get(MCProperties.PROJECT);
         quota = props.getOrDefault(MCProperties.QUOTA, MCProperties.DEFAULT_QUOTA);
 
+        boolean splitCrossPartition =
+                Boolean.parseBoolean(props.getOrDefault(MCProperties.SPLIT_CROSS_PARTITION,
+                MCProperties.DEFAULT_SPLIT_CROSS_PARTITION));
 
         splitStrategy = props.getOrDefault(MCProperties.SPLIT_STRATEGY, MCProperties.DEFAULT_SPLIT_STRATEGY);
         if (splitStrategy.equals(MCProperties.SPLIT_BY_BYTE_SIZE_STRATEGY)) {
             splitByteSize = Long.parseLong(props.getOrDefault(MCProperties.SPLIT_BYTE_SIZE,
                     MCProperties.DEFAULT_SPLIT_BYTE_SIZE));
-
             splitOptions = SplitOptions.newBuilder()
                     .SplitByByteSize(splitByteSize)
-                    .withCrossPartition(false)
+                    .withCrossPartition(splitCrossPartition)
                     .build();
         } else {
             splitRowCount = Long.parseLong(props.getOrDefault(MCProperties.SPLIT_ROW_COUNT,
                     MCProperties.DEFAULT_SPLIT_ROW_COUNT));
             splitOptions = SplitOptions.newBuilder()
                     .SplitByRowOffset()
-                    .withCrossPartition(false)
+                    .withCrossPartition(splitCrossPartition)
                     .build();
         }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index 4ad971a5c64789..9fa22a0fffaab8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -29,6 +29,7 @@
 import org.apache.doris.analysis.SlotRef;
 import org.apache.doris.analysis.TupleDescriptor;
 import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.AnalysisException;
@@ -43,6 +44,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
 import org.apache.doris.nereids.util.DateUtils;
 import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.thrift.TFileFormatType;
@@ -79,15 +81,18 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 public class MaxComputeScanNode extends FileQueryScanNode {
 
     private final MaxComputeExternalTable table;
-    private TableBatchReadSession tableBatchReadSession;
     private Predicate filterPredicate;
-    private static final LocationPath ROW_OFFSET_PATH = new LocationPath("/row_offset", Maps.newHashMap());
-    private static final LocationPath BYTE_SIZE_PATH = new LocationPath("/byte_size", Maps.newHashMap());
+    List<String> requiredPartitionColumns = new ArrayList<>();
+    List<String> orderedRequiredDataColumns = new ArrayList<>();
 
     private int connectTimeout;
     private int readTimeout;
@@ -96,6 +101,10 @@ public class MaxComputeScanNode extends FileQueryScanNode {
     @Setter
     private SelectedPartitions selectedPartitions = null;
 
+    private static final LocationPath ROW_OFFSET_PATH = new LocationPath("/row_offset", Maps.newHashMap());
+    private static final LocationPath BYTE_SIZE_PATH = new LocationPath("/byte_size", Maps.newHashMap());
+
+
     // For new planner
     public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc,
             SelectedPartitions selectedPartitions, boolean needCheckColumnPriv) {
@@ -143,33 +152,17 @@ private void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeS
         rangeDesc.setSize(maxComputeSplit.getLength());
     }
 
-    // Return false if no need to read any partition data.
-    // Return true if need to read partition data.
-    boolean createTableBatchReadSession() throws UserException {
-        List<String> requiredPartitionColumns = new ArrayList<>();
-        List<String> orderedRequiredDataColumns = new ArrayList<>();
-
-        List<PartitionSpec> requiredPartitionSpecs = new ArrayList<>();
-        //if requiredPartitionSpecs is empty, get all partition data.
-        if (!table.getPartitionColumns().isEmpty() && selectedPartitions != SelectedPartitions.NOT_PRUNED) {
-            this.totalPartitionNum = selectedPartitions.totalPartitionNum;
-            this.selectedPartitionNum = selectedPartitions.selectedPartitions.size();
-
-            if (selectedPartitions.selectedPartitions.isEmpty()) {
-                //no need read any partition data.
-                return false;
-            }
-            selectedPartitions.selectedPartitions.forEach(
-                    (key, value) -> requiredPartitionSpecs.add(new PartitionSpec(key))
-            );
-        }
 
+    private void createRequiredColumns() {
         Set<String> requiredSlots =
                 desc.getSlots().stream().map(e -> e.getColumn().getName()).collect(Collectors.toSet());
 
         Set<String> partitionColumns =
                 table.getPartitionColumns().stream().map(Column::getName).collect(Collectors.toSet());
 
+        requiredPartitionColumns.clear();
+        orderedRequiredDataColumns.clear();
+
         for (Column column : table.getColumns()) {
             String columnName =  column.getName();
             if (!requiredSlots.contains(columnName)) {
@@ -181,32 +174,118 @@ boolean createTableBatchReadSession() throws UserException {
                 orderedRequiredDataColumns.add(columnName);
             }
         }
+    }
 
+    /**
+     * For no partition table: request requiredPartitionSpecs is empty
+     * For partition table: if requiredPartitionSpecs is empty, get all partition data.
+     */
+    TableBatchReadSession createTableBatchReadSession(List<PartitionSpec> requiredPartitionSpecs) throws IOException {
+        MaxComputeExternalCatalog mcCatalog = (MaxComputeExternalCatalog) table.getCatalog();
 
+        readTimeout = mcCatalog.getReadTimeout();
+        connectTimeout = mcCatalog.getConnectTimeout();
+        retryTimes = mcCatalog.getRetryTimes();
+
+        TableReadSessionBuilder scanBuilder = new TableReadSessionBuilder();
+        return scanBuilder.identifier(TableIdentifier.of(table.getDbName(), table.getName()))
+                        .withSettings(mcCatalog.getSettings())
+                        .withSplitOptions(mcCatalog.getSplitOption())
+                        .requiredPartitionColumns(requiredPartitionColumns)
+                        .requiredDataColumns(orderedRequiredDataColumns)
+                        .withFilterPredicate(filterPredicate)
+                        .requiredPartitions(requiredPartitionSpecs)
+                        .withArrowOptions(
+                                ArrowOptions.newBuilder()
+                                        .withDatetimeUnit(TimestampUnit.MILLI)
+                                        .withTimestampUnit(TimestampUnit.NANO)
+                                        .build()
+                        ).buildBatchReadSession();
+    }
 
-        MaxComputeExternalCatalog mcCatalog = (MaxComputeExternalCatalog) table.getCatalog();
+    @Override
+    public boolean isBatchMode() {
+        if (table.getPartitionColumns().isEmpty()) {
+            return false;
+        }
 
-        try {
-            TableReadSessionBuilder scanBuilder = new TableReadSessionBuilder();
-            tableBatchReadSession =
-                    scanBuilder.identifier(TableIdentifier.of(table.getDbName(), table.getName()))
-                            .withSettings(mcCatalog.getSettings())
-                            .withSplitOptions(mcCatalog.getSplitOption())
-                            .requiredPartitionColumns(requiredPartitionColumns)
-                            .requiredPartitions(requiredPartitionSpecs)
-                            .requiredDataColumns(orderedRequiredDataColumns)
-                            .withArrowOptions(
-                                    ArrowOptions.newBuilder()
-                                            .withDatetimeUnit(TimestampUnit.MILLI)
-                                            .withTimestampUnit(TimestampUnit.NANO)
-                                            .build()
-                            )
-                            .withFilterPredicate(filterPredicate)
-                            .buildBatchReadSession();
-        } catch (java.io.IOException e) {
-            throw new RuntimeException(e);
+        com.aliyun.odps.Table odpsTable = table.getOdpsTable();
+        if (desc.getSlots().isEmpty() || odpsTable.getFileNum() <= 0) {
+            return false;
         }
-        return true;
+
+        int numPartitions = ConnectContext.get().getSessionVariable().getNumPartitionsInBatchMode();
+        return numPartitions > 0
+                && selectedPartitions != SelectedPartitions.NOT_PRUNED
+                && selectedPartitions.selectedPartitions.size() >= numPartitions;
+    }
+
+    @Override
+    public int numApproximateSplits() {
+        return selectedPartitions.selectedPartitions.size();
+    }
+
+    @Override
+    public void startSplit() {
+        this.totalPartitionNum = selectedPartitions.totalPartitionNum;
+        this.selectedPartitionNum = selectedPartitions.selectedPartitions.size();
+
+        if (selectedPartitions.selectedPartitions.isEmpty()) {
+            //no need read any partition data.
+            return;
+        }
+
+        createRequiredColumns();
+        List<PartitionSpec> requiredPartitionSpecs = new ArrayList<>();
+        selectedPartitions.selectedPartitions.forEach(
+                (key, value) -> requiredPartitionSpecs.add(new PartitionSpec(key))
+        );
+
+
+        int batchNumPartitions = ConnectContext.get().getSessionVariable().getNumPartitionsInBatchMode();
+
+        Executor scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor();
+        AtomicReference<UserException> batchException = new AtomicReference<>(null);
+        AtomicInteger numFinishedPartitions = new AtomicInteger(0);
+
+        CompletableFuture.runAsync(() -> {
+            for (int beginIndex = 0; beginIndex < requiredPartitionSpecs.size(); beginIndex += batchNumPartitions) {
+                int endIndex = Math.min(beginIndex + batchNumPartitions, requiredPartitionSpecs.size());
+                if (batchException.get() != null || splitAssignment.isStop()) {
+                    break;
+                }
+                List<PartitionSpec> requiredBatchPartitionSpecs = requiredPartitionSpecs.subList(beginIndex, endIndex);
+                int curBatchSize = endIndex - beginIndex;
+
+                try {
+                    CompletableFuture.runAsync(() -> {
+                        try {
+                            TableBatchReadSession tableBatchReadSession =
+                                    createTableBatchReadSession(requiredBatchPartitionSpecs);
+                            List<Split> batchSplit = getSplitByTableSession(tableBatchReadSession);
+
+                            splitAssignment.addToQueue(batchSplit);
+                        } catch (IOException e) {
+                            batchException.set(new UserException(e.getMessage(), e));
+                        } finally {
+                            if (batchException.get() != null) {
+                                splitAssignment.setException(batchException.get());
+                            }
+
+                            if (numFinishedPartitions.addAndGet(curBatchSize) == requiredPartitionSpecs.size()) {
+                                splitAssignment.finishSchedule();
+                            }
+                        }
+                    }, scheduleExecutor);
+                } catch (Exception e) {
+                    batchException.set(new UserException(e.getMessage(), e));
+                }
+
+                if (batchException.get() != null) {
+                    splitAssignment.setException(batchException.get());
+                }
+            }
+        });
     }
 
     @Override
@@ -467,67 +546,84 @@ protected Map<String, String> getLocationProperties() throws UserException {
         return new HashMap<>();
     }
 
-    @Override
-    public List<Split> getSplits() throws UserException {
+    List<Split> getSplitByTableSession(TableBatchReadSession tableBatchReadSession) throws java.io.IOException {
         List<Split> result = new ArrayList<>();
-        com.aliyun.odps.Table odpsTable = table.getOdpsTable();
-        if (desc.getSlots().isEmpty() || odpsTable.getFileNum() <= 0) {
-            return result;
-        }
+        String scanSessionSerialize =  serializeSession(tableBatchReadSession);
+        InputSplitAssigner assigner = tableBatchReadSession.getInputSplitAssigner();
+        long modificationTime = table.getOdpsTable().getLastDataModifiedTime().getTime();
 
-        if (!createTableBatchReadSession()) {
-            return result;
-        }
+        MaxComputeExternalCatalog mcCatalog = (MaxComputeExternalCatalog) table.getCatalog();
 
-        try {
-            String scanSessionSerialize =  serializeSession(tableBatchReadSession);
-            InputSplitAssigner assigner = tableBatchReadSession.getInputSplitAssigner();
-            long modificationTime = table.getOdpsTable().getLastDataModifiedTime().getTime();
+        if (mcCatalog.getSplitStrategy().equals(MCProperties.SPLIT_BY_BYTE_SIZE_STRATEGY)) {
 
-            MaxComputeExternalCatalog mcCatalog = (MaxComputeExternalCatalog) table.getCatalog();
+            for (com.aliyun.odps.table.read.split.InputSplit split : assigner.getAllSplits()) {
+                MaxComputeSplit maxComputeSplit =
+                        new MaxComputeSplit(BYTE_SIZE_PATH,
+                                ((IndexedInputSplit) split).getSplitIndex(), -1,
+                                mcCatalog.getSplitByteSize(),
+                                modificationTime, null,
+                                Collections.emptyList());
 
-            readTimeout = mcCatalog.getReadTimeout();
-            connectTimeout = mcCatalog.getConnectTimeout();
-            retryTimes = mcCatalog.getRetryTimes();
 
-            if (mcCatalog.getSplitStrategy().equals(MCProperties.SPLIT_BY_BYTE_SIZE_STRATEGY)) {
+                maxComputeSplit.scanSerialize = scanSessionSerialize;
+                maxComputeSplit.splitType = SplitType.BYTE_SIZE;
+                maxComputeSplit.sessionId = split.getSessionId();
 
-                for (com.aliyun.odps.table.read.split.InputSplit split : assigner.getAllSplits()) {
-                    MaxComputeSplit maxComputeSplit =
-                            new MaxComputeSplit(BYTE_SIZE_PATH,
-                                    ((IndexedInputSplit) split).getSplitIndex(), -1,
-                                    mcCatalog.getSplitByteSize(),
-                                    modificationTime, null,
-                                    Collections.emptyList());
+                result.add(maxComputeSplit);
+            }
+        } else {
+            long totalRowCount =  assigner.getTotalRowCount();
 
+            long recordsPerSplit = mcCatalog.getSplitRowCount();
+            for (long offset = 0; offset < totalRowCount; offset += recordsPerSplit) {
+                recordsPerSplit = Math.min(recordsPerSplit, totalRowCount - offset);
+                com.aliyun.odps.table.read.split.InputSplit split =
+                        assigner.getSplitByRowOffset(offset, recordsPerSplit);
 
-                    maxComputeSplit.scanSerialize = scanSessionSerialize;
-                    maxComputeSplit.splitType = SplitType.BYTE_SIZE;
-                    maxComputeSplit.sessionId = split.getSessionId();
+                MaxComputeSplit maxComputeSplit =
+                        new MaxComputeSplit(ROW_OFFSET_PATH,
+                                offset, recordsPerSplit, totalRowCount, modificationTime, null,
+                                Collections.emptyList());
 
-                    result.add(maxComputeSplit);
-                }
-            } else {
-                long totalRowCount =  assigner.getTotalRowCount();
+                maxComputeSplit.scanSerialize = scanSessionSerialize;
+                maxComputeSplit.splitType = SplitType.ROW_OFFSET;
+                maxComputeSplit.sessionId = split.getSessionId();
+
+                result.add(maxComputeSplit);
+            }
+        }
+        return result;
+    }
 
-                long recordsPerSplit = mcCatalog.getSplitRowCount();
-                for (long offset = 0; offset < totalRowCount; offset += recordsPerSplit) {
-                    recordsPerSplit = Math.min(recordsPerSplit, totalRowCount - offset);
-                    com.aliyun.odps.table.read.split.InputSplit split =
-                            assigner.getSplitByRowOffset(offset, recordsPerSplit);
 
-                    MaxComputeSplit maxComputeSplit =
-                            new MaxComputeSplit(ROW_OFFSET_PATH,
-                            offset, recordsPerSplit, totalRowCount, modificationTime, null,
-                            Collections.emptyList());
+    @Override
+    public List<Split> getSplits() throws UserException {
+        List<Split> result = new ArrayList<>();
+        com.aliyun.odps.Table odpsTable = table.getOdpsTable();
+        if (desc.getSlots().isEmpty() || odpsTable.getFileNum() <= 0) {
+            return result;
+        }
 
-                    maxComputeSplit.scanSerialize = scanSessionSerialize;
-                    maxComputeSplit.splitType = SplitType.ROW_OFFSET;
-                    maxComputeSplit.sessionId = split.getSessionId();
+        createRequiredColumns();
 
-                    result.add(maxComputeSplit);
-                }
+        List<PartitionSpec> requiredPartitionSpecs = new ArrayList<>();
+        //if requiredPartitionSpecs is empty, get all partition data.
+        if (!table.getPartitionColumns().isEmpty() && selectedPartitions != SelectedPartitions.NOT_PRUNED) {
+            this.totalPartitionNum = selectedPartitions.totalPartitionNum;
+            this.selectedPartitionNum = selectedPartitions.selectedPartitions.size();
+
+            if (selectedPartitions.selectedPartitions.isEmpty()) {
+                //no need read any partition data.
+                return result;
             }
+            selectedPartitions.selectedPartitions.forEach(
+                    (key, value) -> requiredPartitionSpecs.add(new PartitionSpec(key))
+            );
+        }
+
+        try {
+            TableBatchReadSession tableBatchReadSession = createTableBatchReadSession(requiredPartitionSpecs);
+            result = getSplitByTableSession(tableBatchReadSession);
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
index efbd01c14777de..3957ffc91bb4b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
@@ -64,6 +64,13 @@ public class MCProperties extends BaseProperties {
     public static final String DEFAULT_READ_TIMEOUT = "120"; // 120s
     public static final String DEFAULT_RETRY_COUNT = "4"; // 4 times
 
+    //withCrossPartition(true):
+    //      Very friendly to scenarios where there are many partitions but each partition is very small.
+    //withCrossPartition(false):
+    //      Very debug friendly.
+    public static final String SPLIT_CROSS_PARTITION = "mc.split_cross_partition";
+    public static final String DEFAULT_SPLIT_CROSS_PARTITION = "true";
+
     public static CloudCredential getCredential(Map<String, String> props) {
         return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN);
     }
diff --git a/regression-test/data/external_table_p2/maxcompute/test_max_compute_partition_prune.out b/regression-test/data/external_table_p2/maxcompute/test_max_compute_partition_prune.out
index 8f443829c4c394..d6c23e6b5abf52 100644
--- a/regression-test/data/external_table_p2/maxcompute/test_max_compute_partition_prune.out
+++ b/regression-test/data/external_table_p2/maxcompute/test_max_compute_partition_prune.out
@@ -123,3 +123,1367 @@
 
 -- !three_partition_11_0 --
 
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
+-- !one_partition_1_1 --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+
+-- !one_partition_2_1 --
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_3_all --
+1	Alice	2024
+2	Bob	2024
+3	Charlie	2024
+4	David	2025
+5	Eva	2025
+
+-- !one_partition_4_all --
+5	Eva	2025
+
+-- !one_partition_5_1 --
+3	Charlie	2024
+
+-- !two_partition_1_1 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+
+-- !two_partition_2_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_3_2 --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+
+-- !two_partition_4_all --
+1	Alice	US	1
+2	Bob	US	1
+3	Charlie	US	1
+4	David	US	2
+5	Eva	US	2
+6	Frank	EU	1
+7	Grace	EU	1
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !two_partition_5_1 --
+
+-- !two_partition_6_1 --
+8	Hannah	EU	2
+9	Ivy	EU	2
+10	Jack	EU	2
+
+-- !three_partition_1_1 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+
+-- !three_partition_2_1 --
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+
+-- !three_partition_3_3 --
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_4_2 --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+6	Frank	US	2025	Q1
+
+-- !three_partition_5_all --
+1	Alice	US	2024	Q1
+2	Bob	US	2024	Q1
+3	Charlie	US	2024	Q1
+4	David	US	2024	Q2
+5	Eva	US	2024	Q2
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_6_1 --
+8	Hannah	EU	2024	Q1
+9	Ivy	EU	2024	Q1
+
+-- !three_partition_7_7 --
+6	Frank	US	2025	Q1
+7	Grace	US	2025	Q2
+10	Jack	EU	2025	Q2
+11	Leo	EU	2025	Q2
+12	Mia	EU	2025	Q3
+13	Nina	AS	2025	Q1
+14	Oscar	AS	2025	Q2
+15	Paul	AS	2025	Q3
+
+-- !three_partition_8_2 --
+7	Grace	US	2025	Q2
+
+-- !one_partition_6_0 --
+
+-- !two_partition_7_0 --
+
+-- !two_partition_8_0 --
+
+-- !three_partition_9_0 --
+
+-- !three_partition_10_0 --
+
+-- !three_partition_11_0 --
+
diff --git a/regression-test/suites/external_table_p2/maxcompute/test_max_compute_partition_prune.groovy b/regression-test/suites/external_table_p2/maxcompute/test_max_compute_partition_prune.groovy
index e34569117a167f..83ebbe2fb29d1a 100644
--- a/regression-test/suites/external_table_p2/maxcompute/test_max_compute_partition_prune.groovy
+++ b/regression-test/suites/external_table_p2/maxcompute/test_max_compute_partition_prune.groovy
@@ -104,179 +104,191 @@ suite("test_max_compute_partition_prune", "p2,external,maxcompute,external_remot
         String mc_db = "mc_datalake"
         String mc_catalog_name = "test_max_compute_partition_prune"
 
-        sql """drop catalog if exists ${mc_catalog_name};"""
-        sql """
-            create catalog if not exists ${mc_catalog_name} properties (
-                "type" = "max_compute",
-                "mc.default.project" = "${mc_db}",
-                "mc.access_key" = "${ak}",
-                "mc.secret_key" = "${sk}",
-                "mc.endpoint" = "http://service.cn-beijing-vpc.maxcompute.aliyun-inc.com/api"
-            );
-        """
-        sql """ switch ${mc_catalog_name} """
-        sql """ use ${mc_db}"""
-
-        qt_one_partition_1_1 one_partition_1_1
-        explain {
-            sql("${one_partition_1_1}")
-            contains "partition=1/2"
-        }
-
-        qt_one_partition_2_1 one_partition_2_1
-        explain {
-            sql("${one_partition_2_1}")
-            contains "partition=1/2"
-        }
-
-        qt_one_partition_3_all one_partition_3_all
-        explain {
-            sql("${one_partition_3_all}")
-            contains "partition=2/2"
-        }
-
-        qt_one_partition_4_all one_partition_4_all
-        explain {
-            sql("${one_partition_4_all}")
-            contains "partition=2/2"
-        }
-
-        qt_one_partition_5_1 one_partition_5_1
-        explain {
-            sql("${one_partition_5_1}")
-            contains "partition=1/2"
-        }
-
-
-        qt_two_partition_1_1 two_partition_1_1
-        explain {
-            sql("${two_partition_1_1}")
-            contains "partition=1/4"
-        }
-
-        qt_two_partition_2_1 two_partition_2_1
-        explain {
-            sql("${two_partition_2_1}")
-            contains "partition=1/4"
-        }
-
-        qt_two_partition_3_2 two_partition_3_2
-        explain {
-            sql("${two_partition_3_2}")
-            contains "partition=2/4"
-        }
-
-        qt_two_partition_4_all two_partition_4_all
-        explain {
-            sql("${two_partition_4_all}")
-            contains "partition=4/4"
-        }
-
-        qt_two_partition_5_1 two_partition_5_1
-        explain {
-            sql("${two_partition_5_1}")
-            contains "partition=1/4"
-        }
-
-        qt_two_partition_6_1 two_partition_6_1
-        explain {
-            sql("${two_partition_6_1}")
-            contains "partition=1/4"
-        }
-
-
-
-        qt_three_partition_1_1 three_partition_1_1
-        explain {
-            sql("${three_partition_1_1}")
-            contains "partition=1/10"
-        }
 
-        qt_three_partition_2_1 three_partition_2_1
-        explain {
-            sql("${three_partition_2_1}")
-            contains "partition=1/10"
+        for (String  enable_profile : ["true","false"] ) {
+            sql """set enable_profile = ${enable_profile} """;
+
+            for (String num_partitions : ["1","10","100"] ) {
+                sql "set num_partitions_in_batch_mode =  ${num_partitions} "
+
+                for (String cross_partition : ["true","false"] ) {
+
+                    sql """drop catalog if exists ${mc_catalog_name};"""
+                    sql """
+                        create catalog if not exists ${mc_catalog_name} properties (
+                            "type" = "max_compute",
+                            "mc.default.project" = "${mc_db}",
+                            "mc.access_key" = "${ak}",
+                            "mc.secret_key" = "${sk}",
+                            "mc.endpoint" = "http://service.cn-beijing-vpc.maxcompute.aliyun-inc.com/api",
+                            "mc.split_cross_partition" = "${cross_partition}"
+                        );
+                    """
+                    sql """ switch ${mc_catalog_name} """
+                    sql """ use ${mc_db}"""
+
+                    qt_one_partition_1_1 one_partition_1_1
+                    explain {
+                        sql("${one_partition_1_1}")
+                        contains "partition=1/2"
+                    }
+
+                    qt_one_partition_2_1 one_partition_2_1
+                    explain {
+                        sql("${one_partition_2_1}")
+                        contains "partition=1/2"
+                    }
+
+                    qt_one_partition_3_all one_partition_3_all
+                    explain {
+                        sql("${one_partition_3_all}")
+                        contains "partition=2/2"
+                    }
+
+                    qt_one_partition_4_all one_partition_4_all
+                    explain {
+                        sql("${one_partition_4_all}")
+                        contains "partition=2/2"
+                    }
+
+                    qt_one_partition_5_1 one_partition_5_1
+                    explain {
+                        sql("${one_partition_5_1}")
+                        contains "partition=1/2"
+                    }
+
+
+                    qt_two_partition_1_1 two_partition_1_1
+                    explain {
+                        sql("${two_partition_1_1}")
+                        contains "partition=1/4"
+                    }
+
+                    qt_two_partition_2_1 two_partition_2_1
+                    explain {
+                        sql("${two_partition_2_1}")
+                        contains "partition=1/4"
+                    }
+
+                    qt_two_partition_3_2 two_partition_3_2
+                    explain {
+                        sql("${two_partition_3_2}")
+                        contains "partition=2/4"
+                    }
+
+                    qt_two_partition_4_all two_partition_4_all
+                    explain {
+                        sql("${two_partition_4_all}")
+                        contains "partition=4/4"
+                    }
+
+                    qt_two_partition_5_1 two_partition_5_1
+                    explain {
+                        sql("${two_partition_5_1}")
+                        contains "partition=1/4"
+                    }
+
+                    qt_two_partition_6_1 two_partition_6_1
+                    explain {
+                        sql("${two_partition_6_1}")
+                        contains "partition=1/4"
+                    }
+
+
+
+                    qt_three_partition_1_1 three_partition_1_1
+                    explain {
+                        sql("${three_partition_1_1}")
+                        contains "partition=1/10"
+                    }
+
+                    qt_three_partition_2_1 three_partition_2_1
+                    explain {
+                        sql("${three_partition_2_1}")
+                        contains "partition=1/10"
+                    }
+
+                    qt_three_partition_3_3 three_partition_3_3
+                    explain {
+                        sql("${three_partition_3_3}")
+                        contains "partition=3/10"
+                    }
+
+                    qt_three_partition_4_2 three_partition_4_2
+                    explain {
+                        sql("${three_partition_4_2}")
+                        contains "partition=2/10"
+                    }
+
+                    qt_three_partition_5_all three_partition_5_all
+                    explain {
+                        sql("${three_partition_5_all}")
+                        contains "partition=10/10"
+                    }
+
+                    qt_three_partition_6_1 three_partition_6_1
+                    explain {
+                        sql("${three_partition_6_1}")
+                        contains "partition=1/10"
+                    }
+
+                    qt_three_partition_7_7 three_partition_7_7
+                    explain {
+                        sql("${three_partition_7_7}")
+                        contains "partition=7/10"
+                    }
+
+                    qt_three_partition_8_2 three_partition_8_2
+                    explain {
+                        sql("${three_partition_8_2}")
+                        contains "partition=2/10"
+                    }
+
+
+                    // 0 partitions
+                    def one_partition_6_0 = """SELECT * FROM one_partition_tb WHERE part1 = 2023 ORDER BY id;"""
+                    qt_one_partition_6_0 one_partition_6_0
+                    explain {
+                        sql("${one_partition_6_0}")
+                        contains "partition=0/2"
+                    }
+
+                    def two_partition_7_0 = """SELECT * FROM two_partition_tb WHERE part1 = 'CN' AND part2 = 1 ORDER BY id;"""
+                    qt_two_partition_7_0 two_partition_7_0
+                    explain {
+                        sql("${two_partition_7_0}")
+                        contains "partition=0/4"
+                    }
+
+                    def two_partition_8_0 = """SELECT * FROM two_partition_tb WHERE part1 = 'US' AND part2 = 3 ORDER BY id;"""
+                    qt_two_partition_8_0 two_partition_8_0
+                    explain {
+                        sql("${two_partition_8_0}")
+                        contains "partition=0/4"
+                    }
+
+                    def three_partition_9_0 = """SELECT * FROM three_partition_tb WHERE part1 = 'US' AND part2 = 2023 AND part3 = 'Q1' ORDER BY id;"""
+                    qt_three_partition_9_0 three_partition_9_0
+                    explain {
+                        sql("${three_partition_9_0}")
+                        contains "partition=0/10"
+                    }
+
+                    def three_partition_10_0 = """SELECT * FROM three_partition_tb WHERE part1 = 'EU' AND part2 = 2024 AND part3 = 'Q4' ORDER BY id;"""
+                    qt_three_partition_10_0 three_partition_10_0
+                    explain {
+                        sql("${three_partition_10_0}")
+                        contains "partition=0/10"
+                    }
+
+                    def three_partition_11_0 = """SELECT * FROM three_partition_tb WHERE part1 = 'AS' AND part2 = 2025 AND part3 = 'Q4' ORDER BY id;"""
+                    qt_three_partition_11_0 three_partition_11_0
+                    explain {
+                        sql("${three_partition_11_0}")
+                        contains "partition=0/10"
+                    }
+                }
+            }
         }
-
-        qt_three_partition_3_3 three_partition_3_3
-        explain {
-            sql("${three_partition_3_3}")
-            contains "partition=3/10"
-        }
-
-        qt_three_partition_4_2 three_partition_4_2
-        explain {
-            sql("${three_partition_4_2}")
-            contains "partition=2/10"
-        }
-
-        qt_three_partition_5_all three_partition_5_all
-        explain {
-            sql("${three_partition_5_all}")
-            contains "partition=10/10"
-        }
-
-        qt_three_partition_6_1 three_partition_6_1
-        explain {
-            sql("${three_partition_6_1}")
-            contains "partition=1/10"
-        }
-
-        qt_three_partition_7_7 three_partition_7_7
-        explain {
-            sql("${three_partition_7_7}")
-            contains "partition=7/10"
-        }
-
-        qt_three_partition_8_2 three_partition_8_2
-        explain {
-            sql("${three_partition_8_2}")
-            contains "partition=2/10"
-        }
-
-
-        // 0 partitions
-        def one_partition_6_0 = """SELECT * FROM one_partition_tb WHERE part1 = 2023 ORDER BY id;"""
-        qt_one_partition_6_0 one_partition_6_0
-        explain {
-            sql("${one_partition_6_0}")
-            contains "partition=0/2"
-        }
-
-        def two_partition_7_0 = """SELECT * FROM two_partition_tb WHERE part1 = 'CN' AND part2 = 1 ORDER BY id;"""
-        qt_two_partition_7_0 two_partition_7_0
-        explain {
-            sql("${two_partition_7_0}")
-            contains "partition=0/4"
-        }
-
-        def two_partition_8_0 = """SELECT * FROM two_partition_tb WHERE part1 = 'US' AND part2 = 3 ORDER BY id;"""
-        qt_two_partition_8_0 two_partition_8_0
-        explain {
-            sql("${two_partition_8_0}")
-            contains "partition=0/4"
-        }
-
-        def three_partition_9_0 = """SELECT * FROM three_partition_tb WHERE part1 = 'US' AND part2 = 2023 AND part3 = 'Q1' ORDER BY id;"""
-        qt_three_partition_9_0 three_partition_9_0
-        explain {
-            sql("${three_partition_9_0}")
-            contains "partition=0/10"
-        }
-
-        def three_partition_10_0 = """SELECT * FROM three_partition_tb WHERE part1 = 'EU' AND part2 = 2024 AND part3 = 'Q4' ORDER BY id;"""
-        qt_three_partition_10_0 three_partition_10_0
-        explain {
-            sql("${three_partition_10_0}")
-            contains "partition=0/10"
-        }
-
-        def three_partition_11_0 = """SELECT * FROM three_partition_tb WHERE part1 = 'AS' AND part2 = 2025 AND part3 = 'Q4' ORDER BY id;"""
-        qt_three_partition_11_0 three_partition_11_0
-        explain {
-            sql("${three_partition_11_0}")
-            contains "partition=0/10"
-        }
-
     }
 }
\ No newline at end of file

From 2b1264c5799cc11396800796e7d6bce8555b161b Mon Sep 17 00:00:00 2001
From: Rijesh Kunhi Parambattu <147430310+rijeshkp@users.noreply.github.com>
Date: Mon, 9 Dec 2024 07:48:43 +0530
Subject: [PATCH 223/399] =?UTF-8?q?=20[Enhancement]=20(nereids)implement?=
 =?UTF-8?q?=20adminCheckTabletsCommand=20in=20nereids=E2=80=A6=20(#44701)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Issue Number: close #42843
Support ADMIN CHECK tabletlist command in nereids
---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  17 +++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../commands/AdminCheckTabletsCommand.java    | 109 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../test_database_management_auth.groovy      |   8 +-
 .../test_nereids_admin_check_tablet.groovy    |  57 +++++++++
 7 files changed, 194 insertions(+), 5 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCheckTabletsCommand.java
 create mode 100644 regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index f229a1b2f4a7b1..c3fe7bb145ba72 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -491,6 +491,7 @@ supportedAdminStatement
     | ADMIN DIAGNOSE TABLET tabletId=INTEGER_VALUE                                  #adminDiagnoseTablet
     | ADMIN SHOW REPLICA STATUS FROM baseTableRef (WHERE STATUS EQ|NEQ STRING_LITERAL)?   #adminShowReplicaStatus
     | ADMIN COMPACT TABLE baseTableRef (WHERE TYPE EQ STRING_LITERAL)?              #adminCompactTable
+    | ADMIN CHECK tabletList properties=propertyClause?                             #adminCheckTablets
     ;
 
 supportedRecoverStatement
@@ -508,7 +509,6 @@ unsupportedAdminStatement
     | ADMIN CANCEL REPAIR TABLE baseTableRef                                        #adminCancelRepairTable
     | ADMIN SET (FRONTEND | (ALL FRONTENDS)) CONFIG
         (LEFT_PAREN propertyItemList RIGHT_PAREN)? ALL?                             #adminSetFrontendConfig
-    | ADMIN CHECK tabletList properties=propertyClause?                             #adminCheckTablets
     | ADMIN REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL
         (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?                              #adminRebalanceDisk
     | ADMIN CANCEL REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 71ea972c4ab2a2..15aeca952a2556 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -50,6 +50,7 @@
 import org.apache.doris.mtmv.MTMVRefreshTriggerInfo;
 import org.apache.doris.nereids.DorisParser;
 import org.apache.doris.nereids.DorisParser.AddConstraintContext;
+import org.apache.doris.nereids.DorisParser.AdminCheckTabletsContext;
 import org.apache.doris.nereids.DorisParser.AdminCompactTableContext;
 import org.apache.doris.nereids.DorisParser.AdminDiagnoseTabletContext;
 import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext;
@@ -470,6 +471,7 @@
 import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
 import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
 import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
@@ -4950,4 +4952,19 @@ public LogicalPlan visitShowCollation(ShowCollationContext ctx) {
         }
         return new ShowCollationCommand(wild);
     }
+
+    @Override
+    public LogicalPlan visitAdminCheckTablets(AdminCheckTabletsContext ctx) {
+        List<Long> tabletIdLists = new ArrayList<>();
+        if (ctx.tabletList() != null) {
+            ctx.tabletList().tabletIdList.stream().forEach(tabletToken -> {
+                tabletIdLists.add(Long.parseLong(tabletToken.getText()));
+            });
+        }
+        Map<String, String> properties = ctx.properties != null
+                ? Maps.newHashMap(visitPropertyClause(ctx.properties))
+                : Maps.newHashMap();
+        return new AdminCheckTabletsCommand(tabletIdLists, properties);
+    }
 }
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index bc8ccf83093b25..8198a8600a60c9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -132,6 +132,7 @@ public enum PlanType {
     PHYSICAL_WINDOW,
 
     // commands
+    ADMIN_CHECK_TABLETS_COMMAND,
     CREATE_POLICY_COMMAND,
     CREATE_TABLE_COMMAND,
     CREATE_SQL_BLOCK_RULE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCheckTabletsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCheckTabletsCommand.java
new file mode 100644
index 00000000000000..bba555d77a2e92
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCheckTabletsCommand.java
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * drop user command
+ */
+public class AdminCheckTabletsCommand extends Command implements ForwardNoSync {
+    private static final Logger LOG = LogManager.getLogger(AdminCheckTabletsCommand.class);
+    private final List<Long> tabletIds;
+    private final Map<String, String> properties;
+
+    /**
+     * Check Type in Properties.
+     * At present only CONSISTENCY is supported
+     */
+    public enum CheckType {
+        CONSISTENCY; // check the consistency of replicas of tablet
+        public static CheckType getTypeFromString(String str) throws AnalysisException {
+            try {
+                return CheckType.valueOf(str.toUpperCase());
+            } catch (Exception e) {
+                throw new AnalysisException("Unknown check type: " + str);
+            }
+        }
+    }
+
+    /**
+     * constructor
+     */
+    // ADMIN CHECK TABLET (id1, id2, ...) PROPERTIES ("type" = "check_consistency");
+    public AdminCheckTabletsCommand(List<Long> tabletIds, Map<String, String> properties) {
+        super(PlanType.ADMIN_CHECK_TABLETS_COMMAND);
+        this.tabletIds = tabletIds;
+        this.properties = properties;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        if (CollectionUtils.isEmpty(tabletIds)) {
+            throw new AnalysisException("Tablet id list is empty");
+        }
+
+        String typeStr = PropertyAnalyzer.analyzeType(properties);
+        if (typeStr == null) {
+            throw new AnalysisException("Should specify 'type' property");
+        }
+        CheckType checkType = CheckType.getTypeFromString(typeStr);
+
+        if (properties != null && !properties.isEmpty()) {
+            throw new AnalysisException("Unknown properties: " + properties.keySet());
+        }
+
+        if (Objects.requireNonNull(checkType) == CheckType.CONSISTENCY) {
+            Env.getCurrentEnv().getConsistencyChecker().addTabletsToCheck(tabletIds);
+        }
+    }
+
+    @Override
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
+        LOG.info("AdminCheckTabletsCommand not supported in cloud mode");
+        throw new DdlException("Unsupported operation");
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAdminCheckTabletsCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 77846ff3ad5328..dea3a004e816be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.visitor;
 
 import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterJobStatusCommand;
@@ -595,4 +596,8 @@ default R visitCreateRoutineLoadCommand(CreateRoutineLoadCommand createRoutineLo
     default R visitShowProcessListCommand(ShowProcessListCommand showProcessListCommand, C context) {
         return visitCommand(showProcessListCommand, context);
     }
+
+    default R visitAdminCheckTabletsCommand(AdminCheckTabletsCommand adminCheckTabletsCommand, C context) {
+        return visitCommand(adminCheckTabletsCommand, context);
+    }
 }
diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy
index 0e1805e20d6840..79114c18e789b5 100644
--- a/regression-test/suites/auth_call/test_database_management_auth.groovy
+++ b/regression-test/suites/auth_call/test_database_management_auth.groovy
@@ -23,14 +23,14 @@ suite("test_database_management_auth","p0,auth_call") {
     String user = 'test_database_management_auth_user'
     String pwd = 'C123_567p'
     String dbName = 'test_database_management_auth_db'
-    def String show_dis_error_msg = "denied"
+    def String error_in_cloud = "denied"
     //cloud-mode
     if (isCloudMode()) {
         def clusters = sql " SHOW CLUSTERS; "
         assertTrue(!clusters.isEmpty())
         def validCluster = clusters[0][0]
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}""";
-        show_dis_error_msg = "Unsupported"
+        error_in_cloud = "Unsupported"
     }
 
     try_sql("DROP USER ${user}")
@@ -79,7 +79,7 @@ suite("test_database_management_auth","p0,auth_call") {
         }
         test {
             sql """SHOW REPLICA DISTRIBUTION FROM tbl;"""
-            exception "${show_dis_error_msg}"
+            exception "${error_in_cloud}"
         }
         test {
             sql """SHOW REPLICA STATUS FROM db1.tbl1;"""
@@ -95,7 +95,7 @@ suite("test_database_management_auth","p0,auth_call") {
         }
         test {
             sql """ADMIN CHECK TABLET (10000, 10001) PROPERTIES("type" = "consistency");"""
-            exception "denied"
+            exception "${error_in_cloud}"
         }
         test {
             sql """SHOW TABLET DIAGNOSIS 0;"""
diff --git a/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy b/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy
new file mode 100644
index 00000000000000..8812e09ea9f6f3
--- /dev/null
+++ b/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_nereids_admin_check_tablet") {
+    def table = "test_nereids_admin_check_tablet"
+    // create table and insert data
+    sql """ drop table if exists ${table} force"""
+    sql """
+    create table ${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+
+    def tabletId
+    def tablets = sql_return_maparray "SHOW TABLETS FROM ${table}"
+
+    for (def tablet : tablets) {
+        //get any partition ID.
+        tabletId = tablet.TabletId;
+        break;
+    }
+
+    logger.info("${tabletId}");
+
+    checkNereidsExecute("ADMIN CHECK TABLET (${tabletId}) PROPERTIES('type' = 'consistency')");
+
+}
+

From f3a30d182150a2ca2a9d31610d8100592dec3ecb Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Mon, 9 Dec 2024 10:44:31 +0800
Subject: [PATCH 224/399] [fix](regression)turn runtime filter off in
 invalid_stats case (#45147)

### What problem does this PR solve?
set "runtime_filter_mode=off" to avoid interference of runtime filter in
the case "invalid_stats.groovy"
---
 .../suites/nereids_p0/stats/invalid_stats/invalid_stats.groovy  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/nereids_p0/stats/invalid_stats/invalid_stats.groovy b/regression-test/suites/nereids_p0/stats/invalid_stats/invalid_stats.groovy
index 51f1a37f118a42..3bda4bc743200a 100644
--- a/regression-test/suites/nereids_p0/stats/invalid_stats/invalid_stats.groovy
+++ b/regression-test/suites/nereids_p0/stats/invalid_stats/invalid_stats.groovy
@@ -17,11 +17,11 @@
 
 suite("invalid_stats") {
     multi_sql """
-        set global enable_auto_analyze=false;
         SET enable_nereids_planner=true;
         SET enable_fallback_to_original_planner=false;
         set disable_nereids_rules=PRUNE_EMPTY_PARTITION;
         set ignore_shape_nodes=PhysicalProject;
+        set runtime_filter_mode=off;
         
         drop table if exists region;
         CREATE TABLE region  (

From a25d457fa8e3a70eb6244ca068ab7bcd84237956 Mon Sep 17 00:00:00 2001
From: shee <13843187+qzsee@users.noreply.github.com>
Date: Mon, 9 Dec 2024 11:00:40 +0800
Subject: [PATCH 225/399]  [opt](lambda) let lambda expression support refer
 outer slot (#44365)

### What problem does this PR solve?

```sql
CREATE TABLE `array_test` (
  `id` int(11) NULL COMMENT "",
  `c_array` ARRAY<int(11)> NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`id`)
COMMENT "OLAP"
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2"
);
```
this sql is ok
```sql
select *, array_map(x -> x+1, c_array) from array_test
```
but lambda expressions in `array_map` do not support field arguments

for example:
```sql
select *, array_map(x -> x+id, c_array) from array_test
```

This pr solves two problems
1. support column arg in lambda expression
2. prevent high memory usage due to large block data

Related PR: #xxx
let lambda expression support refer outer slot
Co-authored-by: garenshi <garenshi@tencent.com>
---
 .../rowset/segment_v2/segment_iterator.cpp    |  10 +-
 .../exprs/lambda_function/lambda_function.h   |   9 +-
 .../lambda_function/varray_map_function.cpp   | 215 +++++++++++++++---
 be/src/vec/exprs/vcolumn_ref.h                |  11 +-
 be/src/vec/exprs/vexpr.h                      |   3 +
 be/src/vec/exprs/vlambda_function_call_expr.h |   1 +
 .../apache/doris/nereids/analyzer/Scope.java  |  11 +-
 .../rules/analysis/BindExpression.java        |   2 +-
 .../rules/analysis/ExpressionAnalyzer.java    |   5 +-
 .../rules/analysis/SubExprAnalyzer.java       |  11 +-
 .../test_array_map_function_with_column.out   |  41 ++++
 .../test_array_map_function.groovy            |   3 +-
 ...test_array_map_function_with_column.groovy |  70 ++++++
 .../datetime_functions/test_convert_tz.groovy |   3 +-
 14 files changed, 335 insertions(+), 60 deletions(-)
 create mode 100644 regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function_with_column.out
 create mode 100644 regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function_with_column.groovy

diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index f5f46e938650e1..abdf9116756f0e 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -2175,11 +2175,11 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
                     if (block->rows() == 0) {
                         vectorized::MutableColumnPtr col0 =
                                 std::move(*block->get_by_position(0).column).mutate();
-                        auto res_column = vectorized::ColumnString::create();
-                        res_column->insert_data("", 0);
-                        auto col_const = vectorized::ColumnConst::create(std::move(res_column),
-                                                                         selected_size);
-                        block->replace_by_position(0, std::move(col_const));
+                        auto tmp_indicator_col =
+                                block->get_by_position(0)
+                                        .type->create_column_const_with_default_value(
+                                                selected_size);
+                        block->replace_by_position(0, std::move(tmp_indicator_col));
                         _output_index_result_column_for_expr(_sel_rowid_idx.data(), selected_size,
                                                              block);
                         block->shrink_char_type_column_suffix_zero(_char_type_idx_no_0);
diff --git a/be/src/vec/exprs/lambda_function/lambda_function.h b/be/src/vec/exprs/lambda_function/lambda_function.h
index 184b4c2cc38abf..07302954688271 100644
--- a/be/src/vec/exprs/lambda_function/lambda_function.h
+++ b/be/src/vec/exprs/lambda_function/lambda_function.h
@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include <fmt/core.h>
+#include <runtime/runtime_state.h>
 
 #include "common/status.h"
 #include "vec/core/block.h"
@@ -31,9 +31,16 @@ class LambdaFunction {
 
     virtual std::string get_name() const = 0;
 
+    virtual doris::Status prepare(RuntimeState* state) {
+        batch_size = state->batch_size();
+        return Status::OK();
+    }
+
     virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block,
                                   int* result_column_id, const DataTypePtr& result_type,
                                   const VExprSPtrs& children) = 0;
+
+    int batch_size;
 };
 
 using LambdaFunctionPtr = std::shared_ptr<LambdaFunction>;
diff --git a/be/src/vec/exprs/lambda_function/varray_map_function.cpp b/be/src/vec/exprs/lambda_function/varray_map_function.cpp
index f80cffa166eac4..f7a864b92d3fb4 100644
--- a/be/src/vec/exprs/lambda_function/varray_map_function.cpp
+++ b/be/src/vec/exprs/lambda_function/varray_map_function.cpp
@@ -15,9 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <vec/data_types/data_type_number.h>
+#include <vec/exprs/vcolumn_ref.h>
+#include <vec/exprs/vslot_ref.h>
+
 #include <memory>
 #include <string>
-#include <utility>
+#include <vector>
 
 #include "common/status.h"
 #include "vec/aggregate_functions/aggregate_function.h"
@@ -39,9 +43,28 @@
 
 namespace doris::vectorized {
 #include "common/compile_check_begin.h"
-
 class VExprContext;
 
+// extend a block with all required parameters
+struct LambdaArgs {
+    // the lambda function need the column ids of all the slots
+    std::vector<int> output_slot_ref_indexs;
+    // which line is extended to the original block
+    int64_t current_row_idx = 0;
+    // when a block is filled, the array may be truncated, recording where it was truncated
+    int64_t current_offset_in_array = 0;
+    // the beginning position of the array
+    size_t array_start = 0;
+    // the size of the array
+    int64_t cur_size = 0;
+    // offset of column array
+    const ColumnArray::Offsets64* offsets_ptr = nullptr;
+    // expend data of repeat times
+    int current_repeat_times = 0;
+    // whether the current row of the original block has been extended
+    bool current_row_eos = false;
+};
+
 class ArrayMapFunction : public LambdaFunction {
     ENABLE_FACTORY_CREATOR(ArrayMapFunction);
 
@@ -57,8 +80,33 @@ class ArrayMapFunction : public LambdaFunction {
     doris::Status execute(VExprContext* context, doris::vectorized::Block* block,
                           int* result_column_id, const DataTypePtr& result_type,
                           const VExprSPtrs& children) override {
-        ///* array_map(lambda,arg1,arg2,.....) *///
+        LambdaArgs args;
+        // collect used slot ref in lambda function body
+        _collect_slot_ref_column_id(children[0], args);
+
+        int gap = 0;
+        if (!args.output_slot_ref_indexs.empty()) {
+            auto max_id = std::max_element(args.output_slot_ref_indexs.begin(),
+                                           args.output_slot_ref_indexs.end());
+            gap = *max_id + 1;
+            _set_column_ref_column_id(children[0], gap);
+        }
 
+        std::vector<std::string> names(gap);
+        DataTypes data_types(gap);
+
+        for (int i = 0; i < gap; ++i) {
+            if (_contains_column_id(args, i)) {
+                names[i] = block->get_by_position(i).name;
+                data_types[i] = block->get_by_position(i).type;
+            } else {
+                // padding some mock data
+                names[i] = "temp";
+                data_types[i] = std::make_shared<DataTypeUInt8>();
+            }
+        }
+
+        ///* array_map(lambda,arg1,arg2,.....) *///
         //1. child[1:end]->execute(src_block)
         doris::vectorized::ColumnNumbers arguments(children.size() - 1);
         for (int i = 1; i < children.size(); ++i) {
@@ -77,14 +125,13 @@ class ArrayMapFunction : public LambdaFunction {
         MutableColumnPtr array_column_offset;
         size_t nested_array_column_rows = 0;
         ColumnPtr first_array_offsets = nullptr;
-
         //2. get the result column from executed expr, and the needed is nested column of array
-        Block lambda_block;
+        std::vector<ColumnPtr> lambda_datas(arguments.size());
+
         for (int i = 0; i < arguments.size(); ++i) {
             const auto& array_column_type_name = block->get_by_position(arguments[i]);
             auto column_array = array_column_type_name.column->convert_to_full_column_if_const();
             auto type_array = array_column_type_name.type;
-
             if (type_array->is_nullable()) {
                 // get the nullmap of nullable column
                 const auto& column_array_nullmap =
@@ -113,6 +160,7 @@ class ArrayMapFunction : public LambdaFunction {
                 const auto& off_data = assert_cast<const ColumnArray::ColumnOffsets&>(
                         col_array.get_offsets_column());
                 array_column_offset = off_data.clone_resized(col_array.get_offsets_column().size());
+                args.offsets_ptr = &col_array.get_offsets();
             } else {
                 // select array_map((x,y)->x+y,c_array1,[0,1,2,3]) from array_test2;
                 // c_array1: [0,1,2,3,4,5,6,7,8,9]
@@ -131,57 +179,164 @@ class ArrayMapFunction : public LambdaFunction {
                             nested_array_column_rows, i + 1, col_array.get_data_ptr()->size());
                 }
             }
-
-            // insert the data column to the new block
-            ColumnWithTypeAndName data_column {col_array.get_data_ptr(), col_type.get_nested_type(),
-                                               "R" + array_column_type_name.name};
-            lambda_block.insert(std::move(data_column));
+            lambda_datas[i] = col_array.get_data_ptr();
+            names.push_back("R" + array_column_type_name.name);
+            data_types.push_back(col_type.get_nested_type());
         }
 
-        //3. child[0]->execute(new_block)
-        RETURN_IF_ERROR(children[0]->execute(context, &lambda_block, result_column_id));
+        ColumnPtr result_col = nullptr;
+        DataTypePtr res_type;
+        std::string res_name;
+
+        //process first row
+        args.array_start = (*args.offsets_ptr)[args.current_row_idx - 1];
+        args.cur_size = (*args.offsets_ptr)[args.current_row_idx] - args.array_start;
+
+        while (args.current_row_idx < block->rows()) {
+            Block lambda_block;
+            for (int i = 0; i < names.size(); i++) {
+                ColumnWithTypeAndName data_column;
+                if (_contains_column_id(args, i) || i >= gap) {
+                    data_column = ColumnWithTypeAndName(data_types[i], names[i]);
+                } else {
+                    data_column = ColumnWithTypeAndName(
+                            data_types[i]->create_column_const_with_default_value(0), data_types[i],
+                            names[i]);
+                }
+                lambda_block.insert(std::move(data_column));
+            }
+
+            MutableColumns columns = lambda_block.mutate_columns();
+            while (columns[gap]->size() < batch_size) {
+                long max_step = batch_size - columns[gap]->size();
+                long current_step =
+                        std::min(max_step, (long)(args.cur_size - args.current_offset_in_array));
+                size_t pos = args.array_start + args.current_offset_in_array;
+                for (int i = 0; i < arguments.size(); ++i) {
+                    columns[gap + i]->insert_range_from(*lambda_datas[i], pos, current_step);
+                }
+                args.current_offset_in_array += current_step;
+                args.current_repeat_times += current_step;
+                if (args.current_offset_in_array >= args.cur_size) {
+                    args.current_row_eos = true;
+                }
+                _extend_data(columns, block, args, gap);
+                if (args.current_row_eos) {
+                    args.current_row_idx++;
+                    args.current_offset_in_array = 0;
+                    if (args.current_row_idx >= block->rows()) {
+                        break;
+                    }
+                    args.current_row_eos = false;
+                    args.array_start = (*args.offsets_ptr)[args.current_row_idx - 1];
+                    args.cur_size = (*args.offsets_ptr)[args.current_row_idx] - args.array_start;
+                }
+            }
+
+            lambda_block.set_columns(std::move(columns));
 
-        auto res_col = lambda_block.get_by_position(*result_column_id)
-                               .column->convert_to_full_column_if_const();
-        auto res_type = lambda_block.get_by_position(*result_column_id).type;
-        auto res_name = lambda_block.get_by_position(*result_column_id).name;
+            //3. child[0]->execute(new_block)
+            RETURN_IF_ERROR(children[0]->execute(context, &lambda_block, result_column_id));
+
+            auto res_col = lambda_block.get_by_position(*result_column_id)
+                                   .column->convert_to_full_column_if_const();
+            res_type = lambda_block.get_by_position(*result_column_id).type;
+            res_name = lambda_block.get_by_position(*result_column_id).name;
+            if (!result_col) {
+                result_col = std::move(res_col);
+            } else {
+                MutableColumnPtr column = (*std::move(result_col)).mutate();
+                column->insert_range_from(*res_col, 0, res_col->size());
+            }
+        }
 
         //4. get the result column after execution, reassemble it into a new array column, and return.
         ColumnWithTypeAndName result_arr;
         if (result_type->is_nullable()) {
             if (res_type->is_nullable()) {
-                result_arr = {ColumnNullable::create(
-                                      ColumnArray::create(res_col, std::move(array_column_offset)),
-                                      std::move(outside_null_map)),
-                              result_type, res_name};
+                result_arr = {
+                        ColumnNullable::create(
+                                ColumnArray::create(result_col, std::move(array_column_offset)),
+                                std::move(outside_null_map)),
+                        result_type, res_name};
             } else {
                 // deal with eg: select array_map(x -> x is null, [null, 1, 2]);
                 // need to create the nested column null map for column array
-                auto nested_null_map = ColumnUInt8::create(res_col->size(), 0);
+                auto nested_null_map = ColumnUInt8::create(result_col->size(), 0);
                 result_arr = {
                         ColumnNullable::create(
-                                ColumnArray::create(
-                                        ColumnNullable::create(res_col, std::move(nested_null_map)),
-                                        std::move(array_column_offset)),
+                                ColumnArray::create(ColumnNullable::create(
+                                                            result_col, std::move(nested_null_map)),
+                                                    std::move(array_column_offset)),
                                 std::move(outside_null_map)),
                         result_type, res_name};
             }
         } else {
             if (res_type->is_nullable()) {
-                result_arr = {ColumnArray::create(res_col, std::move(array_column_offset)),
+                result_arr = {ColumnArray::create(result_col, std::move(array_column_offset)),
                               result_type, res_name};
             } else {
-                auto nested_null_map = ColumnUInt8::create(res_col->size(), 0);
-                result_arr = {ColumnArray::create(
-                                      ColumnNullable::create(res_col, std::move(nested_null_map)),
-                                      std::move(array_column_offset)),
+                auto nested_null_map = ColumnUInt8::create(result_col->size(), 0);
+                result_arr = {ColumnArray::create(ColumnNullable::create(
+                                                          result_col, std::move(nested_null_map)),
+                                                  std::move(array_column_offset)),
                               result_type, res_name};
             }
         }
         block->insert(std::move(result_arr));
         *result_column_id = block->columns() - 1;
+
         return Status::OK();
     }
+
+private:
+    bool _contains_column_id(LambdaArgs& args, int id) {
+        const auto it = std::find(args.output_slot_ref_indexs.begin(),
+                                  args.output_slot_ref_indexs.end(), id);
+        return it != args.output_slot_ref_indexs.end();
+    }
+
+    void _set_column_ref_column_id(VExprSPtr expr, int gap) {
+        for (const auto& child : expr->children()) {
+            if (child->is_column_ref()) {
+                auto* ref = static_cast<VColumnRef*>(child.get());
+                ref->set_gap(gap);
+            } else {
+                _set_column_ref_column_id(child, gap);
+            }
+        }
+    }
+
+    void _collect_slot_ref_column_id(VExprSPtr expr, LambdaArgs& args) {
+        for (const auto& child : expr->children()) {
+            if (child->is_slot_ref()) {
+                const auto* ref = static_cast<VSlotRef*>(child.get());
+                args.output_slot_ref_indexs.push_back(ref->column_id());
+            } else {
+                _collect_slot_ref_column_id(child, args);
+            }
+        }
+    }
+
+    void _extend_data(std::vector<MutableColumnPtr>& columns, Block* block, LambdaArgs& args,
+                      int size) {
+        if (!args.current_repeat_times || !size) {
+            return;
+        }
+        for (int i = 0; i < size; i++) {
+            if (_contains_column_id(args, i)) {
+                auto src_column =
+                        block->get_by_position(i).column->convert_to_full_column_if_const();
+                columns[i]->insert_many_from(*src_column, args.current_row_idx,
+                                             args.current_repeat_times);
+            } else {
+                // must be column const
+                DCHECK(is_column_const(*columns[i]));
+                columns[i]->resize(columns[i]->size() + args.current_repeat_times);
+            }
+        }
+        args.current_repeat_times = 0;
+    }
 };
 
 void register_function_array_map(doris::vectorized::LambdaFunctionFactory& factory) {
diff --git a/be/src/vec/exprs/vcolumn_ref.h b/be/src/vec/exprs/vcolumn_ref.h
index a763797880e361..d58e1375291457 100644
--- a/be/src/vec/exprs/vcolumn_ref.h
+++ b/be/src/vec/exprs/vcolumn_ref.h
@@ -16,6 +16,8 @@
 // under the License.
 
 #pragma once
+#include <atomic>
+
 #include "runtime/descriptors.h"
 #include "runtime/runtime_state.h"
 #include "vec/exprs/vexpr.h"
@@ -57,7 +59,7 @@ class VColumnRef final : public VExpr {
 
     Status execute(VExprContext* context, Block* block, int* result_column_id) override {
         DCHECK(_open_finished || _getting_const_col);
-        *result_column_id = _column_id;
+        *result_column_id = _column_id + _gap;
         return Status::OK();
     }
 
@@ -67,6 +69,12 @@ class VColumnRef final : public VExpr {
 
     const std::string& expr_name() const override { return _column_name; }
 
+    void set_gap(int gap) {
+        if (_gap == 0) {
+            _gap = gap;
+        }
+    }
+
     std::string debug_string() const override {
         std::stringstream out;
         out << "VColumnRef(slot_id: " << _column_id << ",column_name: " << _column_name
@@ -76,6 +84,7 @@ class VColumnRef final : public VExpr {
 
 private:
     int _column_id;
+    std::atomic<int> _gap = 0;
     std::string _column_name;
 };
 } // namespace vectorized
diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h
index 3456fb431a48cb..2580cf7ddc88f9 100644
--- a/be/src/vec/exprs/vexpr.h
+++ b/be/src/vec/exprs/vexpr.h
@@ -148,6 +148,9 @@ class VExpr {
     TypeDescriptor type() { return _type; }
 
     bool is_slot_ref() const { return _node_type == TExprNodeType::SLOT_REF; }
+
+    bool is_column_ref() const { return _node_type == TExprNodeType::COLUMN_REF; }
+
     virtual bool is_literal() const { return false; }
 
     TExprNodeType::type node_type() const { return _node_type; }
diff --git a/be/src/vec/exprs/vlambda_function_call_expr.h b/be/src/vec/exprs/vlambda_function_call_expr.h
index 44d22b1f9ebbea..7b0ea1dcb5a75d 100644
--- a/be/src/vec/exprs/vlambda_function_call_expr.h
+++ b/be/src/vec/exprs/vlambda_function_call_expr.h
@@ -50,6 +50,7 @@ class VLambdaFunctionCallExpr : public VExpr {
             return Status::InternalError("Lambda Function {} is not implemented.",
                                          _fn.name.function_name);
         }
+        RETURN_IF_ERROR(_lambda_function->prepare(state));
         _prepare_finished = true;
         return Status::OK();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java
index dbcbea7c104b5a..7c77c680ff287b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java
@@ -18,7 +18,6 @@
 package org.apache.doris.nereids.analyzer;
 
 import org.apache.doris.nereids.trees.expressions.Slot;
-import org.apache.doris.nereids.trees.expressions.SubqueryExpr;
 import org.apache.doris.nereids.util.Utils;
 
 import com.google.common.base.Suppliers;
@@ -62,20 +61,18 @@ public class Scope {
 
     private final Optional<Scope> outerScope;
     private final List<Slot> slots;
-    private final Optional<SubqueryExpr> ownerSubquery;
     private final Set<Slot> correlatedSlots;
     private final boolean buildNameToSlot;
     private final Supplier<ListMultimap<String, Slot>> nameToSlot;
 
     public Scope(List<? extends Slot> slots) {
-        this(Optional.empty(), slots, Optional.empty());
+        this(Optional.empty(), slots);
     }
 
     /** Scope */
-    public Scope(Optional<Scope> outerScope, List<? extends Slot> slots, Optional<SubqueryExpr> subqueryExpr) {
+    public Scope(Optional<Scope> outerScope, List<? extends Slot> slots) {
         this.outerScope = Objects.requireNonNull(outerScope, "outerScope can not be null");
         this.slots = Utils.fastToImmutableList(Objects.requireNonNull(slots, "slots can not be null"));
-        this.ownerSubquery = Objects.requireNonNull(subqueryExpr, "subqueryExpr can not be null");
         this.correlatedSlots = Sets.newLinkedHashSet();
         this.buildNameToSlot = slots.size() > 500;
         this.nameToSlot = buildNameToSlot ? Suppliers.memoize(this::buildNameToSlot) : null;
@@ -89,10 +86,6 @@ public Optional<Scope> getOuterScope() {
         return outerScope;
     }
 
-    public Optional<SubqueryExpr> getSubquery() {
-        return ownerSubquery;
-    }
-
     public Set<Slot> getCorrelatedSlots() {
         return correlatedSlots;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
index cde659b91021ad..1e481542baec12 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
@@ -1229,7 +1229,7 @@ private <E extends Expression> E checkBoundExceptLambda(E expression, Plan plan)
     private Scope toScope(CascadesContext cascadesContext, List<? extends Slot> slots) {
         Optional<Scope> outerScope = cascadesContext.getOuterScope();
         if (outerScope.isPresent()) {
-            return new Scope(outerScope, slots, outerScope.get().getSubquery());
+            return new Scope(outerScope, slots);
         } else {
             return new Scope(slots);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
index adc68ac6ecac1b..6abbbda447c8ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
@@ -833,8 +833,9 @@ private UnboundFunction bindHighOrderFunction(UnboundFunction unboundFunction, E
                 .map(ArrayItemReference::toSlot)
                 .collect(ImmutableList.toImmutableList());
 
-        ExpressionAnalyzer lambdaAnalyzer = new ExpressionAnalyzer(currentPlan, new Scope(boundedSlots),
-                context == null ? null : context.cascadesContext, true, false) {
+        ExpressionAnalyzer lambdaAnalyzer = new ExpressionAnalyzer(currentPlan, new Scope(Optional.of(getScope()),
+                boundedSlots), context == null ? null : context.cascadesContext,
+                true, true) {
             @Override
             protected void couldNotFoundColumn(UnboundSlot unboundSlot, String tableName) {
                 throw new AnalysisException("Unknown lambda slot '"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
index 7b0ed45708251d..9a70ce24afb8ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
@@ -53,7 +53,6 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -205,19 +204,15 @@ private AnalyzedResult analyzeSubquery(SubqueryExpr expr) {
         }
         CascadesContext subqueryContext = CascadesContext.newContextWithCteContext(
                 cascadesContext, expr.getQueryPlan(), cascadesContext.getCteContext());
-        Scope subqueryScope = genScopeWithSubquery(expr);
+        // don't use `getScope()` because we only need `getScope().getOuterScope()` and `getScope().getSlots()`
+        // otherwise unexpected errors may occur
+        Scope subqueryScope = new Scope(getScope().getOuterScope(), getScope().getSlots());
         subqueryContext.setOuterScope(subqueryScope);
         subqueryContext.newAnalyzer().analyze();
         return new AnalyzedResult((LogicalPlan) subqueryContext.getRewritePlan(),
                 subqueryScope.getCorrelatedSlots());
     }
 
-    private Scope genScopeWithSubquery(SubqueryExpr expr) {
-        return new Scope(getScope().getOuterScope(),
-                getScope().getSlots(),
-                Optional.ofNullable(expr));
-    }
-
     public Scope getScope() {
         return scope;
     }
diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function_with_column.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function_with_column.out
new file mode 100644
index 00000000000000..24d7b53e2b34f1
--- /dev/null
+++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function_with_column.out
@@ -0,0 +1,41 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_1 --
+\N	\N	\N	\N	\N
+1	2	[1, 2, 3, 4]	[null, 2, 3]	[1, 1, 1, 1]
+2	3	[6, 7, null, 9]	[4, null, 6]	[1, 1, null, 1]
+3	4	\N	[4, 5, 6]	\N
+
+-- !select_2 --
+\N	\N	\N	\N	\N
+1	2	[1, 2, 3, 4]	[null, 2, 3]	[1, 1, 1, 1]
+2	3	[6, 7, null, 9]	[4, null, 6]	[1, 1, null, 1]
+3	4	\N	[4, 5, 6]	\N
+
+-- !select_3 --
+\N	\N	\N	\N	\N
+1	2	[1, 2, 3, 4]	[null, 2, 3]	[1, 1, 1, 1]
+2	3	[6, 7, null, 9]	[4, null, 6]	[1, 1, null, 1]
+3	4	\N	[4, 5, 6]	\N
+
+-- !select_4 --
+\N	\N	\N	\N	\N
+1	2	[1, 2, 3, 4]	[null, 2, 3]	[1, 1, 1, 1]
+2	3	[6, 7, null, 9]	[4, null, 6]	[1, 1, null, 1]
+3	4	\N	[4, 5, 6]	\N
+
+-- !select_5 --
+\N	\N	\N	\N	\N
+1	2	[1, 2, 3, 4]	[null, 2, 3]	[1, 1, 1, 1]
+2	3	[6, 7, null, 9]	[4, null, 6]	[1, 1, null, 1]
+3	4	\N	[4, 5, 6]	\N
+
+-- !select_6 --
+\N	\N	\N	\N	\N
+4	5	[6, 7, null, 9]	[4, 5, 6, 7]	[0, 0, null, 0]
+5	6	[10, 11, 12, 13]	[8, 9, null, 11]	[0, 0, null, 0]
+6	7	\N	\N	\N
+
+-- !select_7 --
+4	5	[6, 7, null, 9]	[4, 5, 6, 7]	[0, 0, null, 0]
+5	6	[10, 11, 12, 13]	[8, 9, null, 11]	[0, 0, null, 0]
+
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy
index 311b188e433550..17de93942bfc61 100644
--- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy
@@ -75,8 +75,7 @@ suite("test_array_map_function") {
         test {
             sql"""select c_array1,array_max(array_map(x->countequal(c_array1,x),c_array1)) from array_test2;"""
             check{result, exception, startTime, endTime ->
-                assertTrue(exception != null)
-                logger.info(exception.message)
+                assertTrue(exception == null)
             }
         }
 
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function_with_column.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function_with_column.groovy
new file mode 100644
index 00000000000000..024e4b57a34572
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function_with_column.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_array_map_function_with_column") {
+
+    def tableName = "array_test_with_column"
+    sql "DROP TABLE IF EXISTS ${tableName}"
+    sql """
+        CREATE TABLE IF NOT EXISTS ${tableName} (
+          `k1` int(11) NULL COMMENT "",
+          `k2` int(11) NULL COMMENT "",
+          `c_array1` ARRAY<int(11)> NULL COMMENT "",
+          `c_array2` ARRAY<int(11)> NULL COMMENT ""
+        ) ENGINE=OLAP
+        DISTRIBUTED BY HASH(`k1`,`k2`) BUCKETS 3
+        PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+
+
+    sql """INSERT INTO ${tableName} values
+        (1, 2, [1,2,3,4], [null,2,3]),
+        (2, 3, [6,7,null,9], [4,null,6]),
+        (3, 4, NULL, [4, 5, 6]),
+        (NULL, NULL, NULL, NULL);
+    """
+
+    qt_select_1  "select *,array_map(x->x+k1+k2 > k1*k2,c_array1) from ${tableName} order by k1;"
+
+    sql "set batch_size = 1;"
+    qt_select_2  "select *,array_map(x->x+k1+k2 > k1*k2,c_array1) from ${tableName} order by k1;"
+
+    sql "set batch_size = 4;"
+    qt_select_3  "select *,array_map(x->x+k1+k2 > k1*k2,c_array1) from ${tableName} order by k1;"
+
+    sql "set batch_size = 6;"
+    qt_select_4  "select *,array_map(x->x+k1+k2 > k1*k2,c_array1) from ${tableName} order by k1;"
+
+    sql "set batch_size = 8;"
+    qt_select_5  "select *,array_map(x->x+k1+k2 > k1*k2,c_array1) from ${tableName} order by k1;"
+
+    sql "truncate table ${tableName};"
+
+    sql """INSERT INTO ${tableName} values
+        (4, 5, [6,7,null,9], [4,5,6,7]),
+        (5, 6, [10,11,12,13], [8,9,null,11]),
+        (6, 7, NULL, NULL),
+        (NULL, NULL, NULL, NULL);
+    """
+    qt_select_6  "select *,array_map((x,y)->x+k1+k2 > y+k1*k2,c_array1,c_array2) from ${tableName} order by k1;"
+
+    qt_select_7  "select *,array_map((x,y)->x+k1+k2 > y+k1*k2,c_array1,c_array2) from ${tableName} where array_count((x,y) -> k1*x>y+k2, c_array1, c_array2) > 1 order by k1;"
+
+    sql "DROP TABLE IF EXISTS ${tableName}"
+}
diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy
index 2959c7f136b8e5..e382fe9ae70b84 100644
--- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy
+++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy
@@ -16,6 +16,7 @@
 // under the License.
 
 suite("test_convert_tz") {
+    sql "drop table if exists `cvt_tz`"
     sql """
         CREATE TABLE `cvt_tz` (
             `rowid` int NULL,
@@ -34,4 +35,4 @@ suite("test_convert_tz") {
     order_qt_sql1 """
         select convert_tz(dt, '+00:00', IF(property_value IS NULL, '+00:00', property_value)) from cvt_tz
     """
-}
\ No newline at end of file
+}

From 45168f74c8aeac40464fecab3722706192bc7ab3 Mon Sep 17 00:00:00 2001
From: Uniqueyou <134280716+wyxxxcat@users.noreply.github.com>
Date: Mon, 9 Dec 2024 11:16:02 +0800
Subject: [PATCH 226/399] [fix](sql) Fix error for show create table column
 comment (#44958)

The quotes in the comment should be escaped.
---
 .../main/java/org/apache/doris/catalog/Column.java |  2 +-
 .../doris/analysis/ShowCreateTableStmtTest.java    | 14 ++++++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 916a213027f2f6..0ae6a4f8bdb5eb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -1019,7 +1019,7 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) {
             sb.append(" ON UPDATE ").append(defaultValue).append("");
         }
         if (StringUtils.isNotBlank(comment)) {
-            sb.append(" COMMENT '").append(getComment(true)).append("'");
+            sb.append(" COMMENT \"").append(getComment(true)).append("\"");
         }
         return sb.toString();
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java
index 353fbad9fabfc8..0faf4a8f34e8a8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java
@@ -32,7 +32,7 @@ protected void runBeforeAll() throws Exception {
         createDatabase("test");
         useDatabase("test");
         createTable("create table table1\n"
-                + "(k1 int comment 'test column k1', k2 int comment 'test column k2')  comment 'test table1' "
+                + "(k1 int comment 'test column k1', k2 int comment 'test column k2', `timestamp` DATE NOT NULL COMMENT '[''0000-01-01'', ''9999-12-31'']')  comment 'test table1' "
                 + "PARTITION BY RANGE(`k1`)\n"
                 + "(\n"
                 + "    PARTITION `p01` VALUES LESS THAN (\"10\"),\n"
@@ -48,10 +48,20 @@ public void testNormal() throws Exception {
         String sql = "show create table table1";
         ShowResultSet showResultSet = showCreateTable(sql);
         String showSql = showResultSet.getResultRows().get(0).get(1);
-        Assertions.assertTrue(showSql.contains("`k1` int NULL COMMENT 'test column k1'"));
+        Assertions.assertTrue(showSql.contains("`k1` int NULL COMMENT \"test column k1\""));
         Assertions.assertTrue(showSql.contains("COMMENT 'test table1'"));
     }
 
+    @Test
+    public void testColumnComment() throws Exception {
+        String sql = "show create table table1";
+        ShowResultSet showResultSet = showCreateTable(sql);
+        String showSql = showResultSet.getResultRows().get(0).get(1);
+        Assertions.assertTrue(showSql.contains("`k1` int NULL COMMENT \"test column k1\""));
+        Assertions.assertTrue(showSql.contains("`k2` int NULL COMMENT \"test column k2\""));
+        Assertions.assertTrue(showSql.contains("`timestamp` date NOT NULL COMMENT \"['0000-01-01', '9999-12-31']\""));
+    }
+
     @Test
     public void testBrief() throws Exception {
         String sql = "show brief create table table1";

From 84f9a6339857e50ed9e5c84607ae3e1794f87ae7 Mon Sep 17 00:00:00 2001
From: lihangyu <lihangyu@selectdb.com>
Date: Mon, 9 Dec 2024 11:22:18 +0800
Subject: [PATCH 227/399] [fix](variant) use lock to protect schema from
 threading unsafe (#45026)

---
 be/src/cloud/cloud_tablet.cpp | 1 +
 be/src/olap/base_tablet.h     | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index d545f827bc4e21..93c7128756738c 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -220,6 +220,7 @@ Status CloudTablet::sync_if_not_running() {
 }
 
 TabletSchemaSPtr CloudTablet::merged_tablet_schema() const {
+    std::shared_lock rlock(_meta_lock);
     return _merged_tablet_schema;
 }
 
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index bd46cdbbe14854..c6de447200f87c 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -279,7 +279,10 @@ class BaseTablet {
                                         int64_t limit);
 
     // Return the merged schema of all rowsets
-    virtual TabletSchemaSPtr merged_tablet_schema() const { return _max_version_schema; }
+    virtual TabletSchemaSPtr merged_tablet_schema() const {
+        std::shared_lock rlock(_meta_lock);
+        return _max_version_schema;
+    }
 
     void traverse_rowsets(std::function<void(const RowsetSharedPtr&)> visitor,
                           bool include_stale = false) {

From f4ca6da97ccf6f8195873863020c135524708530 Mon Sep 17 00:00:00 2001
From: zhengyu <zhangzhengyu@selectdb.com>
Date: Mon, 9 Dec 2024 12:17:10 +0800
Subject: [PATCH 228/399] [fix](cloud) fix CHECK failed when transmit from
 non-TTL to normal (#45057)

we need not transmit non-TTL cache to normal, just exclude them.
---
 be/src/io/cache/block_file_cache.cpp | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp
index 528ace90e3e7d7..2a59a5158e46c2 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -1093,15 +1093,16 @@ bool BlockFileCache::remove_if_ttl_file_unlock(const UInt128Wrapper& file_key, b
         _key_to_time.find(file_key) != _key_to_time.end()) {
         if (!remove_directly) {
             for (auto& [_, cell] : _files[file_key]) {
-                if (cell.file_block->cache_type() == FileCacheType::TTL) {
-                    Status st = cell.file_block->update_expiration_time(0);
-                    if (!st.ok()) {
-                        LOG_WARNING("Failed to update expiration time to 0").error(st);
-                    }
+                if (cell.file_block->cache_type() != FileCacheType::TTL) {
+                    continue;
+                }
+                Status st = cell.file_block->update_expiration_time(0);
+                if (!st.ok()) {
+                    LOG_WARNING("Failed to update expiration time to 0").error(st);
                 }
 
                 if (cell.file_block->cache_type() == FileCacheType::NORMAL) continue;
-                auto st = cell.file_block->change_cache_type_between_ttl_and_others(
+                st = cell.file_block->change_cache_type_between_ttl_and_others(
                         FileCacheType::NORMAL);
                 if (st.ok()) {
                     if (cell.queue_iterator) {

From 2bc011f33a7c350832c6fc266d0dfbd81a3a64b6 Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Mon, 9 Dec 2024 12:40:18 +0800
Subject: [PATCH 229/399] [feature](exchange) enable shared exchange sink
 buffer to reduce RPC concurrency. (#44850)

### What problem does this PR solve?

In the past, each exchange sink had its own sink buffer.
If the query concurrency is n, there would be n * n RPCs running
concurrently
in a typical shuffle scenario (each sender instance can send data to all
downstream instances).
Here, we introduce support for shared sink buffers.
This does not reduce the total number of RPCs but can limit the number
of concurrent RPCs.
---
 be/src/pipeline/exec/exchange_sink_buffer.cpp | 144 +++++++------
 be/src/pipeline/exec/exchange_sink_buffer.h   | 115 +++++++---
 .../pipeline/exec/exchange_sink_operator.cpp  |  85 ++++++--
 be/src/pipeline/exec/exchange_sink_operator.h |  34 ++-
 be/src/pipeline/pipeline_fragment_context.cpp |  24 ++-
 be/src/runtime/runtime_state.h                |   5 +
 be/src/vec/sink/vdata_stream_sender.h         |  10 +-
 be/test/vec/exec/exchange_sink_test.cpp       | 196 ++++++++++++++++++
 be/test/vec/exec/exchange_sink_test.h         | 180 ++++++++++++++++
 .../org/apache/doris/qe/SessionVariable.java  |   7 +
 gensrc/thrift/PaloInternalService.thrift      |   2 +-
 11 files changed, 683 insertions(+), 119 deletions(-)
 create mode 100644 be/test/vec/exec/exchange_sink_test.cpp
 create mode 100644 be/test/vec/exec/exchange_sink_test.h

diff --git a/be/src/pipeline/exec/exchange_sink_buffer.cpp b/be/src/pipeline/exec/exchange_sink_buffer.cpp
index 6e6108d13a919f..65e7698737076e 100644
--- a/be/src/pipeline/exec/exchange_sink_buffer.cpp
+++ b/be/src/pipeline/exec/exchange_sink_buffer.cpp
@@ -87,19 +87,22 @@ void BroadcastPBlockHolderMemLimiter::release(const BroadcastPBlockHolder& holde
 } // namespace vectorized
 
 namespace pipeline {
-ExchangeSinkBuffer::ExchangeSinkBuffer(PUniqueId query_id, PlanNodeId dest_node_id, int send_id,
-                                       int be_number, RuntimeState* state,
-                                       ExchangeSinkLocalState* parent)
+ExchangeSinkBuffer::ExchangeSinkBuffer(PUniqueId query_id, PlanNodeId dest_node_id,
+                                       RuntimeState* state,
+                                       const std::vector<InstanceLoId>& sender_ins_ids)
         : HasTaskExecutionCtx(state),
           _queue_capacity(0),
-          _is_finishing(false),
+          _is_failed(false),
           _query_id(std::move(query_id)),
           _dest_node_id(dest_node_id),
-          _sender_id(send_id),
-          _be_number(be_number),
           _state(state),
           _context(state->get_query_ctx()),
-          _parent(parent) {}
+          _exchange_sink_num(sender_ins_ids.size()) {
+    for (auto sender_ins_id : sender_ins_ids) {
+        _queue_deps.emplace(sender_ins_id, nullptr);
+        _parents.emplace(sender_ins_id, nullptr);
+    }
+}
 
 void ExchangeSinkBuffer::close() {
     // Could not clear the queue here, because there maybe a running rpc want to
@@ -110,8 +113,8 @@ void ExchangeSinkBuffer::close() {
     //_instance_to_request.clear();
 }
 
-void ExchangeSinkBuffer::register_sink(TUniqueId fragment_instance_id) {
-    if (_is_finishing) {
+void ExchangeSinkBuffer::construct_request(TUniqueId fragment_instance_id) {
+    if (_is_failed) {
         return;
     }
     auto low_id = fragment_instance_id.lo;
@@ -129,22 +132,27 @@ void ExchangeSinkBuffer::register_sink(TUniqueId fragment_instance_id) {
     finst_id.set_hi(fragment_instance_id.hi);
     finst_id.set_lo(fragment_instance_id.lo);
     _rpc_channel_is_idle[low_id] = true;
-    _instance_to_receiver_eof[low_id] = false;
+    _rpc_channel_is_turn_off[low_id] = false;
     _instance_to_rpc_stats_vec.emplace_back(std::make_shared<RpcInstanceStatistics>(low_id));
     _instance_to_rpc_stats[low_id] = _instance_to_rpc_stats_vec.back().get();
-    _construct_request(low_id, finst_id);
+    _instance_to_request[low_id] = std::make_shared<PTransmitDataParams>();
+    _instance_to_request[low_id]->mutable_finst_id()->CopyFrom(finst_id);
+    _instance_to_request[low_id]->mutable_query_id()->CopyFrom(_query_id);
+
+    _instance_to_request[low_id]->set_node_id(_dest_node_id);
+    _running_sink_count[low_id] = _exchange_sink_num;
 }
 
 Status ExchangeSinkBuffer::add_block(TransmitInfo&& request) {
-    if (_is_finishing) {
+    if (_is_failed) {
         return Status::OK();
     }
-    auto ins_id = request.channel->_fragment_instance_id.lo;
+    auto ins_id = request.channel->dest_ins_id();
     if (!_instance_to_package_queue_mutex.contains(ins_id)) {
         return Status::InternalError("fragment_instance_id {} not do register_sink",
                                      print_id(request.channel->_fragment_instance_id));
     }
-    if (_is_receiver_eof(ins_id)) {
+    if (_rpc_channel_is_turn_off[ins_id]) {
         return Status::EndOfFile("receiver eof");
     }
     bool send_now = false;
@@ -158,12 +166,15 @@ Status ExchangeSinkBuffer::add_block(TransmitInfo&& request) {
         if (request.block) {
             RETURN_IF_ERROR(
                     BeExecVersionManager::check_be_exec_version(request.block->be_exec_version()));
-            COUNTER_UPDATE(_parent->memory_used_counter(), request.block->ByteSizeLong());
+            COUNTER_UPDATE(request.channel->_parent->memory_used_counter(),
+                           request.block->ByteSizeLong());
         }
         _instance_to_package_queue[ins_id].emplace(std::move(request));
         _total_queue_size++;
-        if (_queue_dependency && _total_queue_size > _queue_capacity) {
-            _queue_dependency->block();
+        if (_total_queue_size > _queue_capacity) {
+            for (auto& [_, dep] : _queue_deps) {
+                dep->block();
+            }
         }
     }
     if (send_now) {
@@ -174,15 +185,15 @@ Status ExchangeSinkBuffer::add_block(TransmitInfo&& request) {
 }
 
 Status ExchangeSinkBuffer::add_block(BroadcastTransmitInfo&& request) {
-    if (_is_finishing) {
+    if (_is_failed) {
         return Status::OK();
     }
-    auto ins_id = request.channel->_fragment_instance_id.lo;
+    auto ins_id = request.channel->dest_ins_id();
     if (!_instance_to_package_queue_mutex.contains(ins_id)) {
         return Status::InternalError("fragment_instance_id {} not do register_sink",
                                      print_id(request.channel->_fragment_instance_id));
     }
-    if (_is_receiver_eof(ins_id)) {
+    if (_rpc_channel_is_turn_off[ins_id]) {
         return Status::EndOfFile("receiver eof");
     }
     bool send_now = false;
@@ -209,16 +220,17 @@ Status ExchangeSinkBuffer::add_block(BroadcastTransmitInfo&& request) {
 Status ExchangeSinkBuffer::_send_rpc(InstanceLoId id) {
     std::unique_lock<std::mutex> lock(*_instance_to_package_queue_mutex[id]);
 
-    DCHECK(_rpc_channel_is_idle[id] == false);
-
     std::queue<TransmitInfo, std::list<TransmitInfo>>& q = _instance_to_package_queue[id];
     std::queue<BroadcastTransmitInfo, std::list<BroadcastTransmitInfo>>& broadcast_q =
             _instance_to_broadcast_package_queue[id];
 
-    if (_is_finishing) {
+    if (_is_failed) {
         _turn_off_channel(id, lock);
         return Status::OK();
     }
+    if (_rpc_channel_is_turn_off[id]) {
+        return Status::OK();
+    }
 
     if (!q.empty()) {
         // If we have data to shuffle which is not broadcasted
@@ -226,6 +238,8 @@ Status ExchangeSinkBuffer::_send_rpc(InstanceLoId id) {
         auto& brpc_request = _instance_to_request[id];
         brpc_request->set_eos(request.eos);
         brpc_request->set_packet_seq(_instance_to_seq[id]++);
+        brpc_request->set_sender_id(request.channel->_parent->sender_id());
+        brpc_request->set_be_number(request.channel->_parent->be_number());
         if (request.block && !request.block->column_metas().empty()) {
             brpc_request->set_allocated_block(request.block.get());
         }
@@ -271,14 +285,16 @@ Status ExchangeSinkBuffer::_send_rpc(InstanceLoId id) {
             } else if (!s.ok()) {
                 _failed(id,
                         fmt::format("exchange req success but status isn't ok: {}", s.to_string()));
+                return;
             } else if (eos) {
                 _ended(id);
-            } else {
-                s = _send_rpc(id);
-                if (!s) {
-                    _failed(id, fmt::format("exchange req success but status isn't ok: {}",
-                                            s.to_string()));
-                }
+            }
+            // The eos here only indicates that the current exchange sink has reached eos.
+            // However, the queue still contains data from other exchange sinks, so RPCs need to continue being sent.
+            s = _send_rpc(id);
+            if (!s) {
+                _failed(id,
+                        fmt::format("exchange req success but status isn't ok: {}", s.to_string()));
             }
         });
         {
@@ -296,13 +312,16 @@ Status ExchangeSinkBuffer::_send_rpc(InstanceLoId id) {
             }
         }
         if (request.block) {
-            COUNTER_UPDATE(_parent->memory_used_counter(), -request.block->ByteSizeLong());
+            COUNTER_UPDATE(request.channel->_parent->memory_used_counter(),
+                           -request.block->ByteSizeLong());
             static_cast<void>(brpc_request->release_block());
         }
         q.pop();
         _total_queue_size--;
-        if (_queue_dependency && _total_queue_size <= _queue_capacity) {
-            _queue_dependency->set_ready();
+        if (_total_queue_size <= _queue_capacity) {
+            for (auto& [_, dep] : _queue_deps) {
+                dep->set_ready();
+            }
         }
     } else if (!broadcast_q.empty()) {
         // If we have data to shuffle which is broadcasted
@@ -310,6 +329,8 @@ Status ExchangeSinkBuffer::_send_rpc(InstanceLoId id) {
         auto& brpc_request = _instance_to_request[id];
         brpc_request->set_eos(request.eos);
         brpc_request->set_packet_seq(_instance_to_seq[id]++);
+        brpc_request->set_sender_id(request.channel->_parent->sender_id());
+        brpc_request->set_be_number(request.channel->_parent->be_number());
         if (request.block_holder->get_block() &&
             !request.block_holder->get_block()->column_metas().empty()) {
             brpc_request->set_allocated_block(request.block_holder->get_block());
@@ -352,14 +373,17 @@ Status ExchangeSinkBuffer::_send_rpc(InstanceLoId id) {
             } else if (!s.ok()) {
                 _failed(id,
                         fmt::format("exchange req success but status isn't ok: {}", s.to_string()));
+                return;
             } else if (eos) {
                 _ended(id);
-            } else {
-                s = _send_rpc(id);
-                if (!s) {
-                    _failed(id, fmt::format("exchange req success but status isn't ok: {}",
-                                            s.to_string()));
-                }
+            }
+
+            // The eos here only indicates that the current exchange sink has reached eos.
+            // However, the queue still contains data from other exchange sinks, so RPCs need to continue being sent.
+            s = _send_rpc(id);
+            if (!s) {
+                _failed(id,
+                        fmt::format("exchange req success but status isn't ok: {}", s.to_string()));
             }
         });
         {
@@ -387,16 +411,6 @@ Status ExchangeSinkBuffer::_send_rpc(InstanceLoId id) {
     return Status::OK();
 }
 
-void ExchangeSinkBuffer::_construct_request(InstanceLoId id, PUniqueId finst_id) {
-    _instance_to_request[id] = std::make_shared<PTransmitDataParams>();
-    _instance_to_request[id]->mutable_finst_id()->CopyFrom(finst_id);
-    _instance_to_request[id]->mutable_query_id()->CopyFrom(_query_id);
-
-    _instance_to_request[id]->set_node_id(_dest_node_id);
-    _instance_to_request[id]->set_sender_id(_sender_id);
-    _instance_to_request[id]->set_be_number(_be_number);
-}
-
 void ExchangeSinkBuffer::_ended(InstanceLoId id) {
     if (!_instance_to_package_queue_mutex.template contains(id)) {
         std::stringstream ss;
@@ -411,24 +425,29 @@ void ExchangeSinkBuffer::_ended(InstanceLoId id) {
         __builtin_unreachable();
     } else {
         std::unique_lock<std::mutex> lock(*_instance_to_package_queue_mutex[id]);
-        _turn_off_channel(id, lock);
+        _running_sink_count[id]--;
+        if (_running_sink_count[id] == 0) {
+            _turn_off_channel(id, lock);
+        }
     }
 }
 
 void ExchangeSinkBuffer::_failed(InstanceLoId id, const std::string& err) {
-    _is_finishing = true;
+    _is_failed = true;
     _context->cancel(Status::Cancelled(err));
 }
 
 void ExchangeSinkBuffer::_set_receiver_eof(InstanceLoId id) {
     std::unique_lock<std::mutex> lock(*_instance_to_package_queue_mutex[id]);
-    _instance_to_receiver_eof[id] = true;
+    // When the receiving side reaches eof, it means the receiver has finished early.
+    // The remaining data in the current rpc_channel does not need to be sent,
+    // and the rpc_channel should be turned off immediately.
     _turn_off_channel(id, lock);
     std::queue<BroadcastTransmitInfo, std::list<BroadcastTransmitInfo>>& broadcast_q =
             _instance_to_broadcast_package_queue[id];
     for (; !broadcast_q.empty(); broadcast_q.pop()) {
         if (broadcast_q.front().block_holder->get_block()) {
-            COUNTER_UPDATE(_parent->memory_used_counter(),
+            COUNTER_UPDATE(broadcast_q.front().channel->_parent->memory_used_counter(),
                            -broadcast_q.front().block_holder->get_block()->ByteSizeLong());
         }
     }
@@ -440,7 +459,8 @@ void ExchangeSinkBuffer::_set_receiver_eof(InstanceLoId id) {
     std::queue<TransmitInfo, std::list<TransmitInfo>>& q = _instance_to_package_queue[id];
     for (; !q.empty(); q.pop()) {
         if (q.front().block) {
-            COUNTER_UPDATE(_parent->memory_used_counter(), -q.front().block->ByteSizeLong());
+            COUNTER_UPDATE(q.front().channel->_parent->memory_used_counter(),
+                           -q.front().block->ByteSizeLong());
         }
     }
 
@@ -450,22 +470,22 @@ void ExchangeSinkBuffer::_set_receiver_eof(InstanceLoId id) {
     }
 }
 
-bool ExchangeSinkBuffer::_is_receiver_eof(InstanceLoId id) {
-    std::unique_lock<std::mutex> lock(*_instance_to_package_queue_mutex[id]);
-    return _instance_to_receiver_eof[id];
-}
-
 // The unused parameter `with_lock` is to ensure that the function is called when the lock is held.
 void ExchangeSinkBuffer::_turn_off_channel(InstanceLoId id,
                                            std::unique_lock<std::mutex>& /*with_lock*/) {
     if (!_rpc_channel_is_idle[id]) {
         _rpc_channel_is_idle[id] = true;
     }
-    _instance_to_receiver_eof[id] = true;
-
+    // Ensure that each RPC is turned off only once.
+    if (_rpc_channel_is_turn_off[id]) {
+        return;
+    }
+    _rpc_channel_is_turn_off[id] = true;
     auto weak_task_ctx = weak_task_exec_ctx();
     if (auto pip_ctx = weak_task_ctx.lock()) {
-        _parent->on_channel_finished(id);
+        for (auto& [_, parent] : _parents) {
+            parent->on_channel_finished(id);
+        }
     }
 }
 
@@ -509,7 +529,7 @@ void ExchangeSinkBuffer::update_profile(RuntimeProfile* profile) {
     auto* _max_rpc_timer = ADD_TIMER_WITH_LEVEL(profile, "RpcMaxTime", 1);
     auto* _min_rpc_timer = ADD_TIMER(profile, "RpcMinTime");
     auto* _sum_rpc_timer = ADD_TIMER(profile, "RpcSumTime");
-    auto* _count_rpc = ADD_COUNTER_WITH_LEVEL(profile, "RpcCount", TUnit::UNIT, 1);
+    auto* _count_rpc = ADD_COUNTER(profile, "RpcCount", TUnit::UNIT);
     auto* _avg_rpc_timer = ADD_TIMER(profile, "RpcAvgTime");
 
     int64_t max_rpc_time = 0, min_rpc_time = 0;
diff --git a/be/src/pipeline/exec/exchange_sink_buffer.h b/be/src/pipeline/exec/exchange_sink_buffer.h
index 22a1452f8d545c..b2eb32414feca2 100644
--- a/be/src/pipeline/exec/exchange_sink_buffer.h
+++ b/be/src/pipeline/exec/exchange_sink_buffer.h
@@ -169,13 +169,61 @@ class ExchangeSendCallback : public ::doris::DummyBrpcCallback<Response> {
     bool _eos;
 };
 
-// Each ExchangeSinkOperator have one ExchangeSinkBuffer
-class ExchangeSinkBuffer final : public HasTaskExecutionCtx {
+// ExchangeSinkBuffer can either be shared among multiple ExchangeSinkLocalState instances
+// or be individually owned by each ExchangeSinkLocalState.
+// The following describes the scenario where ExchangeSinkBuffer is shared among multiple ExchangeSinkLocalState instances.
+// Of course, individual ownership can be seen as a special case where only one ExchangeSinkLocalState shares the buffer.
+
+// A sink buffer contains multiple rpc_channels.
+// Each rpc_channel corresponds to a target instance on the receiving side.
+// Data is sent using a ping-pong mode within each rpc_channel,
+// meaning that at most one RPC can exist in a single rpc_channel at a time.
+// The next RPC can only be sent after the previous one has completed.
+//
+// Each exchange sink sends data to all target instances on the receiving side.
+// If the concurrency is 3, a single rpc_channel will be used simultaneously by three exchange sinks.
+
+/*                                                                                                                                                                                                                                                                                                                          
+                          +-----------+          +-----------+        +-----------+      
+                          |dest ins id|          |dest ins id|        |dest ins id|      
+                          |           |          |           |        |           |      
+                          +----+------+          +-----+-----+        +------+----+      
+                               |                       |                     |           
+                               |                       |                     |           
+                      +----------------+      +----------------+     +----------------+  
+                      |                |      |                |     |                |  
+ sink buffer -------- |   rpc_channel  |      |  rpc_channel   |     |  rpc_channel   |  
+                      |                |      |                |     |                |  
+                      +-------+--------+      +----------------+     +----------------+  
+                              |                        |                      |          
+                              |------------------------+----------------------+          
+                              |                        |                      |          
+                              |                        |                      |          
+                     +-----------------+       +-------+---------+    +-------+---------+
+                     |                 |       |                 |    |                 |
+                     |  exchange sink  |       |  exchange sink  |    |  exchange sink  |
+                     |                 |       |                 |    |                 |
+                     +-----------------+       +-----------------+    +-----------------+
+*/
+
+#ifdef BE_TEST
+void transmit_blockv2(PBackendService_Stub& stub,
+                      std::unique_ptr<AutoReleaseClosure<PTransmitDataParams,
+                                                         ExchangeSendCallback<PTransmitDataResult>>>
+                              closure);
+#endif
+class ExchangeSinkBuffer : public HasTaskExecutionCtx {
 public:
-    ExchangeSinkBuffer(PUniqueId query_id, PlanNodeId dest_node_id, int send_id, int be_number,
-                       RuntimeState* state, ExchangeSinkLocalState* parent);
+    ExchangeSinkBuffer(PUniqueId query_id, PlanNodeId dest_node_id, RuntimeState* state,
+                       const std::vector<InstanceLoId>& sender_ins_ids);
+
+#ifdef BE_TEST
+    ExchangeSinkBuffer(RuntimeState* state, int64_t sinknum)
+            : HasTaskExecutionCtx(state), _exchange_sink_num(sinknum) {};
+#endif
     ~ExchangeSinkBuffer() override = default;
-    void register_sink(TUniqueId);
+
+    void construct_request(TUniqueId);
 
     Status add_block(TransmitInfo&& request);
     Status add_block(BroadcastTransmitInfo&& request);
@@ -183,17 +231,18 @@ class ExchangeSinkBuffer final : public HasTaskExecutionCtx {
     void update_rpc_time(InstanceLoId id, int64_t start_rpc_time, int64_t receive_rpc_time);
     void update_profile(RuntimeProfile* profile);
 
-    void set_dependency(std::shared_ptr<Dependency> queue_dependency,
-                        std::shared_ptr<Dependency> finish_dependency) {
-        _queue_dependency = queue_dependency;
-        _finish_dependency = finish_dependency;
-    }
-
-    void set_broadcast_dependency(std::shared_ptr<Dependency> broadcast_dependency) {
-        _broadcast_dependency = broadcast_dependency;
+    void set_dependency(InstanceLoId sender_ins_id, std::shared_ptr<Dependency> queue_dependency,
+                        ExchangeSinkLocalState* local_state) {
+        DCHECK(_queue_deps.contains(sender_ins_id));
+        DCHECK(_parents.contains(sender_ins_id));
+        _queue_deps[sender_ins_id] = queue_dependency;
+        _parents[sender_ins_id] = local_state;
     }
-
+#ifdef BE_TEST
+public:
+#else
 private:
+#endif
     friend class ExchangeSinkLocalState;
 
     phmap::flat_hash_map<InstanceLoId, std::unique_ptr<std::mutex>>
@@ -214,7 +263,10 @@ class ExchangeSinkBuffer final : public HasTaskExecutionCtx {
     // One channel is corresponding to a downstream instance.
     phmap::flat_hash_map<InstanceLoId, bool> _rpc_channel_is_idle;
 
-    phmap::flat_hash_map<InstanceLoId, bool> _instance_to_receiver_eof;
+    // There could be multiple situations that cause an rpc_channel to be turned off,
+    // such as receiving the eof, manual cancellation by the user, or all sinks reaching eos.
+    // Therefore, it is necessary to prevent an rpc_channel from being turned off multiple times.
+    phmap::flat_hash_map<InstanceLoId, bool> _rpc_channel_is_turn_off;
     struct RpcInstanceStatistics {
         RpcInstanceStatistics(InstanceLoId id) : inst_lo_id(id) {}
         InstanceLoId inst_lo_id;
@@ -226,32 +278,43 @@ class ExchangeSinkBuffer final : public HasTaskExecutionCtx {
     std::vector<std::shared_ptr<RpcInstanceStatistics>> _instance_to_rpc_stats_vec;
     phmap::flat_hash_map<InstanceLoId, RpcInstanceStatistics*> _instance_to_rpc_stats;
 
-    std::atomic<bool> _is_finishing;
+    // It is set to true only when an RPC fails. Currently, we do not have an error retry mechanism.
+    // If an RPC error occurs, the query will be canceled.
+    std::atomic<bool> _is_failed;
     PUniqueId _query_id;
     PlanNodeId _dest_node_id;
-    // Sender instance id, unique within a fragment. StreamSender save the variable
-    int _sender_id;
-    int _be_number;
     std::atomic<int64_t> _rpc_count = 0;
     RuntimeState* _state = nullptr;
     QueryContext* _context = nullptr;
 
     Status _send_rpc(InstanceLoId);
-    // must hold the _instance_to_package_queue_mutex[id] mutex to opera
-    void _construct_request(InstanceLoId id, PUniqueId);
+
+#ifndef BE_TEST
     inline void _ended(InstanceLoId id);
     inline void _failed(InstanceLoId id, const std::string& err);
     inline void _set_receiver_eof(InstanceLoId id);
-    inline bool _is_receiver_eof(InstanceLoId id);
     inline void _turn_off_channel(InstanceLoId id, std::unique_lock<std::mutex>& with_lock);
+
+#else
+    virtual void _ended(InstanceLoId id);
+    virtual void _failed(InstanceLoId id, const std::string& err);
+    virtual void _set_receiver_eof(InstanceLoId id);
+    virtual void _turn_off_channel(InstanceLoId id, std::unique_lock<std::mutex>& with_lock);
+#endif
+
     void get_max_min_rpc_time(int64_t* max_time, int64_t* min_time);
     int64_t get_sum_rpc_time();
 
     std::atomic<int> _total_queue_size = 0;
-    std::shared_ptr<Dependency> _queue_dependency = nullptr;
-    std::shared_ptr<Dependency> _finish_dependency = nullptr;
-    std::shared_ptr<Dependency> _broadcast_dependency = nullptr;
-    ExchangeSinkLocalState* _parent = nullptr;
+
+    // _running_sink_count is used to track how many sinks have not finished yet.
+    // It is only decremented when eos is reached.
+    phmap::flat_hash_map<InstanceLoId, int64_t> _running_sink_count;
+    // _queue_deps is used for memory control.
+    phmap::flat_hash_map<InstanceLoId, std::shared_ptr<Dependency>> _queue_deps;
+    // The ExchangeSinkLocalState in _parents is only used in _turn_off_channel.
+    phmap::flat_hash_map<InstanceLoId, ExchangeSinkLocalState*> _parents;
+    const int64_t _exchange_sink_num;
 };
 
 } // namespace pipeline
diff --git a/be/src/pipeline/exec/exchange_sink_operator.cpp b/be/src/pipeline/exec/exchange_sink_operator.cpp
index dfa6df392b74ba..04b9653e9c8299 100644
--- a/be/src/pipeline/exec/exchange_sink_operator.cpp
+++ b/be/src/pipeline/exec/exchange_sink_operator.cpp
@@ -32,6 +32,8 @@
 #include "pipeline/exec/operator.h"
 #include "pipeline/exec/sort_source_operator.h"
 #include "pipeline/local_exchange/local_exchange_sink_operator.h"
+#include "pipeline/local_exchange/local_exchange_source_operator.h"
+#include "pipeline/pipeline_fragment_context.h"
 #include "util/runtime_profile.h"
 #include "util/uid_util.h"
 #include "vec/columns/column_const.h"
@@ -100,6 +102,24 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf
                 fmt::format("WaitForLocalExchangeBuffer{}", i), TUnit ::TIME_NS, timer_name, 1));
     }
     _wait_broadcast_buffer_timer = ADD_CHILD_TIMER(_profile, "WaitForBroadcastBuffer", timer_name);
+
+    size_t local_size = 0;
+    for (int i = 0; i < channels.size(); ++i) {
+        if (channels[i]->is_local()) {
+            local_size++;
+            _last_local_channel_idx = i;
+        }
+    }
+    only_local_exchange = local_size == channels.size();
+
+    if (!only_local_exchange) {
+        _sink_buffer = p.get_sink_buffer(state->fragment_instance_id().lo);
+        register_channels(_sink_buffer.get());
+        _queue_dependency = Dependency::create_shared(_parent->operator_id(), _parent->node_id(),
+                                                      "ExchangeSinkQueueDependency", true);
+        _sink_buffer->set_dependency(state->fragment_instance_id().lo, _queue_dependency, this);
+    }
+
     return Status::OK();
 }
 
@@ -149,20 +169,10 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) {
     id.set_hi(_state->query_id().hi);
     id.set_lo(_state->query_id().lo);
 
-    if (!only_local_exchange) {
-        _sink_buffer = std::make_unique<ExchangeSinkBuffer>(id, p._dest_node_id, _sender_id,
-                                                            _state->be_number(), state, this);
-        register_channels(_sink_buffer.get());
-        _queue_dependency = Dependency::create_shared(_parent->operator_id(), _parent->node_id(),
-                                                      "ExchangeSinkQueueDependency", true);
-        _sink_buffer->set_dependency(_queue_dependency, _finish_dependency);
-    }
-
     if ((_part_type == TPartitionType::UNPARTITIONED || channels.size() == 1) &&
         !only_local_exchange) {
         _broadcast_dependency = Dependency::create_shared(
                 _parent->operator_id(), _parent->node_id(), "BroadcastDependency", true);
-        _sink_buffer->set_broadcast_dependency(_broadcast_dependency);
         _broadcast_pb_mem_limiter =
                 vectorized::BroadcastPBlockHolderMemLimiter::create_shared(_broadcast_dependency);
     } else if (local_size > 0) {
@@ -301,7 +311,8 @@ segment_v2::CompressionTypePB ExchangeSinkLocalState::compression_type() const {
 
 ExchangeSinkOperatorX::ExchangeSinkOperatorX(
         RuntimeState* state, const RowDescriptor& row_desc, int operator_id,
-        const TDataStreamSink& sink, const std::vector<TPlanFragmentDestination>& destinations)
+        const TDataStreamSink& sink, const std::vector<TPlanFragmentDestination>& destinations,
+        const std::vector<TUniqueId>& fragment_instance_ids)
         : DataSinkOperatorX(operator_id, sink.dest_node_id),
           _texprs(sink.output_partition.partition_exprs),
           _row_desc(row_desc),
@@ -315,7 +326,8 @@ ExchangeSinkOperatorX::ExchangeSinkOperatorX(
           _tablet_sink_tuple_id(sink.tablet_sink_tuple_id),
           _tablet_sink_txn_id(sink.tablet_sink_txn_id),
           _t_tablet_sink_exprs(&sink.tablet_sink_exprs),
-          _enable_local_merge_sort(state->enable_local_merge_sort()) {
+          _enable_local_merge_sort(state->enable_local_merge_sort()),
+          _fragment_instance_ids(fragment_instance_ids) {
     DCHECK_GT(destinations.size(), 0);
     DCHECK(sink.output_partition.type == TPartitionType::UNPARTITIONED ||
            sink.output_partition.type == TPartitionType::HASH_PARTITIONED ||
@@ -360,6 +372,11 @@ Status ExchangeSinkOperatorX::open(RuntimeState* state) {
         }
         RETURN_IF_ERROR(vectorized::VExpr::open(_tablet_sink_expr_ctxs, state));
     }
+    std::vector<InstanceLoId> ins_ids;
+    for (auto fragment_instance_id : _fragment_instance_ids) {
+        ins_ids.push_back(fragment_instance_id.lo);
+    }
+    _sink_buffer = _create_buffer(ins_ids);
     return Status::OK();
 }
 
@@ -620,7 +637,7 @@ Status ExchangeSinkOperatorX::sink(RuntimeState* state, vectorized::Block* block
 
 void ExchangeSinkLocalState::register_channels(pipeline::ExchangeSinkBuffer* buffer) {
     for (auto& channel : channels) {
-        channel->register_exchange_buffer(buffer);
+        channel->set_exchange_buffer(buffer);
     }
 }
 
@@ -669,8 +686,8 @@ std::string ExchangeSinkLocalState::debug_string(int indentation_level) const {
         fmt::format_to(debug_string_buffer,
                        ", Sink Buffer: (_is_finishing = {}, blocks in queue: {}, queue capacity: "
                        "{}, queue dep: {}), _reach_limit: {}, working channels: {}",
-                       _sink_buffer->_is_finishing.load(), _sink_buffer->_total_queue_size,
-                       _sink_buffer->_queue_capacity, (void*)_sink_buffer->_queue_dependency.get(),
+                       _sink_buffer->_is_failed.load(), _sink_buffer->_total_queue_size,
+                       _sink_buffer->_queue_capacity, (void*)_queue_dependency.get(),
                        _reach_limit.load(), _working_channels_count.load());
     }
     return fmt::to_string(debug_string_buffer);
@@ -724,4 +741,42 @@ DataDistribution ExchangeSinkOperatorX::required_data_distribution() const {
     return DataSinkOperatorX<ExchangeSinkLocalState>::required_data_distribution();
 }
 
+std::shared_ptr<ExchangeSinkBuffer> ExchangeSinkOperatorX::_create_buffer(
+        const std::vector<InstanceLoId>& sender_ins_ids) {
+    PUniqueId id;
+    id.set_hi(_state->query_id().hi);
+    id.set_lo(_state->query_id().lo);
+    auto sink_buffer =
+            std::make_unique<ExchangeSinkBuffer>(id, _dest_node_id, state(), sender_ins_ids);
+    for (const auto& _dest : _dests) {
+        sink_buffer->construct_request(_dest.fragment_instance_id);
+    }
+    return sink_buffer;
+}
+
+// For a normal shuffle scenario, if the concurrency is n,
+// there can be up to n * n RPCs in the current fragment.
+// Therefore, a shared sink buffer is used here to limit the number of concurrent RPCs.
+// (Note: This does not reduce the total number of RPCs.)
+// In a merge sort scenario, there are only n RPCs, so a shared sink buffer is not needed.
+/// TODO: Modify this to let FE handle the judgment instead of BE.
+std::shared_ptr<ExchangeSinkBuffer> ExchangeSinkOperatorX::get_sink_buffer(
+        InstanceLoId sender_ins_id) {
+    if (!_child) {
+        throw doris::Exception(ErrorCode::INTERNAL_ERROR,
+                               "ExchangeSinkOperatorX did not correctly set the child.");
+    }
+    // When the child is SortSourceOperatorX or LocalExchangeSourceOperatorX,
+    // it is an order-by scenario.
+    // In this case, there is only one target instance, and no n * n RPC concurrency will occur.
+    // Therefore, sharing a sink buffer is not necessary.
+    if (std::dynamic_pointer_cast<SortSourceOperatorX>(_child) ||
+        std::dynamic_pointer_cast<LocalExchangeSourceOperatorX>(_child)) {
+        return _create_buffer({sender_ins_id});
+    }
+    if (_state->enable_shared_exchange_sink_buffer()) {
+        return _sink_buffer;
+    }
+    return _create_buffer({sender_ins_id});
+}
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/exchange_sink_operator.h b/be/src/pipeline/exec/exchange_sink_operator.h
index 91ee1bd27a63e7..8d094b43f613f0 100644
--- a/be/src/pipeline/exec/exchange_sink_operator.h
+++ b/be/src/pipeline/exec/exchange_sink_operator.h
@@ -61,6 +61,14 @@ class ExchangeSinkLocalState final : public PipelineXSinkLocalState<> {
                                              parent->get_name() + "_FINISH_DEPENDENCY", false);
     }
 
+#ifdef BE_TEST
+    ExchangeSinkLocalState(RuntimeState* state) : Base(nullptr, state) {
+        _profile = state->obj_pool()->add(new RuntimeProfile("mock"));
+        _memory_used_counter =
+                _profile->AddHighWaterMarkCounter("MemoryUsage", TUnit::BYTES, "", 1);
+    }
+#endif
+
     std::vector<Dependency*> dependencies() const override {
         std::vector<Dependency*> dep_vec;
         if (_queue_dependency) {
@@ -88,7 +96,12 @@ class ExchangeSinkLocalState final : public PipelineXSinkLocalState<> {
     bool is_finished() const override { return _reach_limit.load(); }
     void set_reach_limit() { _reach_limit = true; };
 
+    // sender_id indicates which instance within a fragment, while be_number indicates which instance
+    // across all fragments. For example, with 3 BEs and 8 instances, the range of sender_id would be 0 to 24,
+    // and the range of be_number would be from n + 0 to n + 24.
+    // Since be_number is a required field, it still needs to be set for compatibility with older code.
     [[nodiscard]] int sender_id() const { return _sender_id; }
+    [[nodiscard]] int be_number() const { return _state->be_number(); }
 
     std::string name_suffix() override;
     segment_v2::CompressionTypePB compression_type() const;
@@ -112,7 +125,7 @@ class ExchangeSinkLocalState final : public PipelineXSinkLocalState<> {
     friend class vectorized::Channel;
     friend class vectorized::BlockSerializer;
 
-    std::unique_ptr<ExchangeSinkBuffer> _sink_buffer = nullptr;
+    std::shared_ptr<ExchangeSinkBuffer> _sink_buffer = nullptr;
     RuntimeProfile::Counter* _serialize_batch_timer = nullptr;
     RuntimeProfile::Counter* _compress_timer = nullptr;
     RuntimeProfile::Counter* _bytes_sent_counter = nullptr;
@@ -197,7 +210,8 @@ class ExchangeSinkOperatorX final : public DataSinkOperatorX<ExchangeSinkLocalSt
 public:
     ExchangeSinkOperatorX(RuntimeState* state, const RowDescriptor& row_desc, int operator_id,
                           const TDataStreamSink& sink,
-                          const std::vector<TPlanFragmentDestination>& destinations);
+                          const std::vector<TPlanFragmentDestination>& destinations,
+                          const std::vector<TUniqueId>& fragment_instance_ids);
     Status init(const TDataSink& tsink) override;
 
     RuntimeState* state() { return _state; }
@@ -209,6 +223,14 @@ class ExchangeSinkOperatorX final : public DataSinkOperatorX<ExchangeSinkLocalSt
     DataDistribution required_data_distribution() const override;
     bool is_serial_operator() const override { return true; }
 
+    // For a normal shuffle scenario, if the concurrency is n,
+    // there can be up to n * n RPCs in the current fragment.
+    // Therefore, a shared sink buffer is used here to limit the number of concurrent RPCs.
+    // (Note: This does not reduce the total number of RPCs.)
+    // In a merge sort scenario, there are only n RPCs, so a shared sink buffer is not needed.
+    /// TODO: Modify this to let FE handle the judgment instead of BE.
+    std::shared_ptr<ExchangeSinkBuffer> get_sink_buffer(InstanceLoId sender_ins_id);
+
 private:
     friend class ExchangeSinkLocalState;
 
@@ -225,6 +247,13 @@ class ExchangeSinkOperatorX final : public DataSinkOperatorX<ExchangeSinkLocalSt
                                      size_t num_channels,
                                      std::vector<std::vector<uint32_t>>& channel2rows,
                                      vectorized::Block* block, bool eos);
+
+    // Use ExchangeSinkOperatorX to create a sink buffer.
+    // The sink buffer can be shared among multiple ExchangeSinkLocalState instances,
+    // or each ExchangeSinkLocalState can have its own sink buffer.
+    std::shared_ptr<ExchangeSinkBuffer> _create_buffer(
+            const std::vector<InstanceLoId>& sender_ins_ids);
+    std::shared_ptr<ExchangeSinkBuffer> _sink_buffer = nullptr;
     RuntimeState* _state = nullptr;
 
     const std::vector<TExpr> _texprs;
@@ -264,6 +293,7 @@ class ExchangeSinkOperatorX final : public DataSinkOperatorX<ExchangeSinkLocalSt
     size_t _data_processed = 0;
     int _writer_count = 1;
     const bool _enable_local_merge_sort;
+    const std::vector<TUniqueId>& _fragment_instance_ids;
 };
 
 } // namespace pipeline
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index 8ceb63eb99324c..8ab0f1d151568d 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -260,7 +260,7 @@ Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& re
         _runtime_state = RuntimeState::create_unique(
                 request.query_id, request.fragment_id, request.query_options,
                 _query_ctx->query_globals, _exec_env, _query_ctx.get());
-
+        _runtime_state->set_task_execution_context(shared_from_this());
         SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_runtime_state->query_mem_tracker());
         if (request.__isset.backend_id) {
             _runtime_state->set_backend_id(request.backend_id);
@@ -296,6 +296,14 @@ Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& re
         if (local_params.__isset.topn_filter_descs) {
             _query_ctx->init_runtime_predicates(local_params.topn_filter_descs);
         }
+
+        // init fragment_instance_ids
+        const auto target_size = request.local_params.size();
+        _fragment_instance_ids.resize(target_size);
+        for (size_t i = 0; i < request.local_params.size(); i++) {
+            auto fragment_instance_id = request.local_params[i].fragment_instance_id;
+            _fragment_instance_ids[i] = fragment_instance_id;
+        }
     }
 
     {
@@ -353,7 +361,6 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag
     _total_tasks = 0;
     const auto target_size = request.local_params.size();
     _tasks.resize(target_size);
-    _fragment_instance_ids.resize(target_size);
     _runtime_filter_states.resize(target_size);
     _task_runtime_states.resize(_pipelines.size());
     for (size_t pip_idx = 0; pip_idx < _pipelines.size(); pip_idx++) {
@@ -365,8 +372,6 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag
     auto pre_and_submit = [&](int i, PipelineFragmentContext* ctx) {
         const auto& local_params = request.local_params[i];
         auto fragment_instance_id = local_params.fragment_instance_id;
-        _fragment_instance_ids[i] = fragment_instance_id;
-
         _runtime_filter_states[i] = RuntimeFilterParamsContext::create(_query_ctx.get());
         std::unique_ptr<RuntimeFilterMgr> runtime_filter_mgr = std::make_unique<RuntimeFilterMgr>(
                 request.query_id, _runtime_filter_states[i], _query_ctx->query_mem_tracker, false);
@@ -1007,7 +1012,8 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS
             return Status::InternalError("Missing data stream sink.");
         }
         _sink.reset(new ExchangeSinkOperatorX(state, row_desc, next_sink_operator_id(),
-                                              thrift_sink.stream_sink, params.destinations));
+                                              thrift_sink.stream_sink, params.destinations,
+                                              _fragment_instance_ids));
         break;
     }
     case TDataSinkType::RESULT_SINK: {
@@ -1134,10 +1140,10 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS
             // 2. create and set sink operator of data stream sender for new pipeline
 
             DataSinkOperatorPtr sink_op;
-            sink_op.reset(
-                    new ExchangeSinkOperatorX(state, *_row_desc, next_sink_operator_id(),
-                                              thrift_sink.multi_cast_stream_sink.sinks[i],
-                                              thrift_sink.multi_cast_stream_sink.destinations[i]));
+            sink_op.reset(new ExchangeSinkOperatorX(
+                    state, *_row_desc, next_sink_operator_id(),
+                    thrift_sink.multi_cast_stream_sink.sinks[i],
+                    thrift_sink.multi_cast_stream_sink.destinations[i], _fragment_instance_ids));
 
             RETURN_IF_ERROR(new_pipeline->set_sink(sink_op));
             {
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index a49567109a3b31..1e7c1e579f7735 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -587,6 +587,11 @@ class RuntimeState {
                _query_options.enable_local_merge_sort;
     }
 
+    bool enable_shared_exchange_sink_buffer() const {
+        return _query_options.__isset.enable_shared_exchange_sink_buffer &&
+               _query_options.enable_shared_exchange_sink_buffer;
+    }
+
     int64_t min_revocable_mem() const {
         if (_query_options.__isset.min_revocable_mem) {
             return std::max(_query_options.min_revocable_mem, (int64_t)1);
diff --git a/be/src/vec/sink/vdata_stream_sender.h b/be/src/vec/sink/vdata_stream_sender.h
index 4999602fdf49a7..16ea49e443c630 100644
--- a/be/src/vec/sink/vdata_stream_sender.h
+++ b/be/src/vec/sink/vdata_stream_sender.h
@@ -76,6 +76,9 @@ namespace vectorized {
 class BlockSerializer {
 public:
     BlockSerializer(pipeline::ExchangeSinkLocalState* parent, bool is_local = true);
+#ifdef BE_TEST
+    BlockSerializer() : _batch_size(0) {};
+#endif
     Status next_serialized_block(Block* src, PBlock* dest, size_t num_receivers, bool* serialized,
                                  bool eos, const std::vector<uint32_t>* rows = nullptr);
     Status serialize_block(PBlock* dest, size_t num_receivers = 1);
@@ -165,10 +168,9 @@ class Channel {
         return Status::OK();
     }
 
-    void register_exchange_buffer(pipeline::ExchangeSinkBuffer* buffer) {
-        _buffer = buffer;
-        _buffer->register_sink(_fragment_instance_id);
-    }
+    void set_exchange_buffer(pipeline::ExchangeSinkBuffer* buffer) { _buffer = buffer; }
+
+    InstanceLoId dest_ins_id() const { return _fragment_instance_id.lo; }
 
     std::shared_ptr<pipeline::ExchangeSendCallback<PTransmitDataResult>> get_send_callback(
             InstanceLoId id, bool eos) {
diff --git a/be/test/vec/exec/exchange_sink_test.cpp b/be/test/vec/exec/exchange_sink_test.cpp
new file mode 100644
index 00000000000000..9576ed71ee2ada
--- /dev/null
+++ b/be/test/vec/exec/exchange_sink_test.cpp
@@ -0,0 +1,196 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exchange_sink_test.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <vector>
+
+#include "pipeline/exec/exchange_sink_buffer.h"
+
+namespace doris::vectorized {
+using namespace pipeline;
+TEST_F(ExchangeSInkTest, test_normal_end) {
+    {
+        auto state = create_runtime_state();
+        auto buffer = create_buffer(state);
+
+        auto sink1 = create_sink(state, buffer);
+        auto sink2 = create_sink(state, buffer);
+        auto sink3 = create_sink(state, buffer);
+
+        EXPECT_EQ(sink1.add_block(dest_ins_id_1, true), Status::OK());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_2, true), Status::OK());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_3, true), Status::OK());
+
+        EXPECT_EQ(sink2.add_block(dest_ins_id_1, true), Status::OK());
+        EXPECT_EQ(sink2.add_block(dest_ins_id_2, true), Status::OK());
+        EXPECT_EQ(sink2.add_block(dest_ins_id_3, true), Status::OK());
+
+        EXPECT_EQ(sink3.add_block(dest_ins_id_1, true), Status::OK());
+        EXPECT_EQ(sink3.add_block(dest_ins_id_2, true), Status::OK());
+        EXPECT_EQ(sink3.add_block(dest_ins_id_3, true), Status::OK());
+
+        for (auto [id, count] : buffer->_running_sink_count) {
+            EXPECT_EQ(count, 3) << "id : " << id;
+        }
+
+        for (auto [id, is_turn_off] : buffer->_rpc_channel_is_turn_off) {
+            EXPECT_EQ(is_turn_off, false) << "id : " << id;
+        }
+
+        pop_block(dest_ins_id_1, PopState::accept);
+        pop_block(dest_ins_id_1, PopState::accept);
+        pop_block(dest_ins_id_1, PopState::accept);
+
+        pop_block(dest_ins_id_2, PopState::accept);
+        pop_block(dest_ins_id_2, PopState::accept);
+        pop_block(dest_ins_id_2, PopState::accept);
+
+        pop_block(dest_ins_id_3, PopState::accept);
+        pop_block(dest_ins_id_3, PopState::accept);
+        pop_block(dest_ins_id_3, PopState::accept);
+
+        for (auto [id, count] : buffer->_running_sink_count) {
+            EXPECT_EQ(count, 0) << "id : " << id;
+        }
+
+        for (auto [id, is_turn_off] : buffer->_rpc_channel_is_turn_off) {
+            EXPECT_EQ(is_turn_off, true) << "id : " << id;
+        }
+        clear_all_done();
+    }
+}
+
+TEST_F(ExchangeSInkTest, test_eof_end) {
+    {
+        auto state = create_runtime_state();
+        auto buffer = create_buffer(state);
+
+        auto sink1 = create_sink(state, buffer);
+        auto sink2 = create_sink(state, buffer);
+        auto sink3 = create_sink(state, buffer);
+
+        EXPECT_EQ(sink1.add_block(dest_ins_id_1, false), Status::OK());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_2, false), Status::OK());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_3, false), Status::OK());
+
+        EXPECT_EQ(sink2.add_block(dest_ins_id_1, true), Status::OK());
+        EXPECT_EQ(sink2.add_block(dest_ins_id_2, true), Status::OK());
+        EXPECT_EQ(sink2.add_block(dest_ins_id_3, true), Status::OK());
+
+        EXPECT_EQ(sink3.add_block(dest_ins_id_1, false), Status::OK());
+        EXPECT_EQ(sink3.add_block(dest_ins_id_2, true), Status::OK());
+        EXPECT_EQ(sink3.add_block(dest_ins_id_3, false), Status::OK());
+
+        for (auto [id, count] : buffer->_running_sink_count) {
+            EXPECT_EQ(count, 3) << "id : " << id;
+        }
+
+        for (auto [id, is_turn_off] : buffer->_rpc_channel_is_turn_off) {
+            EXPECT_EQ(is_turn_off, false) << "id : " << id;
+        }
+
+        pop_block(dest_ins_id_1, PopState::eof);
+        EXPECT_EQ(buffer->_rpc_channel_is_turn_off[dest_ins_id_1], true);
+        EXPECT_TRUE(buffer->_instance_to_package_queue[dest_ins_id_1].empty());
+
+        pop_block(dest_ins_id_2, PopState::accept);
+        pop_block(dest_ins_id_2, PopState::accept);
+        pop_block(dest_ins_id_2, PopState::accept);
+
+        pop_block(dest_ins_id_3, PopState::accept);
+        pop_block(dest_ins_id_3, PopState::accept);
+        pop_block(dest_ins_id_3, PopState::accept);
+
+        EXPECT_EQ(buffer->_rpc_channel_is_turn_off[dest_ins_id_1], true);
+        EXPECT_EQ(buffer->_rpc_channel_is_turn_off[dest_ins_id_2], false) << "not all eos";
+        EXPECT_EQ(buffer->_rpc_channel_is_turn_off[dest_ins_id_3], false) << " not all eos";
+
+        EXPECT_TRUE(sink1.add_block(dest_ins_id_1, true).is<ErrorCode::END_OF_FILE>());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_2, true), Status::OK());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_3, true), Status::OK());
+        pop_block(dest_ins_id_2, PopState::accept);
+        pop_block(dest_ins_id_3, PopState::accept);
+
+        EXPECT_EQ(buffer->_rpc_channel_is_turn_off[dest_ins_id_1], true);
+        EXPECT_EQ(buffer->_rpc_channel_is_turn_off[dest_ins_id_2], true);
+        EXPECT_EQ(buffer->_rpc_channel_is_turn_off[dest_ins_id_3], false);
+        EXPECT_EQ(buffer->_running_sink_count[dest_ins_id_3], 1);
+
+        clear_all_done();
+    }
+}
+
+TEST_F(ExchangeSInkTest, test_error_end) {
+    {
+        auto state = create_runtime_state();
+        auto buffer = create_buffer(state);
+
+        auto sink1 = create_sink(state, buffer);
+        auto sink2 = create_sink(state, buffer);
+        auto sink3 = create_sink(state, buffer);
+
+        EXPECT_EQ(sink1.add_block(dest_ins_id_1, false), Status::OK());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_2, false), Status::OK());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_3, false), Status::OK());
+
+        EXPECT_EQ(sink2.add_block(dest_ins_id_1, false), Status::OK());
+        EXPECT_EQ(sink2.add_block(dest_ins_id_2, false), Status::OK());
+        EXPECT_EQ(sink2.add_block(dest_ins_id_3, false), Status::OK());
+
+        EXPECT_EQ(sink3.add_block(dest_ins_id_1, false), Status::OK());
+        EXPECT_EQ(sink3.add_block(dest_ins_id_2, false), Status::OK());
+        EXPECT_EQ(sink3.add_block(dest_ins_id_3, false), Status::OK());
+
+        for (auto [id, count] : buffer->_running_sink_count) {
+            EXPECT_EQ(count, 3) << "id : " << id;
+        }
+
+        for (auto [id, is_turn_off] : buffer->_rpc_channel_is_turn_off) {
+            EXPECT_EQ(is_turn_off, false) << "id : " << id;
+        }
+
+        pop_block(dest_ins_id_2, PopState::error);
+
+        auto orgin_queue_1_size = done_map[dest_ins_id_1].size();
+        auto orgin_queue_2_size = done_map[dest_ins_id_2].size();
+        auto orgin_queue_3_size = done_map[dest_ins_id_3].size();
+
+        EXPECT_EQ(sink1.add_block(dest_ins_id_1, false), Status::OK());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_2, false), Status::OK());
+        EXPECT_EQ(sink1.add_block(dest_ins_id_3, false), Status::OK());
+
+        EXPECT_EQ(sink2.add_block(dest_ins_id_1, false), Status::OK());
+        EXPECT_EQ(sink2.add_block(dest_ins_id_2, false), Status::OK());
+        EXPECT_EQ(sink2.add_block(dest_ins_id_3, false), Status::OK());
+
+        EXPECT_EQ(sink3.add_block(dest_ins_id_1, false), Status::OK());
+        EXPECT_EQ(sink3.add_block(dest_ins_id_2, false), Status::OK());
+        EXPECT_EQ(sink3.add_block(dest_ins_id_3, false), Status::OK());
+
+        EXPECT_EQ(orgin_queue_1_size, done_map[dest_ins_id_1].size());
+        EXPECT_EQ(orgin_queue_2_size, done_map[dest_ins_id_2].size());
+        EXPECT_EQ(orgin_queue_3_size, done_map[dest_ins_id_3].size());
+
+        clear_all_done();
+    }
+}
+
+} // namespace doris::vectorized
diff --git a/be/test/vec/exec/exchange_sink_test.h b/be/test/vec/exec/exchange_sink_test.h
new file mode 100644
index 00000000000000..253d7b267f91ed
--- /dev/null
+++ b/be/test/vec/exec/exchange_sink_test.h
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <vector>
+
+#include "common/status.h"
+#include "pipeline/exec/exchange_sink_buffer.h"
+#include "pipeline/exec/exchange_sink_operator.h"
+#include "runtime/runtime_state.h"
+#include "udf/udf.h"
+#include "vec/sink/writer/vhive_utils.h"
+
+namespace doris::pipeline {
+
+std::map<int64_t, std::queue<AutoReleaseClosure<PTransmitDataParams,
+                                                ExchangeSendCallback<PTransmitDataResult>>*>>
+
+        done_map;
+
+void add_request(int64_t id, auto* done) {
+    done_map[id].push(done);
+}
+
+void clear_all_done() {
+    for (auto& [id, dones] : done_map) {
+        while (!dones.empty()) {
+            dones.front()->Run();
+            dones.pop();
+        }
+    }
+}
+
+enum PopState : int {
+    eof,
+    error,
+    accept,
+};
+
+void pop_block(int64_t id, PopState state) {
+    if (done_map[id].empty()) {
+        return;
+    }
+    auto* done = done_map[id].front();
+    done_map[id].pop();
+    switch (state) {
+    case PopState::eof: {
+        Status st = Status::EndOfFile("Mock eof");
+        st.to_protobuf(done->response_->mutable_status());
+        done->Run();
+        break;
+    }
+    case error: {
+        done->cntl_->SetFailed("Mock error");
+        done->Run();
+        break;
+    }
+    case accept: {
+        done->Run();
+        break;
+    }
+    }
+}
+void transmit_blockv2(PBackendService_Stub& stub,
+                      std::unique_ptr<AutoReleaseClosure<PTransmitDataParams,
+                                                         ExchangeSendCallback<PTransmitDataResult>>>
+                              closure) {
+    std::cout << "mock transmit_blockv2 dest ins id :" << closure->request_->finst_id().lo()
+              << "\n";
+    add_request(closure->request_->finst_id().lo(), closure.release());
+}
+}; // namespace doris::pipeline
+
+namespace doris::vectorized {
+
+using namespace pipeline;
+class ExchangeSInkTest : public testing::Test {
+public:
+    ExchangeSInkTest() = default;
+    ~ExchangeSInkTest() override = default;
+};
+
+class MockContext : public TaskExecutionContext {};
+
+std::shared_ptr<MockContext> _mock_context = std::make_shared<MockContext>();
+
+auto create_runtime_state() {
+    auto state = RuntimeState::create_shared();
+
+    state->set_task_execution_context(_mock_context);
+    return state;
+}
+constexpr int64_t recvr_fragment_id = 2;
+constexpr int64_t sender_fragment_id = 2;
+
+TUniqueId create_TUniqueId(int64_t hi, int64_t lo) {
+    TUniqueId t {};
+    t.hi = hi;
+    t.lo = lo;
+    return t;
+}
+
+const auto dest_fragment_ins_id_1 = create_TUniqueId(recvr_fragment_id, 1);
+const auto dest_fragment_ins_id_2 = create_TUniqueId(recvr_fragment_id, 2);
+const auto dest_fragment_ins_id_3 = create_TUniqueId(recvr_fragment_id, 3);
+const auto dest_ins_id_1 = dest_fragment_ins_id_1.lo;
+const auto dest_ins_id_2 = dest_fragment_ins_id_2.lo;
+const auto dest_ins_id_3 = dest_fragment_ins_id_3.lo;
+
+class MockSinkBuffer : public ExchangeSinkBuffer {
+public:
+    MockSinkBuffer(RuntimeState* state, int64_t sinknum) : ExchangeSinkBuffer(state, sinknum) {};
+    void _failed(InstanceLoId id, const std::string& err) override {
+        _is_failed = true;
+        std::cout << "_failed\n";
+    }
+};
+
+struct SinkWithChannel {
+    std::shared_ptr<ExchangeSinkLocalState> sink;
+    std::shared_ptr<MockSinkBuffer> buffer;
+    std::map<int64_t, std::shared_ptr<Channel>> channels;
+    Status add_block(int64_t id, bool eos) {
+        auto channel = channels[id];
+        TransmitInfo transmitInfo {.channel = channel.get(),
+                                   .block = std::make_unique<PBlock>(),
+                                   .eos = eos,
+                                   .exec_status = Status::OK()};
+        return buffer->add_block(std::move(transmitInfo));
+    }
+};
+
+auto create_buffer(std::shared_ptr<RuntimeState> state) {
+    auto sink_buffer = std::make_shared<MockSinkBuffer>(state.get(), 3);
+
+    sink_buffer->construct_request(dest_fragment_ins_id_1);
+    sink_buffer->construct_request(dest_fragment_ins_id_2);
+    sink_buffer->construct_request(dest_fragment_ins_id_3);
+    return sink_buffer;
+}
+
+auto create_sink(std::shared_ptr<RuntimeState> state, std::shared_ptr<MockSinkBuffer> sink_buffer) {
+    SinkWithChannel sink_with_channel;
+    sink_with_channel.sink = ExchangeSinkLocalState::create_shared(state.get());
+    sink_with_channel.buffer = sink_buffer;
+    {
+        auto channel = std::make_shared<vectorized::Channel>(
+                sink_with_channel.sink.get(), TNetworkAddress {}, dest_fragment_ins_id_1, 0);
+        sink_with_channel.channels[channel->dest_ins_id()] = channel;
+    }
+    {
+        auto channel = std::make_shared<vectorized::Channel>(
+                sink_with_channel.sink.get(), TNetworkAddress {}, dest_fragment_ins_id_2, 0);
+        sink_with_channel.channels[channel->dest_ins_id()] = channel;
+    }
+    {
+        auto channel = std::make_shared<vectorized::Channel>(
+                sink_with_channel.sink.get(), TNetworkAddress {}, dest_fragment_ins_id_3, 0);
+        sink_with_channel.channels[channel->dest_ins_id()] = channel;
+    }
+    return sink_with_channel;
+}
+
+} // namespace doris::vectorized
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index d07a8f022d7092..380c758e5751c8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -284,6 +284,8 @@ public class SessionVariable implements Serializable, Writable {
 
     public static final String ENABLE_LOCAL_MERGE_SORT = "enable_local_merge_sort";
 
+    public static final String ENABLE_SHARED_EXCHANGE_SINK_BUFFER = "enable_shared_exchange_sink_buffer";
+
     public static final String ENABLE_AGG_STATE = "enable_agg_state";
 
     public static final String ENABLE_RPC_OPT_FOR_PIPELINE = "enable_rpc_opt_for_pipeline";
@@ -1135,6 +1137,9 @@ public enum IgnoreSplitType {
     @VariableMgr.VarAttr(name = ENABLE_LOCAL_MERGE_SORT)
     private boolean enableLocalMergeSort = true;
 
+    @VariableMgr.VarAttr(name = ENABLE_SHARED_EXCHANGE_SINK_BUFFER, fuzzy = true)
+    private boolean enableSharedExchangeSinkBuffer = true;
+
     @VariableMgr.VarAttr(name = ENABLE_AGG_STATE, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL,
             needForward = true)
     public boolean enableAggState = false;
@@ -2370,6 +2375,7 @@ public void initFuzzyModeVariables() {
         this.parallelPrepareThreshold = random.nextInt(32) + 1;
         this.enableCommonExprPushdown = random.nextBoolean();
         this.enableLocalExchange = random.nextBoolean();
+        this.enableSharedExchangeSinkBuffer = random.nextBoolean();
         this.useSerialExchange = random.nextBoolean();
         // This will cause be dead loop, disable it first
         // this.disableJoinReorder = random.nextBoolean();
@@ -3965,6 +3971,7 @@ public TQueryOptions toThrift() {
         tResult.setDataQueueMaxBlocks(dataQueueMaxBlocks);
 
         tResult.setEnableLocalMergeSort(enableLocalMergeSort);
+        tResult.setEnableSharedExchangeSinkBuffer(enableSharedExchangeSinkBuffer);
         tResult.setEnableParallelResultSink(enableParallelResultSink);
         tResult.setEnableParallelOutfile(enableParallelOutfile);
         tResult.setEnableShortCircuitQueryAccessColumnStore(enableShortCircuitQueryAcessColumnStore);
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index 745cb8f21fb627..85abbf9b66ddba 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -359,7 +359,7 @@ struct TQueryOptions {
 
   141: optional bool ignore_runtime_filter_error = false;
   142: optional bool enable_fixed_len_to_uint32_v2 = false;
-
+  143: optional bool enable_shared_exchange_sink_buffer = true;
   // For cloud, to control if the content would be written into file cache
   // In write path, to control if the content would be written into file cache.
   // In read path, read from file cache or remote storage when execute query.

From 107ac9f9908d07694800878c063ce76e37ace3e0 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Mon, 9 Dec 2024 14:41:41 +0800
Subject: [PATCH 230/399] [fix](nereids) common-sub-expression opt: avoid
 computing alias repeatedly  (#45043)

### What problem does this PR solve?
in previous PR#33087, we extract common sub expression, and set
multi-layer projects to save computing effort.
project: f1(h(a)), f2(h(a))
=>
multi-layer:
[
L0: a
L1: a, h(a) as x
L2: a, h(a) as x, f1(x), f2(x)
]
"h(a) as x " is computed at layer L1 and layer L2.
this PR avoids the duplicated computing in L2 by set L2 as
L2: a, x, f1(x), f2(x)
---
 .../post/CommonSubExpressionOpt.java          | 16 ++++-
 .../suites/nereids_rules_p0/cse/cse.groovy    | 72 +++++++++++++++++++
 2 files changed, 85 insertions(+), 3 deletions(-)
 create mode 100644 regression-test/suites/nereids_rules_p0/cse/cse.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionOpt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionOpt.java
index fca84167994d08..72ae4183c3b147 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionOpt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionOpt.java
@@ -29,7 +29,8 @@
 
 import com.google.common.collect.Lists;
 
-import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -67,8 +68,10 @@ private List<List<NamedExpression>> computeMultiLayerProjections(
         for (Expression expr : projects) {
             expr.accept(collector, null);
         }
-        Map<Expression, Alias> aliasMap = new HashMap<>();
+        // use linkedHashMap to make projects order stable
+        Map<Expression, Alias> aliasMap = new LinkedHashMap<>();
         if (!collector.commonExprByDepth.isEmpty()) {
+            Set<Alias> previousAlias = new HashSet<>();
             for (int i = 1; i <= collector.commonExprByDepth.size(); i++) {
                 List<NamedExpression> layer = Lists.newArrayList();
                 layer.addAll(inputSlots);
@@ -87,7 +90,14 @@ private List<List<NamedExpression>> computeMultiLayerProjections(
                         aliasMap.put(expr, rewritten instanceof Alias ? (Alias) rewritten : new Alias(rewritten));
                     }
                 });
-                layer.addAll(aliasMap.values());
+                for (Alias alias : aliasMap.values()) {
+                    if (previousAlias.contains(alias)) {
+                        layer.add(alias.toSlot());
+                    } else {
+                        layer.add(alias);
+                        previousAlias.add(alias);
+                    }
+                }
                 multiLayers.add(layer);
             }
             // final layer
diff --git a/regression-test/suites/nereids_rules_p0/cse/cse.groovy b/regression-test/suites/nereids_rules_p0/cse/cse.groovy
new file mode 100644
index 00000000000000..4e35737674de69
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/cse/cse.groovy
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("cse") {
+    sql """
+    drop table if exists cse;
+    CREATE TABLE `cse` (
+    `k1` int NOT NULL,
+    d   datev2,
+    i  int
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`k1`)
+    DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1",
+    "min_load_replica_num" = "-1",
+    "is_being_synced" = "false",
+    "storage_medium" = "hdd",
+    "storage_format" = "V2",
+    "inverted_index_storage_format" = "V2",
+    "light_schema_change" = "true",
+    "disable_auto_compaction" = "false",
+    "enable_single_replica_compaction" = "false",
+    "group_commit_interval_ms" = "10000",
+    "group_commit_data_bytes" = "134217728"
+    ); 
+
+    insert into cse values (1, '20240101', 100);
+    """
+
+    explain {
+        sql """
+            physical plan
+            select sum(    
+                case when k1 between i and 
+                    cast(from_unixtime(
+                    unix_timestamp(date_add(
+                                from_unixtime(unix_timestamp(cast(d as string), 'yyyyMMdd')),
+                                INTERVAL 5 DAY)), 'yyyyMMdd') as int) 
+                THEN 1
+                ELSE 0
+                end) as c1,
+                sum(
+                        case when k1 between i and 
+                                    cast(from_unixtime(
+                                    unix_timestamp(date_add(
+                                            from_unixtime(unix_timestamp(cast(d as string), 'yyyyMMdd')),
+                                            INTERVAL 2 DAY)), 'yyyyMMdd') as int) 
+                            THEN 1
+                            ELSE 0
+                            end) as c2
+            from cse
+            group by d;
+            """
+        contains("l1([k1#0, d#1, i#2, (k1 >= i)#9, cast(d as TEXT)#10, unix_timestamp(cast(d as TEXT)#10, '%Y%m%d') AS `unix_timestamp(cast(d as TEXT), '%Y%m%d')`#11])")
+    }
+    
+}
+

From 5f0a2392af643fc35b109d4884070bf28f794e93 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Mon, 9 Dec 2024 14:52:52 +0800
Subject: [PATCH 231/399] [fix](memory) Fix init segment map check memory
 exceeds limit (#44092)

### What problem does this PR solve?

Problem Summary:

`RowIdConversion::init_segment_map` may use a lot of memory in
compaction. however, this part of memory is not controlled, the process
may OOM crash.

If process has less than 10M available memory before generating
`rowid_map` for each segment in `init_segment_map`, it will terminate.


![image](https://github.com/user-attachments/assets/652fadc9-bca0-4edd-a03e-04f2546cdf76)
---
 be/src/olap/rowid_conversion.h            | 30 +++++++++++++++--------
 be/src/olap/rowset/beta_rowset_reader.cpp |  3 ++-
 be/test/olap/rowid_conversion_test.cpp    |  6 +++--
 3 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/be/src/olap/rowid_conversion.h b/be/src/olap/rowid_conversion.h
index 01a2cea0d5e308..8f9d96a136aab4 100644
--- a/be/src/olap/rowid_conversion.h
+++ b/be/src/olap/rowid_conversion.h
@@ -37,21 +37,33 @@ class RowIdConversion {
     ~RowIdConversion() { RELEASE_THREAD_MEM_TRACKER(_seg_rowid_map_mem_used); }
 
     // resize segment rowid map to its rows num
-    void init_segment_map(const RowsetId& src_rowset_id, const std::vector<uint32_t>& num_rows) {
-        size_t delta_std_pair_cap = 0;
+    Status init_segment_map(const RowsetId& src_rowset_id, const std::vector<uint32_t>& num_rows) {
         for (size_t i = 0; i < num_rows.size(); i++) {
+            constexpr size_t RESERVED_MEMORY = 10 * 1024 * 1024; // 10M
+            if (doris::GlobalMemoryArbitrator::is_exceed_hard_mem_limit(RESERVED_MEMORY)) {
+                return Status::MemoryLimitExceeded(fmt::format(
+                        "RowIdConversion init_segment_map failed, memory exceed limit, {}, "
+                        "consuming "
+                        "tracker:<{}>, peak used {}, current used {}.",
+                        doris::GlobalMemoryArbitrator::process_limit_exceeded_errmsg_str(),
+                        doris::thread_context()->thread_mem_tracker()->label(),
+                        doris::thread_context()->thread_mem_tracker()->peak_consumption(),
+                        doris::thread_context()->thread_mem_tracker()->consumption()));
+            }
+
             uint32_t id = _segments_rowid_map.size();
             _segment_to_id_map.emplace(std::pair<RowsetId, uint32_t> {src_rowset_id, i}, id);
             _id_to_segment_map.emplace_back(src_rowset_id, i);
             std::vector<std::pair<uint32_t, uint32_t>> vec(
                     num_rows[i], std::pair<uint32_t, uint32_t>(UINT32_MAX, UINT32_MAX));
-            delta_std_pair_cap += vec.capacity();
+
+            //NOTE: manually count _segments_rowid_map's memory here, because _segments_rowid_map could be used by indexCompaction.
+            // indexCompaction is a thridparty code, it's too complex to modify it.
+            // refer compact_column.
+            track_mem_usage(vec.capacity());
             _segments_rowid_map.emplace_back(std::move(vec));
         }
-        //NOTE: manually count _segments_rowid_map's memory here, because _segments_rowid_map could be used by indexCompaction.
-        // indexCompaction is a thridparty code, it's too complex to modify it.
-        // refer compact_column.
-        track_mem_usage(delta_std_pair_cap);
+        return Status::OK();
     }
 
     // set dst rowset id
@@ -124,9 +136,7 @@ class RowIdConversion {
         size_t new_size =
                 _std_pair_cap * sizeof(std::pair<uint32_t, uint32_t>) +
                 _segments_rowid_map.capacity() * sizeof(std::vector<std::pair<uint32_t, uint32_t>>);
-
-        RELEASE_THREAD_MEM_TRACKER(_seg_rowid_map_mem_used);
-        CONSUME_THREAD_MEM_TRACKER(new_size);
+        CONSUME_THREAD_MEM_TRACKER(new_size - _seg_rowid_map_mem_used);
         _seg_rowid_map_mem_used = new_size;
     }
 
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index 042893f1374374..47cf9b820e8562 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -239,7 +239,8 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
         // init segment rowid map for rowid conversion
         std::vector<uint32_t> segment_num_rows;
         RETURN_IF_ERROR(get_segment_num_rows(&segment_num_rows));
-        _read_context->rowid_conversion->init_segment_map(rowset()->rowset_id(), segment_num_rows);
+        RETURN_IF_ERROR(_read_context->rowid_conversion->init_segment_map(rowset()->rowset_id(),
+                                                                          segment_num_rows));
     }
 
     auto [seg_start, seg_end] = _segment_offsets;
diff --git a/be/test/olap/rowid_conversion_test.cpp b/be/test/olap/rowid_conversion_test.cpp
index d48d4150ad3d0d..df56cd0559e4d8 100644
--- a/be/test/olap/rowid_conversion_test.cpp
+++ b/be/test/olap/rowid_conversion_test.cpp
@@ -490,10 +490,12 @@ TEST_F(TestRowIdConversion, Basic) {
     RowIdConversion rowid_conversion;
     src_rowset.init(0);
     std::vector<uint32_t> rs0_segment_num_rows = {4, 3};
-    rowid_conversion.init_segment_map(src_rowset, rs0_segment_num_rows);
+    auto st = rowid_conversion.init_segment_map(src_rowset, rs0_segment_num_rows);
+    EXPECT_EQ(st.ok(), true);
     src_rowset.init(1);
     std::vector<uint32_t> rs1_segment_num_rows = {4};
-    rowid_conversion.init_segment_map(src_rowset, rs1_segment_num_rows);
+    st = rowid_conversion.init_segment_map(src_rowset, rs1_segment_num_rows);
+    EXPECT_EQ(st.ok(), true);
     rowid_conversion.set_dst_rowset_id(dst_rowset);
 
     std::vector<uint32_t> dst_segment_num_rows = {4, 3, 4};

From 9e49bdf012f7fdbc7bfa20606e701141ab2d4fba Mon Sep 17 00:00:00 2001
From: shee <13843187+qzsee@users.noreply.github.com>
Date: Mon, 9 Dec 2024 14:54:21 +0800
Subject: [PATCH 232/399] [Audit](fix) fix audit log lost for npe (#44145)

### What problem does this PR solve?

```
private static void logAuditLogImpl(ConnectContext ctx, String origStmt, StatementBase parsedStmt,
            org.apache.doris.proto.Data.PQueryStatistics statistics, boolean printFuzzyVariables) {

        //........
       // When we execute a statement, we execute the executeQuery method. In this method, we parse the sql using the new optimizer first, and if the parsing fails, we execute handleQueryException to record the audit log. At this time, there is no executor set, resulting in the npe
        if (!Env.getCurrentEnv().isMaster()) {
            if (ctx.executor != null && ctx.executor.isForwardToMaster()) {
                auditEventBuilder.setState(ctx.executor.getProxyStatus());
                int proxyStatusCode = ctx.executor.getProxyStatusCode();
                if (proxyStatusCode != 0) {
                    auditEventBuilder.setErrorCode(proxyStatusCode);
                    auditEventBuilder.setErrorMessage(ctx.executor.getProxyErrMsg());
                }
            }
        }
        if (ctx.getCommand() == MysqlCommand.COM_STMT_PREPARE && ctx.getState().getErrorCode() == null) {
            auditEventBuilder.setState(String.valueOf(MysqlStateType.OK));
        }
        Env.getCurrentEnv().getWorkloadRuntimeStatusMgr().submitFinishQueryToAudit(auditEventBuilder.build());
    }
```

Co-authored-by: garenshi <garenshi@tencent.com>
---
 .../src/main/java/org/apache/doris/qe/AuditLogHelper.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
index 903b72898c9512..f29c617c15861e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
@@ -286,7 +286,7 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme
         auditEventBuilder.setStmtType(getStmtType(parsedStmt));
 
         if (!Env.getCurrentEnv().isMaster()) {
-            if (ctx.executor.isForwardToMaster()) {
+            if (ctx.executor != null && ctx.executor.isForwardToMaster()) {
                 auditEventBuilder.setState(ctx.executor.getProxyStatus());
                 int proxyStatusCode = ctx.executor.getProxyStatusCode();
                 if (proxyStatusCode != 0) {

From 90cbfc5a4e4419ea294d11463fdd0f7cae63c294 Mon Sep 17 00:00:00 2001
From: Rijesh Kunhi Parambattu <147430310+rijeshkp@users.noreply.github.com>
Date: Mon, 9 Dec 2024 12:30:23 +0530
Subject: [PATCH 233/399]  [Enhancement] (nereids)implement showDataSkewCommand
 in nereids #42755 (#44704)

Issue Number: close #42755
---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 .../apache/doris/catalog/MetadataViewer.java  |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |   8 ++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../plans/commands/ShowDataSkewCommand.java   | 109 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../show/test_nereids_show_data_skew.groovy   |  50 ++++++++
 7 files changed, 175 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java
 create mode 100644 regression-test/suites/nereids_p0/show/test_nereids_show_data_skew.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index c3fe7bb145ba72..04ef55d82c7065 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -265,6 +265,7 @@ supportedShowStatement
     | SHOW WHITELIST                                                                #showWhitelist
     | SHOW TABLETS BELONG
         tabletIds+=INTEGER_VALUE (COMMA tabletIds+=INTEGER_VALUE)*                  #showTabletsBelong
+    | SHOW DATA SKEW FROM baseTableRef                                              #showDataSkew
     ;
 
 supportedLoadStatement
@@ -331,7 +332,6 @@ unsupportedShowStatement
     | SHOW ALTER TABLE (ROLLUP | (MATERIALIZED VIEW) | COLUMN)
         ((FROM | IN) database=multipartIdentifier)? wildWhere?
         sortClause? limitClause?                                                    #showAlterTable
-    | SHOW DATA SKEW FROM baseTableRef                                              #showDataSkew
     | SHOW DATA (ALL)? (FROM tableName=multipartIdentifier)?
         sortClause? propertyClause?                                                 #showData
     | SHOW TEMPORARY? PARTITIONS FROM tableName=multipartIdentifier
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
index 501d49e4add8d9..61853bea3d48b7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
@@ -401,7 +401,7 @@ public static List<List<String>> getDataSkew(ShowDataSkewStmt stmt) throws DdlEx
         return getDataSkew(stmt.getDbName(), stmt.getTblName(), stmt.getPartitionNames());
     }
 
-    private static List<List<String>> getDataSkew(String dbName, String tblName, PartitionNames partitionNames)
+    public static List<List<String>> getDataSkew(String dbName, String tblName, PartitionNames partitionNames)
             throws DdlException {
         DecimalFormat df = new DecimalFormat("00.00 %");
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 15aeca952a2556..d5fbd193b4468a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -246,6 +246,7 @@
 import org.apache.doris.nereids.DorisParser.ShowCreateRepositoryContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateTableContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateViewContext;
+import org.apache.doris.nereids.DorisParser.ShowDataSkewContext;
 import org.apache.doris.nereids.DorisParser.ShowDatabaseIdContext;
 import org.apache.doris.nereids.DorisParser.ShowDeleteContext;
 import org.apache.doris.nereids.DorisParser.ShowDiagnoseTabletContext;
@@ -548,6 +549,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowDataSkewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDatabaseIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
@@ -4966,5 +4968,11 @@ public LogicalPlan visitAdminCheckTablets(AdminCheckTabletsContext ctx) {
                 : Maps.newHashMap();
         return new AdminCheckTabletsCommand(tabletIdLists, properties);
     }
+
+    @Override
+    public LogicalPlan visitShowDataSkew(ShowDataSkewContext ctx) {
+        TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef());
+        return new ShowDataSkewCommand(tableRefInfo);
+    }
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 8198a8600a60c9..44d67d7478274a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -212,6 +212,7 @@ public enum PlanType {
     SHOW_CREATE_TABLE_COMMAND,
     SHOW_CREATE_VIEW_COMMAND,
     SHOW_DATABASE_ID_COMMAND,
+    SHOW_DATA_SKEW_COMMAND,
     SHOW_DELETE_COMMAND,
     SHOW_DIAGNOSE_TABLET_COMMAND,
     SHOW_DYNAMIC_PARTITION_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java
new file mode 100644
index 00000000000000..993f7db225e802
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.PartitionNames;
+import org.apache.doris.analysis.RedirectStatus;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MetadataViewer;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * show table id command
+ */
+public class ShowDataSkewCommand extends ShowCommand {
+    public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
+            .add("PartitionName").add("BucketIdx").add("AvgRowCount").add("AvgDataSize")
+            .add("Graph").add("Percent")
+            .build();
+
+    private final TableRefInfo tableRefInfo;
+
+    /**
+     * constructor
+     */
+    public ShowDataSkewCommand(TableRefInfo tableRefInfo) {
+        super(PlanType.SHOW_DATA_SKEW_COMMAND);
+        this.tableRefInfo = tableRefInfo;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        tableRefInfo.analyze(ctx);
+        // disallow external catalog
+        Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName());
+        if (!Env.getCurrentEnv().getAccessManager()
+                .checkTblPriv(ctx, tableRefInfo.getTableNameInfo().getCtl(),
+                    tableRefInfo.getTableNameInfo().getDb(), tableRefInfo.getTableNameInfo().getTbl(),
+                    PrivPredicate.SHOW)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW DATA SKEW",
+                    ctx.getQualifiedUser(),
+                    ctx.getRemoteIP(),
+                    tableRefInfo.getTableNameInfo().getDb() + "." + tableRefInfo.getTableNameInfo().getTbl());
+        }
+        try {
+            tableRefInfo.analyze(ctx);
+            Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(), this.getClass().getSimpleName());
+
+            PartitionNames partitionNames = (tableRefInfo.getPartitionNamesInfo() != null)
+                     ? tableRefInfo.getPartitionNamesInfo().translateToLegacyPartitionNames() : null;
+            List<List<String>> results = MetadataViewer.getDataSkew(tableRefInfo.getTableNameInfo().getDb(),
+                    tableRefInfo.getTableNameInfo().getTbl(), partitionNames);
+            return new ShowResultSet(getMetaData(), results);
+        } catch (DdlException e) {
+            throw new AnalysisException(e.getMessage());
+        }
+
+    }
+
+    public ShowResultSetMetaData getMetaData() {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        for (String title : TITLE_NAMES) {
+            builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
+        }
+        return builder.build();
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowDataSkewCommand(this, context);
+    }
+
+    @Override
+    public RedirectStatus toRedirectStatus() {
+        return RedirectStatus.FORWARD_NO_SYNC;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index dea3a004e816be..81296b0b9b8204 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -92,6 +92,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateRepositoryCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowDataSkewCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDatabaseIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDeleteCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowDiagnoseTabletCommand;
@@ -600,4 +601,8 @@ default R visitShowProcessListCommand(ShowProcessListCommand showProcessListComm
     default R visitAdminCheckTabletsCommand(AdminCheckTabletsCommand adminCheckTabletsCommand, C context) {
         return visitCommand(adminCheckTabletsCommand, context);
     }
+
+    default R visitShowDataSkewCommand(ShowDataSkewCommand showDataSkewCommand, C context) {
+        return visitCommand(showDataSkewCommand, context);
+    }
 }
diff --git a/regression-test/suites/nereids_p0/show/test_nereids_show_data_skew.groovy b/regression-test/suites/nereids_p0/show/test_nereids_show_data_skew.groovy
new file mode 100644
index 00000000000000..5e316f22afcf17
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_nereids_show_data_skew.groovy
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_nereids_show_data_skew") {
+    def table = "test_nereids_show_data_skew"
+    String dbName = "${table}_db"
+    sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+    sql "use ${dbName}"
+    // create table and insert data
+    sql """ drop table if exists ${table} force"""
+    sql """
+      CREATE TABLE ${table} (
+            id int, 
+            name string, 
+            pdate DATETIME) 
+        PARTITION BY RANGE(pdate) ( 
+            FROM ("2023-04-16") TO ("2023-04-20") INTERVAL 1 DAY 
+        ) DISTRIBUTED BY HASH(id) BUCKETS 5 
+        properties("replication_num" = "1");
+        """
+
+    checkNereidsExecute("show data skew from ${dbName}.test_nereids_show_data_skew;")
+    def result = sql """show data skew from ${dbName}.test_nereids_show_data_skew;"""
+    assertTrue(result.size() == 20)
+
+    checkNereidsExecute("show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416);")
+    def result2 = sql """show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416);"""
+    assertTrue(result2.size() == 5)
+
+    checkNereidsExecute("show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416, p_20230418);")
+    def result3 = sql """show data skew from ${dbName}.test_nereids_show_data_skew partition(p_20230416, p_20230418);"""
+    assertTrue(result3.size() == 10)
+
+
+}

From 9c4c0b378f1f73973a449ac5c44d82b5b1192fb8 Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Mon, 9 Dec 2024 12:31:00 +0530
Subject: [PATCH 234/399] [Enhancement] (nereids)implement
 CreateWorkloadGroupCommand in nereids (#44970)

Issue Number: close #42589
---
 .../org/apache/doris/nereids/DorisParser.g4   |   6 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  11 ++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../commands/CreateWorkloadGroupCommand.java  | 100 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../workloadgroup/WorkloadGroupMgr.java       |  10 +-
 .../test_nereids_workload_test.groovy         |   6 +-
 7 files changed, 130 insertions(+), 9 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateWorkloadGroupCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 04ef55d82c7065..eda5dda16984c9 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -183,7 +183,9 @@ supportedCreateStatement
     | CREATE (EXTERNAL)? TABLE (IF NOT EXISTS)? name=multipartIdentifier
         LIKE existedTable=multipartIdentifier
         (WITH ROLLUP (rollupNames=identifierList)?)?                      #createTableLike
-    | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?    #createRole        
+    | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?    #createRole  
+    | CREATE WORKLOAD GROUP (IF NOT EXISTS)?
+        name=identifierOrText properties=propertyClause?                        #createWorkloadGroup          
     | CREATE ROW POLICY (IF NOT EXISTS)? name=identifier
         ON table=multipartIdentifier
         AS type=(RESTRICTIVE | PERMISSIVE)
@@ -764,8 +766,6 @@ unsupportedCreateStatement
         name=identifierOrText properties=propertyClause?                        #createResource
     | CREATE STORAGE VAULT (IF NOT EXISTS)?
         name=identifierOrText properties=propertyClause?                        #createStorageVault
-    | CREATE WORKLOAD GROUP (IF NOT EXISTS)?
-        name=identifierOrText properties=propertyClause?                        #createWorkloadGroup
     | CREATE WORKLOAD POLICY (IF NOT EXISTS)? name=identifierOrText
         (CONDITIONS LEFT_PAREN workloadPolicyConditions RIGHT_PAREN)?
         (ACTIONS LEFT_PAREN workloadPolicyActions RIGHT_PAREN)?
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index d5fbd193b4468a..ff5bed55ebc857 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -104,6 +104,7 @@
 import org.apache.doris.nereids.DorisParser.CreateTableContext;
 import org.apache.doris.nereids.DorisParser.CreateTableLikeContext;
 import org.apache.doris.nereids.DorisParser.CreateViewContext;
+import org.apache.doris.nereids.DorisParser.CreateWorkloadGroupContext;
 import org.apache.doris.nereids.DorisParser.CteContext;
 import org.apache.doris.nereids.DorisParser.DataTypeWithNullableContext;
 import org.apache.doris.nereids.DorisParser.DateCeilContext;
@@ -502,6 +503,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableLikeCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateWorkloadGroupCommand;
 import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand;
 import org.apache.doris.nereids.trees.plans.commands.DeleteFromUsingCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand;
@@ -4862,6 +4864,15 @@ public LogicalPlan visitDropEncryptkey(DropEncryptkeyContext ctx) {
         return new DropEncryptkeyCommand(new EncryptKeyName(nameParts), ctx.EXISTS() != null);
     }
 
+    @Override
+    public LogicalPlan visitCreateWorkloadGroup(CreateWorkloadGroupContext ctx) {
+        String workloadGroupName = stripQuotes(ctx.name.getText());
+        boolean ifNotExists = ctx.EXISTS() != null;
+        Map<String, String> properties = ctx.propertyClause() != null
+                                    ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+        return new CreateWorkloadGroupCommand(workloadGroupName, ifNotExists, properties);
+    }
+
     @Override
     public LogicalPlan visitDropFile(DropFileContext ctx) {
         String dbName = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 44d67d7478274a..fac113564f857e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -247,6 +247,7 @@ public enum PlanType {
     RECOVER_PARTITION_COMMAND,
     REPLAY_COMMAND,
     CREATE_ENCRYPTKEY_COMMAND,
+    CREATE_WORKLOAD_GROUP_COMMAND,
     CREATE_FILE_COMMAND,
     CREATE_ROUTINE_LOAD_COMMAND
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateWorkloadGroupCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateWorkloadGroupCommand.java
new file mode 100644
index 00000000000000..ad4db8cbfdfae8
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateWorkloadGroupCommand.java
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.resource.workloadgroup.WorkloadGroup;
+import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Map;
+
+/**
+ * Create workload group command
+ */
+public class CreateWorkloadGroupCommand extends Command implements ForwardWithSync {
+    private final boolean ifNotExists;
+    private final String workloadGroupName;
+    private final Map<String, String> properties;
+
+    /**
+     * Constructor for CreateWorkloadGroupCommand
+     */
+    public CreateWorkloadGroupCommand(String workloadGroupName, boolean ifNotExists, Map<String, String> properties) {
+        super(PlanType.CREATE_WORKLOAD_GROUP_COMMAND);
+        this.workloadGroupName = workloadGroupName;
+        this.ifNotExists = ifNotExists;
+        this.properties = properties;
+    }
+
+    private void validate(ConnectContext ctx) throws AnalysisException {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        // check name
+        FeNameFormat.checkWorkloadGroupName(workloadGroupName);
+
+        if (properties == null || properties.isEmpty()) {
+            throw new AnalysisException("Workload Group properties can't be empty");
+        }
+
+        if (properties.containsKey(WorkloadGroup.INTERNAL_TYPE)) {
+            throw new AnalysisException(WorkloadGroup.INTERNAL_TYPE + " can not be create or modified ");
+        }
+
+        String tagStr = properties.get(WorkloadGroup.TAG);
+        if (!StringUtils.isEmpty(tagStr) && (WorkloadGroupMgr.DEFAULT_GROUP_NAME.equals(workloadGroupName)
+                || WorkloadGroupMgr.INTERNAL_GROUP_NAME.equals(workloadGroupName))) {
+            throw new AnalysisException(
+                    WorkloadGroupMgr.INTERNAL_GROUP_NAME + " and " + WorkloadGroupMgr.DEFAULT_GROUP_NAME
+                            + " group can not set tag");
+        }
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        // Create workload group
+        WorkloadGroup workloadGroup = WorkloadGroup.create(workloadGroupName, properties);
+        WorkloadGroupMgr workloadGroupMgr = Env.getCurrentEnv().getWorkloadGroupMgr();
+        workloadGroupMgr.createWorkloadGroup(workloadGroup, ifNotExists);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCreateWorkloadGroupCommand(this, context);
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.CREATE;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 81296b0b9b8204..05ee8548cfa93d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -47,6 +47,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableLikeCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateWorkloadGroupCommand;
 import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand;
 import org.apache.doris.nereids.trees.plans.commands.DeleteFromUsingCommand;
 import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand;
@@ -562,6 +563,10 @@ default R visitShowTableIdCommand(ShowTableIdCommand showTableIdCommand, C conte
         return visitCommand(showTableIdCommand, context);
     }
 
+    default R visitCreateWorkloadGroupCommand(CreateWorkloadGroupCommand createWorkloadGroupCommand, C context) {
+        return visitCommand(createWorkloadGroupCommand, context);
+    }
+
     default R visitShowEncryptKeysCommand(ShowEncryptKeysCommand showEncryptKeysCommand, C context) {
         return visitCommand(showEncryptKeysCommand, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
index 22cbcf4a0a1502..94695cc8d5b8ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
@@ -357,13 +357,12 @@ private String getWorkloadGroupNameAndCheckPriv(ConnectContext context) throws A
         return groupName;
     }
 
-    public void createWorkloadGroup(CreateWorkloadGroupStmt stmt) throws DdlException {
-        WorkloadGroup workloadGroup = WorkloadGroup.create(stmt.getWorkloadGroupName(), stmt.getProperties());
+    public void createWorkloadGroup(WorkloadGroup workloadGroup, boolean isIfNotExists) throws DdlException {
         String workloadGroupName = workloadGroup.getName();
         writeLock();
         try {
             if (nameToWorkloadGroup.containsKey(workloadGroupName)) {
-                if (stmt.isIfNotExists()) {
+                if (isIfNotExists) {
                     return;
                 }
                 throw new DdlException("workload group " + workloadGroupName + " already exist");
@@ -382,6 +381,11 @@ public void createWorkloadGroup(CreateWorkloadGroupStmt stmt) throws DdlExceptio
         LOG.info("Create workload group success: {}", workloadGroup);
     }
 
+    public void createWorkloadGroup(CreateWorkloadGroupStmt stmt) throws DdlException {
+        WorkloadGroup workloadGroup = WorkloadGroup.create(stmt.getWorkloadGroupName(), stmt.getProperties());
+        createWorkloadGroup(workloadGroup, stmt.isIfNotExists());
+    }
+
     public void createInternalWorkloadGroup() {
         Map<String, String> properties = Maps.newHashMap();
         // 100 is cgroup v2 default cpu_share value
diff --git a/regression-test/suites/workload_manager_p0/test_nereids_workload_test.groovy b/regression-test/suites/workload_manager_p0/test_nereids_workload_test.groovy
index 3f7c55f5c99cf9..680f5a28ca982e 100644
--- a/regression-test/suites/workload_manager_p0/test_nereids_workload_test.groovy
+++ b/regression-test/suites/workload_manager_p0/test_nereids_workload_test.groovy
@@ -18,10 +18,10 @@
 suite("test_nereids_workload_test") {
     sql "drop workload group if exists test_nereids_wg1;"
     sql "drop workload group if exists test_nereids_wg2;"
-    sql "create workload group test_nereids_wg1 properties('cpu_share'='1024');"
-    sql "create workload group test_nereids_wg2 properties('cpu_share'='1024');"
+    checkNereidsExecute("create workload group test_nereids_wg1 properties('cpu_share'='1024');")
+    checkNereidsExecute("create workload group test_nereids_wg2 properties('cpu_share'='1024');")
     qt_check_workload_check1("select NAME from information_schema.workload_groups where NAME='test_nereids_wg1';")
     checkNereidsExecute("drop workload group  test_nereids_wg1;")
     qt_check_workload_check2("select NAME from information_schema.workload_groups where NAME='test_nereids_wg1';")
     checkNereidsExecute("drop workload group if exists test_nereids_wg2;")
-}
\ No newline at end of file
+}

From b7776a8901ba6ad04ee01dead3b8cfca5d37ce48 Mon Sep 17 00:00:00 2001
From: TengJianPing <tengjianping@selectdb.com>
Date: Mon, 9 Dec 2024 15:08:52 +0800
Subject: [PATCH 235/399] [env](compile) open compile check for table writers
 (#45005)

---
 .../exec/format/table/iceberg/struct_like.h   |  5 +--
 be/src/vec/sink/vrow_distribution.cpp         | 16 ++++++--
 be/src/vec/sink/vrow_distribution.h           |  2 +-
 .../vec/sink/writer/async_result_writer.cpp   |  3 +-
 be/src/vec/sink/writer/async_result_writer.h  |  2 +-
 .../vec/sink/writer/iceberg/partition_data.h  | 15 ++------
 .../writer/iceberg/partition_transformers.h   | 38 +++++++++++++------
 .../writer/iceberg/viceberg_table_writer.cpp  |  1 +
 be/src/vec/sink/writer/vhive_table_writer.cpp |  1 +
 .../vec/sink/writer/vmysql_table_writer.cpp   |  8 ++--
 be/src/vec/sink/writer/vodbc_table_writer.cpp |  1 +
 be/src/vec/sink/writer/vtablet_writer.cpp     | 11 +++---
 be/src/vec/sink/writer/vtablet_writer_v2.cpp  |  3 +-
 13 files changed, 64 insertions(+), 42 deletions(-)

diff --git a/be/src/vec/exec/format/table/iceberg/struct_like.h b/be/src/vec/exec/format/table/iceberg/struct_like.h
index 23d394066fc639..cf02e2d1eb8a25 100644
--- a/be/src/vec/exec/format/table/iceberg/struct_like.h
+++ b/be/src/vec/exec/format/table/iceberg/struct_like.h
@@ -26,11 +26,8 @@ namespace iceberg {
 class StructLike {
 public:
     virtual ~StructLike() = default;
-    virtual int size() const = 0;
 
-    virtual std::any get(int pos) const = 0;
-
-    virtual void set(int pos, const std::any& value) = 0;
+    virtual std::any get(size_t pos) const = 0;
 };
 
 } // namespace iceberg
diff --git a/be/src/vec/sink/vrow_distribution.cpp b/be/src/vec/sink/vrow_distribution.cpp
index f374064c0af767..4a9790a4437272 100644
--- a/be/src/vec/sink/vrow_distribution.cpp
+++ b/be/src/vec/sink/vrow_distribution.cpp
@@ -25,6 +25,7 @@
 #include <memory>
 #include <string>
 
+#include "common/cast_set.h"
 #include "common/logging.h"
 #include "common/status.h"
 #include "runtime/client_cache.h"
@@ -225,7 +226,10 @@ void VRowDistribution::_filter_block_by_skip(vectorized::Block* block,
     auto& partition_ids = row_part_tablet_id.partition_ids;
     auto& tablet_ids = row_part_tablet_id.tablet_ids;
 
-    for (size_t i = 0; i < block->rows(); i++) {
+    auto rows = block->rows();
+    // row count of a block should not exceed UINT32_MAX
+    auto rows_uint32 = cast_set<uint32_t>(rows);
+    for (uint32_t i = 0; i < rows_uint32; i++) {
         if (!_skip[i]) {
             row_ids.emplace_back(i);
             partition_ids.emplace_back(_partitions[i]->id);
@@ -250,7 +254,10 @@ Status VRowDistribution::_filter_block_by_skip_and_where_clause(
     auto& tablet_ids = row_part_tablet_id.tablet_ids;
     if (const auto* nullable_column =
                 vectorized::check_and_get_column<vectorized::ColumnNullable>(*filter_column)) {
-        for (size_t i = 0; i < block->rows(); i++) {
+        auto rows = block->rows();
+        // row count of a block should not exceed UINT32_MAX
+        auto rows_uint32 = cast_set<uint32_t>(rows);
+        for (uint32_t i = 0; i < rows_uint32; i++) {
             if (nullable_column->get_bool_inline(i) && !_skip[i]) {
                 row_ids.emplace_back(i);
                 partition_ids.emplace_back(_partitions[i]->id);
@@ -267,7 +274,10 @@ Status VRowDistribution::_filter_block_by_skip_and_where_clause(
         _filter_block_by_skip(block, row_part_tablet_id);
     } else {
         const auto& filter = assert_cast<const vectorized::ColumnUInt8&>(*filter_column).get_data();
-        for (size_t i = 0; i < block->rows(); i++) {
+        auto rows = block->rows();
+        // row count of a block should not exceed UINT32_MAX
+        auto rows_uint32 = cast_set<uint32_t>(rows);
+        for (uint32_t i = 0; i < rows_uint32; i++) {
             if (filter[i] != 0 && !_skip[i]) {
                 row_ids.emplace_back(i);
                 partition_ids.emplace_back(_partitions[i]->id);
diff --git a/be/src/vec/sink/vrow_distribution.h b/be/src/vec/sink/vrow_distribution.h
index 40202556290ea8..6248a28dba5821 100644
--- a/be/src/vec/sink/vrow_distribution.h
+++ b/be/src/vec/sink/vrow_distribution.h
@@ -48,7 +48,7 @@ class VNodeChannel;
 // <row_idx, partition_id, tablet_id>
 class RowPartTabletIds {
 public:
-    std::vector<int64_t> row_ids;
+    std::vector<uint32_t> row_ids;
     std::vector<int64_t> partition_ids;
     std::vector<int64_t> tablet_ids;
 
diff --git a/be/src/vec/sink/writer/async_result_writer.cpp b/be/src/vec/sink/writer/async_result_writer.cpp
index c17b84b2dbe1de..65210a53ec3b55 100644
--- a/be/src/vec/sink/writer/async_result_writer.cpp
+++ b/be/src/vec/sink/writer/async_result_writer.cpp
@@ -32,6 +32,7 @@ class RowDescriptor;
 class TExpr;
 
 namespace vectorized {
+#include "common/compile_check_begin.h"
 
 AsyncResultWriter::AsyncResultWriter(const doris::vectorized::VExprContextSPtrs& output_expr_ctxs,
                                      std::shared_ptr<pipeline::Dependency> dep,
@@ -225,7 +226,7 @@ void AsyncResultWriter::_return_free_block(std::unique_ptr<Block> b) {
 }
 
 std::unique_ptr<Block> AsyncResultWriter::_get_free_block(doris::vectorized::Block* block,
-                                                          int rows) {
+                                                          size_t rows) {
     std::unique_ptr<Block> b;
     if (!_free_blocks.try_dequeue(b)) {
         b = block->create_same_struct_block(rows, true);
diff --git a/be/src/vec/sink/writer/async_result_writer.h b/be/src/vec/sink/writer/async_result_writer.h
index 36bca48358a8ab..513f2aa7984ca5 100644
--- a/be/src/vec/sink/writer/async_result_writer.h
+++ b/be/src/vec/sink/writer/async_result_writer.h
@@ -76,7 +76,7 @@ class AsyncResultWriter : public ResultWriter {
     Status _projection_block(Block& input_block, Block* output_block);
     const VExprContextSPtrs& _vec_output_expr_ctxs;
 
-    std::unique_ptr<Block> _get_free_block(Block*, int rows);
+    std::unique_ptr<Block> _get_free_block(Block*, size_t rows);
 
     void _return_free_block(std::unique_ptr<Block>);
 
diff --git a/be/src/vec/sink/writer/iceberg/partition_data.h b/be/src/vec/sink/writer/iceberg/partition_data.h
index 512dbd47904108..d3dfb1e8eccb87 100644
--- a/be/src/vec/sink/writer/iceberg/partition_data.h
+++ b/be/src/vec/sink/writer/iceberg/partition_data.h
@@ -21,31 +21,24 @@
 
 namespace doris {
 namespace vectorized {
+#include "common/compile_check_begin.h"
 
 class PartitionData : public iceberg::StructLike {
 public:
     explicit PartitionData(std::vector<std::any> partition_values)
             : _partition_values(std::move(partition_values)) {}
 
-    int size() const override { return _partition_values.size(); }
-
-    std::any get(int pos) const override {
-        if (pos < 0 || pos >= _partition_values.size()) {
+    std::any get(size_t pos) const override {
+        if (pos >= _partition_values.size()) {
             throw std::out_of_range("Index out of range");
         }
         return _partition_values[pos];
     }
 
-    void set(int pos, const std::any& value) override {
-        if (pos < 0 || pos >= _partition_values.size()) {
-            throw std::out_of_range("Index out of range");
-        }
-        _partition_values[pos] = value;
-    }
-
 private:
     std::vector<std::any> _partition_values;
 };
 
 } // namespace vectorized
 } // namespace doris
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/sink/writer/iceberg/partition_transformers.h b/be/src/vec/sink/writer/iceberg/partition_transformers.h
index 84ee3029cdd18f..79eb385b298a8f 100644
--- a/be/src/vec/sink/writer/iceberg/partition_transformers.h
+++ b/be/src/vec/sink/writer/iceberg/partition_transformers.h
@@ -30,6 +30,7 @@ class PartitionField;
 }; // namespace iceberg
 
 namespace vectorized {
+#include "common/compile_check_begin.h"
 
 class IColumn;
 class PartitionColumnTransform;
@@ -174,7 +175,7 @@ class StringTruncatePartitionColumnTransform : public PartitionColumnTransform {
         temp_arguments[0] = 0; // str column
         temp_arguments[1] = 1; // pos
         temp_arguments[2] = 2; // width
-        size_t result_column_id = 3;
+        uint32_t result_column_id = 3;
 
         SubstringUtil::substring_execute(temp_block, temp_arguments, result_column_id,
                                          temp_block.rows());
@@ -623,9 +624,9 @@ class DateBucketPartitionColumnTransform : public PartitionColumnTransform {
             DateV2Value<DateV2ValueType> value =
                     binary_cast<uint32_t, DateV2Value<DateV2ValueType>>(*(UInt32*)p_in);
 
-            int32_t days_from_unix_epoch = value.daynr() - 719528;
-            Int64 long_value = static_cast<Int64>(days_from_unix_epoch);
-            uint32_t hash_value = HashUtil::murmur_hash3_32(&long_value, sizeof(long_value), 0);
+            int64_t days_from_unix_epoch = value.daynr() - 719528;
+            uint32_t hash_value = HashUtil::murmur_hash3_32(&days_from_unix_epoch,
+                                                            sizeof(days_from_unix_epoch), 0);
 
             *p_out = (hash_value & INT32_MAX) % _bucket_num;
             ++p_in;
@@ -836,7 +837,9 @@ class DateYearPartitionColumnTransform : public PartitionColumnTransform {
         while (p_in < end_in) {
             DateV2Value<DateV2ValueType> value =
                     binary_cast<uint32_t, DateV2Value<DateV2ValueType>>(*(UInt32*)p_in);
-            *p_out = datetime_diff<YEAR>(PartitionColumnTransformUtils::epoch_date(), value);
+            // datetime_diff<YEAR> actually returns int
+            *p_out = cast_set<int, int64_t, false>(
+                    datetime_diff<YEAR>(PartitionColumnTransformUtils::epoch_date(), value));
             ++p_in;
             ++p_out;
         }
@@ -906,7 +909,9 @@ class TimestampYearPartitionColumnTransform : public PartitionColumnTransform {
         while (p_in < end_in) {
             DateV2Value<DateTimeV2ValueType> value =
                     binary_cast<uint64_t, DateV2Value<DateTimeV2ValueType>>(*(UInt64*)p_in);
-            *p_out = datetime_diff<YEAR>(PartitionColumnTransformUtils::epoch_datetime(), value);
+            // datetime_diff<YEAR> actually returns int
+            *p_out = cast_set<int, int64_t, false>(
+                    datetime_diff<YEAR>(PartitionColumnTransformUtils::epoch_datetime(), value));
             ++p_in;
             ++p_out;
         }
@@ -976,7 +981,9 @@ class DateMonthPartitionColumnTransform : public PartitionColumnTransform {
         while (p_in < end_in) {
             DateV2Value<DateV2ValueType> value =
                     binary_cast<uint32_t, DateV2Value<DateV2ValueType>>(*(UInt32*)p_in);
-            *p_out = datetime_diff<MONTH>(PartitionColumnTransformUtils::epoch_date(), value);
+            // datetime_diff<MONTH> actually returns int
+            *p_out = cast_set<int, int64_t, false>(
+                    datetime_diff<MONTH>(PartitionColumnTransformUtils::epoch_date(), value));
             ++p_in;
             ++p_out;
         }
@@ -1046,7 +1053,9 @@ class TimestampMonthPartitionColumnTransform : public PartitionColumnTransform {
         while (p_in < end_in) {
             DateV2Value<DateTimeV2ValueType> value =
                     binary_cast<uint64_t, DateV2Value<DateTimeV2ValueType>>(*(UInt64*)p_in);
-            *p_out = datetime_diff<MONTH>(PartitionColumnTransformUtils::epoch_datetime(), value);
+            // datetime_diff<MONTH> actually returns int
+            *p_out = cast_set<int, int64_t, false>(
+                    datetime_diff<MONTH>(PartitionColumnTransformUtils::epoch_datetime(), value));
             ++p_in;
             ++p_out;
         }
@@ -1116,7 +1125,9 @@ class DateDayPartitionColumnTransform : public PartitionColumnTransform {
         while (p_in < end_in) {
             DateV2Value<DateV2ValueType> value =
                     binary_cast<uint32_t, DateV2Value<DateV2ValueType>>(*(UInt32*)p_in);
-            *p_out = datetime_diff<DAY>(PartitionColumnTransformUtils::epoch_date(), value);
+            // datetime_diff<DAY> actually returns int
+            *p_out = cast_set<int, int64_t, false>(
+                    datetime_diff<DAY>(PartitionColumnTransformUtils::epoch_date(), value));
             ++p_in;
             ++p_out;
         }
@@ -1192,7 +1203,9 @@ class TimestampDayPartitionColumnTransform : public PartitionColumnTransform {
         while (p_in < end_in) {
             DateV2Value<DateTimeV2ValueType> value =
                     binary_cast<uint64_t, DateV2Value<DateTimeV2ValueType>>(*(UInt64*)p_in);
-            *p_out = datetime_diff<DAY>(PartitionColumnTransformUtils::epoch_datetime(), value);
+            // datetime_diff<DAY> actually returns int
+            *p_out = cast_set<int, int64_t, false>(
+                    datetime_diff<DAY>(PartitionColumnTransformUtils::epoch_datetime(), value));
             ++p_in;
             ++p_out;
         }
@@ -1267,7 +1280,9 @@ class TimestampHourPartitionColumnTransform : public PartitionColumnTransform {
         while (p_in < end_in) {
             DateV2Value<DateTimeV2ValueType> value =
                     binary_cast<uint64_t, DateV2Value<DateTimeV2ValueType>>(*(UInt64*)p_in);
-            *p_out = datetime_diff<HOUR>(PartitionColumnTransformUtils::epoch_datetime(), value);
+            // hour diff would't overflow int32
+            *p_out = cast_set<int, int64_t, false>(
+                    datetime_diff<HOUR>(PartitionColumnTransformUtils::epoch_datetime(), value));
             ++p_in;
             ++p_out;
         }
@@ -1333,3 +1348,4 @@ class VoidPartitionColumnTransform : public PartitionColumnTransform {
 
 } // namespace vectorized
 } // namespace doris
+#include "common/compile_check_end.h"
diff --git a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
index 280cf8b810724c..29c97b59ea4dba 100644
--- a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
+++ b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
@@ -31,6 +31,7 @@
 
 namespace doris {
 namespace vectorized {
+#include "common/compile_check_begin.h"
 
 VIcebergTableWriter::VIcebergTableWriter(const TDataSink& t_sink,
                                          const VExprContextSPtrs& output_expr_ctxs,
diff --git a/be/src/vec/sink/writer/vhive_table_writer.cpp b/be/src/vec/sink/writer/vhive_table_writer.cpp
index 6eb478c01b7f92..17e6bba326f707 100644
--- a/be/src/vec/sink/writer/vhive_table_writer.cpp
+++ b/be/src/vec/sink/writer/vhive_table_writer.cpp
@@ -28,6 +28,7 @@
 
 namespace doris {
 namespace vectorized {
+#include "common/compile_check_begin.h"
 
 VHiveTableWriter::VHiveTableWriter(const TDataSink& t_sink,
                                    const VExprContextSPtrs& output_expr_ctxs,
diff --git a/be/src/vec/sink/writer/vmysql_table_writer.cpp b/be/src/vec/sink/writer/vmysql_table_writer.cpp
index a0d47ffec1ede9..e4529c59c9abfb 100644
--- a/be/src/vec/sink/writer/vmysql_table_writer.cpp
+++ b/be/src/vec/sink/writer/vmysql_table_writer.cpp
@@ -51,6 +51,7 @@
 
 namespace doris {
 namespace vectorized {
+#include "common/compile_check_begin.h"
 
 std::string MysqlConnInfo::debug_string() const {
     std::stringstream ss;
@@ -124,9 +125,9 @@ Status VMysqlTableWriter::write(RuntimeState* state, vectorized::Block& block) {
 Status VMysqlTableWriter::_insert_row(vectorized::Block& block, size_t row) {
     _insert_stmt_buffer.clear();
     fmt::format_to(_insert_stmt_buffer, "INSERT INTO {} VALUES (", _conn_info.table_name);
-    int num_columns = _vec_output_expr_ctxs.size();
+    size_t num_columns = _vec_output_expr_ctxs.size();
 
-    for (int i = 0; i < num_columns; ++i) {
+    for (size_t i = 0; i < num_columns; ++i) {
         auto& column_ptr = block.get_by_position(i).column;
         auto& type_ptr = block.get_by_position(i).type;
 
@@ -236,8 +237,7 @@ Status VMysqlTableWriter::_insert_row(vectorized::Block& block, size_t row) {
             break;
         }
         case TYPE_DATETIMEV2: {
-            uint32_t int_val =
-                    assert_cast<const vectorized::ColumnUInt64&>(*column).get_data()[row];
+            auto int_val = assert_cast<const vectorized::ColumnUInt64&>(*column).get_data()[row];
             DateV2Value<DateTimeV2ValueType> value =
                     binary_cast<uint64_t, DateV2Value<DateTimeV2ValueType>>(int_val);
 
diff --git a/be/src/vec/sink/writer/vodbc_table_writer.cpp b/be/src/vec/sink/writer/vodbc_table_writer.cpp
index 19cb2e50109890..d99dfc56aaad9d 100644
--- a/be/src/vec/sink/writer/vodbc_table_writer.cpp
+++ b/be/src/vec/sink/writer/vodbc_table_writer.cpp
@@ -29,6 +29,7 @@
 
 namespace doris {
 namespace vectorized {
+#include "common/compile_check_begin.h"
 
 ODBCConnectorParam VOdbcTableWriter::create_connect_param(const doris::TDataSink& t_sink) {
     const TOdbcTableSink& t_odbc_sink = t_sink.odbc_table_sink;
diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp
index 504ffb9cb749bf..55b6845b6bc871 100644
--- a/be/src/vec/sink/writer/vtablet_writer.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer.cpp
@@ -92,6 +92,7 @@ namespace doris {
 class TExpr;
 
 namespace vectorized {
+#include "common/compile_check_begin.h"
 
 bvar::Adder<int64_t> g_sink_write_bytes;
 bvar::PerSecond<bvar::Adder<int64_t>> g_sink_write_bytes_per_second("sink_throughput_byte",
@@ -662,14 +663,14 @@ void VNodeChannel::try_send_pending_block(RuntimeState* state) {
             _send_block_callback->clear_in_flight();
             return;
         }
-        if (compressed_bytes >= double(config::brpc_max_body_size) * 0.95F) {
+        if (double(compressed_bytes) >= double(config::brpc_max_body_size) * 0.95F) {
             LOG(WARNING) << "send block too large, this rpc may failed. send size: "
                          << compressed_bytes << ", threshold: " << config::brpc_max_body_size
                          << ", " << channel_info();
         }
     }
 
-    int remain_ms = _rpc_timeout_ms - _timeout_watch.elapsed_time() / NANOS_PER_MILLIS;
+    auto remain_ms = _rpc_timeout_ms - _timeout_watch.elapsed_time() / NANOS_PER_MILLIS;
     if (UNLIKELY(remain_ms < config::min_load_rpc_timeout_ms)) {
         if (remain_ms <= 0 && !request->eos()) {
             cancel(fmt::format("{}, err: timeout", channel_info()));
@@ -847,7 +848,7 @@ void VNodeChannel::_add_block_success_callback(const PTabletWriterAddBlockResult
     if (result.has_load_channel_profile()) {
         TRuntimeProfileTree tprofile;
         const auto* buf = (const uint8_t*)result.load_channel_profile().data();
-        uint32_t len = result.load_channel_profile().size();
+        auto len = cast_set<uint32_t>(result.load_channel_profile().size());
         auto st = deserialize_thrift_msg(buf, &len, false, &tprofile);
         if (st.ok()) {
             _state->load_channel_profile()->update(tprofile);
@@ -917,7 +918,7 @@ void VNodeChannel::cancel(const std::string& cancel_msg) {
             PTabletWriterCancelRequest,
             DummyBrpcCallback<PTabletWriterCancelResult>>::create_unique(request, cancel_callback);
 
-    int remain_ms = _rpc_timeout_ms - _timeout_watch.elapsed_time() / NANOS_PER_MILLIS;
+    auto remain_ms = _rpc_timeout_ms - _timeout_watch.elapsed_time() / NANOS_PER_MILLIS;
     if (UNLIKELY(remain_ms < config::min_load_rpc_timeout_ms)) {
         remain_ms = config::min_load_rpc_timeout_ms;
     }
@@ -1706,7 +1707,7 @@ void VTabletWriter::_generate_one_index_channel_payload(
 
     size_t row_cnt = row_ids.size();
 
-    for (int i = 0; i < row_ids.size(); i++) {
+    for (size_t i = 0; i < row_ids.size(); i++) {
         // (tablet_id, VNodeChannel) where this tablet locate
         auto it = _channels[index_idx]->_channels_by_tablet.find(tablet_ids[i]);
         DCHECK(it != _channels[index_idx]->_channels_by_tablet.end())
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index cd196a8f2b3df2..f3ad6b1d5e3f0c 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -55,6 +55,7 @@
 #include "vec/sink/vtablet_finder.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 VTabletWriterV2::VTabletWriterV2(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs,
                                  std::shared_ptr<pipeline::Dependency> dep,
@@ -359,7 +360,7 @@ void VTabletWriterV2::_generate_rows_for_tablet(std::vector<RowPartTabletIds>& r
         auto& partition_ids = row_part_tablet_ids[index_idx].partition_ids;
         auto& tablet_ids = row_part_tablet_ids[index_idx].tablet_ids;
 
-        for (int i = 0; i < row_ids.size(); i++) {
+        for (size_t i = 0; i < row_ids.size(); i++) {
             auto& tablet_id = tablet_ids[i];
             auto it = rows_for_tablet.find(tablet_id);
             if (it == rows_for_tablet.end()) {

From 63cc1af7d3a6b292435d24ec9763385abc0181fe Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Mon, 9 Dec 2024 15:16:28 +0800
Subject: [PATCH 236/399] [feat](clone) Speed clone tablet via batch small file
 downloading (#45061)

Downloading small files is too slow and might cause the clone tablet
task to time out. This PR supports a batch downloading API to speed up
the downloading of small files.

Before

```
succeed to copy tablet 10088, total file size: 19256126 B, cost: 78674 ms, rate: 0.244758 MB/s
```

After

```
succeed to copy tablet 30157, total files: 20006, total file size: 19311624 B, cost: 4016 ms, rate: 4.80867 MB/s
```
---
 be/src/common/config.cpp                      |   2 +
 be/src/common/config.h                        |   2 +
 be/src/gutil/strings/stringpiece.h            |   6 +
 be/src/http/action/batch_download_action.cpp  | 216 +++++++++++++++
 be/src/http/action/batch_download_action.h    |  65 +++++
 be/src/http/action/download_binlog_action.cpp |   3 -
 be/src/http/http_channel.cpp                  |  56 +++-
 be/src/http/http_channel.h                    |   8 +
 be/src/http/http_client.cpp                   | 247 +++++++++++++++++-
 be/src/http/http_client.h                     |   5 +
 be/src/http/utils.cpp                         | 130 ++++++++-
 be/src/http/utils.h                           |  15 +-
 be/src/olap/task/engine_clone_task.cpp        | 149 +++++++++--
 be/src/olap/task/engine_clone_task.h          |   3 +
 be/src/service/http_service.cpp               |  11 +
 be/test/http/http_client_test.cpp             |  98 ++++++-
 16 files changed, 975 insertions(+), 41 deletions(-)
 create mode 100644 be/src/http/action/batch_download_action.cpp
 create mode 100644 be/src/http/action/batch_download_action.h

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index b3e7d0bce5ee4d..1528d9091c2f43 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -250,6 +250,8 @@ DEFINE_mInt32(max_download_speed_kbps, "50000");
 DEFINE_mInt32(download_low_speed_limit_kbps, "50");
 // download low speed time(seconds)
 DEFINE_mInt32(download_low_speed_time, "300");
+// whether to download small files in batch
+DEFINE_mBool(enable_batch_download, "false");
 
 DEFINE_String(sys_log_dir, "");
 DEFINE_String(user_function_dir, "${DORIS_HOME}/lib/udf");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 59fc61e8cb3e5a..3bdbeb95edb65d 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -298,6 +298,8 @@ DECLARE_mInt32(max_download_speed_kbps);
 DECLARE_mInt32(download_low_speed_limit_kbps);
 // download low speed time(seconds)
 DECLARE_mInt32(download_low_speed_time);
+// whether to download small files in batch.
+DECLARE_mBool(enable_batch_download);
 
 // deprecated, use env var LOG_DIR in be.conf
 DECLARE_String(sys_log_dir);
diff --git a/be/src/gutil/strings/stringpiece.h b/be/src/gutil/strings/stringpiece.h
index 38e36a27099279..7a4ebabbf098e7 100644
--- a/be/src/gutil/strings/stringpiece.h
+++ b/be/src/gutil/strings/stringpiece.h
@@ -149,6 +149,12 @@ class StringPiece {
         assert(length <= static_cast<size_t>(std::numeric_limits<int>::max()));
         length_ = static_cast<int>(length);
     }
+    StringPiece(std::string_view view) // NOLINT(runtime/explicit)
+            : ptr_(view.data()), length_(0) {
+        size_t length = view.size();
+        assert(length <= static_cast<size_t>(std::numeric_limits<int>::max()));
+        length_ = static_cast<int>(length);
+    }
     StringPiece(const char* offset, int len) : ptr_(offset), length_(len) { assert(len >= 0); }
 
     // Substring of another StringPiece.
diff --git a/be/src/http/action/batch_download_action.cpp b/be/src/http/action/batch_download_action.cpp
new file mode 100644
index 00000000000000..d486883e90be28
--- /dev/null
+++ b/be/src/http/action/batch_download_action.cpp
@@ -0,0 +1,216 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "http/action/batch_download_action.h"
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "common/config.h"
+#include "common/logging.h"
+#include "common/status.h"
+#include "gutil/strings/split.h"
+#include "http/http_channel.h"
+#include "http/http_method.h"
+#include "http/http_request.h"
+#include "http/utils.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/exec_env.h"
+#include "util/security.h"
+
+namespace doris {
+namespace {
+const std::string CHECK_PARAMETER = "check";
+const std::string LIST_PARAMETER = "list";
+const std::string DIR_PARAMETER = "dir";
+const std::string TOKEN_PARAMETER = "token";
+} // namespace
+
+BatchDownloadAction::BatchDownloadAction(
+        ExecEnv* exec_env, std::shared_ptr<bufferevent_rate_limit_group> rate_limit_group,
+        const std::vector<std::string>& allow_dirs)
+        : HttpHandlerWithAuth(exec_env), _rate_limit_group(std::move(rate_limit_group)) {
+    for (const auto& dir : allow_dirs) {
+        std::string p;
+        Status st = io::global_local_filesystem()->canonicalize(dir, &p);
+        if (!st.ok()) {
+            continue;
+        }
+        _allow_paths.emplace_back(std::move(p));
+    }
+}
+
+void BatchDownloadAction::handle(HttpRequest* req) {
+    if (VLOG_CRITICAL_IS_ON) {
+        VLOG_CRITICAL << "accept one batch download request " << req->debug_string();
+    }
+
+    if (req->param(CHECK_PARAMETER) == "true") {
+        // For API support check
+        HttpChannel::send_reply(req, "OK");
+        return;
+    }
+
+    // Get 'dir' parameter, then assembly file absolute path
+    const std::string& dir_path = req->param(DIR_PARAMETER);
+    if (dir_path.empty()) {
+        std::string error_msg =
+                std::string("parameter " + DIR_PARAMETER + " not specified in url.");
+        LOG(WARNING) << "handle batch download request: " << error_msg
+                     << ", url: " << mask_token(req->uri());
+        HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, error_msg);
+        return;
+    }
+
+    if (dir_path.find("..") != std::string::npos) {
+        std::string error_msg = "Not allowed to read relative path: " + dir_path;
+        LOG(WARNING) << "handle batch download request: " << error_msg
+                     << ", url: " << mask_token(req->uri());
+        HttpChannel::send_reply(req, HttpStatus::FORBIDDEN, error_msg);
+        return;
+    }
+
+    Status status;
+    if (config::enable_token_check) {
+        status = _check_token(req);
+        if (!status.ok()) {
+            std::string error_msg = status.to_string();
+            if (status.is<ErrorCode::NOT_AUTHORIZED>()) {
+                HttpChannel::send_reply(req, HttpStatus::UNAUTHORIZED, error_msg);
+                return;
+            } else {
+                HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, error_msg);
+                return;
+            }
+        }
+    }
+
+    status = _check_path_is_allowed(dir_path);
+    if (!status.ok()) {
+        std::string error_msg = status.to_string();
+        if (status.is<ErrorCode::NOT_FOUND>() || status.is<ErrorCode::IO_ERROR>()) {
+            HttpChannel::send_reply(req, HttpStatus::NOT_FOUND, error_msg);
+            return;
+        } else if (status.is<ErrorCode::NOT_AUTHORIZED>()) {
+            HttpChannel::send_reply(req, HttpStatus::UNAUTHORIZED, error_msg);
+            return;
+        } else {
+            HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, error_msg);
+            return;
+        }
+    }
+
+    bool is_dir = false;
+    status = io::global_local_filesystem()->is_directory(dir_path, &is_dir);
+    if (!status.ok()) {
+        LOG(WARNING) << "handle batch download request: " << status.to_string()
+                     << ", url: " << mask_token(req->uri());
+        HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status.to_string());
+        return;
+    }
+
+    if (!is_dir) {
+        std::string error_msg = fmt::format("The requested path is not a directory: {}", dir_path);
+        LOG(WARNING) << "handle batch download request: " << error_msg
+                     << ", url: " << mask_token(req->uri());
+        HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, error_msg);
+        return;
+    }
+
+    _handle(req, dir_path);
+
+    VLOG_CRITICAL << "deal with batch download request finished! ";
+}
+
+void BatchDownloadAction::_handle(HttpRequest* req, const std::string& dir_path) {
+    bool is_list_request = req->param(LIST_PARAMETER) == "true";
+    if (is_list_request) {
+        // return the list of files in the specified directory
+        bool is_acquire_filesize = true;
+        do_dir_response(dir_path, req, is_acquire_filesize);
+    } else {
+        _handle_batch_download(req, dir_path);
+    }
+}
+
+void BatchDownloadAction::_handle_batch_download(HttpRequest* req, const std::string& dir_path) {
+    std::vector<std::string> files =
+            strings::Split(req->get_request_body(), "\n", strings::SkipWhitespace());
+    if (files.empty()) {
+        std::string error_msg = "No file specified in request body.";
+        LOG(WARNING) << "handle batch download request: " << error_msg
+                     << ", url: " << mask_token(req->uri());
+        HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, error_msg);
+        return;
+    }
+
+    if (files.size() > 64) {
+        std::string error_msg =
+                "The number of files to download in a batch should be less than 64.";
+        LOG(WARNING) << "handle batch download request: " << error_msg
+                     << ", url: " << mask_token(req->uri());
+        HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, error_msg);
+        return;
+    }
+
+    for (const auto& file : files) {
+        if (file.find('/') != std::string::npos) {
+            std::string error_msg =
+                    fmt::format("Not allowed to read relative path: {}, dir: {}", file, dir_path);
+            LOG(WARNING) << "handle batch download request: " << error_msg
+                         << ", url: " << mask_token(req->uri());
+            HttpChannel::send_reply(req, HttpStatus::FORBIDDEN, error_msg);
+            return;
+        }
+    }
+
+    HttpChannel::send_files(req, dir_path, std::move(files));
+}
+
+Status BatchDownloadAction::_check_token(HttpRequest* req) {
+    const std::string& token_str = req->param(TOKEN_PARAMETER);
+    if (token_str.empty()) {
+        LOG(WARNING) << "token is not specified in request. url: " << mask_token(req->uri());
+        return Status::NotAuthorized("token is not specified.");
+    }
+
+    const std::string& local_token = _exec_env->token();
+    if (token_str != local_token) {
+        LOG(WARNING) << "invalid download token: " << mask_token(token_str)
+                     << ", local token: " << mask_token(local_token)
+                     << ", url: " << mask_token(req->uri());
+        return Status::NotAuthorized("invalid token {}", mask_token(token_str));
+    }
+
+    return Status::OK();
+}
+
+Status BatchDownloadAction::_check_path_is_allowed(const std::string& file_path) {
+    std::string canonical_file_path;
+    RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize(file_path, &canonical_file_path));
+    for (auto& allow_path : _allow_paths) {
+        if (io::LocalFileSystem::contain_path(allow_path, canonical_file_path)) {
+            return Status::OK();
+        }
+    }
+
+    return Status::NotAuthorized("file path is not allowed: {}", canonical_file_path);
+}
+
+} // end namespace doris
diff --git a/be/src/http/action/batch_download_action.h b/be/src/http/action/batch_download_action.h
new file mode 100644
index 00000000000000..f0b7e3576b9937
--- /dev/null
+++ b/be/src/http/action/batch_download_action.h
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "common/status.h"
+#include "http/http_handler.h"
+#include "http/http_handler_with_auth.h"
+#include "util/threadpool.h"
+
+struct bufferevent_rate_limit_group;
+
+namespace doris {
+
+class ExecEnv;
+class HttpRequest;
+
+// A simple handler that serves incoming HTTP requests of batching file-download to send their
+// respective HTTP responses.
+//
+// We use parameter named 'dir' to specify the static resource path, it is an absolute path.
+//
+// In HEAD request, then this handler will return the list of files in the specified directory.
+//
+// In GET request, the file names to download are specified in the request body as a list of strings,
+// separated by '\n'. To avoid cost resource, the maximum number of files to download in a batch is 64.
+class BatchDownloadAction : public HttpHandlerWithAuth {
+public:
+    BatchDownloadAction(ExecEnv* exec_env,
+                        std::shared_ptr<bufferevent_rate_limit_group> rate_limit_group,
+                        const std::vector<std::string>& allow_dirs);
+
+    ~BatchDownloadAction() override = default;
+
+    void handle(HttpRequest* req) override;
+
+private:
+    Status _check_token(HttpRequest* req);
+    Status _check_path_is_allowed(const std::string& path);
+
+    void _handle(HttpRequest* req, const std::string& dir_path);
+    void _handle_batch_download(HttpRequest* req, const std::string& dir_path);
+
+    std::vector<std::string> _allow_paths;
+    std::shared_ptr<bufferevent_rate_limit_group> _rate_limit_group;
+};
+
+} // end namespace doris
diff --git a/be/src/http/action/download_binlog_action.cpp b/be/src/http/action/download_binlog_action.cpp
index 54701c5e463481..372f840401c4ad 100644
--- a/be/src/http/action/download_binlog_action.cpp
+++ b/be/src/http/action/download_binlog_action.cpp
@@ -21,11 +21,9 @@
 #include <fmt/ranges.h>
 
 #include <cstdint>
-#include <limits>
 #include <stdexcept>
 #include <string_view>
 #include <utility>
-#include <vector>
 
 #include "common/config.h"
 #include "common/logging.h"
@@ -34,7 +32,6 @@
 #include "http/utils.h"
 #include "io/fs/local_file_system.h"
 #include "olap/storage_engine.h"
-#include "olap/tablet.h"
 #include "olap/tablet_manager.h"
 #include "runtime/exec_env.h"
 
diff --git a/be/src/http/http_channel.cpp b/be/src/http/http_channel.cpp
index 96679195316dac..312f1ab9286909 100644
--- a/be/src/http/http_channel.cpp
+++ b/be/src/http/http_channel.cpp
@@ -20,8 +20,8 @@
 #include <event2/buffer.h>
 #include <event2/bufferevent.h>
 #include <event2/http.h>
+#include <event2/http_struct.h>
 
-#include <algorithm>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -57,7 +57,7 @@ void HttpChannel::send_reply(HttpRequest* request, HttpStatus status) {
 }
 
 void HttpChannel::send_reply(HttpRequest* request, HttpStatus status, const std::string& content) {
-    auto evb = evbuffer_new();
+    auto* evb = evbuffer_new();
     std::string compressed_content;
     if (compress_content(request->header(HttpHeaders::ACCEPT_ENCODING), content,
                          &compressed_content)) {
@@ -72,7 +72,7 @@ void HttpChannel::send_reply(HttpRequest* request, HttpStatus status, const std:
 
 void HttpChannel::send_file(HttpRequest* request, int fd, size_t off, size_t size,
                             bufferevent_rate_limit_group* rate_limit_group) {
-    auto evb = evbuffer_new();
+    auto* evb = evbuffer_new();
     evbuffer_add_file(evb, fd, off, size);
     auto* evhttp_request = request->get_evhttp_request();
     if (rate_limit_group) {
@@ -84,6 +84,56 @@ void HttpChannel::send_file(HttpRequest* request, int fd, size_t off, size_t siz
     evbuffer_free(evb);
 }
 
+void HttpChannel::send_files(HttpRequest* request, const std::string& root_dir,
+                             std::vector<std::string> local_files,
+                             bufferevent_rate_limit_group* rate_limit_group) {
+    if (rate_limit_group) {
+        auto* evhttp_request = request->get_evhttp_request();
+        auto* evhttp_connection = evhttp_request_get_connection(evhttp_request);
+        auto* buffer_event = evhttp_connection_get_bufferevent(evhttp_connection);
+        bufferevent_add_to_rate_limit_group(buffer_event, rate_limit_group);
+    }
+
+    send_files(request, root_dir, std::move(local_files));
+}
+
+void HttpChannel::send_files(HttpRequest* request, const std::string& root_dir,
+                             std::vector<std::string> local_files) {
+    std::unique_ptr<evbuffer, decltype(&evbuffer_free)> evb(evbuffer_new(), &evbuffer_free);
+    for (const std::string& file : local_files) {
+        std::string file_path = fmt::format("{}/{}", root_dir, file);
+        int fd = open(file_path.c_str(), O_RDONLY);
+        if (fd < 0) {
+            std::string error_msg = "Failed to open file: " + file_path;
+            LOG(WARNING) << "http channel send files: " << error_msg;
+            HttpChannel::send_reply(request, HttpStatus::NOT_FOUND, error_msg);
+            return;
+        }
+        struct stat st;
+        auto res = fstat(fd, &st);
+        if (res < 0) {
+            close(fd);
+            std::string error_msg = "Failed to open file: " + file_path;
+            LOG(WARNING) << "http channel send files: " << error_msg;
+            HttpChannel::send_reply(request, HttpStatus::NOT_FOUND, error_msg);
+            return;
+        }
+
+        int64_t file_size = st.st_size;
+        VLOG_DEBUG << "http channel send file " << file_path << ", size: " << file_size;
+
+        evbuffer_add_printf(evb.get(), "File-Name: %s\r\n", file.c_str());
+        evbuffer_add_printf(evb.get(), "Content-Length: %ld\r\n", file_size);
+        evbuffer_add_printf(evb.get(), "\r\n");
+        if (file_size > 0) {
+            evbuffer_add_file(evb.get(), fd, 0, file_size);
+        }
+    }
+
+    evhttp_send_reply(request->get_evhttp_request(), HttpStatus::OK,
+                      default_reason(HttpStatus::OK).c_str(), evb.get());
+}
+
 bool HttpChannel::compress_content(const std::string& accept_encoding, const std::string& input,
                                    std::string* output) {
     // Don't bother compressing empty content.
diff --git a/be/src/http/http_channel.h b/be/src/http/http_channel.h
index ee1e6c0888f1d3..0d5e5d4260af8c 100644
--- a/be/src/http/http_channel.h
+++ b/be/src/http/http_channel.h
@@ -20,6 +20,7 @@
 #include <stddef.h>
 
 #include <string>
+#include <vector>
 
 #include "http/http_status.h"
 
@@ -47,6 +48,13 @@ class HttpChannel {
     static void send_file(HttpRequest* request, int fd, size_t off, size_t size,
                           bufferevent_rate_limit_group* rate_limit_group = nullptr);
 
+    static void send_files(HttpRequest* request, const std::string& root_dir,
+                           std::vector<std::string> local_files,
+                           bufferevent_rate_limit_group* rate_limit_group);
+
+    static void send_files(HttpRequest* request, const std::string& root_dir,
+                           std::vector<std::string> local_files);
+
     static bool compress_content(const std::string& accept_encoding, const std::string& input,
                                  std::string* output);
 };
diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp
index fc4c997fce8397..767377cea3f365 100644
--- a/be/src/http/http_client.cpp
+++ b/be/src/http/http_client.cpp
@@ -24,14 +24,225 @@
 #include <ostream>
 
 #include "common/config.h"
+#include "common/status.h"
 #include "http/http_headers.h"
-#include "http/http_status.h"
 #include "runtime/exec_env.h"
 #include "util/security.h"
 #include "util/stack_util.h"
 
 namespace doris {
 
+class MultiFileSplitter {
+public:
+    MultiFileSplitter(std::string local_dir, std::unordered_set<std::string> expected_files)
+            : _local_dir_path(std::move(local_dir)), _expected_files(std::move(expected_files)) {}
+    ~MultiFileSplitter() {
+        if (_fd >= 0) {
+            close(_fd);
+        }
+
+        if (!_status.ok() && !downloaded_files.empty()) {
+            LOG(WARNING) << "download files to " << _local_dir_path << " failed, try remove the "
+                         << downloaded_files.size() << " downloaded files";
+            for (const auto& file : downloaded_files) {
+                remove(file.c_str());
+            }
+        }
+    }
+
+    bool append(const char* data, size_t length) {
+        // Already failed.
+        if (!_status.ok()) {
+            return false;
+        }
+
+        std::string buf;
+        if (!_buffer.empty()) {
+            buf.swap(_buffer);
+            buf.append(data, length);
+            data = buf.data();
+            length = buf.size();
+        }
+        return append_inner(data, length);
+    }
+
+    Status finish() {
+        if (_status.ok()) {
+            _status = finish_inner();
+        }
+
+        return _status;
+    }
+
+private:
+    bool append_inner(const char* data, size_t length) {
+        while (length > 0) {
+            int consumed = 0;
+            if (_is_reading_header) {
+                consumed = parse_header(data, length);
+            } else {
+                consumed = append_file(data, length);
+            }
+
+            if (consumed < 0) {
+                return false;
+            }
+
+            DCHECK(consumed <= length);
+            data += consumed;
+            length -= consumed;
+        }
+        return true;
+    }
+
+    int parse_header(const char* data, size_t length) {
+        DCHECK(_fd < 0);
+
+        std::string_view buf(data, length);
+        size_t pos = buf.find("\r\n\r\n");
+        if (pos == std::string::npos) {
+            _buffer.append(data, length);
+            return static_cast<int>(length);
+        }
+
+        // header already read.
+        _is_reading_header = false;
+
+        bool has_file_name = false;
+        bool has_file_size = false;
+        std::string_view header = buf.substr(0, pos);
+        std::vector<std::string> headers =
+                strings::Split(header, "\r\n", strings::SkipWhitespace());
+        for (auto& s : headers) {
+            size_t header_pos = s.find(':');
+            if (header_pos == std::string::npos) {
+                continue;
+            }
+            std::string_view header_view(s);
+            std::string_view key = header_view.substr(0, header_pos);
+            std::string_view value = header_view.substr(header_pos + 1);
+            if (value.starts_with(' ')) {
+                value.remove_prefix(std::min(value.find_first_not_of(' '), value.size()));
+            }
+            if (key == "File-Name") {
+                _file_name = value;
+                has_file_name = true;
+            } else if (key == "Content-Length") {
+                auto res = std::from_chars(value.data(), value.data() + value.size(), _file_size);
+                if (res.ec != std::errc()) {
+                    std::string error_msg = fmt::format("invalid content length: {}", value);
+                    LOG(WARNING) << "download files to " << _local_dir_path
+                                 << "failed, err=" << error_msg;
+                    _status = Status::HttpError(std::move(error_msg));
+                    return -1;
+                }
+                has_file_size = true;
+            }
+        }
+
+        if (!has_file_name || !has_file_size) {
+            std::string error_msg =
+                    fmt::format("invalid multi part header, has file name: {}, has file size: {}",
+                                has_file_name, has_file_size);
+            LOG(WARNING) << "download files to " << _local_dir_path << "failed, err=" << error_msg;
+            _status = Status::HttpError(std::move(error_msg));
+            return -1;
+        }
+
+        if (!_expected_files.contains(_file_name)) {
+            std::string error_msg = fmt::format("unexpected file: {}", _file_name);
+            LOG(WARNING) << "download files to " << _local_dir_path << "failed, err=" << error_msg;
+            _status = Status::HttpError(std::move(error_msg));
+            return -1;
+        }
+
+        VLOG_DEBUG << "receive file " << _file_name << ", size " << _file_size;
+
+        _written_size = 0;
+        _local_file_path = fmt::format("{}/{}", _local_dir_path, _file_name);
+        _fd = open(_local_file_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
+        if (_fd < 0) {
+            std::string error_msg = "fail to open file to write: " + _local_file_path;
+            LOG(WARNING) << "download files to " << _local_dir_path << "failed, err=" << error_msg;
+            _status = Status::IOError(std::move(error_msg));
+            return -1;
+        }
+        downloaded_files.push_back(_local_file_path);
+
+        return static_cast<int>(pos + 4);
+    }
+
+    int append_file(const char* data, size_t length) {
+        DCHECK(_fd >= 0);
+        DCHECK(_file_size >= _written_size);
+
+        size_t write_size = std::min(length, _file_size - _written_size);
+        if (write_size > 0 && write(_fd, data, write_size) < 0) {
+            auto msg = fmt::format("write file failed, file={}, error={}", _local_file_path,
+                                   strerror(errno));
+            LOG(WARNING) << "download files to " << _local_dir_path << "failed, err=" << msg;
+            _status = Status::HttpError(std::move(msg));
+            return -1;
+        }
+
+        _written_size += write_size;
+        if (_written_size == _file_size) {
+            // This file has been downloaded, switch to the next one.
+            switchToNextFile();
+        }
+
+        return write_size;
+    }
+
+    Status finish_inner() {
+        if (!_is_reading_header && _written_size == _file_size) {
+            switchToNextFile();
+        }
+
+        if (_fd >= 0) {
+            // This file is not completely downloaded.
+            close(_fd);
+            _fd = -1;
+            auto error_msg = fmt::format("file {} is not completely downloaded", _local_file_path);
+            LOG(WARNING) << "download files to " << _local_dir_path << "failed, err=" << error_msg;
+            return Status::HttpError(std::move(error_msg));
+        }
+
+        if (!_expected_files.empty()) {
+            auto error_msg = fmt::format("not all files are downloaded, {} missing files",
+                                         _expected_files.size());
+            LOG(WARNING) << "download files to " << _local_dir_path << "failed, err=" << error_msg;
+            return Status::HttpError(std::move(error_msg));
+        }
+
+        downloaded_files.clear();
+        return Status::OK();
+    }
+
+    void switchToNextFile() {
+        DCHECK(_fd >= 0);
+        DCHECK(_written_size == _file_size);
+
+        close(_fd);
+        _fd = -1;
+        _expected_files.erase(_file_name);
+        _is_reading_header = true;
+    }
+
+    const std::string _local_dir_path;
+    std::string _buffer;
+    std::unordered_set<std::string> _expected_files;
+    Status _status;
+
+    bool _is_reading_header = true;
+    int _fd = -1;
+    std::string _local_file_path;
+    std::string _file_name;
+    size_t _file_size = 0;
+    size_t _written_size = 0;
+    std::vector<std::string> downloaded_files;
+};
+
 static const char* header_error_msg(CURLHcode code) {
     switch (code) {
     case CURLHE_OK:
@@ -174,6 +385,12 @@ void HttpClient::set_method(HttpMethod method) {
     }
 }
 
+void HttpClient::set_speed_limit() {
+    curl_easy_setopt(_curl, CURLOPT_LOW_SPEED_LIMIT, config::download_low_speed_limit_kbps * 1024);
+    curl_easy_setopt(_curl, CURLOPT_LOW_SPEED_TIME, config::download_low_speed_time);
+    curl_easy_setopt(_curl, CURLOPT_MAX_RECV_SPEED_LARGE, config::max_download_speed_kbps * 1024);
+}
+
 size_t HttpClient::on_response_data(const void* data, size_t length) {
     if (*_callback != nullptr) {
         bool is_continue = (*_callback)(data, length);
@@ -184,12 +401,6 @@ size_t HttpClient::on_response_data(const void* data, size_t length) {
     return length;
 }
 
-// Status HttpClient::execute_post_request(const std::string& post_data, const std::function<bool(const void* data, size_t length)>& callback = {}) {
-//     _callback = &callback;
-//     set_post_body(post_data);
-//     return execute(callback);
-// }
-
 Status HttpClient::execute_post_request(const std::string& payload, std::string* response) {
     set_method(POST);
     set_payload(payload);
@@ -234,14 +445,8 @@ Status HttpClient::get_content_md5(std::string* md5) const {
 }
 
 Status HttpClient::download(const std::string& local_path) {
-    // set method to GET
     set_method(GET);
-
-    // TODO(zc) Move this download speed limit outside to limit download speed
-    // at system level
-    curl_easy_setopt(_curl, CURLOPT_LOW_SPEED_LIMIT, config::download_low_speed_limit_kbps * 1024);
-    curl_easy_setopt(_curl, CURLOPT_LOW_SPEED_TIME, config::download_low_speed_time);
-    curl_easy_setopt(_curl, CURLOPT_MAX_RECV_SPEED_LARGE, config::max_download_speed_kbps * 1024);
+    set_speed_limit();
 
     auto fp_closer = [](FILE* fp) { fclose(fp); };
     std::unique_ptr<FILE, decltype(fp_closer)> fp(fopen(local_path.c_str(), "w"), fp_closer);
@@ -270,6 +475,20 @@ Status HttpClient::download(const std::string& local_path) {
     return status;
 }
 
+Status HttpClient::download_multi_files(const std::string& local_dir,
+                                        const std::unordered_set<std::string>& expected_files) {
+    set_speed_limit();
+
+    MultiFileSplitter splitter(local_dir, expected_files);
+    auto callback = [&](const void* data, size_t length) {
+        return splitter.append(reinterpret_cast<const char*>(data), length);
+    };
+    if (auto s = execute(callback); !s.ok()) {
+        return s;
+    }
+    return splitter.finish();
+}
+
 Status HttpClient::execute(std::string* response) {
     auto callback = [response](const void* data, size_t length) {
         response->append((char*)data, length);
diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h
index c0c8863a9b06d4..a6f2f4fdff514b 100644
--- a/be/src/http/http_client.h
+++ b/be/src/http/http_client.h
@@ -24,6 +24,7 @@
 #include <cstdio>
 #include <functional>
 #include <string>
+#include <unordered_set>
 
 #include "common/status.h"
 #include "http/http_headers.h"
@@ -81,6 +82,8 @@ class HttpClient {
         curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, 0L);
     }
 
+    void set_speed_limit();
+
     // TODO(zc): support set header
     // void set_header(const std::string& key, const std::string& value) {
     // _cntl.http_request().SetHeader(key, value);
@@ -141,6 +144,8 @@ class HttpClient {
     // helper function to download a file, you can call this function to download
     // a file to local_path
     Status download(const std::string& local_path);
+    Status download_multi_files(const std::string& local_dir,
+                                const std::unordered_set<std::string>& expected_files);
 
     Status execute_post_request(const std::string& payload, std::string* response);
 
diff --git a/be/src/http/utils.cpp b/be/src/http/utils.cpp
index f91610476b4dc9..ee7a78113e555a 100644
--- a/be/src/http/utils.cpp
+++ b/be/src/http/utils.cpp
@@ -23,6 +23,8 @@
 #include <unistd.h>
 
 #include <ostream>
+#include <string>
+#include <unordered_map>
 #include <vector>
 
 #include "common/config.h"
@@ -30,6 +32,7 @@
 #include "common/status.h"
 #include "common/utils.h"
 #include "http/http_channel.h"
+#include "http/http_client.h"
 #include "http/http_common.h"
 #include "http/http_headers.h"
 #include "http/http_method.h"
@@ -41,10 +44,15 @@
 #include "runtime/exec_env.h"
 #include "util/md5.h"
 #include "util/path_util.h"
+#include "util/security.h"
 #include "util/url_coding.h"
 
 namespace doris {
 
+const uint32_t CHECK_SUPPORT_TIMEOUT = 3;
+const uint32_t DOWNLOAD_FILE_MAX_RETRY = 3;
+const uint32_t LIST_REMOTE_FILE_TIMEOUT = 15;
+
 std::string encode_basic_auth(const std::string& user, const std::string& passwd) {
     std::string auth = user + ":" + passwd;
     std::string encoded_auth;
@@ -190,20 +198,26 @@ void do_file_response(const std::string& file_path, HttpRequest* req,
     HttpChannel::send_file(req, fd, 0, file_size, rate_limit_group);
 }
 
-void do_dir_response(const std::string& dir_path, HttpRequest* req) {
+void do_dir_response(const std::string& dir_path, HttpRequest* req, bool is_acquire_filesize) {
     bool exists = true;
     std::vector<io::FileInfo> files;
     Status st = io::global_local_filesystem()->list(dir_path, true, &files, &exists);
     if (!st.ok()) {
         LOG(WARNING) << "Failed to scan dir. " << st;
         HttpChannel::send_error(req, HttpStatus::INTERNAL_SERVER_ERROR);
+        return;
     }
 
+    VLOG_DEBUG << "list dir: " << dir_path << ", file count: " << files.size();
+
     const std::string FILE_DELIMITER_IN_DIR_RESPONSE = "\n";
 
     std::stringstream result;
     for (auto& file : files) {
         result << file.file_name << FILE_DELIMITER_IN_DIR_RESPONSE;
+        if (is_acquire_filesize) {
+            result << file.file_size << FILE_DELIMITER_IN_DIR_RESPONSE;
+        }
     }
 
     std::string result_str = result.str();
@@ -221,4 +235,118 @@ bool load_size_smaller_than_wal_limit(int64_t content_length) {
     return (content_length < 0.8 * max_available_size);
 }
 
+Status is_support_batch_download(const std::string& endpoint) {
+    std::string url = fmt::format("http://{}/api/_tablet/_batch_download?check=true", endpoint);
+    auto check_support_cb = [&url](HttpClient* client) {
+        RETURN_IF_ERROR(client->init(url));
+        client->set_timeout_ms(CHECK_SUPPORT_TIMEOUT * 1000);
+        client->set_method(HttpMethod::HEAD);
+        std::string response;
+        return client->execute(&response);
+    };
+    return HttpClient::execute_with_retry(DOWNLOAD_FILE_MAX_RETRY, 1, check_support_cb);
+}
+
+Status list_remote_files_v2(const std::string& address, const std::string& token,
+                            const std::string& remote_dir,
+                            std::vector<std::pair<std::string, size_t>>* file_info_list) {
+    std::string remote_url =
+            fmt::format("http://{}/api/_tablet/_batch_download?token={}&dir={}&list=true", address,
+                        token, remote_dir);
+
+    std::string file_list_str;
+    auto list_files_cb = [&](HttpClient* client) {
+        file_list_str.clear();
+        RETURN_IF_ERROR(client->init(remote_url, false));
+        client->set_method(HttpMethod::GET);
+        client->set_timeout_ms(LIST_REMOTE_FILE_TIMEOUT * 1000);
+        return client->execute(&file_list_str);
+    };
+    Status status = HttpClient::execute_with_retry(DOWNLOAD_FILE_MAX_RETRY, 1, list_files_cb);
+    if (!status.ok()) {
+        LOG(WARNING) << "failed to list remote files from " << remote_url
+                     << ", status: " << status.to_string() << ", response: " << file_list_str;
+        return status;
+    }
+
+    std::vector<string> file_list = strings::Split(file_list_str, "\n", strings::SkipWhitespace());
+    if (file_list.size() % 2 != 0) {
+        return Status::InternalError("batch download files: invalid file list, size is not even");
+    }
+
+    VLOG_DEBUG << "list remote files from " << remote_url
+               << ", file count: " << file_list.size() / 2;
+
+    for (size_t i = 0; i < file_list.size(); i += 2) {
+        uint64_t file_size = 0;
+        try {
+            file_size = std::stoull(file_list[i + 1]);
+        } catch (std::exception&) {
+            return Status::InternalError("batch download files: invalid file size format: " +
+                                         file_list[i + 1]);
+        }
+        file_info_list->emplace_back(std::move(file_list[i]), file_size);
+    }
+
+    return Status::OK();
+}
+
+Status download_files_v2(const std::string& address, const std::string& token,
+                         const std::string& remote_dir, const std::string& local_dir,
+                         const std::vector<std::pair<std::string, size_t>>& file_info_list) {
+    std::string remote_url = fmt::format("http://{}/api/_tablet/_batch_download?dir={}&token={}",
+                                         address, remote_dir, token);
+
+    size_t batch_file_size = 0;
+    std::unordered_set<std::string> expected_files;
+    std::stringstream ss;
+    for (const auto& file_info : file_info_list) {
+        ss << file_info.first << "\n";
+        batch_file_size += file_info.second;
+        expected_files.insert(file_info.first);
+    }
+    std::string payload = ss.str();
+
+    uint64_t estimate_timeout = batch_file_size / config::download_low_speed_limit_kbps / 1024;
+    if (estimate_timeout < config::download_low_speed_time) {
+        estimate_timeout = config::download_low_speed_time;
+    }
+
+    LOG(INFO) << "begin to download files from " << remote_url << " to " << local_dir
+              << ", file count: " << file_info_list.size() << ", total size: " << batch_file_size
+              << ", timeout: " << estimate_timeout;
+
+    auto callback = [&](HttpClient* client) -> Status {
+        RETURN_IF_ERROR(client->init(remote_url, false));
+        client->set_method(HttpMethod::POST);
+        client->set_payload(payload);
+        client->set_timeout_ms(estimate_timeout * 1000);
+        RETURN_IF_ERROR(client->download_multi_files(local_dir, expected_files));
+        for (auto&& [file_name, file_size] : file_info_list) {
+            std::string local_file_path = local_dir + "/" + file_name;
+
+            std::error_code ec;
+            // Check file length
+            uint64_t local_file_size = std::filesystem::file_size(local_file_path, ec);
+            if (ec) {
+                LOG(WARNING) << "download file error: " << ec.message();
+                return Status::IOError("can't retrive file_size of {}, due to {}", local_file_path,
+                                       ec.message());
+            }
+            if (local_file_size != file_size) {
+                LOG(WARNING) << "download file length error"
+                             << ", remote_path=" << mask_token(remote_url)
+                             << ", file_name=" << file_name << ", file_size=" << file_size
+                             << ", local_file_size=" << local_file_size;
+                return Status::InternalError("downloaded file size is not equal");
+            }
+            RETURN_IF_ERROR(io::global_local_filesystem()->permission(
+                    local_file_path, io::LocalFileSystem::PERMS_OWNER_RW));
+        }
+
+        return Status::OK();
+    };
+    return HttpClient::execute_with_retry(DOWNLOAD_FILE_MAX_RETRY, 1, callback);
+}
+
 } // namespace doris
diff --git a/be/src/http/utils.h b/be/src/http/utils.h
index 20be6c0fcd7439..b9abb7c6208efb 100644
--- a/be/src/http/utils.h
+++ b/be/src/http/utils.h
@@ -40,9 +40,22 @@ void do_file_response(const std::string& dir_path, HttpRequest* req,
                       bufferevent_rate_limit_group* rate_limit_group = nullptr,
                       bool is_acquire_md5 = false);
 
-void do_dir_response(const std::string& dir_path, HttpRequest* req);
+void do_dir_response(const std::string& dir_path, HttpRequest* req,
+                     bool is_acquire_filesize = false);
 
 std::string get_content_type(const std::string& file_name);
 
 bool load_size_smaller_than_wal_limit(int64_t content_length);
+
+// Whether a backend supports batch download
+Status is_support_batch_download(const std::string& address);
+
+Status list_remote_files_v2(const std::string& address, const std::string& token,
+                            const std::string& remote_dir,
+                            std::vector<std::pair<std::string, size_t>>* file_info_list);
+
+Status download_files_v2(const std::string& address, const std::string& token,
+                         const std::string& remote_dir, const std::string& local_dir,
+                         const std::vector<std::pair<std::string, size_t>>& file_info_list);
+
 } // namespace doris
diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp
index bea1d3b1a91e89..fa8d9b8248e3f4 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -44,6 +44,7 @@
 #include "gutil/strings/split.h"
 #include "gutil/strings/strip.h"
 #include "http/http_client.h"
+#include "http/utils.h"
 #include "io/fs/file_system.h"
 #include "io/fs/local_file_system.h"
 #include "io/fs/path.h"
@@ -399,28 +400,62 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir,
                         .error(st);
             }
         }};
-        std::string remote_url_prefix;
+
+        std::string remote_dir;
         {
             std::stringstream ss;
             if (snapshot_path->back() == '/') {
-                ss << "http://" << get_host_port(src.host, src.http_port) << HTTP_REQUEST_PREFIX
-                   << HTTP_REQUEST_TOKEN_PARAM << token << HTTP_REQUEST_FILE_PARAM << *snapshot_path
-                   << _clone_req.tablet_id << "/" << _clone_req.schema_hash << "/";
+                ss << *snapshot_path << _clone_req.tablet_id << "/" << _clone_req.schema_hash
+                   << "/";
             } else {
-                ss << "http://" << get_host_port(src.host, src.http_port) << HTTP_REQUEST_PREFIX
-                   << HTTP_REQUEST_TOKEN_PARAM << token << HTTP_REQUEST_FILE_PARAM << *snapshot_path
-                   << "/" << _clone_req.tablet_id << "/" << _clone_req.schema_hash << "/";
+                ss << *snapshot_path << "/" << _clone_req.tablet_id << "/" << _clone_req.schema_hash
+                   << "/";
             }
-            remote_url_prefix = ss.str();
+            remote_dir = ss.str();
         }
 
-        status = _download_files(&data_dir, remote_url_prefix, local_data_path);
-        if (!status.ok()) [[unlikely]] {
-            LOG_WARNING("failed to download snapshot from remote BE")
-                    .tag("url", mask_token(remote_url_prefix))
-                    .error(status);
-            continue; // Try another BE
+        std::string address = get_host_port(src.host, src.http_port);
+        if (config::enable_batch_download && is_support_batch_download(address).ok()) {
+            // download files via batch api.
+            LOG_INFO("remote BE supports batch download, use batch file download")
+                    .tag("address", address)
+                    .tag("remote_dir", remote_dir);
+            status = _batch_download_files(&data_dir, address, remote_dir, local_data_path);
+            if (!status.ok()) [[unlikely]] {
+                LOG_WARNING("failed to download snapshot from remote BE in batch")
+                        .tag("address", address)
+                        .tag("remote_dir", remote_dir)
+                        .error(status);
+                continue; // Try another BE
+            }
+        } else {
+            if (config::enable_batch_download) {
+                LOG_INFO("remote BE does not support batch download, use single file download")
+                        .tag("address", address)
+                        .tag("remote_dir", remote_dir);
+            } else {
+                LOG_INFO("batch download is disabled, use single file download")
+                        .tag("address", address)
+                        .tag("remote_dir", remote_dir);
+            }
+
+            std::string remote_url_prefix;
+            {
+                std::stringstream ss;
+                ss << "http://" << address << HTTP_REQUEST_PREFIX << HTTP_REQUEST_TOKEN_PARAM
+                   << token << HTTP_REQUEST_FILE_PARAM << remote_dir;
+                remote_url_prefix = ss.str();
+            }
+
+            status = _download_files(&data_dir, remote_url_prefix, local_data_path);
+            if (!status.ok()) [[unlikely]] {
+                LOG_WARNING("failed to download snapshot from remote BE")
+                        .tag("url", mask_token(remote_url_prefix))
+                        .error(status);
+                continue; // Try another BE
+            }
         }
+
         // No need to try again with another BE
         _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids(
                 local_data_path, _clone_req.tablet_id, _clone_req.replica_id, _clone_req.table_id,
@@ -514,7 +549,7 @@ Status EngineCloneTask::_download_files(DataDir* data_dir, const std::string& re
     // If the header file is not exist, the table couldn't loaded by olap engine.
     // Avoid of data is not complete, we copy the header file at last.
     // The header file's name is end of .hdr.
-    for (int i = 0; i < file_name_list.size() - 1; ++i) {
+    for (int i = 0; i + 1 < file_name_list.size(); ++i) {
         if (file_name_list[i].ends_with(".hdr")) {
             std::swap(file_name_list[i], file_name_list[file_name_list.size() - 1]);
             break;
@@ -593,13 +628,91 @@ Status EngineCloneTask::_download_files(DataDir* data_dir, const std::string& re
     }
     _copy_size = (int64_t)total_file_size;
     _copy_time_ms = (int64_t)total_time_ms;
-    LOG(INFO) << "succeed to copy tablet " << _signature << ", total file size: " << total_file_size
-              << " B"
-              << ", cost: " << total_time_ms << " ms"
+    LOG(INFO) << "succeed to copy tablet " << _signature
+              << ", total files: " << file_name_list.size()
+              << ", total file size: " << total_file_size << " B, cost: " << total_time_ms << " ms"
               << ", rate: " << copy_rate << " MB/s";
     return Status::OK();
 }
 
+Status EngineCloneTask::_batch_download_files(DataDir* data_dir, const std::string& address,
+                                              const std::string& remote_dir,
+                                              const std::string& local_dir) {
+    constexpr size_t BATCH_FILE_SIZE = 64 << 20; // 64MB
+    constexpr size_t BATCH_FILE_NUM = 64;
+
+    // Check local path exist, if exist, remove it, then create the dir
+    // local_file_full_path = tabletid/clone， for a specific tablet, there should be only one folder
+    // if this folder exists, then should remove it
+    // for example, BE clone from BE 1 to download file 1 with version (2,2), but clone from BE 1 failed
+    // then it will try to clone from BE 2, but it will find the file 1 already exist, but file 1 with same
+    // name may have different versions.
+    RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(local_dir));
+    RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(local_dir));
+
+    const std::string& token = _cluster_info->token;
+    std::vector<std::pair<std::string, size_t>> file_info_list;
+    RETURN_IF_ERROR(list_remote_files_v2(address, token, remote_dir, &file_info_list));
+
+    // If the header file is not exist, the table couldn't loaded by olap engine.
+    // Avoid of data is not complete, we copy the header file at last.
+    // The header file's name is end of .hdr.
+    for (int i = 0; i + 1 < file_info_list.size(); ++i) {
+        if (file_info_list[i].first.ends_with(".hdr")) {
+            std::swap(file_info_list[i], file_info_list[file_info_list.size() - 1]);
+            break;
+        }
+    }
+
+    MonotonicStopWatch watch;
+    watch.start();
+
+    size_t total_file_size = 0;
+    size_t total_files = file_info_list.size();
+    std::vector<std::pair<std::string, size_t>> batch_files;
+    for (size_t i = 0; i < total_files;) {
+        size_t batch_file_size = 0;
+        for (size_t j = i; j < total_files; j++) {
+            // Split batchs by file number and file size,
+            if (BATCH_FILE_NUM <= batch_files.size() || BATCH_FILE_SIZE <= batch_file_size ||
+                // ... or separate the last .hdr file into a single batch.
+                (j + 1 == total_files && !batch_files.empty())) {
+                break;
+            }
+            batch_files.push_back(file_info_list[j]);
+            batch_file_size += file_info_list[j].second;
+        }
+
+        // check disk capacity
+        if (data_dir->reach_capacity_limit(batch_file_size)) {
+            return Status::Error<EXCEEDED_LIMIT>(
+                    "reach the capacity limit of path {}, file_size={}", data_dir->path(),
+                    batch_file_size);
+        }
+
+        RETURN_IF_ERROR(download_files_v2(address, token, remote_dir, local_dir, batch_files));
+
+        total_file_size += batch_file_size;
+        i += batch_files.size();
+        batch_files.clear();
+    }
+
+    uint64_t total_time_ms = watch.elapsed_time() / 1000 / 1000;
+    total_time_ms = total_time_ms > 0 ? total_time_ms : 0;
+    double copy_rate = 0.0;
+    if (total_time_ms > 0) {
+        copy_rate = total_file_size / ((double)total_time_ms) / 1000;
+    }
+    _copy_size = (int64_t)total_file_size;
+    _copy_time_ms = (int64_t)total_time_ms;
+    LOG(INFO) << "succeed to copy tablet " << _signature
+              << ", total files: " << file_info_list.size()
+              << ", total file size: " << total_file_size << " B, cost: " << total_time_ms << " ms"
+              << ", rate: " << copy_rate << " MB/s";
+
+    return Status::OK();
+}
+
 /// This method will only be called if tablet already exist in this BE when doing clone.
 /// This method will do the following things:
 /// 1. Link all files from CLONE dir to tablet dir if file does not exist in tablet dir
diff --git a/be/src/olap/task/engine_clone_task.h b/be/src/olap/task/engine_clone_task.h
index a11d4c742f4bcc..e2ced28f03c88d 100644
--- a/be/src/olap/task/engine_clone_task.h
+++ b/be/src/olap/task/engine_clone_task.h
@@ -79,6 +79,9 @@ class EngineCloneTask final : public EngineTask {
     Status _download_files(DataDir* data_dir, const std::string& remote_url_prefix,
                            const std::string& local_path);
 
+    Status _batch_download_files(DataDir* data_dir, const std::string& endpoint,
+                                 const std::string& remote_dir, const std::string& local_dir);
+
     Status _make_snapshot(const std::string& ip, int port, TTableId tablet_id,
                           TSchemaHash schema_hash, int timeout_s,
                           const std::vector<Version>& missing_versions, std::string* snapshot_path,
diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp
index 912f9f5ff403e7..c4b01a641987e6 100644
--- a/be/src/service/http_service.cpp
+++ b/be/src/service/http_service.cpp
@@ -32,6 +32,7 @@
 #include "common/status.h"
 #include "http/action/adjust_log_level.h"
 #include "http/action/adjust_tracing_dump.h"
+#include "http/action/batch_download_action.h"
 #include "http/action/be_proc_thread_action.h"
 #include "http/action/calc_file_crc_action.h"
 #include "http/action/check_rpc_channel_action.h"
@@ -309,6 +310,16 @@ void HttpService::register_local_handler(StorageEngine& engine) {
                                       tablet_download_action);
     _ev_http_server->register_handler(HttpMethod::GET, "/api/_tablet/_download",
                                       tablet_download_action);
+
+    BatchDownloadAction* batch_download_action =
+            _pool.add(new BatchDownloadAction(_env, _rate_limit_group, allow_paths));
+    _ev_http_server->register_handler(HttpMethod::HEAD, "/api/_tablet/_batch_download",
+                                      batch_download_action);
+    _ev_http_server->register_handler(HttpMethod::GET, "/api/_tablet/_batch_download",
+                                      batch_download_action);
+    _ev_http_server->register_handler(HttpMethod::POST, "/api/_tablet/_batch_download",
+                                      batch_download_action);
+
     if (config::enable_single_replica_load) {
         DownloadAction* single_replica_download_action = _pool.add(new DownloadAction(
                 _env, nullptr, allow_paths, config::single_replica_load_download_num_workers));
diff --git a/be/test/http/http_client_test.cpp b/be/test/http/http_client_test.cpp
index c98328d7c8e37c..84e4d259ff5ccd 100644
--- a/be/test/http/http_client_test.cpp
+++ b/be/test/http/http_client_test.cpp
@@ -25,6 +25,7 @@
 #include <unistd.h>
 
 #include <boost/algorithm/string/predicate.hpp>
+#include <filesystem>
 
 #include "gtest/gtest_pred_impl.h"
 #include "http/ev_http_server.h"
@@ -102,14 +103,32 @@ class HttpDownloadFileHandler : public HttpHandler {
     }
 };
 
+class HttpBatchDownloadFileHandler : public HttpHandler {
+public:
+    void handle(HttpRequest* req) override {
+        if (req->param("check") == "true") {
+            HttpChannel::send_reply(req, "OK");
+        } else if (req->param("list") == "true") {
+            do_dir_response(req->param("dir"), req, true);
+        } else {
+            std::vector<std::string> acquire_files =
+                    strings::Split(req->get_request_body(), "\n", strings::SkipWhitespace());
+            HttpChannel::send_files(req, req->param("dir"), acquire_files);
+        }
+    }
+};
+
 static EvHttpServer* s_server = nullptr;
 static int real_port = 0;
 static std::string hostname = "";
+static std::string address = "";
+constexpr std::string_view TMP_DIR = "./http_test_tmp";
 
 static HttpClientTestSimpleGetHandler s_simple_get_handler;
 static HttpClientTestSimplePostHandler s_simple_post_handler;
 static HttpNotFoundHandler s_not_found_handler;
 static HttpDownloadFileHandler s_download_file_handler;
+static HttpBatchDownloadFileHandler s_batch_download_file_handler;
 
 class HttpClientTest : public testing::Test {
 public:
@@ -123,10 +142,17 @@ class HttpClientTest : public testing::Test {
         s_server->register_handler(POST, "/simple_post", &s_simple_post_handler);
         s_server->register_handler(GET, "/not_found", &s_not_found_handler);
         s_server->register_handler(HEAD, "/download_file", &s_download_file_handler);
+        s_server->register_handler(HEAD, "/api/_tablet/_batch_download",
+                                   &s_batch_download_file_handler);
+        s_server->register_handler(GET, "/api/_tablet/_batch_download",
+                                   &s_batch_download_file_handler);
+        s_server->register_handler(POST, "/api/_tablet/_batch_download",
+                                   &s_batch_download_file_handler);
         static_cast<void>(s_server->start());
         real_port = s_server->get_real_port();
         EXPECT_NE(0, real_port);
-        hostname = "http://127.0.0.1:" + std::to_string(real_port);
+        address = "127.0.0.1:" + std::to_string(real_port);
+        hostname = "http://" + address;
     }
 
     static void TearDownTestCase() { delete s_server; }
@@ -571,4 +597,74 @@ TEST_F(HttpClientTest, enable_http_auth) {
     }
 }
 
+TEST_F(HttpClientTest, batch_download) {
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(TMP_DIR).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(TMP_DIR).ok());
+
+    std::string root_dir(TMP_DIR);
+    std::string remote_related_dir = root_dir + "/source";
+    std::string local_dir = root_dir + "/target";
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(remote_related_dir).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(local_dir).ok());
+
+    std::string remote_dir;
+    EXPECT_TRUE(io::global_local_filesystem()->canonicalize(remote_related_dir, &remote_dir).ok());
+
+    // 0. create dir source and prepare a large file exceeds 1MB
+    {
+        std::string large_file = remote_dir + "/a_large_file";
+        int fd = open(large_file.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
+        ASSERT_TRUE(fd >= 0);
+        std::string buf = "0123456789";
+        for (int i = 0; i < 10; i++) {
+            buf += buf;
+        }
+        for (int i = 0; i < 1024; i++) {
+            ASSERT_TRUE(write(fd, buf.c_str(), buf.size()) > 0);
+        }
+        close(fd);
+
+        // create some small files.
+        for (int i = 0; i < 32; i++) {
+            std::string small_file = remote_dir + "/small_file_" + std::to_string(i);
+            fd = open(small_file.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
+            ASSERT_TRUE(fd >= 0);
+            ASSERT_TRUE(write(fd, buf.c_str(), buf.size()) > 0);
+            close(fd);
+        }
+
+        // create a empty file
+        std::string empty_file = remote_dir + "/empty_file";
+        fd = open(empty_file.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
+        ASSERT_TRUE(fd >= 0);
+        close(fd);
+
+        empty_file = remote_dir + "/zzzz";
+        fd = open(empty_file.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
+        ASSERT_TRUE(fd >= 0);
+        close(fd);
+    }
+
+    // 1. check remote support batch download
+    Status st = is_support_batch_download(address);
+    EXPECT_TRUE(st.ok());
+
+    // 2. list remote files
+    std::vector<std::pair<std::string, size_t>> file_info_list;
+    st = list_remote_files_v2(address, "token", remote_dir, &file_info_list);
+    EXPECT_TRUE(st.ok());
+
+    // 3. download files
+    if (file_info_list.size() > 64) {
+        file_info_list.resize(64);
+    }
+
+    // sort file info list by file name
+    std::sort(file_info_list.begin(), file_info_list.end(),
+              [](const auto& a, const auto& b) { return a.first < b.first; });
+
+    st = download_files_v2(address, "token", remote_dir, local_dir, file_info_list);
+    EXPECT_TRUE(st.ok());
+}
+
 } // namespace doris

From 3d9597ec71ab0955f2ed56e8794da3018f5eedb1 Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Mon, 9 Dec 2024 15:16:58 +0800
Subject: [PATCH 237/399] [Improve](columns) update columns and ut  (#42269)

## Proposed changes
this pr is aim to standardization Column behavior in UT, and we should
add this ut case if we add a new Column type to make sure the semantics
of interface defined in Column which is right
Issue Number: close #xxx

<!--Describe your changes.-->
---
 be/src/vec/columns/column.h                   |   20 +-
 be/src/vec/columns/column_array.cpp           |    2 +-
 be/src/vec/columns/column_impl.h              |    1 +
 be/src/vec/columns/column_vector.cpp          |    1 +
 be/src/vec/columns/column_vector.h            |    8 +-
 be/src/vec/columns/columns_common.cpp         |   14 +
 be/src/vec/data_types/data_type.h             |    2 -
 be/src/vec/data_types/data_type_object.h      |    1 -
 be/src/vec/functions/function_cast.h          |    1 +
 .../functions/function_variant_element.cpp    |    3 +
 be/test/vec/columns/column_ip_test.cpp        |  315 +++
 be/test/vec/columns/common_column_test.h      | 1973 +++++++++++++++++
 be/test/vec/core/column_array_test.cpp        |    9 +-
 13 files changed, 2329 insertions(+), 21 deletions(-)
 create mode 100644 be/test/vec/columns/column_ip_test.cpp
 create mode 100644 be/test/vec/columns/common_column_test.h

diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index 24653f878555f5..729e5470c97ad5 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -126,7 +126,7 @@ class IColumn : public COW<IColumn> {
     // for non-str col, will reach here(do nothing). only ColumnStr will really shrink itself.
     virtual void shrink_padding_chars() {}
 
-    /// Some columns may require finalization before using of other operations.
+    // Only used in ColumnObject to handle lifecycle of variant. Other columns would do nothing.
     virtual void finalize() {}
 
     // Only used on ColumnDictionary
@@ -208,6 +208,8 @@ class IColumn : public COW<IColumn> {
     }
 
     /// Appends one element from other column with the same type multiple times.
+    /// we should make sure position is less than src's size and length is less than src's size,
+    /// and position + length is less than src's size
     virtual void insert_many_from(const IColumn& src, size_t position, size_t length) {
         for (size_t i = 0; i < length; ++i) {
             insert_from(src, position);
@@ -221,6 +223,7 @@ class IColumn : public COW<IColumn> {
                                           const std::vector<size_t>& positions) = 0;
 
     /// Appends a batch elements from other column with the same type
+    /// Also here should make sure indices_end is bigger than indices_begin
     /// indices_begin + indices_end represent the row indices of column src
     virtual void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
                                      const uint32_t* indices_end) = 0;
@@ -460,7 +463,7 @@ class IColumn : public COW<IColumn> {
 
     /** Returns a permutation that sorts elements of this column,
       *  i.e. perm[i]-th element of source column should be i-th element of sorted column.
-      * reverse - reverse ordering (ascending).
+      * reverse - true: descending order, false: ascending order.
       * limit - if isn't 0, then only first limit elements of the result column could be sorted.
       * nan_direction_hint - see above.
       */
@@ -476,13 +479,6 @@ class IColumn : public COW<IColumn> {
       */
     virtual Ptr replicate(const Offsets& offsets) const = 0;
 
-    /// Appends one field multiple times. Can be optimized in inherited classes.
-    virtual void insert_many(const Field& field, size_t length) {
-        for (size_t i = 0; i < length; ++i) {
-            insert(field);
-        }
-    }
-
     /** Split column to smaller columns. Each value goes to column index, selected by corresponding element of 'selector'.
       * Selector must contain values from 0 to num_columns - 1.
       * For default implementation, see column_impl.h
@@ -511,6 +507,7 @@ class IColumn : public COW<IColumn> {
 
     /// Resize memory for specified amount of elements. If reservation isn't possible, does nothing.
     /// It affects performance only (not correctness).
+    /// Note. resize means not only change column self but also sub-columns if have.
     virtual void resize(size_t /*n*/) {}
 
     /// Size of column data in memory (may be approximate) - for profiling. Zero, if could not be determined.
@@ -647,8 +644,7 @@ class IColumn : public COW<IColumn> {
       */
     String dump_structure() const;
 
-    // only used in agg value replace
-    // ColumnString should replace according to 0,1,2... ,size,0,1,2...
+    // only used in agg value replace for column which is not variable length, eg.BlockReader::_copy_value_data
     // usage: self_column.replace_column_data(other_column, other_column's row index, self_column's row index)
     virtual void replace_column_data(const IColumn&, size_t row, size_t self_row = 0) = 0;
     // replace data to default value if null, used to avoid null data output decimal check failure
@@ -742,4 +738,4 @@ struct ColumnPtrWrapper {
 
     ColumnPtrWrapper(vectorized::ColumnPtr col) : column_ptr(std::move(col)) {}
 };
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/vec/columns/column_array.cpp b/be/src/vec/columns/column_array.cpp
index 0c5a53cdb24ca5..0df0ddcb0f30d5 100644
--- a/be/src/vec/columns/column_array.cpp
+++ b/be/src/vec/columns/column_array.cpp
@@ -1062,4 +1062,4 @@ ColumnPtr ColumnArray::permute(const Permutation& perm, size_t limit) const {
     return res;
 }
 
-} // namespace doris::vectorized
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/columns/column_impl.h b/be/src/vec/columns/column_impl.h
index f7abbf2a023ba4..45724d4364bccc 100644
--- a/be/src/vec/columns/column_impl.h
+++ b/be/src/vec/columns/column_impl.h
@@ -43,6 +43,7 @@ void IColumn::append_data_by_selector_impl(MutablePtr& res, const Selector& sele
                                selector.size(), num_rows);
     }
     DCHECK_GE(end, begin);
+    DCHECK_LE(end, selector.size());
     // here wants insert some value from this column, and the nums is (end - begin)
     // and many be this column num_rows is 4096, but only need insert num is (1 - 0) = 1
     // so can't call res->reserve(num_rows), it's will be too mush waste memory
diff --git a/be/src/vec/columns/column_vector.cpp b/be/src/vec/columns/column_vector.cpp
index 396edcc2ad4c3c..7a23156063d4e1 100644
--- a/be/src/vec/columns/column_vector.cpp
+++ b/be/src/vec/columns/column_vector.cpp
@@ -290,6 +290,7 @@ MutableColumnPtr ColumnVector<T>::clone_resized(size_t size) const {
 template <typename T>
 void ColumnVector<T>::insert_range_from(const IColumn& src, size_t start, size_t length) {
     const ColumnVector& src_vec = assert_cast<const ColumnVector&>(src);
+    //  size_t(start)  start > src_vec.data.size() || length > src_vec.data.size() should not be negative which cause overflow
     if (start + length > src_vec.data.size()) {
         throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
                                "Parameters start = {}, length = {}, are out of bound in "
diff --git a/be/src/vec/columns/column_vector.h b/be/src/vec/columns/column_vector.h
index 8006d76c1f98be..c14361c6572245 100644
--- a/be/src/vec/columns/column_vector.h
+++ b/be/src/vec/columns/column_vector.h
@@ -335,7 +335,13 @@ class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
 
     void resize(size_t n) override { data.resize(n); }
 
-    std::string get_name() const override { return TypeName<T>::get(); }
+    std::string get_name() const override {
+        // however we have a conflict type of number and other can store in number type such as ipv4 and uint32
+        if (std::is_same_v<T, IPv4>) {
+            return "IPv4";
+        }
+        return TypeName<T>::get();
+    }
 
     MutableColumnPtr clone_resized(size_t size) const override;
 
diff --git a/be/src/vec/columns/columns_common.cpp b/be/src/vec/columns/columns_common.cpp
index 0671e9abd85393..6297be2ce28e58 100644
--- a/be/src/vec/columns/columns_common.cpp
+++ b/be/src/vec/columns/columns_common.cpp
@@ -363,6 +363,8 @@ INSTANTIATE(UInt32, IColumn::Offset)
 INSTANTIATE(UInt32, ColumnArray::Offset64)
 INSTANTIATE(UInt64, IColumn::Offset)
 INSTANTIATE(UInt64, ColumnArray::Offset64)
+INSTANTIATE(UInt128, IColumn::Offset)
+INSTANTIATE(UInt128, ColumnArray::Offset64)
 INSTANTIATE(Int8, IColumn::Offset)
 INSTANTIATE(Int8, ColumnArray::Offset64)
 INSTANTIATE(Int16, IColumn::Offset)
@@ -371,10 +373,22 @@ INSTANTIATE(Int32, IColumn::Offset)
 INSTANTIATE(Int32, ColumnArray::Offset64)
 INSTANTIATE(Int64, IColumn::Offset)
 INSTANTIATE(Int64, ColumnArray::Offset64)
+INSTANTIATE(Int128, IColumn::Offset)
+INSTANTIATE(Int128, ColumnArray::Offset64)
 INSTANTIATE(Float32, IColumn::Offset)
 INSTANTIATE(Float32, ColumnArray::Offset64)
 INSTANTIATE(Float64, IColumn::Offset)
 INSTANTIATE(Float64, ColumnArray::Offset64)
+INSTANTIATE(Decimal32, IColumn::Offset)
+INSTANTIATE(Decimal32, ColumnArray::Offset64)
+INSTANTIATE(Decimal64, IColumn::Offset)
+INSTANTIATE(Decimal64, ColumnArray::Offset64)
+INSTANTIATE(Decimal128V2, IColumn::Offset)
+INSTANTIATE(Decimal128V2, ColumnArray::Offset64)
+INSTANTIATE(Decimal128V3, IColumn::Offset)
+INSTANTIATE(Decimal128V3, ColumnArray::Offset64)
+INSTANTIATE(Decimal256, IColumn::Offset)
+INSTANTIATE(Decimal256, ColumnArray::Offset64)
 
 #undef INSTANTIATE
 
diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
index e193856c05187d..7f1ee0cd85032f 100644
--- a/be/src/vec/data_types/data_type.h
+++ b/be/src/vec/data_types/data_type.h
@@ -151,8 +151,6 @@ class IDataType : private boost::noncopyable {
       */
     virtual bool is_value_represented_by_number() const { return false; }
 
-    virtual bool is_object() const { return false; }
-
     /** Values are unambiguously identified by contents of contiguous memory region,
       *  that can be obtained by IColumn::get_data_at method.
       * Examples: numbers, Date, DateTime, String, FixedString,
diff --git a/be/src/vec/data_types/data_type_object.h b/be/src/vec/data_types/data_type_object.h
index 7723d1c53774ad..f3336067419fa1 100644
--- a/be/src/vec/data_types/data_type_object.h
+++ b/be/src/vec/data_types/data_type_object.h
@@ -65,7 +65,6 @@ class DataTypeObject : public IDataType {
         return doris::FieldType::OLAP_FIELD_TYPE_VARIANT;
     }
     MutableColumnPtr create_column() const override { return ColumnObject::create(is_nullable); }
-    bool is_object() const override { return true; }
     bool equals(const IDataType& rhs) const override;
     bool have_subtypes() const override { return true; };
     int64_t get_uncompressed_serialized_bytes(const IColumn& column,
diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
index 3ad456d60d201f..48619ff85f83c8 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -1871,6 +1871,7 @@ class FunctionCast final : public IFunctionBase {
             auto& variant = assert_cast<const ColumnObject&>(*col_from);
             ColumnPtr col_to = data_type_to->create_column();
             if (!variant.is_finalized()) {
+                // ColumnObject should be finalized before parsing, finalize maybe modify original column structure
                 variant.assume_mutable()->finalize();
             }
             // It's important to convert as many elements as possible in this context. For instance,
diff --git a/be/src/vec/functions/function_variant_element.cpp b/be/src/vec/functions/function_variant_element.cpp
index e04ff54dbe9d7c..9d0aab7685424e 100644
--- a/be/src/vec/functions/function_variant_element.cpp
+++ b/be/src/vec/functions/function_variant_element.cpp
@@ -127,6 +127,7 @@ class FunctionVariantElement : public IFunction {
             *result = ColumnObject::create(true);
             // src subcolumns empty but src row count may not be 0
             (*result)->assume_mutable()->insert_many_defaults(src.size());
+            // ColumnObject should be finalized before parsing, finalize maybe modify original column structure
             (*result)->assume_mutable()->finalize();
             return Status::OK();
         }
@@ -152,6 +153,7 @@ class FunctionVariantElement : public IFunction {
                 }
             }
             *result = ColumnObject::create(true, type, std::move(result_column));
+            // ColumnObject should be finalized before parsing, finalize maybe modify original column structure
             (*result)->assume_mutable()->finalize();
             return Status::OK();
         } else {
@@ -194,6 +196,7 @@ class FunctionVariantElement : public IFunction {
                 result_col->insert_many_defaults(src.size());
             }
             *result = result_col->get_ptr();
+            // ColumnObject should be finalized before parsing, finalize maybe modify original column structure
             (*result)->assume_mutable()->finalize();
             VLOG_DEBUG << "dump new object "
                        << static_cast<const ColumnObject*>(result_col.get())->debug_string()
diff --git a/be/test/vec/columns/column_ip_test.cpp b/be/test/vec/columns/column_ip_test.cpp
new file mode 100644
index 00000000000000..5f56763c5aff1e
--- /dev/null
+++ b/be/test/vec/columns/column_ip_test.cpp
@@ -0,0 +1,315 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
+
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/columns_number.h"
+#include "vec/columns/common_column_test.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+
+// this test is gonna to make a template ColumnTest
+// for example column_ip should test these functions
+
+namespace doris::vectorized {
+class ColumnIPTest : public CommonColumnTest {
+protected:
+    void SetUp() override {
+        // insert from data csv and assert insert result
+        column_ipv4 = ColumnIPv4::create();
+        column_ipv6 = ColumnIPv6::create();
+
+        // insert from data csv and assert insert result with
+        // regression-test/data/nereids_function_p0/fn_test_ip_invalid.csv
+        // regression-test/data/nereids_function_p0/fn_test_ip_normal.csv
+        // regression-test/data/nereids_function_p0/fn_test_ip_nullable.csv
+        // regression-test/data/nereids_function_p0/fn_test_ip_special.csv
+        // regression-test/data/nereids_function_p0/fn_test_ip_special_no_null.csv
+        data_files = {"regression-test/data/nereids_function_p0/fn_test_ip_invalid.csv",
+                      "regression-test/data/nereids_function_p0/fn_test_ip_normal.csv",
+                      "regression-test/data/nereids_function_p0/fn_test_ip_nullable.csv",
+                      "regression-test/data/nereids_function_p0/fn_test_ip_special.csv",
+                      "regression-test/data/nereids_function_p0/fn_test_ip_special_no_null.csv"};
+        MutableColumns ip_cols;
+        ip_cols.push_back(column_ipv4->get_ptr());
+        ip_cols.push_back(column_ipv6->get_ptr());
+        //        check_data(ip_cols, serde, ';', {1, 2},
+        //                   "regression-test/data/nereids_function_p0/fn_test_ip_invalid.csv",
+        //                   assert_insert_from_callback);
+    }
+
+    DataTypePtr dt_ipv4 =
+            DataTypeFactory::instance().create_data_type(FieldType::OLAP_FIELD_TYPE_IPV4, 0, 0);
+    DataTypePtr dt_ipv6 =
+            DataTypeFactory::instance().create_data_type(FieldType::OLAP_FIELD_TYPE_IPV6, 0, 0);
+    DataTypePtr dt_ipv4_nullable = std::make_shared<vectorized::DataTypeNullable>(dt_ipv4);
+    DataTypePtr dt_ipv6_nullable = std::make_shared<vectorized::DataTypeNullable>(dt_ipv6);
+
+    ColumnIPv4::MutablePtr column_ipv4;
+    ColumnIPv6::MutablePtr column_ipv6;
+    ColumnConst::MutablePtr column_ipv4_const;
+    ColumnConst::MutablePtr column_ipv6_const;
+    DataTypeSerDeSPtrs serde = {dt_ipv4->get_serde(), dt_ipv6->get_serde()};
+
+    // common ip data
+    std::vector<string> data_files;
+};
+
+TEST_F(ColumnIPTest, InsertRangeFromTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_insert_range_from_callback);
+}
+
+TEST_F(ColumnIPTest, InsertManyFromTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_insert_many_from_callback);
+}
+
+TEST_F(ColumnIPTest, InsertIndicesFromTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_insert_indices_from_callback);
+}
+
+TEST_F(ColumnIPTest, InsertDefaultTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    // ipv4 default value is '0.0.0.0' and ipv6 default value is '::'
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_insert_default_callback);
+}
+
+TEST_F(ColumnIPTest, InsertManyDefaultsTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_insert_many_defaults_callback);
+}
+
+TEST_F(ColumnIPTest, GetDataAtTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_get_data_at_callback);
+}
+
+TEST_F(ColumnIPTest, FieldTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_field_callback);
+}
+
+TEST_F(ColumnIPTest, GetRawDataTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, {serde[1]}, ';', {2}, data_files[0], assert_get_raw_data_callback<IPv6>);
+}
+
+TEST_F(ColumnIPTest, GetBoolTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_get_bool_callback);
+}
+
+TEST_F(ColumnIPTest, GetIntTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_get_int_callback);
+}
+
+TEST_F(ColumnIPTest, SerDeVecTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    load_data_from_csv(serde, ip_cols, data_files[0], ';', {1, 2});
+    ser_deser_vec(ip_cols, {dt_ipv4, dt_ipv6});
+}
+
+TEST_F(ColumnIPTest, serDeserializeWithArenaImpl) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+
+    load_data_from_csv(serde, ip_cols, data_files[0], ';', {1, 2});
+    ser_deserialize_with_arena_impl(ip_cols, {dt_ipv4, dt_ipv6});
+}
+
+TEST_F(ColumnIPTest, SizeTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_size_callback);
+}
+
+TEST_F(ColumnIPTest, ByteSizeTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_byte_size_callback);
+}
+
+TEST_F(ColumnIPTest, AllocateBytesTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_allocated_bytes_callback);
+}
+
+TEST_F(ColumnIPTest, PopbackTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_pop_back_callback);
+}
+
+TEST_F(ColumnIPTest, CloneTest) {
+    // we test the column with clone_resize, clone_empty for assert size and ptr
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    load_data_from_csv(serde, ip_cols, data_files[0], ';', {1, 2});
+    assert_clone_empty(column_ipv4->assume_mutable_ref());
+    assert_clone_empty(column_ipv6->assume_mutable_ref());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_clone_resized_callback);
+}
+
+TEST_F(ColumnIPTest, CutTest) {
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    load_data_from_csv(serde, ip_cols, data_files[0], ';', {1, 2});
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_cut_callback);
+}
+
+TEST_F(ColumnIPTest, ResizeTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_resize_callback);
+}
+
+TEST_F(ColumnIPTest, ReserveTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_reserve_callback);
+}
+
+TEST_F(ColumnIPTest, ReplicateTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+
+    load_data_from_csv(serde, ip_cols, data_files[0], ';', {1, 2});
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_replicate_callback);
+}
+
+TEST_F(ColumnIPTest, ReplaceColumnTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+
+    load_data_from_csv(serde, ip_cols, data_files[0], ';', {1, 2});
+    // replace_column_data
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_replace_column_data_callback);
+    // replace_nullable_column_data
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0],
+               assert_replace_column_null_data_callback);
+}
+
+TEST_F(ColumnIPTest, AppendDataBySelectorTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_append_data_by_selector_callback);
+}
+
+TEST_F(ColumnIPTest, PermutationAndSortTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    load_data_from_csv(serde, ip_cols, data_files[1], ';', {1, 2});
+    assert_column_permutations(column_ipv4->assume_mutable_ref(), dt_ipv4);
+    assert_column_permutations(column_ipv6->assume_mutable_ref(), dt_ipv6);
+}
+
+TEST_F(ColumnIPTest, FilterTest) {
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_filter_callback);
+}
+
+// HASH Interfaces
+TEST_F(ColumnIPTest, HashTest) {
+    // XXHash
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+
+    load_data_from_csv(serde, ip_cols, data_files[0], ';', {1, 2});
+    // update_hashes_with_value
+    check_data(ip_cols, serde, ';', {1, 2}, data_files[0],
+               assert_update_hashes_with_value_callback);
+    // CrcHash
+    std::vector<PrimitiveType> pts = {PrimitiveType::TYPE_IPV4, PrimitiveType::TYPE_IPV6};
+    assert_update_crc_hashes_callback(ip_cols, serde, pts);
+};
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/columns/common_column_test.h b/be/test/vec/columns/common_column_test.h
new file mode 100644
index 00000000000000..8e1b86c0168f99
--- /dev/null
+++ b/be/test/vec/columns/common_column_test.h
@@ -0,0 +1,1973 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+
+#include "olap/schema.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/columns/columns_number.h"
+#include "vec/core/field.h"
+#include "vec/core/sort_block.h"
+#include "vec/core/sort_description.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_map.h"
+
+// this test is gonna to be a column test template for all column which should make ut test to coverage the function defined in column
+// for example column_array should test this function:
+// size, reserve, resize, empty, byte_size, allocated_bytes, clone_resized,
+// get_shrinked_column, filter, filter_by_selector, serialize_vec, deserialize_vec, get_max_row_byte_size
+//
+namespace doris::vectorized {
+
+static bool gen_check_data_in_assert = true;
+
+class CommonColumnTest : public ::testing::Test {
+protected:
+    // Helper function to load data from CSV, with index which splited by spliter and load to columns
+    void load_data_from_csv(const DataTypeSerDeSPtrs serders, MutableColumns& columns,
+                            const std::string& file_path, const char spliter = ';',
+                            const std::set<int> idxes = {0}) {
+        ASSERT_EQ(serders.size(), columns.size())
+                << "serder size: " << serders.size() << " column size: " << columns.size();
+        ASSERT_EQ(serders.size(), idxes.size())
+                << "serder size: " << serders.size() << " idxes size: " << idxes.size();
+        ASSERT_EQ(serders.size(), *idxes.end())
+                << "serder size: " << serders.size() << " idxes size: " << *idxes.end();
+        std::ifstream file(file_path);
+        if (!file) {
+            throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "can not open the file: {} ",
+                                   file_path);
+        }
+
+        std::string line;
+        DataTypeSerDe::FormatOptions options;
+        while (std::getline(file, line)) {
+            std::stringstream lineStream(line);
+            //            std::cout << "whole : " << lineStream.str() << std::endl;
+            std::string value;
+            int l_idx = 0;
+            int c_idx = 0;
+            while (std::getline(lineStream, value, spliter)) {
+                if (idxes.contains(l_idx)) {
+                    Slice string_slice(value.data(), value.size());
+                    std::cout << string_slice << std::endl;
+                    if (auto st = serders[c_idx]->deserialize_one_cell_from_json(
+                                *columns[c_idx], string_slice, options);
+                        !st.ok()) {
+                        //                        std::cout << "error in deserialize but continue: " << st.to_string()
+                        //                                  << std::endl;
+                    }
+                    ++c_idx;
+                }
+                ++l_idx;
+            }
+        }
+    }
+
+    //// this is very helpful function to check data in column against expected results according different function in assert function
+    //// such as run regress tests
+    ////  if gen_check_data_in_assert is true, we will generate a file for check data, otherwise we will read the file to check data
+    ////  so the key point is we should how we write assert callback function to check data,
+    ///   and when check data is generated, we should check result to statisfy the semantic of the function
+    static void check_res_file(string function_name, std::vector<std::vector<std::string>>& res) {
+        string filename = "./res_" + function_name + ".csv";
+        if (gen_check_data_in_assert) {
+            std::ofstream res_file(filename);
+            std::cout << "gen check data: " << res.size() << " with file: " << filename
+                      << std::endl;
+            if (!res_file.is_open()) {
+                throw std::ios_base::failure("Failed to open file.");
+            }
+
+            for (const auto& row : res) {
+                for (size_t i = 0; i < row.size(); ++i) {
+                    auto cell = row[i];
+                    res_file << cell;
+                    if (i < row.size() - 1) {
+                        res_file << ";"; // Add semicolon between columns
+                    }
+                }
+                res_file << "\n"; // Newline after each row
+            }
+
+            res_file.close();
+        } else {
+            // we read generate file to check result
+            std::cout << "check data: " << res.size() << " with file: " << filename << std::endl;
+            std::ifstream file(filename);
+            if (!file) {
+                throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "can not open the file: {} ",
+                                       filename);
+            }
+
+            std::string line;
+            std::vector<std::vector<std::string>> assert_res;
+            while (std::getline(file, line)) {
+                std::vector<std::string> row;
+                std::stringstream lineStream(line);
+                std::string value;
+                while (std::getline(lineStream, value, ';')) {
+                    row.push_back(value);
+                }
+                assert_res.push_back(row);
+            }
+
+            // we just do check here
+            for (size_t i = 0; i < res.size(); ++i) {
+                for (size_t j = 0; j < res[i].size(); ++j) {
+                    EXPECT_EQ(res[i][j], assert_res[i][j]);
+                }
+            }
+        }
+    }
+
+public:
+    void SetUp() override {
+        col_str = ColumnString::create();
+        col_str->insert_data("aaa", 3);
+        col_str->insert_data("bb", 2);
+        col_str->insert_data("cccc", 4);
+
+        col_int = ColumnInt64::create();
+        col_int->insert_value(1);
+        col_int->insert_value(2);
+        col_int->insert_value(3);
+
+        col_dcm = ColumnDecimal64::create(0, 3);
+        col_dcm->insert_value(1.23);
+        col_dcm->insert_value(4.56);
+        col_dcm->insert_value(7.89);
+
+        col_arr = ColumnArray::create(ColumnInt64::create(), ColumnArray::ColumnOffsets::create());
+        Array array1 = {1, 2, 3};
+        Array array2 = {4};
+        col_arr->insert(array1);
+        col_arr->insert(Array());
+        col_arr->insert(array2);
+
+        col_map = ColumnMap::create(ColumnString::create(), ColumnInt64::create(),
+                                    ColumnArray::ColumnOffsets::create());
+        Array k1 = {"a", "b", "c"};
+        Array v1 = {1, 2, 3};
+        Array k2 = {"d"};
+        Array v2 = {4};
+        Array a = Array();
+        Map map1, map2, map3;
+        map1.push_back(k1);
+        map1.push_back(v1);
+        col_map->insert(map1);
+        map3.push_back(a);
+        map3.push_back(a);
+        col_map->insert(map3);
+        map2.push_back(k2);
+        map2.push_back(v2);
+        col_map->insert(map2);
+    }
+
+    ColumnString::MutablePtr col_str;
+    ColumnInt64::MutablePtr col_int;
+    ColumnDecimal64::MutablePtr col_dcm;
+    ColumnArray::MutablePtr col_arr;
+    ColumnMap::MutablePtr col_map;
+
+    ////==================================================================================================================
+    // this is common function to check data in column against expected results according different function in assert function
+    // which can be used in all column test
+    // such as run regress tests
+    //  step1. we can set gen_check_data_in_assert to true, then we will generate a file for check data, otherwise we will read the file to check data
+    //  step2. we should write assert callback function to check data
+    void check_data(MutableColumns& columns, DataTypeSerDeSPtrs serders, char col_spliter,
+                    std::set<int> idxes, const std::string& column_data_file,
+                    std::function<void(MutableColumns& load_cols, DataTypeSerDeSPtrs serders)>
+                            assert_callback) {
+        ASSERT_EQ(serders.size(), columns.size());
+        // Step 1: Insert data from `column_data_file` into the column and check result with `check_data_file`
+        // Load column data and expected data from CSV files
+        std::vector<std::vector<std::string>> res;
+        struct stat buff;
+        if (stat(column_data_file.c_str(), &buff) == 0) {
+            if (S_ISREG(buff.st_mode)) {
+                // file
+                load_data_from_csv(serders, columns, column_data_file, col_spliter, idxes);
+            } else if (S_ISDIR(buff.st_mode)) {
+                // dir
+                std::filesystem::path fs_path(column_data_file);
+                for (const auto& entry : std::filesystem::directory_iterator(fs_path)) {
+                    std::string file_path = entry.path().string();
+                    std::cout << "load data from file: " << file_path << std::endl;
+                    load_data_from_csv(serders, columns, file_path, col_spliter, idxes);
+                }
+            }
+        }
+
+        // Step 2: Validate the data in `column` matches `expected_data`
+        assert_callback(columns, serders);
+    }
+    ////==================================================================================================================
+
+    void check_columns(MutableColumns& columns, DataTypeSerDeSPtrs serders, DataTypes dataTypes,
+                       char col_spliter, std::set<int> idxes, const std::string& column_data_file,
+                       const std::string& check_data_file, MutableColumns& check_columns,
+                       DataTypeSerDeSPtrs check_serders, char check_col_spliter,
+                       std::set<int> check_idxes,
+                       std::function<void(MutableColumns& load_cols, MutableColumns& assert_columns,
+                                          DataTypes dataTypes)>
+                               assert_callback) {
+        // Load column data and expected data from CSV files
+        load_data_from_csv(serders, columns, column_data_file, col_spliter, idxes);
+        load_data_from_csv(check_serders, check_columns, check_data_file, col_spliter, idxes);
+
+        // Step 2: Validate the data in `column` matches `expected_data`
+        assert_callback(columns, check_columns, dataTypes);
+    }
+
+    static void checkColumn(const IColumn& col1, const IColumn& col2, const IDataType& dataType,
+                            size_t column_size) {
+        for (size_t i = 0; i < column_size; ++i) {
+            Field f1;
+            Field f2;
+            col1.get(i, f1);
+            col2.get(i, f2);
+            EXPECT_EQ(f1, f2);
+        }
+    }
+
+    void printColumn(const IColumn& column, const IDataType& dataType) {
+        std::cout << "colum: " << column.get_name() << " total size: " << column.size()
+                  << std::endl;
+        auto serde = dataType.get_serde(0);
+        auto serde_col = ColumnString::create();
+        auto option = DataTypeSerDe::FormatOptions();
+        serde_col->reserve(column.size());
+        VectorBufferWriter buffer_writer(*serde_col.get());
+        for (size_t i = 0; i < column.size(); ++i) {
+            if (auto st = serde->serialize_one_cell_to_json(column, i, buffer_writer, option);
+                !st) {
+                std::cerr << "Failed to serialize column at row " << i << std::endl;
+                break;
+            }
+            buffer_writer.commit();
+            std::cout << serde_col->get_data_at(i).to_string() << std::endl;
+        }
+    }
+
+    // assert insert_from
+    // Define the custom assert callback function to verify insert_from behavior
+    static void assert_insert_from_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders,
+                                            std::vector<std::vector<std::string>>& assert_res) {
+        // Create an empty column to verify `insert_from` functionality
+        MutableColumns verify_columns;
+        for (auto& col : load_cols) {
+            verify_columns.push_back(col->clone_empty());
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+        // Insert data from `load_cols` to `verify_columns` using `insert_from`
+        std::vector<std::vector<std::string>> res;
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            auto& target_column = verify_columns[i];
+            std::vector<std::string> data;
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                target_column->insert_from(*source_column, j);
+            }
+
+            // Verify the inserted data matches the expected results in `assert_res`
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(target_column->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            for (size_t j = 0; j < target_column->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*target_column, j,
+                                                                     buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                buffer_writer.commit();
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("insert_from", assert_res);
+    }
+
+    // assert insert_range_from
+    // Define the custom assert callback function to verify insert_range_from behavior
+    static void assert_insert_range_from_callback(MutableColumns& load_cols,
+                                                  DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `insert_range_from` functionality
+        MutableColumns verify_columns;
+        for (auto& col : load_cols) {
+            verify_columns.push_back(col->clone_empty());
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+
+        std::vector<size_t> check_length = {0, 10, 100, 1000, 10000, 100000, 1000000};
+        // Insert data from `load_cols` to `verify_columns` using `insert_range_from`
+        std::vector<std::vector<std::string>> res;
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                std::vector<std::string> data;
+                std::cout << "==== now test insert_range_from with col "
+                          << verify_columns[i]->get_name() << std::endl;
+                auto& source_column = load_cols[i];
+                auto& target_column = verify_columns[i];
+                std::vector<size_t> check_start_pos = {0, source_column->size(),
+                                                       (source_column->size() + 1) >> 1};
+                // size_t(-1) may cause overflow, but here we have compiler to check it
+                std::vector<size_t> err_start_pos = {source_column->size() + 1};
+                for (auto pos = err_start_pos.begin(); pos < err_start_pos.end(); ++pos) {
+                    std::cout << "error insert_range_from from " << *pos << " with length " << *cl
+                              << std::endl;
+                    std::cout << *pos + *cl << " > " << source_column->size() << std::endl;
+                    EXPECT_THROW(target_column->insert_range_from(*source_column, *pos, *cl),
+                                 Exception);
+                }
+                for (auto pos = check_start_pos.begin(); pos < check_start_pos.end(); ++pos) {
+                    target_column->clear();
+                    std::cout << "now insert_range_from from " << *pos << " with length " << *cl
+                              << " with source size: " << source_column->size() << std::endl;
+                    if (*pos + *cl > source_column->size()) {
+                        EXPECT_THROW(target_column->insert_range_from(*source_column, *pos, *cl),
+                                     Exception);
+                        continue;
+                    } else {
+                        target_column->insert_range_from(*source_column, *pos, *cl);
+                    }
+
+                    // Verify the inserted data matches the expected results in `assert_res`
+                    auto ser_col = ColumnString::create();
+                    ser_col->reserve(target_column->size());
+                    VectorBufferWriter buffer_writer(*ser_col.get());
+                    for (size_t j = 0; j < target_column->size(); ++j) {
+                        if (auto st = serders[i]->serialize_one_cell_to_json(*target_column, j,
+                                                                             buffer_writer, option);
+                            !st) {
+                            std::cerr << "Failed to serialize column " << i << " at row " << j
+                                      << std::endl;
+                            break;
+                        }
+                        buffer_writer.commit();
+                        data.push_back(ser_col->get_data_at(j).to_string());
+                    }
+                    res.push_back(data);
+                }
+            }
+        }
+        check_res_file("insert_range_from", res);
+    }
+
+    // assert insert_range_from_ignore_overflow which happened in columnStr<UInt32> want to insert from ColumnStr<UInt64> for more column string to be inserted not just limit to the 4G
+    // Define the custom assert callback function to verify insert_range_from_ignore_overflow behavior
+
+    // assert insert_many_from (used in join situation, which handle left table col to expand for right table : such as A[1,2,3] inner join B[2,2,4,4] => A[2,2] )
+    // Define the custom assert callback function to verify insert_many_from behavior
+    static void assert_insert_many_from_callback(MutableColumns& load_cols,
+                                                 DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `insert_many_from` functionality
+        std::cout << "now we are in assert_insert_many_from_callback" << std::endl;
+        MutableColumns verify_columns;
+        for (auto& col : load_cols) {
+            verify_columns.push_back(col->clone_empty());
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<std::vector<string>> actual_res;
+
+        std::vector<size_t> check_length = {0, 10, 100, 1000, 10000, 100000, 1000000};
+        // Insert data from `load_cols` to `verify_columns` using `insert_many_from`
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                auto& source_column = load_cols[i];
+                auto& target_column = verify_columns[i];
+                // size_t(-1) may cause overflow, but here we have compiler to check it
+                std::vector<size_t> check_start_pos = {0, source_column->size(),
+                                                       source_column->size() + 1,
+                                                       (source_column->size() + 1) >> 1};
+                for (auto pos = check_start_pos.begin(); pos < check_start_pos.end(); ++pos) {
+                    if (*pos > source_column->size() || *cl > source_column->size()) {
+                        // insert_range_from now we have no any exception error data to handle, so here will meet crash
+                        continue;
+                    } else if (*pos + *cl > source_column->size()) {
+                        target_column->clear();
+                        // insert_range_from now we have no any exception error data to handle and also no crash
+                        std::cout << "we expect exception insert_many_from from " << *pos
+                                  << " with length " << *cl
+                                  << "with source size: " << source_column->size() << std::endl;
+                        target_column->insert_many_from(*source_column, *pos, *cl);
+                    }
+
+                    // Verify the inserted data matches the expected results in `assert_res`
+                    auto ser_col = ColumnString::create();
+                    ser_col->reserve(target_column->size());
+                    VectorBufferWriter buffer_writer(*ser_col.get());
+                    std::vector<std::string> data;
+                    for (size_t j = 0; j < target_column->size(); ++j) {
+                        if (auto st = serders[i]->serialize_one_cell_to_json(*target_column, j,
+                                                                             buffer_writer, option);
+                            !st) {
+                            std::cerr << "Failed to serialize column " << i << " at row " << j
+                                      << std::endl;
+                            break;
+                        }
+                        buffer_writer.commit();
+                        std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                        data.push_back(actual_str_value);
+                    }
+                    actual_res.push_back(data);
+                }
+            }
+        }
+        // Generate or check the actual result file
+        check_res_file("insert_many_from", actual_res);
+    }
+
+    static void assert_insert_indices_from_callback(MutableColumns& load_cols,
+                                                    DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `insert_indices_from` functionality
+        MutableColumns verify_columns;
+        for (auto& col : load_cols) {
+            verify_columns.push_back(col->clone_empty());
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<std::vector<string>> res;
+        // Insert data from `load_cols` to `verify_columns` using `insert_indices_from`
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            auto& target_column = verify_columns[i];
+
+            // uint32_t(-1) now we have compiler to make sure it will not appear
+            // and this function
+            std::vector<uint32_t> check_indices = {0, uint32_t(source_column->size()),
+                                                   uint32_t(source_column->size() + 1),
+                                                   uint32_t((source_column->size() + 1) >> 1)};
+            for (auto from_idx = check_indices.begin(); from_idx < check_indices.end();
+                 ++from_idx) {
+                for (auto end_idx = check_indices.begin(); end_idx < check_indices.end();
+                     ++end_idx) {
+                    target_column->clear();
+                    // Insert data from `load_cols` to `verify_columns` using `insert_indices_from`
+                    if (*from_idx > *end_idx || *from_idx >= source_column->size() ||
+                        *end_idx >= source_column->size()) {
+                        //                        EXPECT_ANY_THROW(target_column->insert_indices_from(*source_column, &(*from_idx), &(*end_idx)));
+                        // now we do not to check it but we should make sure pass the arguments correctly, if we do not
+                        // here we will meet `heap-buffer-overflow on address`
+                        continue;
+                    } else {
+                        target_column->insert_indices_from(*source_column, &(*from_idx),
+                                                           &(*end_idx));
+                    }
+                    std::cout << source_column->get_name() << " now insert_indices_from from "
+                              << *from_idx << " to " << *end_idx << std::endl;
+                    // Verify the inserted data matches the expected results in `assert_res`
+                    auto ser_col = ColumnString::create();
+                    ser_col->reserve(target_column->size());
+                    VectorBufferWriter buffer_writer(*ser_col.get());
+                    std::vector<string> data;
+                    for (size_t j = 0; j < target_column->size(); ++j) {
+                        if (auto st = serders[i]->serialize_one_cell_to_json(*target_column, j,
+                                                                             buffer_writer, option);
+                            !st) {
+                            std::cerr << "Failed to serialize column " << i << " at row " << j
+                                      << std::endl;
+                            break;
+                        }
+                        buffer_writer.commit();
+                        std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                        std::cout << "actual_str_value: " << actual_str_value << std::endl;
+                        data.push_back(actual_str_value);
+                    }
+                    res.push_back(data);
+                }
+            }
+        }
+        check_res_file("insert_indices_from", res);
+    }
+
+    static void assert_insert_data_from_callback(MutableColumns& load_cols,
+                                                 DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `insert_data` functionality
+        MutableColumns verify_columns;
+        for (auto& col : load_cols) {
+            verify_columns.push_back(col->clone_empty());
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+        // Insert data from `load_cols` to `verify_columns` using `insert_data`
+        std::vector<std::vector<string>> res;
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            auto& target_column = verify_columns[i];
+
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                target_column->insert_data(source_column->get_data_at(j).data,
+                                           source_column->get_data_at(j).size);
+            }
+
+            // Verify the inserted data matches the expected results in `assert_res`
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(target_column->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            for (size_t j = 0; j < target_column->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*target_column, j,
+                                                                     buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("insert_data", res);
+    }
+
+    static void assert_insert_many_raw_data_from_callback(MutableColumns& load_cols,
+                                                          DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `insert_many_raw_data` functionality
+        MutableColumns verify_columns;
+        for (auto& col : load_cols) {
+            verify_columns.push_back(col->clone_empty());
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<std::vector<string>> res;
+        // Insert data from `load_cols` to `verify_columns` using `insert_many_raw_data`
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            auto& target_column = verify_columns[i];
+
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                target_column->insert_many_raw_data(source_column->get_data_at(j).data,
+                                                    source_column->get_data_at(j).size);
+            }
+
+            // Verify the inserted data matches the expected results in `assert_res`
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(target_column->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            for (size_t j = 0; j < target_column->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*target_column, j,
+                                                                     buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("insert_many_raw_data", res);
+    }
+
+    // assert insert_default
+    // Define the custom assert callback function to verify insert_default behavior
+    static void assert_insert_default_callback(MutableColumns& load_cols,
+                                               DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `insert_default` functionality
+        MutableColumns verify_columns;
+        for (auto& col : load_cols) {
+            verify_columns.push_back(col->clone_empty());
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<std::vector<string>> res;
+        // Insert data from `load_cols` to `verify_columns` using `insert_default`
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& target_column = verify_columns[i];
+            target_column->insert_default();
+
+            // Verify the inserted data matches the expected results in `assert_res`
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(target_column->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            for (size_t j = 0; j < target_column->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*target_column, j,
+                                                                     buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                buffer_writer.commit();
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("insert_default", res);
+    }
+
+    // assert insert_many_default
+    // Define the custom assert callback function to verify insert_many_defaults behavior
+    static void assert_insert_many_defaults_callback(MutableColumns& load_cols,
+                                                     DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `insert_many_defaults` functionality
+        MutableColumns verify_columns;
+        for (auto& col : load_cols) {
+            verify_columns.push_back(col->clone_empty());
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<std::vector<string>> res;
+        std::vector<size_t> check_length = {0, 10, 100, 1000, 10000, 100000, 1000000};
+        // Insert data from `load_cols` to `verify_columns` using `insert_many_defaults`
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                auto& target_column = verify_columns[i];
+                target_column->insert_many_defaults(*cl);
+
+                // Verify the inserted data matches the expected results in `assert_res`
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(target_column->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                std::vector<string> data;
+                for (size_t j = 0; j < target_column->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*target_column, j,
+                                                                         buffer_writer, option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+                res.push_back(data);
+            }
+        }
+        check_res_file("insert_many_defaults", res);
+    }
+
+    //// data access interfaces:
+    // virtual StringRef
+    //get_data_at (size_t n) const = 0
+    // if we implement the get_data_at, we should know the data is stored in the column, and we can get the data by the index
+    static void assert_get_data_at_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        std::vector<std::vector<std::string>> res;
+        // just check cols get_data_at is the same as assert_res
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            std::vector<std::string> data;
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                auto actual_str_value = source_column->get_data_at(j).to_string();
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("get_data_at", res);
+    }
+
+    //virtual void
+    // field is memory layout of the data , maybe we can use get from base column and insert into same type column to check this behavior is right
+    //get (size_t n, Field &res) const = 0
+    //Like the previous one, but avoids extra copying if Field is in a container, for example.
+    //virtual Field
+    //operator[] (size_t n) const = 0
+    static void assert_field_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        MutableColumns assert_cols(load_cols.size());
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            assert_cols[i] = load_cols[i]->clone_empty();
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<std::vector<std::string>> res;
+        // just check cols get is the same as assert_res
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            std::cout << " new insert field for column : " << assert_cols[i]->get_name()
+                      << " with size : " << assert_cols[i]->size() << std::endl;
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                Field f;
+                source_column->get(j, f);
+                std::cout << "field: " << f.get_type() << std::endl;
+                assert_cols[i]->insert(f);
+            }
+        }
+        // Verify the inserted data matches the expected results in `assert_res`
+        for (size_t i = 0; i < assert_cols.size(); ++i) {
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(load_cols[i]->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            for (size_t j = 0; j < assert_cols[i]->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*assert_cols[i], j,
+                                                                     buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                buffer_writer.commit();
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                std::cout << "actual_str_value: " << actual_str_value << std::endl;
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("get_field", res);
+        // just check cols operator [] to get field same with field
+        std::vector<std::vector<std::string>> res2;
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                Field f = source_column->operator[](j);
+                assert_cols[i]->insert(f);
+            }
+        }
+
+        // Verify the inserted data matches the expected results in `assert_res`
+        for (size_t i = 0; i < assert_cols.size(); ++i) {
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(load_cols[i]->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            for (size_t j = 0; j < assert_cols[i]->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*assert_cols[i], j,
+                                                                     buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                buffer_writer.commit();
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                std::cout << "actual_str_value: " << actual_str_value << std::endl;
+                data.push_back(actual_str_value);
+            }
+            res2.push_back(data);
+        }
+        check_res_file("get_field_operator", res2);
+    }
+    //
+    //virtual StringRef
+    //get_raw_data () const which is continues memory layout of the data,
+    // we can use this to check the data is stored in the column
+    template <class T>
+    static void assert_get_raw_data_callback(MutableColumns& load_cols,
+                                             DataTypeSerDeSPtrs serders) {
+        // just check cols get_raw_data is the same as assert_res
+        std::cout << "now we are in assert_get_raw_data_callback" << std::endl;
+        std::vector<std::vector<string>> res;
+        MutableColumns assert_cols(load_cols.size());
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            assert_cols[i] = load_cols[i]->clone_empty();
+        }
+        auto option = DataTypeSerDe::FormatOptions();
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            const T* rd = (T*)source_column->get_raw_data().data;
+            for (size_t j = 0; j < source_column->size(); j++) {
+                Field f;
+                source_column->get(j, f);
+                ASSERT_EQ(f, Field(rd[j]));
+                // insert field to assert column
+                assert_cols[i]->insert(f);
+            }
+        }
+
+        // Verify the inserted data matches the expected results in `assert_res`
+        for (size_t i = 0; i < assert_cols.size(); ++i) {
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(load_cols[i]->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            for (size_t j = 0; j < assert_cols[i]->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*assert_cols[i], j,
+                                                                     buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                buffer_writer.commit();
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                std::cout << "actual_str_value: " << actual_str_value << std::endl;
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("get_raw_data", res);
+    }
+
+    //If returns the underlying data array, otherwise throws an exception.
+    //virtual Int64
+    //get_int (size_t) const
+    static void assert_get_int_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        std::vector<std::vector<string>> res;
+        // just check cols get_int is the same as assert_res
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            std::vector<string> data;
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                auto actual_str_value = std::to_string(source_column->get_int(j));
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("get_int", res);
+    }
+    //virtual bool
+    //get_bool (size_t) const
+    static void assert_get_bool_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        std::vector<std::vector<string>> res;
+        // just check cols get_bool is the same as assert_res
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            std::vector<string> data;
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                auto actual_str_value = std::to_string(source_column->get_bool(j));
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("get_bool", res);
+    }
+
+    //// column meta interfaces:
+    // virtual std::string
+    //get_name () const , simple assert to make sure name
+    static void getNameAssert(IColumn& column, const string expect_name) {
+        ASSERT_EQ(expect_name, column.get_name());
+    }
+
+    // size related we can check from checked file to make sure the size is right
+    //
+    //Returns number of values in column.
+    //virtual size_t
+    //size_of_value_if_fixed () const
+    //
+    //If values_have_fixed_size, returns size of value, otherwise throw an exception.
+    //virtual size_t
+    //byte_size () const =0
+    //
+    //Size of column data in memory (may be approximate) - for profiling. Zero, if could not be determined.
+    //
+    //
+    //virtual size_t
+    //allocated_bytes () const =0
+    //
+    static void assert_size_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        std::vector<std::vector<string>> res;
+        // just check cols size is the same as assert_res
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            std::vector<string> data;
+            auto& source_column = load_cols[i];
+            auto actual_str_value = std::to_string(source_column->size());
+            data.push_back(actual_str_value);
+            res.push_back(data);
+        }
+        check_res_file("size", res);
+    }
+
+    // assert byte_size
+    // Define the custom assert callback function to verify byte_size behavior
+    static void assert_byte_size_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        // just check cols byte_size is the same as assert_res
+        std::vector<std::vector<string>> res;
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            std::vector<string> data;
+            auto& source_column = load_cols[i];
+            auto actual_str_value = std::to_string(source_column->byte_size());
+            data.push_back(actual_str_value);
+            res.push_back(data);
+        }
+        check_res_file("byte_size", res);
+    }
+
+    // assert allocated_bytes Define the custom assert callback function to verify allocated_bytes behavior
+    static void assert_allocated_bytes_callback(MutableColumns& load_cols,
+                                                DataTypeSerDeSPtrs serders) {
+        // just check cols allocated_bytes is the same as assert_res
+        std::cout << "now we are in assert_allocated_bytes_callback" << std::endl;
+        std::vector<std::vector<string>> res;
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            std::vector<string> data;
+            auto& source_column = load_cols[i];
+            auto actual_str_value = std::to_string(source_column->allocated_bytes());
+            data.push_back(actual_str_value);
+            res.push_back(data);
+        }
+        check_res_file("allocated_bytes", res);
+    }
+
+    // is_exclusive assert should assert the column which has multiple columnPtr in the column
+
+    //// data manage interfaces
+    //virtual void
+    // pop_back (size_t n) = 0
+    // assert pop_back
+    static void assert_pop_back_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `pop_back` functionality
+        // check pop_back with different n
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        // size_t(-1) 会导致pod_array 溢出
+        std::vector<size_t> check_length = {0, 1, 10, 100, 1000, 10000, 100000};
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                auto& source_column = load_cols[i];
+                if (*cl > source_column->size()) {
+                    // now we do not check in popback, but we should make sure the arguments are passed correctly,
+                    // otherwise we should meet `Check failed: false Amount of memory requested to allocate is more than allowed`
+                    std::cout << "now we are in pop_back column : " << load_cols[i]->get_name()
+                              << " for column size : " << source_column->size() << std::endl;
+                    source_column->pop_back(source_column->size());
+                } else {
+                    std::cout << "now we are in pop_back column : " << load_cols[i]->get_name()
+                              << " with check length: " << *cl
+                              << " for column size : " << source_column->size() << std::endl;
+                    source_column->pop_back(*cl);
+                }
+
+                // Verify the pop back data matches the expected results in `assert_res`
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(load_cols[i]->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                std::vector<string> data;
+                for (size_t j = 0; j < source_column->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*source_column, j,
+                                                                         buffer_writer, option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+                res.push_back(data);
+            }
+        }
+        check_res_file("pop_back", res);
+    }
+
+    //virtual MutablePtr
+    // Creates empty column with the same type.
+    //clone_empty () const this is clone ,we should also check if the size is 0 after clone and ptr is not the same
+    void assert_clone_empty(IColumn& column) {
+        auto ptr = column.clone_empty();
+        EXPECT_EQ(ptr->size(), 0);
+        EXPECT_NE(ptr.get(), &column);
+    }
+
+    //virtual MutablePtr
+    //clone_resized (size_t s) const
+    static void assert_clone_resized_callback(MutableColumns& load_cols,
+                                              DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `clone_resized` functionality
+        // check clone_resized with different size
+        // size_t(-1) 会导致pod_array 溢出
+        std::vector<size_t> check_length = {0, 1, 10, 100, 1000, 10000, 100000};
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<std::vector<string>> res;
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                auto& source_column = load_cols[i];
+                auto ptr = source_column->clone_resized(*cl);
+                // check size
+                EXPECT_EQ(ptr->size(), *cl);
+                // check ptr is not the same
+                EXPECT_NE(ptr.get(), source_column);
+
+                // check after clone_resized with assert_res
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(ptr->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                std::vector<string> data;
+                data.push_back("column: " + source_column->get_name() + " with check size: " +
+                               std::to_string(*cl) + " with ptr: " + std::to_string(ptr->size()));
+                for (size_t j = 0; j < ptr->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*ptr, j, buffer_writer,
+                                                                         option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+                res.push_back(data);
+            }
+        }
+        check_res_file("clone_resized", res);
+    }
+
+    // is_exclusive() means to check the ptr is not shared with other, and we can just clean it. so it's important to check the column is exclusive or not
+
+    //virtual Ptr
+    //cut (size_t start, size_t length) const final will call clone_empty and insert_range_from
+    static void assert_cut_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `cut` functionality
+        // check cut with different start and length
+        // size_t(-1) 会导致pod_array 溢出
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        //        std::vector<size_t> cut_start = {0, 1, 10, 100, 1000, 10000, 100000};
+        std::vector<size_t> check_length = {0, 1, 10, 100, 1000, 10000, 100000};
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                auto& source_column = load_cols[i];
+                ColumnPtr ptr;
+                size_t insert_size = *cl > source_column->size() ? source_column->size() : *cl;
+                // now we do not check in cut, but we should make sure the arguments are passed correctly,
+                // otherwise we should meet `Check failed: false Amount of memory requested to allocate is more than allowed`
+                std::cout << "now we are in cut column : " << load_cols[i]->get_name()
+                          << " with check length: " << insert_size
+                          << " for column size : " << source_column->size() << std::endl;
+                ptr = source_column->cut(0, insert_size);
+                // check size
+                EXPECT_EQ(ptr->size(), insert_size);
+                // check ptr is not the same
+                EXPECT_NE(ptr.get(), source_column);
+                // check after cut with assert_res
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(ptr->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                std::vector<string> data;
+                data.push_back("column: " + source_column->get_name() + " with check size: " +
+                               std::to_string(*cl) + " with ptr: " + std::to_string(ptr->size()));
+                for (size_t j = 0; j < ptr->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*ptr, j, buffer_writer,
+                                                                         option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+            }
+        }
+        check_res_file("cut", res);
+    }
+
+    //virtual Ptr
+    //shrink (size_t length) const final  shrink many has different ptr for the column, because of
+    // here has some improvement according the origin column which use_count is 1, we can just return the origin column to avoid the copy
+    // but we should make sure the column data is absolutely the same as expand or cut with the data
+    static void assert_shrink_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `cut` functionality
+        // check shrink with different start and length
+        // size_t(-1) 会导致pod_array 溢出
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        //        std::vector<size_t> cut_start = {0, 1, 10, 100, 1000, 10000, 100000};
+        // less check_length: cut , more check_length: expand
+        std::vector<size_t> check_length = {0, 1, 10, 100, 1000, 10000, 100000};
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                auto& source_column = load_cols[i];
+                ColumnPtr ptr;
+                size_t insert_size = *cl > source_column->size() ? source_column->size() : *cl;
+                // now we do not check in cut, but we should make sure the arguments are passed correctly,
+                // otherwise we should meet `Check failed: false Amount of memory requested to allocate is more than allowed`
+                std::cout << "now we are in shrink column : " << load_cols[i]->get_name()
+                          << " with check length: " << insert_size
+                          << " for column size : " << source_column->size() << std::endl;
+                ptr = source_column->shrink(insert_size);
+                std::cout << "use_count : " << source_column->use_count() << std::endl;
+                // check size
+                EXPECT_EQ(ptr->size(), insert_size);
+                // check ptr is not the same
+                EXPECT_NE(ptr.get(), source_column);
+                // check after cut with assert_res
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(ptr->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                std::vector<string> data;
+                data.push_back("column: " + source_column->get_name() + " with check size: " +
+                               std::to_string(*cl) + " with ptr: " + std::to_string(ptr->size()));
+                for (size_t j = 0; j < ptr->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*ptr, j, buffer_writer,
+                                                                         option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+            }
+        }
+        check_res_file("shrink", res);
+    }
+    //
+    //
+    //cut or expand inplace. this would be moved, only the return value is available.
+    //virtual void
+    //reserve (size_t)
+    static void assert_reserve_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `reserve` functionality
+        // check reserve with different size
+        // size_t(-1) 会导致pod_array 溢出: Check failed: false Amount of memory requested to allocate is more than allowed
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<size_t> check_length = {0, 1, 10, 100, 1000, 10000, 100000};
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                auto assert_origin_size = load_cols[i]->size();
+                auto& source_column = load_cols[i];
+                std::cout << "now we are in reserve column : " << load_cols[i]->get_name()
+                          << " with check length: " << *cl
+                          << " for column size : " << source_column->size() << std::endl;
+                source_column->reserve(*cl);
+                // check size no changed after reserve
+                EXPECT_EQ(source_column->size(), assert_origin_size);
+                // check after reserve with assert_res
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(source_column->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                std::vector<string> data;
+                data.push_back("column: " + source_column->get_name() +
+                               " with check size: " + std::to_string(*cl) +
+                               " with ptr: " + std::to_string(source_column->size()));
+                for (size_t j = 0; j < source_column->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*source_column, j,
+                                                                         buffer_writer, option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+                res.push_back(data);
+            }
+        }
+        check_res_file("reserve", res);
+    }
+
+    //virtual void
+    //resize (size_t) means we should really resize the column, include the all sub columns, like data column in column array
+    static void assert_resize_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `resize` functionality
+        // check resize with different size
+        // size_t(-1) 会导致pod_array 溢出: Check failed: false Amount of memory requested to allocate is more than allowed
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<size_t> check_length = {0, 1, 10, 100, 1000, 10000, 100000};
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                auto& source_column = load_cols[i];
+                std::cout << "now we are in resize column : " << load_cols[i]->get_name()
+                          << " with check length: " << *cl
+                          << " for column size : " << source_column->size() << std::endl;
+                source_column->resize(*cl);
+                // check size
+                EXPECT_EQ(source_column->size(), *cl);
+                // check after resize with assert_res
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(source_column->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                std::vector<string> data;
+                data.push_back("column: " + source_column->get_name() +
+                               " with check size: " + std::to_string(*cl) +
+                               " with ptr: " + std::to_string(source_column->size()));
+                for (size_t j = 0; j < source_column->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*source_column, j,
+                                                                         buffer_writer, option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+                res.push_back(data);
+            }
+        }
+        check_res_file("resize", res);
+    }
+
+    //virtual Ptr
+    //replicate (const Offsets &offsets)
+    // 1. used in ColumnConst.convert_to_full_column,
+    //   we should make a situation that the column is not full column, and then we can use replicate to make it full column
+    // 2. used in some agg calculate
+    static void assert_replicate_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `replicate` functionality
+        // check replicate with different offsets
+        // size_t(-1) 会导致pod_array 溢出: Check failed: false Amount of memory requested to allocate is more than allowed
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<size_t> check_length = {0, 1, 10, 100, 1000, 10000, 100000};
+        //       std::vector<size_t> check_length = {10, 9};
+        //       size_t sum = std::reduce(check_length.begin(), check_length.end(), 0, std::plus<size_t>());
+        IColumn::Offsets offsets;
+        for (size_t i = 0; i < check_length.size(); i++) {
+            offsets.push_back(check_length[i]);
+        }
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            //               auto origin_size = load_cols[0]->size();
+            // here will heap_use_after_free
+            //               ColumnConst* const_col = ColumnConst::create(load_cols[i]->clone_resized(1), *cl);
+            if (load_cols[i]->size() != check_length.size()) {
+                EXPECT_ANY_THROW(load_cols[i]->replicate(offsets));
+            }
+            auto source_column = load_cols[i]->shrink(check_length.size());
+            std::cout << "now we are in replicate column : " << load_cols[i]->get_name()
+                      << " for column size : " << source_column->size() << std::endl;
+            //               auto ptr = const_col->convert_to_full_column();
+            // here will return different ptr
+            auto ptr = source_column->replicate(offsets);
+            // check ptr
+            EXPECT_NE(ptr.get(), source_column.get());
+            // check after replicate with assert_res
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(ptr->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            data.push_back("column: " + source_column->get_name() +
+                           " with generate col size: " + std::to_string(ptr->size()));
+            for (size_t j = 0; j < ptr->size(); ++j) {
+                // check size
+                if (auto st =
+                            serders[i]->serialize_one_cell_to_json(*ptr, j, buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                buffer_writer.commit();
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("replicate", res);
+    }
+    //virtual void
+    //for_each_subcolumn (ColumnCallback)
+    //virtual void
+    //replace_column_data (const IColumn &, size_t row, size_t self_row=0) used in BlockReader(VerticalBlockReader)::_copy_agg_data() for agg value data
+    // the passed column must be non-variable length column: like columnVector...
+    static void assert_replace_column_data_callback(MutableColumns& load_cols,
+                                                    DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `replace_column_data` functionality
+        // check replace_column_data with different row and self_row
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<size_t> check_length = {0, 1, 7, 10, 100, 1000};
+        for (auto cl = check_length.begin(); cl < check_length.end(); ++cl) {
+            for (size_t i = 0; i < load_cols.size(); ++i) {
+                if (load_cols[i]->is_variable_length()) {
+                    throw std::runtime_error(
+                            "replace_column_data only support non-variable length column");
+                }
+                auto& source_column = load_cols[i];
+                if (*cl > source_column->size()) {
+                    // if replace row is bigger than the source column size here meet pod coredump
+                    continue;
+                }
+                std::cout << "now we are in replace_column_data column : "
+                          << load_cols[i]->get_name() << " with check length: " << *cl
+                          << " for column size : " << source_column->size() << std::endl;
+                source_column->replace_column_data(*source_column, *cl);
+                // check after replace_column_data: the first data is same with the data in source column's *cl row
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(source_column->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                std::vector<string> data;
+                data.push_back("column: " + source_column->get_name() +
+                               " with check size: " + std::to_string(*cl) +
+                               " with ptr: " + std::to_string(source_column->size()));
+                for (size_t j = 0; j < source_column->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*source_column, j,
+                                                                         buffer_writer, option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+                res.push_back(data);
+            }
+        }
+        check_res_file("replace_column_data", res);
+    }
+
+    //
+    //virtual void
+    //replace_column_null_data (const uint8_t *__restrict null_map)
+    // which is only for column-vector/decimal with null data, like columnNullable<columnDecimal>, so other column will do nothing
+    // some situation: we just calculate the decimal column data, then set the null_map. when we look at null row, maybe here meet an overflow decimal-value or just a random value but nullmap[row] is true
+    // so we should make nullmap[row] = 1 but data[row] is default value for this kind of column
+    static void assert_replace_column_null_data_callback(MutableColumns& load_cols,
+                                                         DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `replace_column_null_data` functionality
+        // check replace_column_null_data with different null_map
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        const NullMap null_map = {1, 1, 0, 1};
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            std::cout << "now we are in replace_column_null_data column : "
+                      << load_cols[i]->get_name() << " for column size : " << source_column->size()
+                      << "with nullmap" << null_map.data() << std::endl;
+            source_column->replace_column_null_data(null_map.data());
+
+            // check after replace_column_null_data: 1 in nullmap present the load cols data is null and data should be default value
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(source_column->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            data.push_back("column: " + source_column->get_name() +
+                           " with nullmap: " + std::to_string(*null_map.data()) +
+                           " with ptr: " + std::to_string(source_column->size()));
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*source_column, j,
+                                                                     buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                buffer_writer.commit();
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("replace_column_null_data", res);
+    }
+
+    //
+    //virtual void
+    //append_data_by_selector (MutablePtr &res, const Selector &selector, size_t begin, size_t end) const =0
+    static void assert_append_data_by_selector_callback(MutableColumns& load_cols,
+                                                        DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `append_data_by_selector` functionality
+        // check append_data_by_selector with different selector
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            MutableColumnPtr res_col = source_column->clone_empty();
+            // selector is range for the column, contain values from 0 to num_columns - 1.
+            // selector size should bigger than begin and end ,
+            // because selector[i], i in range(begin,end),  Make a DCHECK for this
+            const ColumnArray::Selector selector = {1, 2, 3, 0};
+            std::cout << "now we are in append_data_by_selector column : "
+                      << load_cols[i]->get_name() << " for column size : " << source_column->size()
+                      << " with selector size: " << selector.size() << std::endl;
+            source_column->append_data_by_selector(res_col, selector, 0, 4);
+            // check after append_data_by_selector: 1 in selector present the load cols data is selected and data should be default value
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(res_col->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            data.push_back("column: " + source_column->get_name() +
+                           " with selector size : " + std::to_string(selector.size()) +
+                           " with ptr: " + std::to_string(res_col->size()));
+            for (size_t j = 0; j < res_col->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*res_col, j, buffer_writer,
+                                                                     option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                buffer_writer.commit();
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("append_data_by_selector", res);
+    }
+
+    //
+    //virtual bool
+    //structure_equals (const IColumn &) const
+    //void
+    //copy_date_types (const IColumn &col)
+    //String
+    //dump_structure () const
+    //MutablePtr
+    //mutate () const &&
+    //virtual void
+    //clear ()=0
+    //
+    //Clear data of column, just like vector clear.
+
+    // Column Calculate Interface: filter, compare, permute, sort
+    // filter (const Filter &filt, ssize_t result_size_hint) const =0 with a result_size_hint to pass, but we should make sure the result_size_hint is not bigger than the source column size
+    //  Filter is a array contains 0 or 1 to present the row is selected or not, so it should be the same size with the source column
+    static void assert_filter_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `filter` functionality
+        // check filter with different filter
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            auto source_size = source_column->size();
+            const ColumnArray::Filter all_filtered(source_size, 0);
+            const ColumnArray::Filter no_filtered(source_size, 1);
+            // invalid data -1 will also make data without be filtered ??
+            ColumnArray::Filter invalid_filter(source_size - 1, 1);
+            invalid_filter.emplace_back(-1);
+            std::vector<string> data;
+            std::cout << "now we are in filter column : " << load_cols[i]->get_name()
+                      << " for column size : " << source_column->size() << std::endl;
+            {
+                auto ptr = source_column->filter(all_filtered, source_column->size());
+                // check filter res
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(ptr->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                data.push_back("column: " + source_column->get_name() +
+                               " with all filtered with ptr: " + std::to_string(ptr->size()));
+                for (size_t j = 0; j < ptr->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*ptr, j, buffer_writer,
+                                                                         option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+                res.push_back(data);
+            }
+
+            {
+                auto ptr = source_column->filter(no_filtered, source_column->size());
+                // check filter res
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(ptr->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                data.clear();
+                data.push_back("column: " + source_column->get_name() +
+                               " with no filtered with ptr: " + std::to_string(ptr->size()));
+                for (size_t j = 0; j < ptr->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*ptr, j, buffer_writer,
+                                                                         option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+                res.push_back(data);
+            }
+            {
+                // check filter with invalid filter
+                auto ptr = source_column->filter(invalid_filter, source_column->size());
+                // check filter res
+                auto ser_col = ColumnString::create();
+                ser_col->reserve(ptr->size());
+                VectorBufferWriter buffer_writer(*ser_col.get());
+                data.clear();
+                data.push_back("column: " + source_column->get_name() +
+                               " with invalid filtered with ptr: " + std::to_string(ptr->size()));
+                for (size_t j = 0; j < ptr->size(); ++j) {
+                    if (auto st = serders[i]->serialize_one_cell_to_json(*ptr, j, buffer_writer,
+                                                                         option);
+                        !st) {
+                        std::cerr << "Failed to serialize column " << i << " at row " << j
+                                  << std::endl;
+                        break;
+                    }
+                    buffer_writer.commit();
+                    std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                    data.push_back(actual_str_value);
+                }
+                res.push_back(data);
+            }
+        }
+        check_res_file("filter", res);
+    }
+
+    // Compare
+
+    // Column Hash Interfaces:
+    // update_hashes_with_value (size_t, size_t, Hashes &hashes) const : which inner just use xxhash for column data
+    static void assert_update_hashes_with_value_callback(MutableColumns& load_cols,
+                                                         DataTypeSerDeSPtrs serders) {
+        // Create an empty column to verify `update_hashes_with_value` functionality
+        // check update_hashes_with_value with different hashes
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            std::vector<uint64_t> xx_hash_vals(source_column->size());
+            std::cout << "now we are in update_hashes_with_value column : "
+                      << load_cols[i]->get_name() << " for column size : " << source_column->size()
+                      << std::endl;
+            auto* __restrict xx_hashes = xx_hash_vals.data();
+            EXPECT_NO_FATAL_FAILURE(source_column->update_hashes_with_value(xx_hashes));
+            // check after update_hashes_with_value: 1 in selector present the load cols data is selected and data should be default value
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(source_column->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            data.push_back("column: " + source_column->get_name() +
+                           " with hashes: " + std::to_string(*xx_hashes) +
+                           " with ptr: " + std::to_string(source_column->size()));
+            res.push_back(data);
+        }
+        check_res_file("update_hashes_with_value", res);
+    }
+
+    //virtual void
+    //update_hashes (size_t, size_t, Hashes &hashes) const
+    static void assert_update_crc_hashes_callback(MutableColumns& load_cols,
+                                                  DataTypeSerDeSPtrs serders,
+                                                  std::vector<PrimitiveType> pts) {
+        // Create an empty column to verify `update_hashes` functionality
+        // check update_hashes with different hashes
+        std::vector<std::vector<string>> res;
+        auto option = DataTypeSerDe::FormatOptions();
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            auto& source_column = load_cols[i];
+            std::vector<uint32_t> crc_hash_vals(source_column->size());
+            std::cout << "now we are in update_hashes column : " << load_cols[i]->get_name()
+                      << " for column size : " << source_column->size() << std::endl;
+            EXPECT_NO_FATAL_FAILURE(source_column->update_crcs_with_value(
+                    crc_hash_vals.data(), pts[i], source_column->size()));
+            // check after update_hashes: 1 in selector present the load cols data is selected and data should be default value
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(source_column->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            data.push_back("column: " + source_column->get_name() +
+                           " with hashes: " + std::to_string(*crc_hash_vals.data()) +
+                           " with ptr: " + std::to_string(source_column->size()));
+            res.push_back(data);
+        }
+        check_res_file("update_crcs_hashes", res);
+    }
+
+    // column size changed calculation:
+    //  size, reserve, resize, empty, byte_size, allocated_bytes, clone_resized, get_shrinked_column
+    //  cut(LIMIT operation), shrink
+
+    // get_shrinked_column should only happened in char-type column or nested char-type column,
+    // other column just return the origin column without any data changed, so check file content should be the same as the origin column
+    //  just shrink the end zeros for char-type column which happened in segmentIterator
+    //    eg. column_desc: char(6), insert into char(3), the char(3) will padding the 3 zeros at the end for writing to disk.
+    //       but we select should just print the char(3) without the padding zeros
+    //  limit and topN operation will trigger this function call
+    void shrink_padding_chars_callback(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
+        auto option = DataTypeSerDe::FormatOptions();
+        std::vector<std::vector<string>> res;
+        for (size_t i = 0; i < load_cols.size(); i++) {
+            auto& source_column = load_cols[i];
+            std::cout << "now we are in shrink_padding_chars column : " << load_cols[i]->get_name()
+                      << " for column size : " << source_column->size() << std::endl;
+            source_column->shrink_padding_chars();
+            // check after get_shrinked_column: 1 in selector present the load cols data is selected and data should be default value
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(source_column->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            std::vector<string> data;
+            data.push_back("column: " + source_column->get_name() +
+                           " with shrinked column size: " + std::to_string(source_column->size()));
+            for (size_t j = 0; j < source_column->size(); ++j) {
+                if (auto st = serders[i]->serialize_one_cell_to_json(*source_column, j,
+                                                                     buffer_writer, option);
+                    !st) {
+                    std::cerr << "Failed to serialize column " << i << " at row " << j << std::endl;
+                    break;
+                }
+                buffer_writer.commit();
+                std::string actual_str_value = ser_col->get_data_at(j).to_string();
+                data.push_back(actual_str_value);
+            }
+            res.push_back(data);
+        }
+        check_res_file("shrink_padding_chars", res);
+    }
+
+    void assert_size_eq(MutableColumnPtr col, size_t expect_size) {
+        EXPECT_EQ(col->size(), expect_size);
+    }
+
+    // empty just use size() == 0 to impl as default behavior
+    void assert_empty(MutableColumnPtr col) { EXPECT_EQ(col->size(), 0); }
+
+    // reserve, resize, byte_size, allocated_bytes, clone_resized, get_shrinked_column
+    void assert_reserve_size(MutableColumnPtr col, size_t reserve_size, size_t expect_size) {
+        col->reserve(reserve_size);
+        EXPECT_EQ(col->size(), expect_size);
+    }
+
+    //  cut(LIMIT operation) will cut the column with the given from and to, and return the new column
+    //  notice return column is clone from origin column
+    void assert_cut(MutableColumnPtr col, size_t from, size_t to) {
+        auto ori = col->size();
+        auto ptr = col->cut(from, to);
+        EXPECT_EQ(ptr->size(), to - from);
+        EXPECT_EQ(col->size(), ori);
+    }
+
+    // shrink is cut/append the column with the given size, which called from Block::set_num_rows
+    // and some Operator may call this set_num_rows to make rows satisfied, like limit operation
+    // but different from cut behavior which
+    // return column is mutate from origin column
+    void assert_shrink(MutableColumnPtr col, size_t shrink_size) {
+        auto ptr = col->shrink(shrink_size);
+        EXPECT_EQ(ptr->size(), shrink_size);
+        EXPECT_EQ(col->size(), shrink_size);
+    }
+
+    // resize has fixed-column implementation and variable-column implementation
+    // like string column, the resize will resize the offsets column but not the data column (because it doesn't matter the size of data column, all operation for string column is based on the offsets column)
+    // like vector column, the resize will resize the data column
+    // like array column, the resize will resize the offsets column and the data column (which in creator we have check staff for the size of data column is the same as the size of offsets column)
+    void assert_resize(MutableColumnPtr col, size_t expect_size) {
+        col->resize(expect_size);
+        EXPECT_EQ(col->size(), expect_size);
+    }
+
+    // replicate is clone with new column from the origin column, always from ColumnConst to expand the column
+    void assert_replicate(MutableColumnPtr col, IColumn::Offsets& offsets) {
+        auto new_col = col->replicate(offsets);
+        EXPECT_EQ(new_col->size(), offsets.back());
+    }
+
+    // byte size is just appriximate size of the column
+    //  as fixed column type, like column_vector, the byte size is sizeof(columnType) * size()
+    //  as variable column type, like column_string, the byte size is sum of chars size() and offsets size * sizeof(offsetType)
+    void assert_byte_size(MutableColumnPtr col, size_t expect_size) {
+        EXPECT_EQ(col->byte_size(), expect_size);
+    }
+
+    // allocated bytes is the real size of the column
+    void assert_allocated_bytes(MutableColumnPtr col, size_t expect_size) {
+        EXPECT_EQ(col->allocated_bytes(), expect_size);
+    }
+
+    // clone_resized will clone the column and cut/append to the new column with the size of the original column
+    void assert_clone_resized(MutableColumnPtr col, size_t expect_size) {
+        auto new_col = col->clone_resized(expect_size);
+        EXPECT_EQ(new_col->size(), expect_size);
+    }
+
+    //serialize and deserialize which usually used in AGG function:
+    //  serialize_value_into_arena, deserialize_and_insert_from_arena (called by AggregateFunctionDistinctMultipleGenericData, group_array_intersect, nested-types serder like: DataTypeArraySerDe::write_one_cell_to_jsonb)
+    void ser_deserialize_with_arena_impl(MutableColumns& columns, const DataTypes& data_types) {
+        size_t rows = columns[0]->size();
+        for (auto& column : columns) {
+            if (column->size() > rows) {
+                std::cerr << "Column size mismatch: " << column->size() << " vs " << rows
+                          << std::endl;
+                column->pop_back(rows - column->size());
+            } else if (column->size() < rows) {
+                std::cerr << "Column size mismatch: " << column->size() << " vs " << rows
+                          << std::endl;
+                column->insert_many_defaults(rows - column->size());
+            }
+        }
+        /// check serialization is reversible.
+        Arena arena;
+        MutableColumns argument_columns(data_types.size());
+        const char* pos = nullptr;
+        StringRef key(pos, 0);
+        {
+            // serialize
+            for (size_t r = 0; r < columns[0]->size(); ++r) {
+                for (size_t i = 0; i < columns.size(); ++i) {
+                    auto cur_ref = columns[i]->serialize_value_into_arena(r, arena, pos);
+                    key.data = cur_ref.data - key.size;
+                    key.size += cur_ref.size;
+                    //                    printColumn(*columns[i], *data_types[i]);
+                }
+            }
+        }
+
+        {
+            // deserialize
+            for (size_t i = 0; i < data_types.size(); ++i) {
+                argument_columns[i] = data_types[i]->create_column();
+            }
+            const char* begin = key.data;
+            for (size_t r = 0; r < columns[0]->size(); ++r) {
+                for (size_t i = 0; i < argument_columns.size(); ++i) {
+                    begin = argument_columns[i]->deserialize_and_insert_from_arena(begin);
+                    //                    printColumn(*argument_columns[i], *data_types[i]);
+                }
+            }
+        }
+        {
+            // check column data equal
+            for (size_t i = 0; i < columns.size(); ++i) {
+                EXPECT_EQ(columns[i]->size(), argument_columns[i]->size());
+                checkColumn(*columns[i], *argument_columns[i], *data_types[i], columns[0]->size());
+            }
+        }
+    }
+
+    //  serialize_vec, deserialize_vec (called by MethodSerialized.init_serialized_keys), here are some scenarios:
+    //    1/ AggState: groupby key column which be serialized to hash-table key, eg.AggLocalState::_emplace_into_hash_table
+    //    2/ JoinState: hash join key column which be serialized to hash-table key, or probe column which be serialized to hash-table key, eg.ProcessHashTableBuild, ProcessHashTableProbe<JoinOpType>::probe_side_output_column
+    //  serialize_vec_with_null_map, deserialize_vec_with_null_map which only called by ColumnNullable serialize_vec and deserialize_vec, and derived by other columns
+    //  get_max_row_byte_size used in MethodSerialized which calculating the memory size for vectorized serialization of aggregation keys.
+    void ser_deser_vec(MutableColumns& columns, DataTypes dataTypes) {
+        // step1. make input_keys with given rows for a block
+        size_t rows = columns[0]->size();
+        std::vector<StringRef> input_keys;
+        input_keys.resize(rows);
+        MutableColumns check_columns(columns.size());
+        int c = 0;
+        for (auto& column : columns) {
+            if (column->size() > rows) {
+                std::cerr << "Column size mismatch: " << column->size() << " vs " << rows
+                          << std::endl;
+                column->pop_back(rows - column->size());
+            } else if (column->size() < rows) {
+                std::cerr << "Column size mismatch: " << column->size() << " vs " << rows
+                          << std::endl;
+                column->insert_many_defaults(rows - column->size());
+            }
+            check_columns[c] = column->clone_empty();
+            ++c;
+        }
+
+        // step2. calculate the needed memory size for vectorized serialization of aggregation keys
+        size_t max_one_row_byte_size = 0;
+        for (const auto& column : columns) {
+            max_one_row_byte_size += column->get_max_row_byte_size();
+        }
+        size_t memory_size = max_one_row_byte_size * rows;
+        Arena arena(memory_size);
+        auto* serialized_key_buffer = reinterpret_cast<uint8_t*>(arena.alloc(memory_size));
+
+        // serialize the keys into arena
+        {
+            // step3. serialize the keys into arena
+            for (size_t i = 0; i < rows; ++i) {
+                input_keys[i].data =
+                        reinterpret_cast<char*>(serialized_key_buffer + i * max_one_row_byte_size);
+                input_keys[i].size = 0;
+            }
+            std::cout << "max_one_row_byte_size : " << max_one_row_byte_size << std::endl;
+            for (const auto& column : columns) {
+                std::cout << "now serialize_vec for column:" << column->get_name()
+                          << " with column size: " << column->size() << std::endl;
+                column->serialize_vec(input_keys, rows, max_one_row_byte_size);
+            }
+        }
+        // deserialize the keys from arena into columns
+        {
+            // step4. deserialize the keys from arena into columns
+            for (auto& column : check_columns) {
+                column->deserialize_vec(input_keys, rows);
+            }
+        }
+        // check the deserialized columns
+        {
+            for (size_t i = 0; i < columns.size(); ++i) {
+                EXPECT_EQ(columns[i]->size(), check_columns[i]->size());
+                EXPECT_EQ(columns[i]->size(), check_columns[i]->size());
+                checkColumn(*columns[i], *check_columns[i], *dataTypes[i], rows);
+            }
+        }
+    }
+
+    PaddedPODArray<UInt8> create_filter(std::vector<uint8_t> data) {
+        PaddedPODArray<UInt8> filter;
+        filter.insert(filter.end(), data.begin(), data.end());
+        return filter;
+    }
+
+    // filter calculation:
+    //  filter (called in Block::filter_block_internal to filter data with filter filled with 0or1 array, like: [0,1,0,1])
+    //   used in join to filter next block by row_ids, and filter column by row_ids in first read which called in SegmentIterator
+    void assert_filter(MutableColumnPtr col, std::vector<uint8_t> filter, size_t expect_size) {
+        EXPECT_EQ(col->size(), filter.size());
+        auto filted_col = col->filter(create_filter(filter), expect_size);
+        EXPECT_EQ(filted_col->size(), expect_size);
+    }
+
+    //  filter_by_selector (called SegmentIterator::copy_column_data_by_selector,
+    //  now just used in filter column, according to the selector to
+    //  select size of row_ids for column by given column, which only used for predict_column and column_dictionary, column_nullable sometimes in Schema::get_predicate_column_ptr() also will return)
+    void assert_filter_by_selector(vectorized::IColumn::MutablePtr col,
+                                   std::vector<uint16_t> selector, DataTypeSerDeSPtr serder,
+                                   MutableColumnPtr should_sel_col, size_t expect_size) {
+        // for every data type should assert behavior in own UT case
+        DataTypeSerDe::FormatOptions option;
+        col->clear();
+        col->insert_many_defaults(should_sel_col->size());
+        std::cout << "col size:" << col->size() << std::endl;
+        Status st = col->filter_by_selector(selector.data(), expect_size, should_sel_col.get());
+        EXPECT_EQ(st, Status::OK());
+        std::vector<std::vector<string>> res;
+        std::vector<string> data;
+        auto ser_col = ColumnString::create();
+        ser_col->reserve(should_sel_col->size());
+        VectorBufferWriter buffer_writer(*ser_col.get());
+        data.push_back("column: " + col->get_name() +
+                       " with selector: " + std::to_string(selector.size()) +
+                       " with ptr: " + std::to_string(should_sel_col->size()));
+        for (size_t j = 0; j < should_sel_col->size(); ++j) {
+            if (auto ret = serder->serialize_one_cell_to_json(*should_sel_col, j, buffer_writer,
+                                                              option);
+                !ret) {
+                std::cerr << "Failed to serialize column "
+                          << " at row " << j << std::endl;
+                break;
+            }
+            buffer_writer.commit();
+            std::string actual_str_value = ser_col->get_data_at(j).to_string();
+            data.push_back(actual_str_value);
+        }
+        res.push_back(data);
+        check_res_file("filter_by_selector-" + col->get_name(), res);
+    }
+
+    void assert_permutations_with_limit(const IColumn::Permutation& lhs,
+                                        const IColumn::Permutation& rhs, size_t limit) {
+        std::cout << "lhs size: " << lhs.size() << " rhs size: " << rhs.size()
+                  << " limit: " << limit << std::endl;
+        if (limit == 0) {
+            limit = lhs.size();
+        }
+
+        for (size_t i = 0; i < limit; ++i) {
+            ASSERT_EQ(lhs[i], rhs[i]) << "i: " << i << "limit: " << limit;
+        }
+    }
+
+    // this function helps to check sort permutation behavior for column
+    void stable_get_column_permutation(const IColumn& column, bool ascending, size_t limit,
+                                       int nan_direction_hint,
+                                       IColumn::Permutation& out_permutation) {
+        (void)(limit);
+
+        size_t size = column.size();
+        out_permutation.resize(size);
+        std::iota(out_permutation.begin(), out_permutation.end(),
+                  IColumn::Permutation::value_type(0));
+
+        std::stable_sort(out_permutation.begin(), out_permutation.end(),
+                         [&](size_t lhs, size_t rhs) {
+                             int res = column.compare_at(lhs, rhs, column, nan_direction_hint);
+                             // to check element in column is sorted or not
+                             if (ascending)
+                                 return res < 0;
+                             else
+                                 return res > 0;
+                         });
+    }
+
+    // sort calculation: (which used in sort_block )
+    //   get_permutation
+    // this function helps check permutation result with sort & limit
+    //  by given ColumnValueGetter which how to generate a column value
+    void assert_column_permutations(vectorized::IColumn& column, DataTypePtr dataType) {
+        IColumn::Permutation actual_permutation;
+        IColumn::Permutation expected_permutation;
+
+        static constexpr size_t limit_parts = 4;
+        printColumn(column, *dataType);
+
+        size_t column_size = column.size();
+        size_t column_limit_part = (column_size / limit_parts) + 1;
+        std::cout << "column size: " << column_size << std::endl;
+        for (size_t limit = 0; limit < column_size; limit += column_limit_part) {
+            assert_column_permutation(column, true, limit, -1, actual_permutation,
+                                      expected_permutation);
+            assert_column_permutation(column, true, limit, 1, actual_permutation,
+                                      expected_permutation);
+
+            assert_column_permutation(column, false, limit, -1, actual_permutation,
+                                      expected_permutation);
+            assert_column_permutation(column, false, limit, 1, actual_permutation,
+                                      expected_permutation);
+        }
+    }
+
+    // sort calculation: (which used in sort_block )
+    //    get_permutation means sort data in Column as sort order
+    //    limit should be set to limit the sort result
+    //    nan_direction_hint deal with null|NaN value
+    void assert_column_permutation(const IColumn& column, bool ascending, size_t limit,
+                                   int nan_direction_hint, IColumn::Permutation& actual_permutation,
+                                   IColumn::Permutation& expected_permutation) {
+        std::cout << "assertColumnPermutation start, limit: " << limit
+                  << " ascending: " << ascending << " nan_direction_hint: " << nan_direction_hint
+                  << " column size: " << column.size()
+                  << " actual_permutation size: " << actual_permutation.size()
+                  << " expected_permutation size: " << expected_permutation.size() << std::endl;
+        // step1. get expect permutation as stabled sort
+        stable_get_column_permutation(column, ascending, limit, nan_direction_hint,
+                                      expected_permutation);
+        std::cout << "expected_permutation size: " << expected_permutation.size() << std::endl;
+        for (size_t i = 0; i < expected_permutation.size(); i++) {
+            std::cout << "expected_permutation: " << expected_permutation[i] << std::endl;
+        }
+        // step2. get permutation by column
+        column.get_permutation(!ascending, limit, nan_direction_hint, actual_permutation);
+
+        std::cout << "actual_permutation size: " << actual_permutation.size() << std::endl;
+        for (size_t i = 0; i < actual_permutation.size(); i++) {
+            std::cout << "actual_permutation: " << actual_permutation[i] << std::endl;
+        }
+
+        if (limit == 0) {
+            limit = actual_permutation.size();
+        }
+
+        // step3. check the permutation result
+        assert_permutations_with_limit(actual_permutation, expected_permutation, limit);
+        std::cout << "assertColumnPermutation done" << std::endl;
+    }
+
+    //  permute()
+    //   1/ Key topN set read_orderby_key_reverse = true; SegmentIterator::next_batch will permute the column by the given permutation(which reverse the rows of current segment)
+    //  should check rows with the given permutation
+    void assert_permute(MutableColumns& cols, IColumn::Permutation& permutation, size_t num_rows) {
+        std::vector<ColumnPtr> res_permuted;
+        for (auto& col : cols) {
+            res_permuted.emplace_back(col->permute(permutation, num_rows));
+        }
+        // check the permutation result for rowsize
+        size_t res_rows = res_permuted[0]->size();
+        for (auto& col : res_permuted) {
+            EXPECT_EQ(col->size(), res_rows);
+        }
+    }
+};
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/core/column_array_test.cpp b/be/test/vec/core/column_array_test.cpp
index 0c8a486b5bf06c..26d1d3267f1060 100644
--- a/be/test/vec/core/column_array_test.cpp
+++ b/be/test/vec/core/column_array_test.cpp
@@ -58,7 +58,7 @@ void check_array_data(const IColumn& arr, const std::vector<std::string>& data)
     }
 }
 
-TEST(ColumnArrayTest, IntArrayTest) {
+TEST(ColumnArrayOldTest, IntArrayTest) {
     auto off_column = ColumnVector<ColumnArray::Offset64>::create();
     auto data_column = ColumnVector<int32_t>::create();
     // init column array with [[1,2,3],[],[4]]
@@ -83,7 +83,7 @@ TEST(ColumnArrayTest, IntArrayTest) {
     }
 }
 
-TEST(ColumnArrayTest, StringArrayTest) {
+TEST(ColumnArrayOldTest, StringArrayTest) {
     auto off_column = ColumnVector<ColumnArray::Offset64>::create();
     auto data_column = ColumnString::create();
     // init column array with [["abc","d"],["ef"],[], [""]];
@@ -181,7 +181,7 @@ TEST(ColumnArrayTest, IntArrayPermuteTest) {
     check_array_data<int32_t>(*res2, {5, 6, 4, 1, 2, 3});
 }
 
-TEST(ColumnArrayTest, StringArrayPermuteTest) {
+TEST(ColumnArrayOldTest, StringArrayPermuteTest) {
     auto off_column = ColumnVector<ColumnArray::Offset64>::create();
     auto data_column = ColumnString::create();
     // init column array with [["abc","d"],["ef"],[], [""]];
@@ -198,6 +198,7 @@ TEST(ColumnArrayTest, StringArrayPermuteTest) {
     IColumn::Permutation perm = {3, 2, 1, 0};
     // return array column: [[""],[]];
     auto res1 = array_column.permute(perm, 2);
+
     check_array_offsets(*res1, {1, 1});
     check_array_data<std::string>(*res1, {""});
 
@@ -207,7 +208,7 @@ TEST(ColumnArrayTest, StringArrayPermuteTest) {
     check_array_data<std::string>(*res2, {"", "ef", "abc", "d"});
 }
 
-TEST(ColumnArrayTest, EmptyArrayPermuteTest) {
+TEST(ColumnArrayOldTest, EmptyArrayPermuteTest) {
     auto off_column = ColumnVector<ColumnArray::Offset64>::create();
     auto data_column = ColumnVector<int32_t>::create();
     // init column array with [[],[],[],[]]

From 2c279f267ea2a36fa3757715fc03b1278ebea81a Mon Sep 17 00:00:00 2001
From: qiye <luen@selectdb.com>
Date: Mon, 9 Dec 2024 15:21:23 +0800
Subject: [PATCH 238/399] [test](index compaction)Add exception cases for index
 compaction (#45056)

Problem Summary:

1. Add more exception cases for index compaction
2. Refactor index compaction test code, add a util class for all cases
3. Merge index_compaction_delete case to index compaction test
---
 .../compaction/index_compaction_test.cpp      | 963 ++++++++++++------
 .../index_compaction_with_deleted_term.cpp    | 671 ------------
 .../util/index_compaction_utils.cpp           | 621 +++++++++++
 3 files changed, 1248 insertions(+), 1007 deletions(-)
 delete mode 100644 be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_with_deleted_term.cpp
 create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp

diff --git a/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp
index 0f1b27fd4fa11f..264786570e731a 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp
@@ -15,14 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
+#include <gmock/gmock.h>
 
-#include "olap/cumulative_compaction.h"
-#include "olap/rowset/beta_rowset_writer.h"
-#include "olap/rowset/rowset_factory.h"
-#include "olap/rowset/segment_v2/inverted_index/query/query_factory.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
-#include "olap/storage_engine.h"
+#include "util/index_compaction_utils.cpp"
 
 namespace doris {
 
@@ -31,14 +26,6 @@ using namespace doris::vectorized;
 constexpr static uint32_t MAX_PATH_LEN = 1024;
 constexpr static std::string_view dest_dir = "./ut_dir/inverted_index_test";
 constexpr static std::string_view tmp_dir = "./ut_dir/tmp";
-static int64_t inc_id = 1000;
-
-struct DataRow {
-    int key;
-    std::string word;
-    std::string url;
-    int num;
-};
 
 class IndexCompactionTest : public ::testing::Test {
 protected:
@@ -46,8 +33,8 @@ class IndexCompactionTest : public ::testing::Test {
         // absolute dir
         char buffer[MAX_PATH_LEN];
         EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
-        _curreent_dir = std::string(buffer);
-        _absolute_dir = _curreent_dir + std::string(dest_dir);
+        _current_dir = std::string(buffer);
+        _absolute_dir = _current_dir + std::string(dest_dir);
         EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
         EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok());
 
@@ -57,7 +44,8 @@ class IndexCompactionTest : public ::testing::Test {
         std::vector<StorePath> paths;
         paths.emplace_back(std::string(tmp_dir), 1024000000);
         auto tmp_file_dirs = std::make_unique<segment_v2::TmpFileDirs>(paths);
-        EXPECT_TRUE(tmp_file_dirs->init().ok());
+        Status st = tmp_file_dirs->init();
+        EXPECT_TRUE(st.ok()) << st.to_json();
         ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs));
 
         // storage engine
@@ -73,24 +61,22 @@ class IndexCompactionTest : public ::testing::Test {
         schema_pb.set_keys_type(KeysType::DUP_KEYS);
         schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
 
-        construct_column(schema_pb.add_column(), schema_pb.add_index(), 10000, "key_index", 0,
-                         "INT", "key");
-        construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001, "v1_index", 1,
-                         "STRING", "v1");
-        construct_column(schema_pb.add_column(), schema_pb.add_index(), 10002, "v2_index", 2,
-                         "STRING", "v2", true);
-        construct_column(schema_pb.add_column(), schema_pb.add_index(), 10003, "v3_index", 3, "INT",
-                         "v3");
-
-        _tablet_schema.reset(new TabletSchema);
+        IndexCompactionUtils::construct_column(schema_pb.add_column(), schema_pb.add_index(), 10000,
+                                               "key_index", 0, "INT", "key");
+        IndexCompactionUtils::construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001,
+                                               "v1_index", 1, "STRING", "v1");
+        IndexCompactionUtils::construct_column(schema_pb.add_column(), schema_pb.add_index(), 10002,
+                                               "v2_index", 2, "STRING", "v2", true);
+        IndexCompactionUtils::construct_column(schema_pb.add_column(), schema_pb.add_index(), 10003,
+                                               "v3_index", 3, "INT", "v3");
+        _tablet_schema = std::make_shared<TabletSchema>();
         _tablet_schema->init_from_pb(schema_pb);
 
         // tablet
         TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
 
-        _tablet.reset(new Tablet(*_engine_ref, tablet_meta, _data_dir.get()));
+        _tablet = std::make_shared<Tablet>(*_engine_ref, tablet_meta, _data_dir.get());
         EXPECT_TRUE(_tablet->init().ok());
-        config::inverted_index_compaction_enable = true;
     }
     void TearDown() override {
         EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
@@ -100,41 +86,6 @@ class IndexCompactionTest : public ::testing::Test {
         ExecEnv::GetInstance()->set_storage_engine(nullptr);
     }
 
-    void construct_column(ColumnPB* column_pb, TabletIndexPB* tablet_index, int64_t index_id,
-                          const std::string& index_name, int32_t col_unique_id,
-                          const std::string& column_type, const std::string& column_name,
-                          bool parser = false) {
-        column_pb->set_unique_id(col_unique_id);
-        column_pb->set_name(column_name);
-        column_pb->set_type(column_type);
-        column_pb->set_is_key(false);
-        column_pb->set_is_nullable(true);
-        tablet_index->set_index_id(index_id);
-        tablet_index->set_index_name(index_name);
-        tablet_index->set_index_type(IndexType::INVERTED);
-        tablet_index->add_col_unique_id(col_unique_id);
-        if (parser) {
-            auto* properties = tablet_index->mutable_properties();
-            (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_UNICODE;
-        }
-    }
-
-    RowsetWriterContext rowset_writer_context() {
-        RowsetWriterContext context;
-        RowsetId rowset_id;
-        rowset_id.init(inc_id);
-        context.rowset_id = rowset_id;
-        context.rowset_type = BETA_ROWSET;
-        context.data_dir = _data_dir.get();
-        context.rowset_state = VISIBLE;
-        context.tablet_schema = _tablet_schema;
-        context.tablet_path = _tablet->tablet_path();
-        context.version = Version(inc_id, inc_id);
-        context.max_rows_per_segment = 200;
-        inc_id++;
-        return context;
-    }
-
     IndexCompactionTest() = default;
     ~IndexCompactionTest() override = default;
 
@@ -144,300 +95,640 @@ class IndexCompactionTest : public ::testing::Test {
     std::unique_ptr<DataDir> _data_dir = nullptr;
     TabletSharedPtr _tablet = nullptr;
     std::string _absolute_dir;
-    std::string _curreent_dir;
+    std::string _current_dir;
 };
 
-std::vector<DataRow> read_data(const std::string file_name) {
-    std::ifstream file(file_name);
-    EXPECT_TRUE(file.is_open());
-
-    std::string line;
-    std::vector<DataRow> data;
-
-    while (std::getline(file, line)) {
-        std::stringstream ss(line);
-        std::string item;
-        DataRow row;
-        EXPECT_TRUE(std::getline(ss, item, ','));
-        row.key = std::stoi(item);
-        EXPECT_TRUE(std::getline(ss, item, ','));
-        row.word = item;
-        EXPECT_TRUE(std::getline(ss, item, ','));
-        row.url = item;
-        EXPECT_TRUE(std::getline(ss, item, ','));
-        row.num = std::stoi(item);
-        data.emplace_back(std::move(row));
-    }
+TEST_F(IndexCompactionTest, tes_write_index_normally) {
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 4); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 2);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(1));
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
+    EXPECT_TRUE(st.ok()) << st.to_string();
+
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    auto custom_check_normal = [](const BaseCompaction& compaction,
+                                  const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 0);
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    RowsetSharedPtr output_rowset_normal;
+    st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, false,
+                                             output_rowset_normal, custom_check_normal);
+    EXPECT_TRUE(st.ok()) << st.to_string();
+    const auto& seg_path_normal = output_rowset_normal->segment_path(0);
+    EXPECT_TRUE(seg_path_normal.has_value()) << seg_path_normal.error();
+    auto inverted_index_file_reader_normal = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_normal, seg_path_normal.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+    // check index file terms
+    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, "");
+    EXPECT_TRUE(dir_idx_compaction.has_value()) << dir_idx_compaction.error();
+    auto dir_normal_compaction = inverted_index_file_reader_normal->_open(10001, "");
+    EXPECT_TRUE(dir_normal_compaction.has_value()) << dir_normal_compaction.error();
+    std::ostringstream oss;
+    IndexCompactionUtils::check_terms_stats(dir_idx_compaction->get(), oss);
+    std::string output = oss.str();
+    EXPECT_EQ(output, expected_output);
+    oss.str("");
+    oss.clear();
+    IndexCompactionUtils::check_terms_stats(dir_normal_compaction->get(), oss);
+    output = oss.str();
+    EXPECT_EQ(output, expected_output);
+
+    st = IndexCompactionUtils::check_idx_file_correctness(dir_idx_compaction->get(),
+                                                          dir_normal_compaction->get());
+    EXPECT_TRUE(st.ok()) << st.to_string();
 
-    file.close();
-    return data;
+    // check meta and file
+    std::map<int, QueryData> query_map = {
+            {0, {{"99", "66", "56", "87", "85", "96", "20000"}, {21, 25, 22, 18, 14, 18, 0}}},
+            {3, {{"99", "66", "56", "87", "85", "96", "10000"}, {12, 20, 25, 23, 16, 24, 0}}},
+            {1, {{"good", "maybe", "great", "null"}, {197, 191, 194, 0}}},
+            {2, {{"musicstream.com", "http", "https", "null"}, {191, 799, 1201, 0}}}};
+    IndexCompactionUtils::check_meta_and_file(output_rowset_index, _tablet_schema, query_map);
+    IndexCompactionUtils::check_meta_and_file(output_rowset_normal, _tablet_schema, query_map);
 }
 
-bool query_bkd(const TabletIndex* index,
-               std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
-               const std::vector<int>& query_data, const std::vector<int>& query_result) {
-    const auto& idx_reader = BkdIndexReader::create_shared(index, inverted_index_file_reader);
-    const auto& index_searcher_builder = std::make_unique<BKDIndexSearcherBuilder>();
-    auto dir = inverted_index_file_reader->open(index);
-    EXPECT_TRUE(dir.has_value());
-    auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
-    EXPECT_TRUE(searcher_result.has_value());
-    auto bkd_searcher = std::get_if<BKDIndexSearcherPtr>(&searcher_result.value());
-    EXPECT_TRUE(bkd_searcher != nullptr);
-    idx_reader->_type_info = get_scalar_type_info((FieldType)(*bkd_searcher)->type);
-    EXPECT_TRUE(idx_reader->_type_info != nullptr);
-    idx_reader->_value_key_coder = get_key_coder(idx_reader->_type_info->type());
-
-    for (int i = 0; i < query_data.size(); i++) {
-        vectorized::Field param_value = Int32(query_data[i]);
-        std::unique_ptr<segment_v2::InvertedIndexQueryParamFactory> query_param = nullptr;
-        EXPECT_TRUE(segment_v2::InvertedIndexQueryParamFactory::create_query_value(
-                            PrimitiveType::TYPE_INT, &param_value, query_param)
-                            .ok());
-        auto result = std::make_shared<roaring::Roaring>();
-        EXPECT_TRUE(idx_reader
-                            ->invoke_bkd_query(query_param->get_value(),
-                                               InvertedIndexQueryType::EQUAL_QUERY, *bkd_searcher,
-                                               result)
-                            .ok());
-        EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i];
-    }
-    return true;
+TEST_F(IndexCompactionTest, test_col_unique_ids_empty) {
+    // clear column unique id in tablet index 10001 and rebuild tablet_schema
+    TabletSchemaPB schema_pb;
+    _tablet_schema->to_schema_pb(&schema_pb);
+    auto* index_pb = schema_pb.mutable_index(1);
+    index_pb->clear_col_unique_id();
+    _tablet_schema->init_from_pb(schema_pb);
+
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 3); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        // only index id 10002 will do index compaction
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 1);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
+    EXPECT_TRUE(st.ok()) << st.to_string();
+
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    // check index file
+    // index 10001 cannot be found in idx file
+    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, "");
+    EXPECT_TRUE(!dir_idx_compaction.has_value()) << dir_idx_compaction.error();
+    EXPECT_THAT(dir_idx_compaction.error().to_string(),
+                testing::HasSubstr("No index with id 10001 found"));
 }
 
-bool query_string(const TabletIndex* index,
-                  std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
-                  const std::string& column_name, const std::vector<std::string>& query_data,
-                  const std::vector<int>& query_result) {
-    const auto& idx_reader =
-            StringTypeInvertedIndexReader::create_shared(index, inverted_index_file_reader);
-    const auto& index_searcher_builder = std::make_unique<FulltextIndexSearcherBuilder>();
-    auto dir = inverted_index_file_reader->open(index);
-    EXPECT_TRUE(dir.has_value());
-    auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
-    EXPECT_TRUE(searcher_result.has_value());
-    auto string_searcher = std::get_if<FulltextIndexSearcherPtr>(&searcher_result.value());
-    EXPECT_TRUE(string_searcher != nullptr);
-    std::wstring column_name_ws = StringUtil::string_to_wstring(column_name);
-
-    for (int i = 0; i < query_data.size(); i++) {
-        TQueryOptions queryOptions;
-        auto query = QueryFactory::create(InvertedIndexQueryType::EQUAL_QUERY, *string_searcher,
-                                          queryOptions, nullptr);
-        EXPECT_TRUE(query != nullptr);
-        InvertedIndexQueryInfo query_info;
-        query_info.field_name = column_name_ws;
-        query_info.terms.emplace_back(query_data[i]);
-        query->add(query_info);
-        auto result = std::make_shared<roaring::Roaring>();
-        query->search(*result);
-        EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i];
-    }
-    return true;
+TEST_F(IndexCompactionTest, test_tablet_index_id_not_equal) {
+    // replace unique id from 2 to 1 in tablet index 10002 and rebuild tablet_schema
+    TabletSchemaPB schema_pb;
+    _tablet_schema->to_schema_pb(&schema_pb);
+    auto* index_pb = schema_pb.mutable_index(2);
+    index_pb->set_col_unique_id(0, 1);
+    _tablet_schema->init_from_pb(schema_pb);
+
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 3); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        // only index id 10001 will do index compaction
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 1);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(1));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
+    EXPECT_TRUE(st.ok()) << st.to_string();
+
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    // check index file
+    // index 10002 cannot be found in idx file
+    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10002, "");
+    EXPECT_TRUE(!dir_idx_compaction.has_value()) << dir_idx_compaction.error();
+    EXPECT_THAT(dir_idx_compaction.error().to_string(),
+                testing::HasSubstr("No index with id 10002 found"));
 }
 
-bool query_fulltext(const TabletIndex* index,
-                    std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
-                    const std::string& column_name, const std::vector<std::string>& query_data,
-                    const std::vector<int>& query_result) {
-    const auto& idx_reader = FullTextIndexReader::create_shared(index, inverted_index_file_reader);
-    const auto& index_searcher_builder = std::make_unique<FulltextIndexSearcherBuilder>();
-    auto dir = inverted_index_file_reader->open(index);
-    EXPECT_TRUE(dir.has_value());
-    auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
-    EXPECT_TRUE(searcher_result.has_value());
-    auto string_searcher = std::get_if<FulltextIndexSearcherPtr>(&searcher_result.value());
-    EXPECT_TRUE(string_searcher != nullptr);
-    std::wstring column_name_ws = StringUtil::string_to_wstring(column_name);
-
-    for (int i = 0; i < query_data.size(); i++) {
-        TQueryOptions queryOptions;
-        auto query = QueryFactory::create(InvertedIndexQueryType::MATCH_ANY_QUERY, *string_searcher,
-                                          queryOptions, nullptr);
-        EXPECT_TRUE(query != nullptr);
-        InvertedIndexQueryInfo query_info;
-        query_info.field_name = column_name_ws;
-        query_info.terms.emplace_back(query_data[i]);
-        query->add(query_info);
-        auto result = std::make_shared<roaring::Roaring>();
-        query->search(*result);
-        EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i];
-    }
-    return true;
+TEST_F(IndexCompactionTest, test_tablet_schema_tablet_index_is_null) {
+    // set index suffix in tablet index 10001 and rebuild tablet_schema
+    // simulate the case that index is null, tablet_schema->inverted_index(1) will return nullptr
+    TabletSchemaPB schema_pb;
+    _tablet_schema->to_schema_pb(&schema_pb);
+    auto* index_pb = schema_pb.mutable_index(1);
+    index_pb->set_index_suffix_name("mock");
+    _tablet_schema->init_from_pb(schema_pb);
+
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 3); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        // only index id 10002 will do index compaction
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 1);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
+    EXPECT_TRUE(st.ok()) << st.to_string();
+
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    // check index file
+    // index 10001 cannot be found in idx file
+    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, "");
+    EXPECT_TRUE(!dir_idx_compaction.has_value()) << dir_idx_compaction.error();
+    EXPECT_THAT(dir_idx_compaction.error().to_string(),
+                testing::HasSubstr("No index with id 10001 found"));
 }
 
-TEST_F(IndexCompactionTest, write_index_test) {
+TEST_F(IndexCompactionTest, test_rowset_schema_tablet_index_is_null) {
     EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
     EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
     std::string data_file1 =
-            _curreent_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
     std::string data_file2 =
-            _curreent_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
-
-    std::vector<std::vector<DataRow>> data;
-    data.emplace_back(read_data(data_file1));
-    data.emplace_back(read_data(data_file2));
-
-    std::vector<RowsetSharedPtr> rowsets(data.size());
-    for (int i = 0; i < data.size(); i++) {
-        const auto& res =
-                RowsetFactory::create_rowset_writer(*_engine_ref, rowset_writer_context(), false);
-        EXPECT_TRUE(res.has_value()) << res.error();
-        const auto& rowset_writer = res.value();
-
-        Block block = _tablet_schema->create_block();
-        auto columns = block.mutate_columns();
-        for (const auto& row : data[i]) {
-            vectorized::Field key = Int32(row.key);
-            vectorized::Field v1(row.word);
-            vectorized::Field v2(row.url);
-            vectorized::Field v3 = Int32(row.num);
-            columns[0]->insert(key);
-            columns[1]->insert(v1);
-            columns[2]->insert(v2);
-            columns[3]->insert(v3);
-        }
-        EXPECT_TRUE(rowset_writer->add_block(&block).ok());
-        EXPECT_TRUE(rowset_writer->flush().ok());
-        const auto& dst_writer = dynamic_cast<BaseBetaRowsetWriter*>(rowset_writer.get());
-
-        // inverted index file writer
-        for (const auto& [seg_id, idx_file_writer] : dst_writer->_idx_files.get_file_writers()) {
-            EXPECT_TRUE(idx_file_writer->_closed);
-        }
-
-        EXPECT_TRUE(rowset_writer->build(rowsets[i]).ok());
-        EXPECT_TRUE(_tablet->add_rowset(rowsets[i]).ok());
-        EXPECT_TRUE(rowsets[i]->num_segments() == 5);
-
-        // check rowset meta and file
-        for (int seg_id = 0; seg_id < rowsets[i]->num_segments(); seg_id++) {
-            const auto& index_info = rowsets[i]->_rowset_meta->inverted_index_file_info(seg_id);
-            EXPECT_TRUE(index_info.has_index_size());
-            const auto& fs = rowsets[i]->_rowset_meta->fs();
-            const auto& file_name = fmt::format("{}/{}_{}.idx", rowsets[i]->tablet_path(),
-                                                rowsets[i]->rowset_id().to_string(), seg_id);
-            int64_t file_size = 0;
-            EXPECT_TRUE(fs->file_size(file_name, &file_size).ok());
-            EXPECT_EQ(index_info.index_size(), file_size);
-
-            const auto& seg_path = rowsets[i]->segment_path(seg_id);
-            EXPECT_TRUE(seg_path.has_value());
-            const auto& index_file_path_prefix =
-                    InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value());
-            auto inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
-                    fs, std::string(index_file_path_prefix),
-                    _tablet_schema->get_inverted_index_storage_format(), index_info);
-            EXPECT_TRUE(inverted_index_file_reader->init().ok());
-            const auto& dirs = inverted_index_file_reader->get_all_directories();
-            EXPECT_TRUE(dirs.has_value());
-            EXPECT_EQ(dirs.value().size(), 4);
-        }
-    }
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 4); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        // only index id 10002 will do index compaction
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 1);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    // set index suffix in tablet index 10001 and rebuild tablet_schema
+    // simulate the case that index is null, tablet_schema->inverted_index(1) will return nullptr
+    TabletSchemaPB schema_pb;
+    TabletSchemaSPtr mock_schema = std::make_shared<TabletSchema>();
+    _tablet_schema->to_schema_pb(&schema_pb);
+    auto* index_pb = schema_pb.mutable_index(1);
+    index_pb->set_index_suffix_name("mock");
+    mock_schema->init_from_pb(schema_pb);
+    rowsets[0]->_schema = mock_schema;
+
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
+    EXPECT_TRUE(st.ok()) << st.to_string();
 
-    CumulativeCompaction compaction(*_engine_ref, _tablet);
-    compaction._input_rowsets = std::move(rowsets);
-    compaction.build_basic_info();
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    // check index file
+    // index 10001 cannot be found in idx file
+    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, "");
+    EXPECT_TRUE(dir_idx_compaction.has_value()) << dir_idx_compaction.error();
+    // check index 10001 term stats
+    std::ostringstream oss;
+    IndexCompactionUtils::check_terms_stats(dir_idx_compaction.value().get(), oss);
+    std::string output = oss.str();
+    EXPECT_EQ(output, expected_output);
+}
 
-    std::vector<RowsetReaderSharedPtr> input_rs_readers;
-    input_rs_readers.reserve(compaction._input_rowsets.size());
-    for (auto& rowset : compaction._input_rowsets) {
-        RowsetReaderSharedPtr rs_reader;
-        EXPECT_TRUE(rowset->create_reader(&rs_reader).ok());
-        input_rs_readers.push_back(std::move(rs_reader));
-    }
+TEST_F(IndexCompactionTest, test_tablet_index_properties_not_equal) {
+    // add mock property in tablet index 10001 and rebuild tablet_schema
+    // simulate the case that index properties not equal among input rowsets
+    TabletSchemaSPtr mock_schema = std::make_shared<TabletSchema>();
+    TabletSchemaPB schema_pb;
+    _tablet_schema->to_schema_pb(&schema_pb);
+    auto* index_pb = schema_pb.mutable_index(1);
+    (*index_pb->mutable_properties())["mock_key"] = "mock_value";
+    mock_schema->init_from_pb(schema_pb);
 
-    RowsetWriterContext ctx;
-    EXPECT_TRUE(compaction.construct_output_rowset_writer(ctx).ok());
-
-    // col word
-    EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(1));
-    // col url
-    EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
-
-    compaction._stats.rowid_conversion = compaction._rowid_conversion.get();
-    EXPECT_TRUE(Merger::vertical_merge_rowsets(_tablet, compaction.compaction_type(),
-                                               *(compaction._cur_tablet_schema), input_rs_readers,
-                                               compaction._output_rs_writer.get(), 100000, 5,
-                                               &compaction._stats)
-                        .ok());
-    const auto& dst_writer =
-            dynamic_cast<BaseBetaRowsetWriter*>(compaction._output_rs_writer.get());
-    for (const auto& [seg_id, idx_file_writer] : dst_writer->_idx_files.get_file_writers()) {
-        EXPECT_FALSE(idx_file_writer->_closed);
-    }
-    auto st = compaction.do_inverted_index_compaction();
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 4); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        // only index id 10002 will do index compaction
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 1);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    // set mock_schema to the first input rowset
+    rowsets[0]->_schema = mock_schema;
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
     EXPECT_TRUE(st.ok()) << st.to_string();
 
-    st = compaction._output_rs_writer->build(compaction._output_rowset);
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    // check index file
+    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, "");
+    EXPECT_TRUE(dir_idx_compaction.has_value()) << dir_idx_compaction.error();
+
+    // check index 10001 term stats
+    std::ostringstream oss;
+    IndexCompactionUtils::check_terms_stats(dir_idx_compaction.value().get(), oss);
+    std::string output = oss.str();
+    EXPECT_EQ(output, expected_output);
+}
+
+TEST_F(IndexCompactionTest, test_is_skip_index_compaction_not_empty) {
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 4); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        // only index id 10002 will do index compaction
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 1);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    // set col_unique_id=1(index_id=10001) to skip index compaction
+    rowsets[0]->set_skip_index_compaction(1);
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
     EXPECT_TRUE(st.ok()) << st.to_string();
 
-    for (const auto& [seg_id, idx_file_writer] : dst_writer->_idx_files.get_file_writers()) {
-        EXPECT_TRUE(idx_file_writer->_closed);
-    }
-    EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
-
-    const auto& output_rowset = compaction._output_rowset;
-
-    // check rowset meta and file
-    for (int seg_id = 0; seg_id < output_rowset->num_segments(); seg_id++) {
-        // meta
-        const auto& index_info = output_rowset->_rowset_meta->inverted_index_file_info(seg_id);
-        EXPECT_TRUE(index_info.has_index_size());
-        const auto& fs = output_rowset->_rowset_meta->fs();
-        const auto& file_name = fmt::format("{}/{}_{}.idx", output_rowset->tablet_path(),
-                                            output_rowset->rowset_id().to_string(), seg_id);
-        int64_t file_size = 0;
-        EXPECT_TRUE(fs->file_size(file_name, &file_size).ok());
-        EXPECT_EQ(index_info.index_size(), file_size);
-
-        // file
-        const auto& seg_path = output_rowset->segment_path(seg_id);
-        EXPECT_TRUE(seg_path.has_value());
-        const auto& index_file_path_prefix =
-                InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value());
-        auto inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
-                fs, std::string(index_file_path_prefix),
-                _tablet_schema->get_inverted_index_storage_format(), index_info);
-        EXPECT_TRUE(inverted_index_file_reader->init().ok());
-        const auto& dirs = inverted_index_file_reader->get_all_directories();
-        EXPECT_TRUE(dirs.has_value());
-        EXPECT_EQ(dirs.value().size(), 4);
-
-        // read col key
-        const auto& key = _tablet_schema->column_by_uid(0);
-        const auto* key_index = _tablet_schema->inverted_index(key);
-        EXPECT_TRUE(key_index != nullptr);
-        std::vector<int> query_data {99, 66, 56, 87, 85, 96, 20000};
-        std::vector<int> query_result {21, 25, 22, 18, 14, 18, 0};
-        EXPECT_TRUE(query_bkd(key_index, inverted_index_file_reader, query_data, query_result));
-
-        // read col v3
-        const auto& v3_column = _tablet_schema->column_by_uid(3);
-        const auto* v3_index = _tablet_schema->inverted_index(v3_column);
-        EXPECT_TRUE(v3_index != nullptr);
-        std::vector<int> query_data3 {99, 66, 56, 87, 85, 96, 10000};
-        std::vector<int> query_result3 {12, 20, 25, 23, 16, 24, 0};
-        EXPECT_TRUE(query_bkd(v3_index, inverted_index_file_reader, query_data3, query_result3));
-
-        // read col v1
-        const auto& v1_column = _tablet_schema->column_by_uid(1);
-        const auto* v1_index = _tablet_schema->inverted_index(v1_column);
-        EXPECT_TRUE(v1_index != nullptr);
-        std::vector<std::string> query_data1 {"good", "maybe", "great", "null"};
-        std::vector<int> query_result1 {197, 191, 194, 0};
-        EXPECT_TRUE(query_string(v1_index, inverted_index_file_reader, "1", query_data1,
-                                 query_result1));
-
-        // read col v2
-        const auto& v2_column = _tablet_schema->column_by_uid(2);
-        const auto* v2_index = _tablet_schema->inverted_index(v2_column);
-        EXPECT_TRUE(v2_index != nullptr);
-        std::vector<std::string> query_data2 {"musicstream.com", "http", "https", "null"};
-        std::vector<int> query_result2 {191, 799, 1201, 0};
-        EXPECT_TRUE(query_fulltext(v2_index, inverted_index_file_reader, "2", query_data2,
-                                   query_result2));
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    // check index file
+    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, "");
+    EXPECT_TRUE(dir_idx_compaction.has_value()) << dir_idx_compaction.error();
+
+    // check index 10001 term stats
+    std::ostringstream oss;
+    IndexCompactionUtils::check_terms_stats(dir_idx_compaction.value().get(), oss);
+    std::string output = oss.str();
+    EXPECT_EQ(output, expected_output);
+}
+
+TEST_F(IndexCompactionTest, test_rowset_fs_nullptr) {
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 4); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        // only index id 10002 will do index compaction
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 1);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    // set mock_id to resource_id to simulate getting fs nullptr
+    RowsetMetaSharedPtr mock_rowset_meta = std::make_shared<RowsetMeta>();
+    RowsetMetaPB rs_meta_pb;
+    rowsets[0]->to_rowset_pb(&rs_meta_pb);
+    rs_meta_pb.set_resource_id("mock_id");
+    mock_rowset_meta->init_from_pb(rs_meta_pb);
+    rowsets[0]->_rowset_meta = mock_rowset_meta;
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
+    EXPECT_TRUE(!st.ok());
+    EXPECT_THAT(st.to_string(), testing::HasSubstr("[E-206]get fs failed"));
+}
+
+TEST_F(IndexCompactionTest, test_input_row_num_zero) {
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 4); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        // only index id 10002 will do index compaction
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 2);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(1));
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    // set num_rows to 0 to simulate input_row_num = 0
+    for (auto rowset : rowsets) {
+        RowsetMetaSharedPtr mock_rowset_meta = std::make_shared<RowsetMeta>();
+        RowsetMetaPB rs_meta_pb;
+        rowset->to_rowset_pb(&rs_meta_pb);
+        rs_meta_pb.set_num_rows(0);
+        mock_rowset_meta->init_from_pb(rs_meta_pb);
+        rowset->_rowset_meta = mock_rowset_meta;
     }
+
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
+    EXPECT_TRUE(st.ok());
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    // check index file
+    // index 10001 cannot be found in idx file
+    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, "");
+    EXPECT_TRUE(!dir_idx_compaction.has_value()) << dir_idx_compaction.error();
+    EXPECT_THAT(dir_idx_compaction.error().to_string(),
+                testing::HasSubstr("No index with id 10001 found"));
+}
+
+TEST_F(IndexCompactionTest, test_cols_to_do_index_compaction_empty) {
+    // add mock property in tablet index 10001, 10002 and rebuild tablet_schema
+    // simulate the case that index properties not equal among input rowsets
+    // the two cols will skip index compaction and make ctx.columns_to_do_index_compaction empty
+    TabletSchemaSPtr mock_schema = std::make_shared<TabletSchema>();
+    TabletSchemaPB schema_pb;
+    _tablet_schema->to_schema_pb(&schema_pb);
+    auto* index_pb_1 = schema_pb.mutable_index(1);
+    (*index_pb_1->mutable_properties())["mock_key"] = "mock_value";
+    auto* index_pb_2 = schema_pb.mutable_index(2);
+    (*index_pb_2->mutable_properties())["mock_key"] = "mock_value";
+    mock_schema->init_from_pb(schema_pb);
+
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 4); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        // none index will do index compaction
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 0);
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    // set mock_schema to the first input rowset
+    rowsets[0]->_schema = mock_schema;
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
+    EXPECT_TRUE(st.ok()) << st.to_string();
+
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    // check index file
+    auto dir_idx_compaction_1 = inverted_index_file_reader_index->_open(10001, "");
+    EXPECT_TRUE(dir_idx_compaction_1.has_value()) << dir_idx_compaction_1.error();
+
+    // check index 10001 term stats
+    std::ostringstream oss;
+    IndexCompactionUtils::check_terms_stats(dir_idx_compaction_1.value().get(), oss);
+    std::string output = oss.str();
+    EXPECT_EQ(output, expected_output);
+
+    auto dir_idx_compaction_2 = inverted_index_file_reader_index->_open(10002, "");
+    EXPECT_TRUE(dir_idx_compaction_2.has_value()) << dir_idx_compaction_2.error();
+}
+
+TEST_F(IndexCompactionTest, test_index_compaction_with_delete) {
+    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+    std::string data_file1 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
+    std::string data_file2 =
+            _current_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
+    std::vector<std::string> data_files;
+    data_files.push_back(data_file1);
+    data_files.push_back(data_file2);
+
+    std::vector<RowsetSharedPtr> rowsets(data_files.size());
+    auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 4); };
+    IndexCompactionUtils::build_rowsets(_data_dir, _tablet_schema, _tablet, _engine_ref, rowsets,
+                                        data_files, custom_check_build_rowsets);
+
+    // create delete predicate rowset and add to tablet
+    auto delete_rowset = IndexCompactionUtils::create_delete_predicate_rowset(
+            _tablet_schema, "v1='great'", inc_id++);
+    EXPECT_TRUE(_tablet->add_rowset(delete_rowset).ok());
+    EXPECT_TRUE(_tablet->rowset_map().size() == 3);
+    rowsets.push_back(delete_rowset);
+    EXPECT_TRUE(rowsets.size() == 3);
+
+    auto custom_check_index = [](const BaseCompaction& compaction, const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 2);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(1));
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    RowsetSharedPtr output_rowset_index;
+    auto st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, true,
+                                                  output_rowset_index, custom_check_index);
+    EXPECT_TRUE(st.ok()) << st.to_string();
+
+    const auto& seg_path = output_rowset_index->segment_path(0);
+    EXPECT_TRUE(seg_path.has_value()) << seg_path.error();
+    auto inverted_index_file_reader_index = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_index, seg_path.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+
+    auto custom_check_normal = [](const BaseCompaction& compaction,
+                                  const RowsetWriterContext& ctx) {
+        EXPECT_EQ(compaction._cur_tablet_schema->inverted_indexes().size(), 4);
+        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 0);
+        EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
+    };
+
+    RowsetSharedPtr output_rowset_normal;
+    st = IndexCompactionUtils::do_compaction(rowsets, _engine_ref, _tablet, false,
+                                             output_rowset_normal, custom_check_normal);
+    EXPECT_TRUE(st.ok()) << st.to_string();
+    const auto& seg_path_normal = output_rowset_normal->segment_path(0);
+    EXPECT_TRUE(seg_path_normal.has_value()) << seg_path_normal.error();
+    auto inverted_index_file_reader_normal = IndexCompactionUtils::init_index_file_reader(
+            output_rowset_normal, seg_path_normal.value(),
+            _tablet_schema->get_inverted_index_storage_format());
+    // check index file terms
+    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, "");
+    EXPECT_TRUE(dir_idx_compaction.has_value()) << dir_idx_compaction.error();
+    auto dir_normal_compaction = inverted_index_file_reader_normal->_open(10001, "");
+    EXPECT_TRUE(dir_normal_compaction.has_value()) << dir_normal_compaction.error();
+    std::ostringstream oss;
+    IndexCompactionUtils::check_terms_stats(dir_idx_compaction->get(), oss);
+    std::string output = oss.str();
+    EXPECT_EQ(output, expected_delete_output);
+    oss.str("");
+    oss.clear();
+    IndexCompactionUtils::check_terms_stats(dir_normal_compaction->get(), oss);
+    output = oss.str();
+    EXPECT_EQ(output, expected_delete_output);
+
+    st = IndexCompactionUtils::check_idx_file_correctness(dir_idx_compaction->get(),
+                                                          dir_normal_compaction->get());
+    EXPECT_TRUE(st.ok()) << st.to_string();
+
+    // check meta and file
+    std::map<int, QueryData> query_map = {
+            {0, {{"99", "66", "56", "87", "85", "96", "20000"}, {19, 21, 21, 16, 14, 18, 0}}},
+            {3, {{"99", "66", "56", "87", "85", "96", "10000"}, {12, 18, 22, 21, 16, 20, 0}}},
+            {1, {{"good", "maybe", "great", "null"}, {197, 191, 0, 0}}},
+            {2, {{"musicstream.com", "http", "https", "null"}, {176, 719, 1087, 0}}}};
+    IndexCompactionUtils::check_meta_and_file(output_rowset_index, _tablet_schema, query_map);
+    IndexCompactionUtils::check_meta_and_file(output_rowset_normal, _tablet_schema, query_map);
 }
 
 } // namespace doris
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_with_deleted_term.cpp b/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_with_deleted_term.cpp
deleted file mode 100644
index a46f5f210dfc15..00000000000000
--- a/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_with_deleted_term.cpp
+++ /dev/null
@@ -1,671 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <iostream>
-#include <memory>
-
-#include "CLucene/StdHeader.h"
-#include "CLucene/config/repl_wchar.h"
-#include "json2pb/json_to_pb.h"
-#include "json2pb/pb_to_json.h"
-#include "olap/base_compaction.h"
-#include "olap/rowset/beta_rowset.h"
-#include "olap/rowset/beta_rowset_writer.h"
-#include "olap/rowset/rowset_factory.h"
-#include "olap/rowset/segment_v2/inverted_index/query/query_factory.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
-#include "olap/storage_engine.h"
-
-namespace doris {
-
-using namespace doris::vectorized;
-
-constexpr static uint32_t MAX_PATH_LEN = 1024;
-constexpr static std::string_view dest_dir = "/ut_dir/inverted_index_test";
-constexpr static std::string_view tmp_dir = "./ut_dir/tmp";
-static int64_t inc_id = 1000;
-
-struct DataRow {
-    int key;
-    std::string word;
-    std::string url;
-    int num;
-};
-
-static std::vector<DataRow> read_data(const std::string file_name) {
-    std::ifstream file(file_name);
-    EXPECT_TRUE(file.is_open());
-
-    std::string line;
-    std::vector<DataRow> data;
-
-    while (std::getline(file, line)) {
-        std::stringstream ss(line);
-        std::string item;
-        DataRow row;
-        EXPECT_TRUE(std::getline(ss, item, ','));
-        row.key = std::stoi(item);
-        EXPECT_TRUE(std::getline(ss, item, ','));
-        row.word = item;
-        EXPECT_TRUE(std::getline(ss, item, ','));
-        row.url = item;
-        EXPECT_TRUE(std::getline(ss, item, ','));
-        row.num = std::stoi(item);
-        data.emplace_back(std::move(row));
-    }
-
-    file.close();
-    return data;
-}
-
-static bool query_bkd(const TabletIndex* index,
-                      std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
-                      const std::vector<int>& query_data, const std::vector<int>& query_result) {
-    const auto& idx_reader = BkdIndexReader::create_shared(index, inverted_index_file_reader);
-    const auto& index_searcher_builder = std::make_unique<BKDIndexSearcherBuilder>();
-    auto dir = inverted_index_file_reader->open(index);
-    EXPECT_TRUE(dir.has_value());
-    auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
-    EXPECT_TRUE(searcher_result.has_value());
-    auto bkd_searcher = std::get_if<BKDIndexSearcherPtr>(&searcher_result.value());
-    EXPECT_TRUE(bkd_searcher != nullptr);
-    idx_reader->_type_info = get_scalar_type_info((FieldType)(*bkd_searcher)->type);
-    EXPECT_TRUE(idx_reader->_type_info != nullptr);
-    idx_reader->_value_key_coder = get_key_coder(idx_reader->_type_info->type());
-
-    for (int i = 0; i < query_data.size(); i++) {
-        vectorized::Field param_value = Int32(query_data[i]);
-        std::unique_ptr<segment_v2::InvertedIndexQueryParamFactory> query_param = nullptr;
-        EXPECT_TRUE(segment_v2::InvertedIndexQueryParamFactory::create_query_value(
-                            PrimitiveType::TYPE_INT, &param_value, query_param)
-                            .ok());
-        auto result = std::make_shared<roaring::Roaring>();
-        EXPECT_TRUE(idx_reader
-                            ->invoke_bkd_query(query_param->get_value(),
-                                               InvertedIndexQueryType::EQUAL_QUERY, *bkd_searcher,
-                                               result)
-                            .ok());
-        EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i];
-    }
-    return true;
-}
-
-static bool query_string(const TabletIndex* index,
-                         std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
-                         const std::string& column_name, const std::vector<std::string>& query_data,
-                         const std::vector<int>& query_result) {
-    const auto& idx_reader =
-            StringTypeInvertedIndexReader::create_shared(index, inverted_index_file_reader);
-    const auto& index_searcher_builder = std::make_unique<FulltextIndexSearcherBuilder>();
-    auto dir = inverted_index_file_reader->open(index);
-    EXPECT_TRUE(dir.has_value());
-    auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
-    EXPECT_TRUE(searcher_result.has_value());
-    auto string_searcher = std::get_if<FulltextIndexSearcherPtr>(&searcher_result.value());
-    EXPECT_TRUE(string_searcher != nullptr);
-    std::wstring column_name_ws = StringUtil::string_to_wstring(column_name);
-
-    for (int i = 0; i < query_data.size(); i++) {
-        TQueryOptions queryOptions;
-        auto query = QueryFactory::create(InvertedIndexQueryType::EQUAL_QUERY, *string_searcher,
-                                          queryOptions, nullptr);
-        EXPECT_TRUE(query != nullptr);
-        InvertedIndexQueryInfo query_info;
-        query_info.field_name = column_name_ws;
-        query_info.terms.emplace_back(query_data[i]);
-        query->add(query_info);
-        auto result = std::make_shared<roaring::Roaring>();
-        query->search(*result);
-        EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i];
-    }
-    return true;
-}
-
-static bool query_fulltext(const TabletIndex* index,
-                           std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
-                           const std::string& column_name,
-                           const std::vector<std::string>& query_data,
-                           const std::vector<int>& query_result) {
-    const auto& idx_reader = FullTextIndexReader::create_shared(index, inverted_index_file_reader);
-    const auto& index_searcher_builder = std::make_unique<FulltextIndexSearcherBuilder>();
-    auto dir = inverted_index_file_reader->open(index);
-    EXPECT_TRUE(dir.has_value());
-    auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
-    EXPECT_TRUE(searcher_result.has_value());
-    auto string_searcher = std::get_if<FulltextIndexSearcherPtr>(&searcher_result.value());
-    EXPECT_TRUE(string_searcher != nullptr);
-    std::wstring column_name_ws = StringUtil::string_to_wstring(column_name);
-
-    for (int i = 0; i < query_data.size(); i++) {
-        TQueryOptions queryOptions;
-        auto query = QueryFactory::create(InvertedIndexQueryType::MATCH_ANY_QUERY, *string_searcher,
-                                          queryOptions, nullptr);
-        EXPECT_TRUE(query != nullptr);
-        InvertedIndexQueryInfo query_info;
-        query_info.field_name = column_name_ws;
-        query_info.terms.emplace_back(query_data[i]);
-        query->add(query_info);
-        auto result = std::make_shared<roaring::Roaring>();
-        query->search(*result);
-        EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i];
-    }
-    return true;
-}
-
-static void check_terms_stats(lucene::store::Directory* dir) {
-    IndexReader* r = IndexReader::open(dir);
-
-    printf("Max Docs: %d\n", r->maxDoc());
-    printf("Num Docs: %d\n", r->numDocs());
-
-    int64_t ver = r->getCurrentVersion(dir);
-    printf("Current Version: %f\n", (float_t)ver);
-
-    TermEnum* te = r->terms();
-    int32_t nterms;
-    for (nterms = 0; te->next(); nterms++) {
-        /* empty */
-        std::string token =
-                lucene_wcstoutf8string(te->term(false)->text(), te->term(false)->textLength());
-        std::string field = lucene_wcstoutf8string(te->term(false)->field(),
-                                                   lenOfString(te->term(false)->field()));
-
-        printf("Field: %s ", field.c_str());
-        printf("Term: %s ", token.c_str());
-        printf("Freq: %d\n", te->docFreq());
-        if (false) {
-            TermDocs* td = r->termDocs(te->term());
-            while (td->next()) {
-                printf("DocID: %d ", td->doc());
-                printf("TermFreq: %d\n", td->freq());
-            }
-            _CLLDELETE(td);
-        }
-    }
-    printf("Term count: %d\n\n", nterms);
-    te->close();
-    _CLLDELETE(te);
-
-    r->close();
-    _CLLDELETE(r);
-}
-static Status check_idx_file_correctness(lucene::store::Directory* index_reader,
-                                         lucene::store::Directory* tmp_index_reader) {
-    lucene::index::IndexReader* idx_reader = lucene::index::IndexReader::open(index_reader);
-    lucene::index::IndexReader* tmp_idx_reader = lucene::index::IndexReader::open(tmp_index_reader);
-
-    // compare numDocs
-    if (idx_reader->numDocs() != tmp_idx_reader->numDocs()) {
-        return Status::InternalError(
-                "index compaction correctness check failed, numDocs not equal, idx_numDocs={}, "
-                "tmp_idx_numDocs={}",
-                idx_reader->numDocs(), tmp_idx_reader->numDocs());
-    }
-
-    lucene::index::TermEnum* term_enum = idx_reader->terms();
-    lucene::index::TermEnum* tmp_term_enum = tmp_idx_reader->terms();
-    lucene::index::TermDocs* term_docs = nullptr;
-    lucene::index::TermDocs* tmp_term_docs = nullptr;
-
-    // iterate TermEnum
-    while (term_enum->next() && tmp_term_enum->next()) {
-        std::string token = lucene_wcstoutf8string(term_enum->term(false)->text(),
-                                                   term_enum->term(false)->textLength());
-        std::string field = lucene_wcstoutf8string(term_enum->term(false)->field(),
-                                                   lenOfString(term_enum->term(false)->field()));
-        std::string tmp_token = lucene_wcstoutf8string(tmp_term_enum->term(false)->text(),
-                                                       tmp_term_enum->term(false)->textLength());
-        std::string tmp_field =
-                lucene_wcstoutf8string(tmp_term_enum->term(false)->field(),
-                                       lenOfString(tmp_term_enum->term(false)->field()));
-        // compare token and field
-        if (field != tmp_field) {
-            return Status::InternalError(
-                    "index compaction correctness check failed, fields not equal, field={}, "
-                    "tmp_field={}",
-                    field, field);
-        }
-        if (token != tmp_token) {
-            return Status::InternalError(
-                    "index compaction correctness check failed, tokens not equal, token={}, "
-                    "tmp_token={}",
-                    token, tmp_token);
-        }
-
-        // get term's docId and freq
-        term_docs = idx_reader->termDocs(term_enum->term(false));
-        tmp_term_docs = tmp_idx_reader->termDocs(tmp_term_enum->term(false));
-
-        // compare term's docId and freq
-        while (term_docs->next() && tmp_term_docs->next()) {
-            if (term_docs->doc() != tmp_term_docs->doc() ||
-                term_docs->freq() != tmp_term_docs->freq()) {
-                return Status::InternalError(
-                        "index compaction correctness check failed, docId or freq not equal, "
-                        "docId={}, tmp_docId={}, freq={}, tmp_freq={}",
-                        term_docs->doc(), tmp_term_docs->doc(), term_docs->freq(),
-                        tmp_term_docs->freq());
-            }
-        }
-
-        // check if there are remaining docs
-        if (term_docs->next() || tmp_term_docs->next()) {
-            return Status::InternalError(
-                    "index compaction correctness check failed, number of docs not equal for "
-                    "term={}, tmp_term={}",
-                    token, tmp_token);
-        }
-        if (term_docs) {
-            term_docs->close();
-            _CLLDELETE(term_docs);
-        }
-        if (tmp_term_docs) {
-            tmp_term_docs->close();
-            _CLLDELETE(tmp_term_docs);
-        }
-    }
-
-    // check if there are remaining terms
-    if (term_enum->next() || tmp_term_enum->next()) {
-        return Status::InternalError(
-                "index compaction correctness check failed, number of terms not equal");
-    }
-    if (term_enum) {
-        term_enum->close();
-        _CLLDELETE(term_enum);
-    }
-    if (tmp_term_enum) {
-        tmp_term_enum->close();
-        _CLLDELETE(tmp_term_enum);
-    }
-    if (idx_reader) {
-        idx_reader->close();
-        _CLLDELETE(idx_reader);
-    }
-    if (tmp_idx_reader) {
-        tmp_idx_reader->close();
-        _CLLDELETE(tmp_idx_reader);
-    }
-    return Status::OK();
-}
-
-static RowsetSharedPtr do_compaction(std::vector<RowsetSharedPtr> rowsets,
-                                     StorageEngine* engine_ref, TabletSharedPtr tablet,
-                                     bool is_index_compaction) {
-    config::inverted_index_compaction_enable = is_index_compaction;
-    // only base compaction can handle delete predicate
-    BaseCompaction compaction(*engine_ref, tablet);
-    compaction._input_rowsets = std::move(rowsets);
-    compaction.build_basic_info();
-
-    std::vector<RowsetReaderSharedPtr> input_rs_readers;
-    input_rs_readers.reserve(compaction._input_rowsets.size());
-    for (auto& rowset : compaction._input_rowsets) {
-        RowsetReaderSharedPtr rs_reader;
-        EXPECT_TRUE(rowset->create_reader(&rs_reader).ok());
-        input_rs_readers.push_back(std::move(rs_reader));
-    }
-
-    RowsetWriterContext ctx;
-    EXPECT_TRUE(compaction.construct_output_rowset_writer(ctx).ok());
-
-    if (is_index_compaction) {
-        EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 2);
-        // col v1
-        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(1));
-        // col v2
-        EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2));
-    }
-
-    compaction._stats.rowid_conversion = compaction._rowid_conversion.get();
-    EXPECT_TRUE(Merger::vertical_merge_rowsets(tablet, compaction.compaction_type(),
-                                               *(compaction._cur_tablet_schema), input_rs_readers,
-                                               compaction._output_rs_writer.get(), 100000, 5,
-                                               &compaction._stats)
-                        .ok());
-    const auto& dst_writer =
-            dynamic_cast<BaseBetaRowsetWriter*>(compaction._output_rs_writer.get());
-    for (const auto& [seg_id, idx_file_writer] : dst_writer->_idx_files.get_file_writers()) {
-        EXPECT_FALSE(idx_file_writer->_closed);
-    }
-    Status st = compaction.do_inverted_index_compaction();
-    EXPECT_TRUE(st.ok()) << st.to_string();
-
-    st = compaction._output_rs_writer->build(compaction._output_rowset);
-    EXPECT_TRUE(st.ok()) << st.to_string();
-
-    for (const auto& [seg_id, idx_file_writer] : dst_writer->_idx_files.get_file_writers()) {
-        EXPECT_TRUE(idx_file_writer->_closed);
-    }
-    EXPECT_TRUE(compaction._output_rowset->num_segments() == 1);
-
-    return compaction._output_rowset;
-}
-
-class IndexCompactionDeleteTest : public ::testing::Test {
-protected:
-    void SetUp() override {
-        // absolute dir
-        char buffer[MAX_PATH_LEN];
-        EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
-        _curreent_dir = std::string(buffer);
-        _absolute_dir = _curreent_dir + std::string(dest_dir);
-        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
-        EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok());
-
-        // tmp dir
-        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok());
-        EXPECT_TRUE(io::global_local_filesystem()->create_directory(tmp_dir).ok());
-        std::vector<StorePath> paths;
-        paths.emplace_back(std::string(tmp_dir), 1024000000);
-        auto tmp_file_dirs = std::make_unique<segment_v2::TmpFileDirs>(paths);
-        Status st = tmp_file_dirs->init();
-        EXPECT_TRUE(st.ok()) << st.to_json();
-        ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs));
-
-        // storage engine
-        doris::EngineOptions options;
-        auto engine = std::make_unique<StorageEngine>(options);
-        _engine_ref = engine.get();
-        _data_dir = std::make_unique<DataDir>(*_engine_ref, _absolute_dir);
-        static_cast<void>(_data_dir->update_capacity());
-        ExecEnv::GetInstance()->set_storage_engine(std::move(engine));
-
-        // tablet_schema
-        TabletSchemaPB schema_pb;
-        schema_pb.set_keys_type(KeysType::DUP_KEYS);
-        schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
-
-        construct_column(schema_pb.add_column(), schema_pb.add_index(), 10000, "key_index", 0,
-                         "INT", "key");
-        construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001, "v1_index", 1,
-                         "STRING", "v1");
-        construct_column(schema_pb.add_column(), schema_pb.add_index(), 10002, "v2_index", 2,
-                         "STRING", "v2", true);
-        construct_column(schema_pb.add_column(), schema_pb.add_index(), 10003, "v3_index", 3, "INT",
-                         "v3");
-
-        _tablet_schema.reset(new TabletSchema);
-        _tablet_schema->init_from_pb(schema_pb);
-
-        // tablet
-        TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
-
-        _tablet.reset(new Tablet(*_engine_ref, tablet_meta, _data_dir.get()));
-        EXPECT_TRUE(_tablet->init().ok());
-    }
-    void TearDown() override {
-        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
-        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
-        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok());
-        _engine_ref = nullptr;
-        ExecEnv::GetInstance()->set_storage_engine(nullptr);
-    }
-
-    void init_rs_meta(RowsetMetaSharedPtr& rs_meta, int64_t start, int64_t end) {
-        std::string json_rowset_meta = R"({
-            "rowset_id": 540081,
-            "tablet_id": 15673,
-            "partition_id": 10000,
-            "tablet_schema_hash": 567997577,
-            "rowset_type": "BETA_ROWSET",
-            "rowset_state": "VISIBLE",
-            "empty": false
-        })";
-        RowsetMetaPB rowset_meta_pb;
-        json2pb::JsonToProtoMessage(json_rowset_meta, &rowset_meta_pb);
-        rowset_meta_pb.set_start_version(start);
-        rowset_meta_pb.set_end_version(end);
-        rs_meta->init_from_pb(rowset_meta_pb);
-    }
-
-    RowsetSharedPtr create_delete_predicate_rowset(const TabletSchemaSPtr& schema, std::string pred,
-                                                   int64_t version) {
-        DeletePredicatePB del_pred;
-        del_pred.add_sub_predicates(pred);
-        del_pred.set_version(1);
-        RowsetMetaSharedPtr rsm(new RowsetMeta());
-        init_rs_meta(rsm, version, version);
-        RowsetId id;
-        id.init(version);
-        rsm->set_rowset_id(id);
-        rsm->set_delete_predicate(std::move(del_pred));
-        rsm->set_tablet_schema(schema);
-        return std::make_shared<BetaRowset>(schema, rsm, "");
-    }
-
-    void construct_column(ColumnPB* column_pb, TabletIndexPB* tablet_index, int64_t index_id,
-                          const std::string& index_name, int32_t col_unique_id,
-                          const std::string& column_type, const std::string& column_name,
-                          bool parser = false) {
-        column_pb->set_unique_id(col_unique_id);
-        column_pb->set_name(column_name);
-        column_pb->set_type(column_type);
-        column_pb->set_is_key(false);
-        column_pb->set_is_nullable(true);
-        tablet_index->set_index_id(index_id);
-        tablet_index->set_index_name(index_name);
-        tablet_index->set_index_type(IndexType::INVERTED);
-        tablet_index->add_col_unique_id(col_unique_id);
-        if (parser) {
-            auto* properties = tablet_index->mutable_properties();
-            (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_UNICODE;
-        }
-    }
-
-    void check_meta_and_file(RowsetSharedPtr output_rowset) {
-        CHECK_EQ(output_rowset->num_segments(), 1);
-        // check rowset meta and file
-        int seg_id = 0;
-        // meta
-        const auto& index_info = output_rowset->_rowset_meta->inverted_index_file_info(seg_id);
-        EXPECT_TRUE(index_info.has_index_size());
-        const auto& fs = output_rowset->_rowset_meta->fs();
-        const auto& file_name = fmt::format("{}/{}_{}.idx", output_rowset->tablet_path(),
-                                            output_rowset->rowset_id().to_string(), seg_id);
-        int64_t file_size = 0;
-        EXPECT_TRUE(fs->file_size(file_name, &file_size).ok());
-        EXPECT_EQ(index_info.index_size(), file_size);
-
-        // file
-        const auto& seg_path = output_rowset->segment_path(seg_id);
-        EXPECT_TRUE(seg_path.has_value());
-        const auto& index_file_path_prefix =
-                InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value());
-        auto inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
-                fs, std::string(index_file_path_prefix),
-                _tablet_schema->get_inverted_index_storage_format(), index_info);
-        EXPECT_TRUE(inverted_index_file_reader->init().ok());
-        const auto& dirs = inverted_index_file_reader->get_all_directories();
-        EXPECT_TRUE(dirs.has_value());
-        EXPECT_EQ(dirs.value().size(), 4);
-
-        // read col key
-        const auto& key = _tablet_schema->column_by_uid(0);
-        const auto* key_index = _tablet_schema->inverted_index(key);
-        EXPECT_TRUE(key_index != nullptr);
-        std::vector<int> query_data {99, 66, 56, 87, 85, 96, 20000};
-        std::vector<int> query_result {19, 21, 21, 16, 14, 18, 0};
-        EXPECT_TRUE(query_bkd(key_index, inverted_index_file_reader, query_data, query_result));
-
-        // read col v3
-        const auto& v3_column = _tablet_schema->column_by_uid(3);
-        const auto* v3_index = _tablet_schema->inverted_index(v3_column);
-        EXPECT_TRUE(v3_index != nullptr);
-        std::vector<int> query_data3 {99, 66, 56, 87, 85, 96, 10000};
-        std::vector<int> query_result3 {12, 18, 22, 21, 16, 20, 0};
-        EXPECT_TRUE(query_bkd(v3_index, inverted_index_file_reader, query_data3, query_result3));
-
-        // read col v1
-        const auto& v1_column = _tablet_schema->column_by_uid(1);
-        const auto* v1_index = _tablet_schema->inverted_index(v1_column);
-        EXPECT_TRUE(v1_index != nullptr);
-        std::vector<std::string> query_data1 {"good", "maybe", "great", "null"};
-        std::vector<int> query_result1 {197, 191, 0, 0};
-        EXPECT_TRUE(query_string(v1_index, inverted_index_file_reader, "1", query_data1,
-                                 query_result1));
-
-        // read col v2
-        const auto& v2_column = _tablet_schema->column_by_uid(2);
-        const auto* v2_index = _tablet_schema->inverted_index(v2_column);
-        EXPECT_TRUE(v2_index != nullptr);
-        std::vector<std::string> query_data2 {"musicstream.com", "http", "https", "null"};
-        std::vector<int> query_result2 {176, 719, 1087, 0};
-        EXPECT_TRUE(query_fulltext(v2_index, inverted_index_file_reader, "2", query_data2,
-                                   query_result2));
-    }
-
-    RowsetWriterContext rowset_writer_context() {
-        RowsetWriterContext context;
-        RowsetId rowset_id;
-        rowset_id.init(inc_id);
-        context.rowset_id = rowset_id;
-        context.rowset_type = BETA_ROWSET;
-        context.data_dir = _data_dir.get();
-        context.rowset_state = VISIBLE;
-        context.tablet_schema = _tablet_schema;
-        context.tablet_path = _tablet->tablet_path();
-        context.version = Version(inc_id, inc_id);
-        context.max_rows_per_segment = 200;
-        inc_id++;
-        return context;
-    }
-
-    IndexCompactionDeleteTest() = default;
-    ~IndexCompactionDeleteTest() override = default;
-
-private:
-    TabletSchemaSPtr _tablet_schema = nullptr;
-    StorageEngine* _engine_ref = nullptr;
-    std::unique_ptr<DataDir> _data_dir = nullptr;
-    TabletSharedPtr _tablet = nullptr;
-    std::string _absolute_dir;
-    std::string _curreent_dir;
-};
-
-TEST_F(IndexCompactionDeleteTest, delete_index_test) {
-    EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
-    EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
-    std::string data_file1 =
-            _curreent_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv";
-    std::string data_file2 =
-            _curreent_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv";
-
-    std::vector<std::vector<DataRow>> data;
-    data.emplace_back(read_data(data_file1));
-    data.emplace_back(read_data(data_file2));
-
-    std::vector<RowsetSharedPtr> rowsets(data.size());
-    for (int i = 0; i < data.size(); i++) {
-        const auto& res =
-                RowsetFactory::create_rowset_writer(*_engine_ref, rowset_writer_context(), false);
-        EXPECT_TRUE(res.has_value()) << res.error();
-        const auto& rowset_writer = res.value();
-
-        Block block = _tablet_schema->create_block();
-        auto columns = block.mutate_columns();
-        for (const auto& row : data[i]) {
-            vectorized::Field key = Int32(row.key);
-            vectorized::Field v1(row.word);
-            vectorized::Field v2(row.url);
-            vectorized::Field v3 = Int32(row.num);
-            columns[0]->insert(key);
-            columns[1]->insert(v1);
-            columns[2]->insert(v2);
-            columns[3]->insert(v3);
-        }
-        EXPECT_TRUE(rowset_writer->add_block(&block).ok());
-        EXPECT_TRUE(rowset_writer->flush().ok());
-        const auto& dst_writer = dynamic_cast<BaseBetaRowsetWriter*>(rowset_writer.get());
-
-        // inverted index file writer
-        for (const auto& [seg_id, idx_file_writer] : dst_writer->_idx_files.get_file_writers()) {
-            EXPECT_TRUE(idx_file_writer->_closed);
-        }
-
-        EXPECT_TRUE(rowset_writer->build(rowsets[i]).ok());
-        EXPECT_TRUE(_tablet->add_rowset(rowsets[i]).ok());
-        EXPECT_TRUE(rowsets[i]->num_segments() == 5);
-
-        // check rowset meta and file
-        for (int seg_id = 0; seg_id < rowsets[i]->num_segments(); seg_id++) {
-            const auto& index_info = rowsets[i]->_rowset_meta->inverted_index_file_info(seg_id);
-            EXPECT_TRUE(index_info.has_index_size());
-            const auto& fs = rowsets[i]->_rowset_meta->fs();
-            const auto& file_name = fmt::format("{}/{}_{}.idx", rowsets[i]->tablet_path(),
-                                                rowsets[i]->rowset_id().to_string(), seg_id);
-            int64_t file_size = 0;
-            EXPECT_TRUE(fs->file_size(file_name, &file_size).ok());
-            EXPECT_EQ(index_info.index_size(), file_size);
-
-            const auto& seg_path = rowsets[i]->segment_path(seg_id);
-            EXPECT_TRUE(seg_path.has_value());
-            const auto& index_file_path_prefix =
-                    InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value());
-            auto inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
-                    fs, std::string(index_file_path_prefix),
-                    _tablet_schema->get_inverted_index_storage_format(), index_info);
-            EXPECT_TRUE(inverted_index_file_reader->init().ok());
-            const auto& dirs = inverted_index_file_reader->get_all_directories();
-            EXPECT_TRUE(dirs.has_value());
-            EXPECT_EQ(dirs.value().size(), 4);
-        }
-    }
-
-    // create delete predicate rowset and add to tablet
-    auto delete_rowset = create_delete_predicate_rowset(_tablet_schema, "v1='great'", inc_id++);
-    EXPECT_TRUE(_tablet->add_rowset(delete_rowset).ok());
-    EXPECT_TRUE(_tablet->rowset_map().size() == 3);
-    rowsets.push_back(delete_rowset);
-    EXPECT_TRUE(rowsets.size() == 3);
-
-    auto output_rowset_index = do_compaction(rowsets, _engine_ref, _tablet, true);
-    const auto& seg_path = output_rowset_index->segment_path(0);
-    EXPECT_TRUE(seg_path.has_value());
-    const auto& index_file_path_prefix =
-            InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value());
-    auto inverted_index_file_reader_index = std::make_shared<InvertedIndexFileReader>(
-            output_rowset_index->_rowset_meta->fs(), std::string(index_file_path_prefix),
-            _tablet_schema->get_inverted_index_storage_format());
-    EXPECT_TRUE(inverted_index_file_reader_index->init().ok());
-
-    auto output_rowset_normal = do_compaction(rowsets, _engine_ref, _tablet, false);
-    const auto& seg_path_normal = output_rowset_normal->segment_path(0);
-    EXPECT_TRUE(seg_path_normal.has_value());
-    const auto& index_file_path_prefix_normal =
-            InvertedIndexDescriptor::get_index_file_path_prefix(seg_path_normal.value());
-    auto inverted_index_file_reader_normal = std::make_shared<InvertedIndexFileReader>(
-            output_rowset_normal->_rowset_meta->fs(), std::string(index_file_path_prefix_normal),
-            _tablet_schema->get_inverted_index_storage_format());
-    EXPECT_TRUE(inverted_index_file_reader_normal->init().ok());
-
-    // check index file terms
-    auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, "");
-    auto dir_normal_compaction = inverted_index_file_reader_normal->_open(10001, "");
-    check_terms_stats(dir_idx_compaction->get());
-    check_terms_stats(dir_normal_compaction->get());
-    auto st = check_idx_file_correctness(dir_idx_compaction->get(), dir_normal_compaction->get());
-    EXPECT_TRUE(st.ok()) << st.to_string();
-
-    // check meta and file
-    check_meta_and_file(output_rowset_index);
-    check_meta_and_file(output_rowset_normal);
-}
-
-} // namespace doris
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp b/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp
new file mode 100644
index 00000000000000..530dca8054c19a
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp
@@ -0,0 +1,621 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <vector>
+
+#include "CLucene/StdHeader.h"
+#include "CLucene/config/repl_wchar.h"
+#include "json2pb/json_to_pb.h"
+#include "json2pb/pb_to_json.h"
+#include "olap/base_compaction.h"
+#include "olap/rowset/beta_rowset.h"
+#include "olap/rowset/beta_rowset_writer.h"
+#include "olap/rowset/rowset_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/query/query_factory.h"
+#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
+#include "olap/storage_engine.h"
+
+namespace doris {
+
+static int64_t inc_id = 1000;
+const static std::string expected_output =
+        "Max Docs: 2000\n"
+        "Num Docs: 2000\n"
+        "Field: 1 Term: bad Freq: 196\n"
+        "Field: 1 Term: excellent Freq: 227\n"
+        "Field: 1 Term: fine Freq: 190\n"
+        "Field: 1 Term: good Freq: 197\n"
+        "Field: 1 Term: great Freq: 194\n"
+        "Field: 1 Term: maybe Freq: 191\n"
+        "Field: 1 Term: no Freq: 205\n"
+        "Field: 1 Term: ok Freq: 175\n"
+        "Field: 1 Term: terrible Freq: 205\n"
+        "Field: 1 Term: yes Freq: 220\n"
+        "Term count: 10\n\n";
+const static std::string expected_delete_output =
+        "Max Docs: 1806\n"
+        "Num Docs: 1806\n"
+        "Field: 1 Term: bad Freq: 196\n"
+        "Field: 1 Term: excellent Freq: 227\n"
+        "Field: 1 Term: fine Freq: 190\n"
+        "Field: 1 Term: good Freq: 197\n"
+        "Field: 1 Term: maybe Freq: 191\n"
+        "Field: 1 Term: no Freq: 205\n"
+        "Field: 1 Term: ok Freq: 175\n"
+        "Field: 1 Term: terrible Freq: 205\n"
+        "Field: 1 Term: yes Freq: 220\n"
+        "Term count: 9\n\n";
+
+using QueryData = std::pair<std::vector<std::string>, std::vector<int>>;
+
+class IndexCompactionUtils {
+    struct DataRow {
+        int key;
+        std::string word;
+        std::string url;
+        int num;
+    };
+
+    static std::vector<DataRow> read_data(const std::string file_name) {
+        std::ifstream file(file_name);
+        EXPECT_TRUE(file.is_open());
+
+        std::string line;
+        std::vector<DataRow> data;
+
+        while (std::getline(file, line)) {
+            std::stringstream ss(line);
+            std::string item;
+            DataRow row;
+            EXPECT_TRUE(std::getline(ss, item, ','));
+            row.key = std::stoi(item);
+            EXPECT_TRUE(std::getline(ss, item, ','));
+            row.word = item;
+            EXPECT_TRUE(std::getline(ss, item, ','));
+            row.url = item;
+            EXPECT_TRUE(std::getline(ss, item, ','));
+            row.num = std::stoi(item);
+            data.emplace_back(std::move(row));
+        }
+
+        file.close();
+        return data;
+    }
+
+    static bool query_bkd(const TabletIndex* index,
+                          std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
+                          const std::vector<int>& query_data,
+                          const std::vector<int>& query_result) {
+        const auto& idx_reader = BkdIndexReader::create_shared(index, inverted_index_file_reader);
+        const auto& index_searcher_builder = std::make_unique<BKDIndexSearcherBuilder>();
+        auto dir = inverted_index_file_reader->open(index);
+        EXPECT_TRUE(dir.has_value());
+        auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
+        EXPECT_TRUE(searcher_result.has_value());
+        auto bkd_searcher = std::get_if<BKDIndexSearcherPtr>(&searcher_result.value());
+        EXPECT_TRUE(bkd_searcher != nullptr);
+        idx_reader->_type_info = get_scalar_type_info((FieldType)(*bkd_searcher)->type);
+        EXPECT_TRUE(idx_reader->_type_info != nullptr);
+        idx_reader->_value_key_coder = get_key_coder(idx_reader->_type_info->type());
+
+        for (int i = 0; i < query_data.size(); i++) {
+            vectorized::Field param_value = int32_t(query_data[i]);
+            std::unique_ptr<segment_v2::InvertedIndexQueryParamFactory> query_param = nullptr;
+            EXPECT_TRUE(segment_v2::InvertedIndexQueryParamFactory::create_query_value(
+                                PrimitiveType::TYPE_INT, &param_value, query_param)
+                                .ok());
+            auto result = std::make_shared<roaring::Roaring>();
+            EXPECT_TRUE(idx_reader
+                                ->invoke_bkd_query(query_param->get_value(),
+                                                   InvertedIndexQueryType::EQUAL_QUERY,
+                                                   *bkd_searcher, result)
+                                .ok());
+            EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i];
+        }
+        return true;
+    }
+
+    static bool query_string(const TabletIndex* index,
+                             std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
+                             const std::string& column_name,
+                             const std::vector<std::string>& query_data,
+                             const std::vector<int>& query_result) {
+        const auto& idx_reader =
+                StringTypeInvertedIndexReader::create_shared(index, inverted_index_file_reader);
+        const auto& index_searcher_builder = std::make_unique<FulltextIndexSearcherBuilder>();
+        auto dir = inverted_index_file_reader->open(index);
+        EXPECT_TRUE(dir.has_value());
+        auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
+        EXPECT_TRUE(searcher_result.has_value());
+        auto string_searcher = std::get_if<FulltextIndexSearcherPtr>(&searcher_result.value());
+        EXPECT_TRUE(string_searcher != nullptr);
+        std::wstring column_name_ws = StringUtil::string_to_wstring(column_name);
+
+        for (int i = 0; i < query_data.size(); i++) {
+            TQueryOptions queryOptions;
+            auto query = QueryFactory::create(InvertedIndexQueryType::EQUAL_QUERY, *string_searcher,
+                                              queryOptions, nullptr);
+            EXPECT_TRUE(query != nullptr);
+            InvertedIndexQueryInfo query_info;
+            query_info.field_name = column_name_ws;
+            query_info.terms.emplace_back(query_data[i]);
+            query->add(query_info);
+            auto result = std::make_shared<roaring::Roaring>();
+            query->search(*result);
+            EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i];
+        }
+        return true;
+    }
+
+    static bool query_fulltext(const TabletIndex* index,
+                               std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
+                               const std::string& column_name,
+                               const std::vector<std::string>& query_data,
+                               const std::vector<int>& query_result) {
+        const auto& idx_reader =
+                FullTextIndexReader::create_shared(index, inverted_index_file_reader);
+        const auto& index_searcher_builder = std::make_unique<FulltextIndexSearcherBuilder>();
+        auto dir = inverted_index_file_reader->open(index);
+        EXPECT_TRUE(dir.has_value());
+        auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
+        EXPECT_TRUE(searcher_result.has_value());
+        auto string_searcher = std::get_if<FulltextIndexSearcherPtr>(&searcher_result.value());
+        EXPECT_TRUE(string_searcher != nullptr);
+        std::wstring column_name_ws = StringUtil::string_to_wstring(column_name);
+
+        for (int i = 0; i < query_data.size(); i++) {
+            TQueryOptions queryOptions;
+            auto query = QueryFactory::create(InvertedIndexQueryType::MATCH_ANY_QUERY,
+                                              *string_searcher, queryOptions, nullptr);
+            EXPECT_TRUE(query != nullptr);
+            InvertedIndexQueryInfo query_info;
+            query_info.field_name = column_name_ws;
+            query_info.terms.emplace_back(query_data[i]);
+            query->add(query_info);
+            auto result = std::make_shared<roaring::Roaring>();
+            query->search(*result);
+            EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i];
+        }
+        return true;
+    }
+
+    static void check_terms_stats(lucene::store::Directory* dir, std::ostream& os = std::cout) {
+        IndexReader* r = IndexReader::open(dir);
+
+        os << "Max Docs: " << r->maxDoc() << "\n";
+        os << "Num Docs: " << r->numDocs() << "\n";
+
+        TermEnum* te = r->terms();
+        int32_t nterms;
+        for (nterms = 0; te->next(); nterms++) {
+            std::string token =
+                    lucene_wcstoutf8string(te->term(false)->text(), te->term(false)->textLength());
+            std::string field = lucene_wcstoutf8string(te->term(false)->field(),
+                                                       lenOfString(te->term(false)->field()));
+
+            os << "Field: " << field << " ";
+            os << "Term: " << token << " ";
+            os << "Freq: " << te->docFreq() << "\n";
+            if (false) {
+                TermDocs* td = r->termDocs(te->term());
+                while (td->next()) {
+                    os << "DocID: " << td->doc() << " ";
+                    os << "TermFreq: " << td->freq() << "\n";
+                }
+                _CLLDELETE(td);
+            }
+        }
+        os << "Term count: " << nterms << "\n\n";
+        te->close();
+        _CLLDELETE(te);
+
+        r->close();
+        _CLLDELETE(r);
+    }
+    static Status check_idx_file_correctness(lucene::store::Directory* index_reader,
+                                             lucene::store::Directory* tmp_index_reader) {
+        lucene::index::IndexReader* idx_reader = lucene::index::IndexReader::open(index_reader);
+        lucene::index::IndexReader* tmp_idx_reader =
+                lucene::index::IndexReader::open(tmp_index_reader);
+
+        // compare numDocs
+        if (idx_reader->numDocs() != tmp_idx_reader->numDocs()) {
+            return Status::InternalError(
+                    "index compaction correctness check failed, numDocs not equal, idx_numDocs={}, "
+                    "tmp_idx_numDocs={}",
+                    idx_reader->numDocs(), tmp_idx_reader->numDocs());
+        }
+
+        lucene::index::TermEnum* term_enum = idx_reader->terms();
+        lucene::index::TermEnum* tmp_term_enum = tmp_idx_reader->terms();
+        lucene::index::TermDocs* term_docs = nullptr;
+        lucene::index::TermDocs* tmp_term_docs = nullptr;
+
+        // iterate TermEnum
+        while (term_enum->next() && tmp_term_enum->next()) {
+            std::string token = lucene_wcstoutf8string(term_enum->term(false)->text(),
+                                                       term_enum->term(false)->textLength());
+            std::string field = lucene_wcstoutf8string(
+                    term_enum->term(false)->field(), lenOfString(term_enum->term(false)->field()));
+            std::string tmp_token = lucene_wcstoutf8string(
+                    tmp_term_enum->term(false)->text(), tmp_term_enum->term(false)->textLength());
+            std::string tmp_field =
+                    lucene_wcstoutf8string(tmp_term_enum->term(false)->field(),
+                                           lenOfString(tmp_term_enum->term(false)->field()));
+            // compare token and field
+            if (field != tmp_field) {
+                return Status::InternalError(
+                        "index compaction correctness check failed, fields not equal, field={}, "
+                        "tmp_field={}",
+                        field, field);
+            }
+            if (token != tmp_token) {
+                return Status::InternalError(
+                        "index compaction correctness check failed, tokens not equal, token={}, "
+                        "tmp_token={}",
+                        token, tmp_token);
+            }
+
+            // get term's docId and freq
+            term_docs = idx_reader->termDocs(term_enum->term(false));
+            tmp_term_docs = tmp_idx_reader->termDocs(tmp_term_enum->term(false));
+
+            // compare term's docId and freq
+            while (term_docs->next() && tmp_term_docs->next()) {
+                if (term_docs->doc() != tmp_term_docs->doc() ||
+                    term_docs->freq() != tmp_term_docs->freq()) {
+                    return Status::InternalError(
+                            "index compaction correctness check failed, docId or freq not equal, "
+                            "docId={}, tmp_docId={}, freq={}, tmp_freq={}",
+                            term_docs->doc(), tmp_term_docs->doc(), term_docs->freq(),
+                            tmp_term_docs->freq());
+                }
+            }
+
+            // check if there are remaining docs
+            if (term_docs->next() || tmp_term_docs->next()) {
+                return Status::InternalError(
+                        "index compaction correctness check failed, number of docs not equal for "
+                        "term={}, tmp_term={}",
+                        token, tmp_token);
+            }
+            if (term_docs) {
+                term_docs->close();
+                _CLLDELETE(term_docs);
+            }
+            if (tmp_term_docs) {
+                tmp_term_docs->close();
+                _CLLDELETE(tmp_term_docs);
+            }
+        }
+
+        // check if there are remaining terms
+        if (term_enum->next() || tmp_term_enum->next()) {
+            return Status::InternalError(
+                    "index compaction correctness check failed, number of terms not equal");
+        }
+        if (term_enum) {
+            term_enum->close();
+            _CLLDELETE(term_enum);
+        }
+        if (tmp_term_enum) {
+            tmp_term_enum->close();
+            _CLLDELETE(tmp_term_enum);
+        }
+        if (idx_reader) {
+            idx_reader->close();
+            _CLLDELETE(idx_reader);
+        }
+        if (tmp_idx_reader) {
+            tmp_idx_reader->close();
+            _CLLDELETE(tmp_idx_reader);
+        }
+        return Status::OK();
+    }
+
+    static Status do_compaction(
+            const std::vector<RowsetSharedPtr>& rowsets, StorageEngine* engine_ref,
+            const TabletSharedPtr& tablet, bool is_index_compaction, RowsetSharedPtr& rowset_ptr,
+            const std::function<void(const BaseCompaction&, const RowsetWriterContext&)>
+                    custom_check = nullptr) {
+        config::inverted_index_compaction_enable = is_index_compaction;
+        // only base compaction can handle delete predicate
+        BaseCompaction compaction(*engine_ref, tablet);
+        compaction._input_rowsets = std::move(rowsets);
+        compaction.build_basic_info();
+
+        std::vector<RowsetReaderSharedPtr> input_rs_readers;
+        create_input_rowsets_readers(compaction, input_rs_readers);
+
+        RowsetWriterContext ctx;
+        RETURN_IF_ERROR(compaction.construct_output_rowset_writer(ctx));
+
+        compaction._stats.rowid_conversion = compaction._rowid_conversion.get();
+        RETURN_IF_ERROR(Merger::vertical_merge_rowsets(
+                tablet, compaction.compaction_type(), *(compaction._cur_tablet_schema),
+                input_rs_readers, compaction._output_rs_writer.get(), 100000, 5,
+                &compaction._stats));
+
+        const auto& dst_writer =
+                dynamic_cast<BaseBetaRowsetWriter*>(compaction._output_rs_writer.get());
+        check_idx_file_writer_closed(dst_writer, false);
+
+        RETURN_IF_ERROR(compaction.do_inverted_index_compaction());
+
+        RETURN_IF_ERROR(compaction._output_rs_writer->build(compaction._output_rowset));
+
+        check_idx_file_writer_closed(dst_writer, true);
+
+        if (custom_check) {
+            custom_check(compaction, ctx);
+        }
+
+        rowset_ptr = std::move(compaction._output_rowset);
+        return Status::OK();
+    }
+
+    static void create_input_rowsets_readers(const BaseCompaction& compaction,
+                                             std::vector<RowsetReaderSharedPtr>& input_rs_readers) {
+        input_rs_readers.reserve(compaction._input_rowsets.size());
+        for (auto& rowset : compaction._input_rowsets) {
+            RowsetReaderSharedPtr rs_reader;
+            EXPECT_TRUE(rowset->create_reader(&rs_reader).ok());
+            input_rs_readers.push_back(std::move(rs_reader));
+        }
+    }
+
+    static void check_idx_file_writer_closed(BaseBetaRowsetWriter* writer, bool closed) {
+        for (const auto& [seg_id, idx_file_writer] : writer->inverted_index_file_writers()) {
+            EXPECT_EQ(idx_file_writer->_closed, closed);
+        }
+    }
+
+    static void init_rs_meta(RowsetMetaSharedPtr& rs_meta, int64_t start, int64_t end) {
+        std::string json_rowset_meta = R"({
+            "rowset_id": 540081,
+            "tablet_id": 15673,
+            "partition_id": 10000,
+            "tablet_schema_hash": 567997577,
+            "rowset_type": "BETA_ROWSET",
+            "rowset_state": "VISIBLE",
+            "empty": false
+        })";
+        RowsetMetaPB rowset_meta_pb;
+        json2pb::JsonToProtoMessage(json_rowset_meta, &rowset_meta_pb);
+        rowset_meta_pb.set_start_version(start);
+        rowset_meta_pb.set_end_version(end);
+        rs_meta->init_from_pb(rowset_meta_pb);
+    }
+
+    static RowsetSharedPtr create_delete_predicate_rowset(const TabletSchemaSPtr& schema,
+                                                          std::string pred, int64_t version) {
+        DeletePredicatePB del_pred;
+        del_pred.add_sub_predicates(pred);
+        del_pred.set_version(1);
+        RowsetMetaSharedPtr rsm(new RowsetMeta());
+        init_rs_meta(rsm, version, version);
+        RowsetId id;
+        id.init(version);
+        rsm->set_rowset_id(id);
+        rsm->set_delete_predicate(std::move(del_pred));
+        rsm->set_tablet_schema(schema);
+        return std::make_shared<BetaRowset>(schema, rsm, "");
+    }
+
+    static void construct_column(ColumnPB* column_pb, TabletIndexPB* tablet_index, int64_t index_id,
+                                 const std::string& index_name, int32_t col_unique_id,
+                                 const std::string& column_type, const std::string& column_name,
+                                 bool parser = false) {
+        column_pb->set_unique_id(col_unique_id);
+        column_pb->set_name(column_name);
+        column_pb->set_type(column_type);
+        column_pb->set_is_key(false);
+        column_pb->set_is_nullable(true);
+        tablet_index->set_index_id(index_id);
+        tablet_index->set_index_name(index_name);
+        tablet_index->set_index_type(IndexType::INVERTED);
+        tablet_index->add_col_unique_id(col_unique_id);
+        if (parser) {
+            auto* properties = tablet_index->mutable_properties();
+            (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_UNICODE;
+        }
+    }
+
+    static void construct_column(ColumnPB* column_pb, int32_t col_unique_id,
+                                 const std::string& column_type, const std::string& column_name) {
+        column_pb->set_unique_id(col_unique_id);
+        column_pb->set_name(column_name);
+        column_pb->set_type(column_type);
+        column_pb->set_is_key(false);
+        column_pb->set_is_nullable(true);
+    }
+
+    static void construct_index(TabletIndexPB* tablet_index, int64_t index_id,
+                                const std::string& index_name, int32_t col_unique_id,
+                                bool parser = false) {
+        tablet_index->set_index_id(index_id);
+        tablet_index->set_index_name(index_name);
+        tablet_index->set_index_type(IndexType::INVERTED);
+        tablet_index->add_col_unique_id(col_unique_id);
+        if (parser) {
+            auto* properties = tablet_index->mutable_properties();
+            (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_UNICODE;
+        }
+    }
+
+    static void check_meta_and_file(const RowsetSharedPtr& output_rowset,
+                                    const TabletSchemaSPtr& tablet_schema,
+                                    const std::map<int, QueryData>& query_map) {
+        CHECK_EQ(output_rowset->num_segments(), 1);
+        // check rowset meta and file
+        int seg_id = 0;
+        // meta
+        const auto& index_info = output_rowset->_rowset_meta->inverted_index_file_info(seg_id);
+        EXPECT_TRUE(index_info.has_index_size());
+        const auto& fs = output_rowset->_rowset_meta->fs();
+        const auto& file_name = fmt::format("{}/{}_{}.idx", output_rowset->tablet_path(),
+                                            output_rowset->rowset_id().to_string(), seg_id);
+        int64_t file_size = 0;
+        EXPECT_TRUE(fs->file_size(file_name, &file_size).ok());
+        EXPECT_EQ(index_info.index_size(), file_size);
+
+        // file
+        const auto& seg_path = output_rowset->segment_path(seg_id);
+        EXPECT_TRUE(seg_path.has_value());
+        const auto& index_file_path_prefix =
+                InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value());
+        auto inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
+                fs, std::string(index_file_path_prefix),
+                tablet_schema->get_inverted_index_storage_format(), index_info);
+        EXPECT_TRUE(inverted_index_file_reader->init().ok());
+        const auto& dirs = inverted_index_file_reader->get_all_directories();
+        EXPECT_TRUE(dirs.has_value());
+        EXPECT_EQ(dirs.value().size(), 4);
+
+        for (const auto& [col_uid, query_data] : query_map) {
+            const auto& column = tablet_schema->column_by_uid(col_uid);
+            const auto* index = tablet_schema->inverted_index(column);
+            EXPECT_TRUE(index != nullptr);
+
+            if (col_uid == 0 || col_uid == 3) {
+                // BKD index
+                std::vector<int> query_data_int;
+                for (const auto& data : query_data.first) {
+                    query_data_int.push_back(std::stoi(data));
+                }
+                EXPECT_TRUE(query_bkd(index, inverted_index_file_reader, query_data_int,
+                                      query_data.second));
+            } else if (col_uid == 1) {
+                // String index
+                EXPECT_TRUE(query_string(index, inverted_index_file_reader, std::to_string(col_uid),
+                                         query_data.first, query_data.second));
+            } else if (col_uid == 2) {
+                // Fulltext index
+                EXPECT_TRUE(query_fulltext(index, inverted_index_file_reader,
+                                           std::to_string(col_uid), query_data.first,
+                                           query_data.second));
+            }
+        }
+    }
+
+    static RowsetWriterContext rowset_writer_context(const std::unique_ptr<DataDir>& data_dir,
+                                                     const TabletSchemaSPtr& schema,
+                                                     const std::string& tablet_path) {
+        RowsetWriterContext context;
+        RowsetId rowset_id;
+        rowset_id.init(inc_id);
+        context.rowset_id = rowset_id;
+        context.rowset_type = BETA_ROWSET;
+        context.data_dir = data_dir.get();
+        context.rowset_state = VISIBLE;
+        context.tablet_schema = schema;
+        context.tablet_path = tablet_path;
+        context.version = Version(inc_id, inc_id);
+        context.max_rows_per_segment = 200;
+        inc_id++;
+        return context;
+    }
+
+    static void build_rowsets(const std::unique_ptr<DataDir>& data_dir,
+                              const TabletSchemaSPtr& schema, const TabletSharedPtr& tablet,
+                              StorageEngine* engine_ref, std::vector<RowsetSharedPtr>& rowsets,
+                              const std::vector<std::string>& data_files,
+                              const std::function<void(const int32_t&)> custom_check = nullptr) {
+        std::vector<std::vector<DataRow>> data;
+        for (auto file : data_files) {
+            data.emplace_back(read_data(file));
+        }
+        for (int i = 0; i < data.size(); i++) {
+            const auto& res = RowsetFactory::create_rowset_writer(
+                    *engine_ref, rowset_writer_context(data_dir, schema, tablet->tablet_path()),
+                    false);
+            EXPECT_TRUE(res.has_value()) << res.error();
+            const auto& rowset_writer = res.value();
+
+            vectorized::Block block = schema->create_block();
+            auto columns = block.mutate_columns();
+            for (const auto& row : data[i]) {
+                vectorized::Field key = int32_t(row.key);
+                vectorized::Field v1(row.word);
+                vectorized::Field v2(row.url);
+                vectorized::Field v3 = int32_t(row.num);
+                columns[0]->insert(key);
+                columns[1]->insert(v1);
+                columns[2]->insert(v2);
+                columns[3]->insert(v3);
+            }
+            EXPECT_TRUE(rowset_writer->add_block(&block).ok());
+            EXPECT_TRUE(rowset_writer->flush().ok());
+            const auto& dst_writer = dynamic_cast<BaseBetaRowsetWriter*>(rowset_writer.get());
+
+            check_idx_file_writer_closed(dst_writer, true);
+
+            EXPECT_TRUE(rowset_writer->build(rowsets[i]).ok());
+            EXPECT_TRUE(tablet->add_rowset(rowsets[i]).ok());
+            EXPECT_TRUE(rowsets[i]->num_segments() == 5);
+
+            // check rowset meta and file
+            for (int seg_id = 0; seg_id < rowsets[i]->num_segments(); seg_id++) {
+                const auto& index_info = rowsets[i]->_rowset_meta->inverted_index_file_info(seg_id);
+                EXPECT_TRUE(index_info.has_index_size());
+                const auto& fs = rowsets[i]->_rowset_meta->fs();
+                const auto& file_name = fmt::format("{}/{}_{}.idx", rowsets[i]->tablet_path(),
+                                                    rowsets[i]->rowset_id().to_string(), seg_id);
+                int64_t file_size = 0;
+                EXPECT_TRUE(fs->file_size(file_name, &file_size).ok());
+                EXPECT_EQ(index_info.index_size(), file_size);
+
+                const auto& seg_path = rowsets[i]->segment_path(seg_id);
+                EXPECT_TRUE(seg_path.has_value());
+                const auto& index_file_path_prefix =
+                        InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value());
+                auto inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
+                        fs, std::string(index_file_path_prefix),
+                        schema->get_inverted_index_storage_format(), index_info);
+                EXPECT_TRUE(inverted_index_file_reader->init().ok());
+                const auto& dirs = inverted_index_file_reader->get_all_directories();
+                EXPECT_TRUE(dirs.has_value());
+                if (custom_check) {
+                    custom_check(dirs.value().size());
+                }
+            }
+        }
+    }
+
+    static std::shared_ptr<InvertedIndexFileReader> init_index_file_reader(
+            const RowsetSharedPtr& output_rowset, const std::string& seg_path,
+            const InvertedIndexStorageFormatPB& index_storage_format) {
+        const auto& index_file_path_prefix =
+                InvertedIndexDescriptor::get_index_file_path_prefix(seg_path);
+        auto inverted_index_file_reader_index = std::make_shared<InvertedIndexFileReader>(
+                output_rowset->_rowset_meta->fs(), std::string(index_file_path_prefix),
+                index_storage_format);
+        auto st = inverted_index_file_reader_index->init();
+        EXPECT_TRUE(st.ok()) << st.to_string();
+
+        return inverted_index_file_reader_index;
+    }
+};
+
+} // namespace doris
\ No newline at end of file

From 0ec35de23a7ae2a0e12fb6ec5be3f5d5196ee8dd Mon Sep 17 00:00:00 2001
From: Xujian Duan <50550370+DarvenDuan@users.noreply.github.com>
Date: Mon, 9 Dec 2024 17:08:31 +0800
Subject: [PATCH 239/399] [bug](s3) fix S3 file system gets absolute path
 (#44965)

### What problem does this PR solve?

Issue Number: close https://github.com/apache/doris/issues/44902

Related PR: #xxx

###  Problem Summary
If we create a S3 resource for cooldown data storage and the
s3.root.path has a leading slash(/), such as: /root, the remote S3 data
file will not be deleted if we drop the cooldown tablet from Doris.

### Reason
Doris will get the path of all files of a tablet and then delete those
files, The AWS `S3Client:ListObjectsV2` uses objects prefix to get the
object files, but if the prefix has a leading slash(/),
the`S3Client:ListObjectsV2` gets empty result.

### Solution
use _prefix instead of _root_path in S3FileSystem for getting absolute
path, _prefix is normalized in constructor and it is removed the first
and last '/'.

### test case
1. create a s3 resource with a leading '/' and a storage policy base on
it:
```
CREATE RESOURCE "test_resource"
PROPERTIES
(
    "type" = "s3",
    "s3.endpoint" = "xxx",
    "s3.region" = "xxx",
    "s3.bucket" = "xxx",
    "s3.root.path" = "/tmp",
    "s3.access_key" = "xx",
    "s3.secret_key" = "xx"
);

CREATE STORAGE POLICY test_policy PROPERTIES (
    "storage_resource" = "test_resource",
    "cooldown_ttl" = "1"
)
```
2. create a table and set the storage_policy to `test_policy` and insert
test data:
```
CREATE TABLE `test_table` (
  `k` bigint,
  `v` bigint
) ENGINE=OLAP
DUPLICATE KEY(`k`)
DISTRIBUTED BY HASH(`k`) BUCKETS 1
PROPERTIES (
"replication_num" = "1",
"storage_policy" = "test_policy"
);

insert into test_table values (1,2),(2,3);
```
3. wait for the rowset cooldown to s3.
4. truncate test table
```
truncate table test_table force;
```
5. If we get a log like ` delete remote rowsets of tablet.
root_path="/tmp", tablet_id=xxx` in be.INFO, the remote tablet file
should be deleted.

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [x] Regression test
    - [x] Unit Test
    - [x] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [x] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 be/src/io/fs/s3_file_system.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h
index 61967a63e44379..f6efa5053324ff 100644
--- a/be/src/io/fs/s3_file_system.h
+++ b/be/src/io/fs/s3_file_system.h
@@ -121,7 +121,7 @@ class S3FileSystem final : public RemoteFileSystem {
             abs_path = path;
         } else {
             // path with no schema
-            abs_path = _root_path / path;
+            abs_path = _prefix / path;
         }
         return Status::OK();
     }

From 8c5307e40d340446c765836bf4635fa5c65b477a Mon Sep 17 00:00:00 2001
From: Siyang Tang <tangsiyang2001@foxmail.com>
Date: Mon, 9 Dec 2024 17:51:16 +0800
Subject: [PATCH 240/399] [fix](meta-service) Avoid rowset meta exceeds 2G
 result in protobuf fatal (#44780)

Rowset meta may exceed limit of 2GB incase of table of extremely wide
variant type. Check the size before serializing.
---
 cloud/src/meta-service/meta_service.cpp | 18 ++++-
 cloud/test/txn_lazy_commit_test.cpp     | 90 ++++++++++++++++++++++++-
 2 files changed, 104 insertions(+), 4 deletions(-)

diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index 9da5750d8d83f9..33c109d19bcc31 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -1328,15 +1328,27 @@ void internal_get_rowset(Transaction* txn, int64_t start, int64_t end,
 
         while (it->has_next()) {
             auto [k, v] = it->next();
-            auto rs = response->add_rowset_meta();
+            auto* rs = response->add_rowset_meta();
+            auto byte_size = rs->ByteSizeLong();
+            TEST_SYNC_POINT_CALLBACK("get_rowset:meta_exceed_limit", &byte_size);
+            if (byte_size + v.size() > std::numeric_limits<int32_t>::max()) {
+                code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+                msg = fmt::format(
+                        "rowset meta exceeded 2G, unable to serialize, key={}. byte_size={}",
+                        hex(k), byte_size);
+                LOG(WARNING) << msg;
+                return;
+            }
             if (!rs->ParseFromArray(v.data(), v.size())) {
                 code = MetaServiceCode::PROTOBUF_PARSE_ERR;
-                msg = "malformed rowset meta, unable to deserialize";
+                msg = "malformed rowset meta, unable to serialize";
                 LOG(WARNING) << msg << " key=" << hex(k);
                 return;
             }
             ++num_rowsets;
-            if (!it->has_next()) key0 = k;
+            if (!it->has_next()) {
+                key0 = k;
+            }
         }
         key0.push_back('\x00'); // Update to next smallest key for iteration
     } while (it->more());
diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp
index 9a7679f3dd9e23..0f284508a3f34e 100644
--- a/cloud/test/txn_lazy_commit_test.cpp
+++ b/cloud/test/txn_lazy_commit_test.cpp
@@ -25,7 +25,9 @@
 
 #include <atomic>
 #include <condition_variable>
+#include <cstddef>
 #include <cstdint>
+#include <limits>
 #include <memory>
 #include <random>
 #include <string>
@@ -1812,4 +1814,90 @@ TEST(TxnLazyCommitTest, ConcurrentCommitTxnEventuallyCase4Test) {
     ASSERT_TRUE(abort_timeout_txn_hit);
     ASSERT_EQ(txn_id, txn_info_pb.txn_id());
 }
-} // namespace doris::cloud
\ No newline at end of file
+
+TEST(TxnLazyCommitTest, RowsetMetaSizeExceedTest) {
+    auto txn_kv = get_mem_txn_kv();
+
+    int64_t db_id = 5252025;
+    int64_t table_id = 35201043384;
+    int64_t index_id = 256439;
+    int64_t partition_id = 732536259;
+
+    auto meta_service = get_meta_service(txn_kv, true);
+    int64_t tablet_id = 25910248;
+
+    {
+        create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, partition_id,
+                                 tablet_id);
+    }
+    {
+        int tmp_txn_id = 0;
+        {
+            brpc::Controller cntl;
+            BeginTxnRequest req;
+            req.set_cloud_unique_id("test_cloud_unique_id");
+            TxnInfoPB txn_info_pb;
+            txn_info_pb.set_db_id(db_id);
+            txn_info_pb.set_label("test_label_32ae213dasg3");
+            txn_info_pb.add_table_ids(table_id);
+            txn_info_pb.set_timeout_ms(36000);
+            req.mutable_txn_info()->CopyFrom(txn_info_pb);
+            BeginTxnResponse res;
+            meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
+                                    &req, &res, nullptr);
+            ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+            tmp_txn_id = res.txn_id();
+            ASSERT_GT(res.txn_id(), 0);
+        }
+        {
+            auto tmp_rowset = create_rowset(tmp_txn_id, tablet_id, partition_id);
+            CreateRowsetResponse res;
+            commit_rowset(meta_service.get(), tmp_rowset, res);
+            ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+        }
+        {
+            brpc::Controller cntl;
+            CommitTxnRequest req;
+            req.set_cloud_unique_id("test_cloud_unique_id");
+            req.set_db_id(db_id);
+            req.set_txn_id(tmp_txn_id);
+            req.set_is_2pc(false);
+            req.set_enable_txn_lazy_commit(true);
+            CommitTxnResponse res;
+            meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
+                                     &req, &res, nullptr);
+            ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+        }
+    }
+
+    auto* sp = SyncPoint::get_instance();
+    sp->set_call_back("get_rowset:meta_exceed_limit", [](auto&& args) {
+        auto* byte_size = try_any_cast<size_t*>(args[0]);
+        *byte_size = std::numeric_limits<int32_t>::max();
+        ++(*byte_size);
+    });
+
+    sp->enable_processing();
+    {
+        brpc::Controller cntl;
+        GetRowsetRequest req;
+        req.set_cloud_unique_id("test_cloud_unique_id");
+        auto* tablet_idx = req.mutable_idx();
+        tablet_idx->set_table_id(table_id);
+        tablet_idx->set_index_id(index_id);
+        tablet_idx->set_partition_id(partition_id);
+        tablet_idx->set_tablet_id(tablet_id);
+        req.set_start_version(0);
+        req.set_end_version(-1);
+        req.set_cumulative_compaction_cnt(0);
+        req.set_base_compaction_cnt(0);
+        req.set_cumulative_point(2);
+
+        GetRowsetResponse res;
+        meta_service->get_rowset(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
+                                 &res, nullptr);
+        ASSERT_EQ(res.status().code(), MetaServiceCode::PROTOBUF_PARSE_ERR);
+    }
+}
+
+} // namespace doris::cloud

From 0a0c50253649f8a5dc1f377162df7c9549347b36 Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Mon, 9 Dec 2024 18:44:24 +0800
Subject: [PATCH 241/399] [fix](be-ut) fix compile warning: declaration shadows
 a local variable (#45204)

### What problem does this PR solve?
Problem Summary:
```text
FAILED: src/vec/CMakeFiles/Vec.dir/sink/writer/vtablet_writer.cpp.o
/root/doris/be/src/vec/sink/writer/vtablet_writer.cpp:1402:23: error: declaration shadows a local variable [-Werror,-Wshadow]
    } else if (Status st = ich.check_tablet_filtered_rows_consistency(); !st.ok()) {
/root/doris/be/src/vec/sink/writer/vtablet_writer.cpp:1400:23: note: previous declaration is here
    } else if (Status st = ich.check_tablet_received_rows_consistency(); !st.ok()) {
```
---
 be/src/vec/sink/writer/vtablet_writer.cpp | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp
index 55b6845b6bc871..0de868c00b995e 100644
--- a/be/src/vec/sink/writer/vtablet_writer.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer.cpp
@@ -1394,13 +1394,12 @@ static Status cancel_channel_and_check_intolerable_failure(Status status,
     nch.cancel(err_msg);
 
     // check if index has intolerable failure
-    Status index_st = ich.check_intolerable_failure();
-    if (!index_st.ok()) {
+    if (Status index_st = ich.check_intolerable_failure(); !index_st.ok()) {
         status = std::move(index_st);
-    } else if (Status st = ich.check_tablet_received_rows_consistency(); !st.ok()) {
-        status = std::move(st);
-    } else if (Status st = ich.check_tablet_filtered_rows_consistency(); !st.ok()) {
-        status = std::move(st);
+    } else if (Status receive_st = ich.check_tablet_received_rows_consistency(); !receive_st.ok()) {
+        status = std::move(receive_st);
+    } else if (Status filter_st = ich.check_tablet_filtered_rows_consistency(); !filter_st.ok()) {
+        status = std::move(filter_st);
     }
     return status;
 }

From 0a73618313c3fd3dfcb377a95c97049673277ecd Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Mon, 9 Dec 2024 19:14:53 +0800
Subject: [PATCH 242/399] [feature](udf) Support for IP types in Java UDF.
 (#44871)

### What problem does this PR solve?

Add support for IPV types in Java UDFs.
doc https://github.com/apache/doris-website/pull/1444
---
 be/src/vec/exec/jni_connector.cpp             |  12 +-
 .../common/jni/utils/JavaUdfDataType.java     |   8 +
 .../common/jni/utils/TypeNativeBytes.java     |  16 ++
 .../doris/common/jni/utils/UdfUtils.java      |   6 +
 .../doris/common/jni/vec/ColumnType.java      |  20 +++
 .../doris/common/jni/vec/VectorColumn.java    |  79 ++++++++++
 .../java/org/apache/doris/catalog/Type.java   |   4 +-
 .../nereids_p0/javaudf/test_javaudf_ip.out    |  42 +++++
 .../org/apache/doris/udf/IPV4TypeTest.java    |  74 +++++++++
 .../org/apache/doris/udf/IPV6TypeTest.java    |  73 +++++++++
 .../java/org/apache/doris/udf/MySumIP.java    |  76 +++++++++
 .../nereids_p0/javaudf/test_javaudf_ip.groovy | 146 ++++++++++++++++++
 12 files changed, 554 insertions(+), 2 deletions(-)
 create mode 100644 regression-test/data/nereids_p0/javaudf/test_javaudf_ip.out
 create mode 100644 regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IPV4TypeTest.java
 create mode 100644 regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IPV6TypeTest.java
 create mode 100644 regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MySumIP.java
 create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_ip.groovy

diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp
index 4c977b69ad6a42..a87ccf987ac7af 100644
--- a/be/src/vec/exec/jni_connector.cpp
+++ b/be/src/vec/exec/jni_connector.cpp
@@ -63,7 +63,9 @@ namespace doris::vectorized {
     M(TypeIndex::Date, ColumnVector<Int64>, Int64)                  \
     M(TypeIndex::DateV2, ColumnVector<UInt32>, UInt32)              \
     M(TypeIndex::DateTime, ColumnVector<Int64>, Int64)              \
-    M(TypeIndex::DateTimeV2, ColumnVector<UInt64>, UInt64)
+    M(TypeIndex::DateTimeV2, ColumnVector<UInt64>, UInt64)          \
+    M(TypeIndex::IPv4, ColumnVector<IPv4>, IPv4)                    \
+    M(TypeIndex::IPv6, ColumnVector<IPv6>, IPv6)
 
 Status JniConnector::open(RuntimeState* state, RuntimeProfile* profile) {
     _state = state;
@@ -450,6 +452,10 @@ std::string JniConnector::get_jni_type(const DataTypePtr& data_type) {
         return "float";
     case TYPE_DOUBLE:
         return "double";
+    case TYPE_IPV4:
+        return "ipv4";
+    case TYPE_IPV6:
+        return "ipv6";
     case TYPE_VARCHAR:
         [[fallthrough]];
     case TYPE_CHAR:
@@ -534,6 +540,10 @@ std::string JniConnector::get_jni_type(const TypeDescriptor& desc) {
         return "float";
     case TYPE_DOUBLE:
         return "double";
+    case TYPE_IPV4:
+        return "ipv4";
+    case TYPE_IPV6:
+        return "ipv6";
     case TYPE_VARCHAR: {
         buffer << "varchar(" << desc.len << ")";
         return buffer.str();
diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JavaUdfDataType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JavaUdfDataType.java
index 9f973543b29413..6077f713e8319d 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JavaUdfDataType.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JavaUdfDataType.java
@@ -27,6 +27,7 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Set;
@@ -55,6 +56,9 @@ public class JavaUdfDataType {
     public static final JavaUdfDataType DECIMAL64 = new JavaUdfDataType("DECIMAL64", TPrimitiveType.DECIMAL64, 8);
     public static final JavaUdfDataType DECIMAL128 = new JavaUdfDataType("DECIMAL128", TPrimitiveType.DECIMAL128I,
             16);
+
+    public static final JavaUdfDataType IPV4 = new JavaUdfDataType("IPV4", TPrimitiveType.IPV4, 4);
+    public static final JavaUdfDataType IPV6 = new JavaUdfDataType("IPV6", TPrimitiveType.IPV6, 16);
     public static final JavaUdfDataType ARRAY_TYPE = new JavaUdfDataType("ARRAY_TYPE", TPrimitiveType.ARRAY, 0);
     public static final JavaUdfDataType MAP_TYPE = new JavaUdfDataType("MAP_TYPE", TPrimitiveType.MAP, 0);
     public static final JavaUdfDataType STRUCT_TYPE = new JavaUdfDataType("STRUCT_TYPE", TPrimitiveType.STRUCT, 0);
@@ -83,6 +87,8 @@ public class JavaUdfDataType {
         JavaUdfDataTypeSet.add(ARRAY_TYPE);
         JavaUdfDataTypeSet.add(MAP_TYPE);
         JavaUdfDataTypeSet.add(STRUCT_TYPE);
+        JavaUdfDataTypeSet.add(IPV4);
+        JavaUdfDataTypeSet.add(IPV6);
     }
 
     private final String description;
@@ -156,6 +162,8 @@ public static Set<JavaUdfDataType> getCandidateTypes(Class<?> c) {
             return Sets.newHashSet(JavaUdfDataType.ARRAY_TYPE, JavaUdfDataType.STRUCT_TYPE);
         } else if (c == java.util.HashMap.class) {
             return Sets.newHashSet(JavaUdfDataType.MAP_TYPE);
+        } else if (c == InetAddress.class) {
+            return Sets.newHashSet(JavaUdfDataType.IPV4, JavaUdfDataType.IPV6);
         }
         return Sets.newHashSet(JavaUdfDataType.INVALID_TYPE);
     }
diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java
index c6c3c28d228bd7..7474ce4467d31f 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java
@@ -20,6 +20,8 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.math.RoundingMode;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.time.DateTimeException;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -58,6 +60,20 @@ public static BigInteger getBigInteger(byte[] bytes) {
         return new BigInteger(originalBytes);
     }
 
+    public static InetAddress getInetAddress(byte[] bytes) {
+        // Convert the byte order back if necessary
+        byte[] originalBytes = convertByteOrder(bytes);
+        try {
+            return InetAddress.getByAddress(originalBytes);
+        } catch (UnknownHostException e) {
+            return null;
+        }
+    }
+
+    public static byte[] getInetAddressBytes(InetAddress v) {
+        return convertByteOrder(v.getAddress());
+    }
+
     public static byte[] getDecimalBytes(BigDecimal v, int scale, int size) {
         BigDecimal retValue = v.setScale(scale, RoundingMode.HALF_EVEN);
         BigInteger data = retValue.unscaledValue();
diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
index 2ef1956118b26a..d5f6e746132443 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java
@@ -197,6 +197,12 @@ public static Pair<Boolean, JavaUdfDataType[]> setArgTypes(Type[] parameterTypes
                 StructType structType = (StructType) parameterTypes[finalI];
                 ArrayList<StructField> fields = structType.getFields();
                 inputArgTypes[i].setFields(fields);
+            } else if (parameterTypes[finalI].isIP()) {
+                if (parameterTypes[finalI].isIPv4()) {
+                    inputArgTypes[i] = new JavaUdfDataType(JavaUdfDataType.IPV4);
+                } else {
+                    inputArgTypes[i] = new JavaUdfDataType(JavaUdfDataType.IPV6);
+                }
             }
             if (res.length == 0) {
                 return Pair.of(false, inputArgTypes);
diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
index 37a58075978c18..f234015c9bfc5e 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
@@ -56,6 +56,8 @@ public enum Type {
         DECIMAL32(4),
         DECIMAL64(8),
         DECIMAL128(16),
+        IPV4(4),
+        IPV6(16),
         STRING(-1),
         ARRAY(-1),
         MAP(-1),
@@ -155,6 +157,18 @@ public boolean isArray() {
         return type == Type.ARRAY;
     }
 
+    public boolean isIpv4() {
+        return type == Type.IPV4;
+    }
+
+    public boolean isIpv6() {
+        return type == Type.IPV6;
+    }
+
+    public boolean isIp() {
+        return isIpv4() || isIpv6();
+    }
+
     public boolean isMap() {
         return type == Type.MAP;
     }
@@ -287,6 +301,12 @@ public static ColumnType parseType(String columnName, String hiveType) {
             case "double":
                 type = Type.DOUBLE;
                 break;
+            case "ipv4":
+                type = Type.IPV4;
+                break;
+            case "ipv6":
+                type = Type.IPV6;
+                break;
             case "datev1":
                 type = Type.DATE;
                 break;
diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
index 839027b03b61c5..940a6ce4f16481 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
@@ -28,6 +28,8 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -78,6 +80,19 @@ public class VectorColumn {
     // todo: support pruned struct fields
     private List<Integer> structFieldIndex;
 
+
+    public static final InetAddress DEFAULT_IPV4;
+    public static final InetAddress DEFAULT_IPV6;
+
+    static {
+        try {
+            DEFAULT_IPV4 = InetAddress.getByName("127.0.0.1");
+            DEFAULT_IPV6 = InetAddress.getByName("::1");
+        } catch (UnknownHostException e) {
+            throw new RuntimeException("Failed to initialize default InetAddress values", e);
+        }
+    }
+
     // Create writable column
     private VectorColumn(ColumnType columnType, int capacity) {
         this.isConst = false;
@@ -376,6 +391,10 @@ public int appendNull(ColumnType.Type typeValue) {
                 return appendLong(0);
             case LARGEINT:
                 return appendBigInteger(BigInteger.ZERO);
+            case IPV4:
+                return appendInetAddress(DEFAULT_IPV4);
+            case IPV6:
+                return appendInetAddress(DEFAULT_IPV6);
             case FLOAT:
                 return appendFloat(0);
             case DOUBLE:
@@ -857,6 +876,56 @@ public BigInteger[] getBigIntegerColumn(int start, int end) {
         return result;
     }
 
+    public byte[] getInetAddressBytes(int rowId) {
+        int typeSize = columnType.getTypeSize();
+        byte[] bytes = new byte[typeSize];
+        OffHeap.copyMemory(null, data + (long) rowId * typeSize, bytes, OffHeap.BYTE_ARRAY_OFFSET, typeSize);
+        return bytes;
+    }
+
+    public InetAddress getInetAddress(int rowId) {
+        return TypeNativeBytes.getInetAddress(getInetAddressBytes(rowId));
+    }
+
+    public InetAddress[] getInetAddressColumn(int start, int end) {
+        InetAddress[] result = new InetAddress[end - start];
+        for (int i = start; i < end; ++i) {
+            if (!isNullAt(i)) {
+                result[i - start] = getInetAddress(i);
+            }
+        }
+        return result;
+    }
+
+    public int appendInetAddress(InetAddress v) {
+        reserve(appendIndex + 1);
+        putInetAddress(appendIndex, v);
+        return appendIndex++;
+    }
+
+    public void appendInetAddress(InetAddress[] batch, boolean isNullable) {
+        reserve(appendIndex + batch.length);
+        for (InetAddress v : batch) {
+            if (v == null) {
+                putNull(appendIndex);
+                if (columnType.isIpv4()) {
+                    putInetAddress(appendIndex, DEFAULT_IPV4);
+                } else {
+                    putInetAddress(appendIndex, DEFAULT_IPV6);
+                }
+            } else {
+                putInetAddress(appendIndex, v);
+            }
+            appendIndex++;
+        }
+    }
+
+    private void putInetAddress(int rowId, InetAddress v) {
+        int typeSize = columnType.getTypeSize();
+        byte[] bytes = TypeNativeBytes.getInetAddressBytes(v);
+        OffHeap.copyMemory(bytes, OffHeap.BYTE_ARRAY_OFFSET, null, data + (long) rowId * typeSize, typeSize);
+    }
+
     public int appendDecimal(BigDecimal v) {
         reserve(appendIndex + 1);
         putDecimal(appendIndex, v);
@@ -1382,6 +1451,9 @@ public Object[] newObjectContainerArray(ColumnType.Type type, int size) {
                 return new Long[size];
             case LARGEINT:
                 return new BigInteger[size];
+            case IPV4:
+            case IPV6:
+                return new InetAddress[size];
             case FLOAT:
                 return new Float[size];
             case DOUBLE:
@@ -1431,6 +1503,10 @@ public void appendObjectColumn(Object[] batch, boolean isNullable) {
             case LARGEINT:
                 appendBigInteger((BigInteger[]) batch, isNullable);
                 break;
+            case IPV4:
+            case IPV6:
+                appendInetAddress((InetAddress[]) batch, isNullable);
+                break;
             case FLOAT:
                 appendFloat((Float[]) batch, isNullable);
                 break;
@@ -1493,6 +1569,9 @@ public Object[] getObjectColumn(int start, int end) {
                 return getLongColumn(start, end);
             case LARGEINT:
                 return getBigIntegerColumn(start, end);
+            case IPV4:
+            case IPV6:
+                return getInetAddressColumn(start, end);
             case FLOAT:
                 return getFloatColumn(start, end);
             case DOUBLE:
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 7a8dda5aabedef..7dfcfd15ebec84 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -36,6 +36,7 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.net.InetAddress;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
@@ -321,7 +322,8 @@ public abstract class Type {
                     .put(PrimitiveType.FLOAT, Sets.newHashSet(Float.class, float.class))
                     .put(PrimitiveType.DOUBLE, Sets.newHashSet(Double.class, double.class))
                     .put(PrimitiveType.BIGINT, Sets.newHashSet(Long.class, long.class))
-                    .put(PrimitiveType.IPV4, Sets.newHashSet(Integer.class, int.class))
+                    .put(PrimitiveType.IPV4, Sets.newHashSet(InetAddress.class))
+                    .put(PrimitiveType.IPV6, Sets.newHashSet(InetAddress.class))
                     .put(PrimitiveType.STRING, Sets.newHashSet(String.class))
                     .put(PrimitiveType.DATE, DATE_SUPPORTED_JAVA_TYPE)
                     .put(PrimitiveType.DATEV2, DATE_SUPPORTED_JAVA_TYPE)
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_ip.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_ip.out
new file mode 100644
index 00000000000000..b7eb66cab9cc01
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_ip.out
@@ -0,0 +1,42 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_ipv4_1 --
+1	0.0.0.123	/0.0.0.123
+2	0.0.12.42	/0.0.12.42
+3	0.119.130.67	/0.119.130.67
+4	\N	null
+
+-- !select_ipv4_2 --
+1	0.0.0.123	0.0.0.123
+2	0.0.0.123	0.0.0.123
+
+-- !select_ipv4_3 --
+nulludf/0.0.0.123udf/0.0.12.42udf/0.119.130.67udf
+
+-- !select_ipv4_4 --
+1	["127.0.0.1", "127.0.0.1", "127.0.0.1", null, null, "127.0.0.1"]
+2	["127.0.0.1", "127.0.0.1", "127.0.0.1", null, null, "127.0.0.1"]
+3	["127.0.0.1", "127.0.0.1", "127.0.0.1", null, null, "127.0.0.1"]
+4	["127.0.0.1", "127.0.0.1", "127.0.0.1", null, null, "127.0.0.1"]
+
+-- !select_ipv6_1 --
+1	::855d	/0:0:0:0:0:0:0:855d
+2	::0.4.221.183	/0:0:0:0:0:0:4:ddb7
+3	::a:7429:d0d6:6e08:9f5f	/0:0:0:a:7429:d0d6:6e08:9f5f
+4	\N	null
+
+-- !select_ipv6_2 --
+3	2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D	2001:db8:ac10:fe01:feed:babe:cafe:f00d
+4	2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D	2001:db8:ac10:fe01:feed:babe:cafe:f00d
+
+-- !select_ipv6_3 --
+nulludf/0:0:0:0:0:0:0:855dudf/0:0:0:0:0:0:4:ddb7udf/0:0:0:a:7429:d0d6:6e08:9f5fudf
+
+-- !select_ipv6_4 --
+1	["::1", "::1", "::1", null, null, "::1"]
+2	["::1", "::1", "::1", null, null, "::1"]
+3	["::1", "::1", "::1", null, null, "::1"]
+4	["::1", "::1", "::1", null, null, "::1"]
+
+-- !select_ipv6_5 --
+/0:0:0:0:0:0:0:855d, /0:0:0:0:0:0:4:ddb7, /0:0:0:a:7429:d0d6:6e08:9f5f
+
diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IPV4TypeTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IPV4TypeTest.java
new file mode 100644
index 00000000000000..65d6ea2ab1a31a
--- /dev/null
+++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IPV4TypeTest.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.udf;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+
+public class IPV4TypeTest {
+    // input ipv4
+    public String evaluate(InetAddress x) {
+        if (x == null) {
+            return "null";
+        }
+        return x.toString();
+    }
+
+    // output ipv4
+    public InetAddress evaluate(String s) {
+        try {
+            InetAddress ipv4Address = InetAddress.getByName(s);
+            if (ipv4Address.getAddress().length == 4) {
+                return ipv4Address;
+            } else {
+                return null;
+            }
+        } catch (UnknownHostException e) {
+            return null;
+        }
+    }
+
+    // input array<ipv4>
+    public String evaluate(ArrayList<InetAddress> s) {
+        String ret = "";
+        for (InetAddress ip : s) {
+            ret += evaluate(ip) + "udf";
+        }
+        return ret;
+    }
+
+    // output array<ipv4>
+    public ArrayList<InetAddress> evaluate() {
+        ArrayList<InetAddress> ret = new ArrayList<InetAddress>();
+        InetAddress DEFAULT_IPV = null;
+        try {
+            DEFAULT_IPV = InetAddress.getByName("127.0.0.1");
+        } catch (UnknownHostException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+
+        ret.add(DEFAULT_IPV);
+        ret.add(DEFAULT_IPV);
+        ret.add(DEFAULT_IPV);
+        ret.add(null);
+        ret.add(null);
+        ret.add(DEFAULT_IPV);
+        return ret;
+    }
+}
diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IPV6TypeTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IPV6TypeTest.java
new file mode 100644
index 00000000000000..f4345cbab5de6e
--- /dev/null
+++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IPV6TypeTest.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.udf;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+
+public class IPV6TypeTest {
+    // input ipv6
+    public String evaluate(InetAddress x) {
+        if (x == null) {
+            return "null";
+        }
+        return x.toString();
+    }
+
+    // output ipv6
+    public InetAddress evaluate(String s) {
+        try {
+            InetAddress ipv6Address = InetAddress.getByName(s);
+            if (ipv6Address.getAddress().length == 16) {
+                return ipv6Address;
+            } else {
+                return null;
+            }
+        } catch (UnknownHostException e) {
+            return null;
+        }
+    }
+
+    // input array<ipv6>
+    public String evaluate(ArrayList<InetAddress> s) {
+        String ret = "";
+        for (InetAddress ip : s) {
+            ret += evaluate(ip) + "udf";
+        }
+        return ret;
+    }
+
+    // output array<ipv6>
+    public ArrayList<InetAddress> evaluate() {
+        ArrayList<InetAddress> ret = new ArrayList<InetAddress>();
+        InetAddress DEFAULT_IPV = null;
+        try {
+            DEFAULT_IPV = InetAddress.getByName("::1");
+        } catch (UnknownHostException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+        ret.add(DEFAULT_IPV);
+        ret.add(DEFAULT_IPV);
+        ret.add(DEFAULT_IPV);
+        ret.add(null);
+        ret.add(null);
+        ret.add(DEFAULT_IPV);
+        return ret;
+    }
+}
diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MySumIP.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MySumIP.java
new file mode 100644
index 00000000000000..b57418ec04c37b
--- /dev/null
+++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MySumIP.java
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.udf;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.stream.Collectors;
+
+public class MySumIP {
+    public static class State {
+        public ArrayList<String> list = new ArrayList<>();
+    }
+
+    public State create() {
+        return new State();
+    }
+
+    public void destroy(State state) {
+    }
+
+    public void add(State state, InetAddress val1) {
+        if (val1 == null)
+            return;
+        state.list.add(val1.toString());
+    }
+
+    public void serialize(State state, DataOutputStream out) throws IOException {
+        out.writeInt(state.list.size());
+
+        for (String str : state.list) {
+            out.writeUTF(str);
+        }
+    }
+
+    public void deserialize(State state, DataInputStream in) throws IOException {
+        int size = in.readInt();
+
+        state.list = new ArrayList<>(size);
+
+        for (int i = 0; i < size; i++) {
+            state.list.add(in.readUTF());
+        }
+    }
+
+    public void merge(State state, State rhs) {
+        for (String s : rhs.list) {
+            state.list.add(s);
+        }
+    }
+
+    public String getValue(State state) {
+        state.list.sort(Comparator.nullsLast(String::compareTo));
+        String result = state.list.stream()
+                .filter(s -> s != null)
+                .collect(Collectors.joining(", "));
+        return result;
+    }
+}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_ip.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_ip.groovy
new file mode 100644
index 00000000000000..aa079af16d049d
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_ip.groovy
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_ip") {
+    sql 'set enable_nereids_planner=true'
+    sql 'set enable_fallback_to_original_planner=false'
+
+    def jarPath = """${context.file.parent}/../../javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar"""
+    scp_udf_file_to_all_be(jarPath)
+
+    try {
+        sql """ DROP FUNCTION IF EXISTS java_udf_ipv4_test1(ipv4);"""
+        sql """ CREATE FUNCTION java_udf_ipv4_test1(ipv4) RETURNS string PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.IPV4TypeTest",
+            "type"="JAVA_UDF"
+        ); """
+        sql """ DROP FUNCTION IF EXISTS java_udf_ipv4_test2(string);"""
+        sql """ CREATE FUNCTION java_udf_ipv4_test2(string) RETURNS ipv4 PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.IPV4TypeTest",
+            "type"="JAVA_UDF"
+        ); """
+        sql """ DROP FUNCTION IF EXISTS java_udf_ipv4_test3(array<ipv4>);"""
+        sql """ CREATE FUNCTION java_udf_ipv4_test3(array<ipv4>) RETURNS string PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.IPV4TypeTest",
+            "type"="JAVA_UDF"
+        ); """
+        sql """ DROP FUNCTION IF EXISTS java_udf_ipv4_test4();"""
+        sql """ CREATE FUNCTION java_udf_ipv4_test4() RETURNS array<ipv4> PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.IPV4TypeTest",
+            "type"="JAVA_UDF"
+        ); """
+
+        sql """DROP TABLE IF EXISTS test_udf_ip;"""
+        sql """
+                CREATE TABLE test_udf_ip
+                (
+                    k1 BIGINT ,
+                    k4 ipv4 ,
+                    k6 ipv6 ,
+                    s string 
+                )
+                DISTRIBUTED BY HASH(k1) BUCKETS 1
+                PROPERTIES("replication_num" = "1");
+        """
+        sql """ insert into test_udf_ip values(1,123,34141,"0.0.0.123") , (2,3114,318903,"0.0.0.123") , (3,7832131,192837891738927931231,"2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D"),(4,null,null,"2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D"); """
+        qt_select_ipv4_1 """ select k1,k4,java_udf_ipv4_test1(k4) from test_udf_ip order by k1   """
+        qt_select_ipv4_2 """ select k1,s,java_udf_ipv4_test2(s) from test_udf_ip where IS_IPV4_STRING(s) order by k1  """
+        qt_select_ipv4_3 """ select java_udf_ipv4_test3(array_sort(array_agg(k4))) from test_udf_ip   """
+        qt_select_ipv4_4 """ select k1, java_udf_ipv4_test4() from test_udf_ip order by k1   """
+        
+    } finally {
+        try_sql("DROP FUNCTION IF EXISTS java_udf_ipv4_test1(ipv4);")
+        try_sql("DROP FUNCTION IF EXISTS java_udf_ipv4_test2(string);")
+        try_sql("DROP FUNCTION IF EXISTS java_udf_ipv4_test3(array<ipv4>);")
+        try_sql("DROP FUNCTION IF EXISTS java_udf_ipv4_test4();")
+    }
+
+    try {
+        sql """ DROP FUNCTION IF EXISTS java_udf_ipv6_test1(ipv6);"""
+        sql """ CREATE FUNCTION java_udf_ipv6_test1(ipv6) RETURNS string PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.IPV6TypeTest",
+            "type"="JAVA_UDF"
+        ); """
+        sql """ DROP FUNCTION IF EXISTS java_udf_ipv6_test2(string);"""
+        sql """ CREATE FUNCTION java_udf_ipv6_test2(string) RETURNS ipv6 PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.IPV6TypeTest",
+            "type"="JAVA_UDF"
+        ); """
+        sql """ DROP FUNCTION IF EXISTS java_udf_ipv6_test3(array<ipv6>);"""
+        sql """ CREATE FUNCTION java_udf_ipv6_test3(array<ipv6>) RETURNS string PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.IPV6TypeTest",
+            "type"="JAVA_UDF"
+        ); """
+        sql """ DROP FUNCTION IF EXISTS java_udf_ipv6_test4();"""
+        sql """ CREATE FUNCTION java_udf_ipv6_test4() RETURNS array<ipv6> PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.IPV6TypeTest",
+            "type"="JAVA_UDF"
+        ); """
+
+        sql """DROP TABLE IF EXISTS test_udf_ip;"""
+        sql """
+                CREATE TABLE test_udf_ip
+                (
+                    k1 BIGINT ,
+                    k4 ipv4 ,
+                    k6 ipv6 ,
+                    s string 
+                )
+                DISTRIBUTED BY HASH(k1) BUCKETS 1
+                PROPERTIES("replication_num" = "1");
+        """
+        sql """ insert into test_udf_ip values(1,123,34141,"0.0.0.123") , (2,3114,318903,"0.0.0.123") , (3,7832131,192837891738927931231,"2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D"),(4,null,null,"2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D"); """
+        qt_select_ipv6_1 """ select k1,k6,java_udf_ipv6_test1(k6) from test_udf_ip order by k1   """
+        qt_select_ipv6_2 """ select k1,s,java_udf_ipv6_test2(s) from test_udf_ip where IS_IPV6_STRING(s) order by k1  """
+        qt_select_ipv6_3 """ select java_udf_ipv6_test3(array_sort(array_agg(k6))) from test_udf_ip   """
+        qt_select_ipv6_4 """ select k1, java_udf_ipv6_test4() from test_udf_ip order by k1   """
+
+
+
+
+        sql """ CREATE AGGREGATE FUNCTION udaf_my_sum_ip(ipv6) RETURNS string PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.MySumIP",
+            "always_nullable"="false",
+            "type"="JAVA_UDF"
+        ); """
+
+
+        qt_select_ipv6_5  """ select udaf_my_sum_ip(k6) from test_udf_ip; """
+        
+    } finally {
+        try_sql("DROP FUNCTION IF EXISTS java_udf_ipv6_test1(ipv4);")
+        try_sql("DROP FUNCTION IF EXISTS java_udf_ipv6_test2(string);")
+        try_sql("DROP FUNCTION IF EXISTS java_udf_ipv6_test3(array<ipv4>);")
+        try_sql("DROP FUNCTION IF EXISTS java_udf_ipv6_test4();")
+        try_sql("DROP FUNCTION IF EXISTS udaf_my_sum_ip(ipv6);")
+    }
+}

From 7b5b8db1bc477720a1565ee58ce5a2cf33b31c13 Mon Sep 17 00:00:00 2001
From: Rijesh Kunhi Parambattu <147430310+rijeshkp@users.noreply.github.com>
Date: Mon, 9 Dec 2024 19:08:28 +0530
Subject: [PATCH 243/399]  [Enhancement] (nereids)implement
 showTableCreationCommand in nereids #42776  (#44703)

Issue Number: close #42776
---
 .../org/apache/doris/nereids/DorisParser.g4   |   3 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  20 +++-
 .../doris/nereids/trees/plans/PlanType.java   |   3 +-
 .../commands/ShowTableCreationCommand.java    | 101 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../test_nereids_show_table_creation.groovy   |  70 ++++++++++++
 6 files changed, 198 insertions(+), 4 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCreationCommand.java
 create mode 100644 regression-test/suites/nereids_p0/show/test_nereids_show_table_creation.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index eda5dda16984c9..3bc884408c55ce 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -268,6 +268,8 @@ supportedShowStatement
     | SHOW TABLETS BELONG
         tabletIds+=INTEGER_VALUE (COMMA tabletIds+=INTEGER_VALUE)*                  #showTabletsBelong
     | SHOW DATA SKEW FROM baseTableRef                                              #showDataSkew
+    | SHOW TABLE CREATION ((FROM | IN) database=multipartIdentifier)?
+        (LIKE STRING_LITERAL)?                                                      #showTableCreation
     ;
 
 supportedLoadStatement
@@ -359,7 +361,6 @@ unsupportedShowStatement
     | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL                                 #showQueryProfile
     | SHOW CACHE HOTSPOT tablePath=STRING_LITERAL                                   #showCacheHotSpot
     | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)?                     #showSyncJob
-    | SHOW TABLE CREATION ((FROM | IN) database=multipartIdentifier)? wildWhere?    #showTableCreation
     | SHOW CATALOG RECYCLE BIN wildWhere?                                           #showCatalogRecycleBin
     | SHOW QUERY STATS ((FOR database=identifier)
             | (FROM tableName=multipartIdentifier (ALL VERBOSE?)?))?                #showQueryStats
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index ff5bed55ebc857..8c964e997069a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -271,6 +271,7 @@
 import org.apache.doris.nereids.DorisParser.ShowSmallFilesContext;
 import org.apache.doris.nereids.DorisParser.ShowSqlBlockRuleContext;
 import org.apache.doris.nereids.DorisParser.ShowStorageEnginesContext;
+import org.apache.doris.nereids.DorisParser.ShowTableCreationContext;
 import org.apache.doris.nereids.DorisParser.ShowTableIdContext;
 import org.apache.doris.nereids.DorisParser.ShowTabletsBelongContext;
 import org.apache.doris.nereids.DorisParser.ShowTrashContext;
@@ -574,6 +575,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTabletsBelongCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTrashCommand;
@@ -1402,7 +1404,7 @@ public BrokerDesc visitWithRemoteStorageSystem(WithRemoteStorageSystemContext ct
     @Override
     public List<Pair<LogicalPlan, StatementContext>> visitMultiStatements(MultiStatementsContext ctx) {
         List<Pair<LogicalPlan, StatementContext>> logicalPlans = Lists.newArrayList();
-        for (org.apache.doris.nereids.DorisParser.StatementContext statement : ctx.statement()) {
+        for (DorisParser.StatementContext statement : ctx.statement()) {
             StatementContext statementContext = new StatementContext();
             ConnectContext connectContext = ConnectContext.get();
             if (connectContext != null) {
@@ -1527,7 +1529,7 @@ public LogicalPlan visitLoad(DorisParser.LoadContext ctx) {
      * ******************************************************************************************** */
 
     /**
-     * process lateral view, add a {@link org.apache.doris.nereids.trees.plans.logical.LogicalGenerate} on plan.
+     * process lateral view, add a {@link LogicalGenerate} on plan.
      */
     protected LogicalPlan withGenerate(LogicalPlan plan, LateralViewContext ctx) {
         if (ctx.LATERAL() == null) {
@@ -4985,5 +4987,19 @@ public LogicalPlan visitShowDataSkew(ShowDataSkewContext ctx) {
         TableRefInfo tableRefInfo = visitBaseTableRefContext(ctx.baseTableRef());
         return new ShowDataSkewCommand(tableRefInfo);
     }
+
+    @Override
+    public LogicalPlan visitShowTableCreation(ShowTableCreationContext ctx) {
+        String dbName = null;
+        String wild = null;
+        if (ctx.database != null) {
+            List<String> nameParts = visitMultipartIdentifier(ctx.database);
+            dbName = nameParts.get(0); // only one entry possible
+        }
+        if (ctx.STRING_LITERAL() != null) {
+            wild = ctx.STRING_LITERAL().getText();
+        }
+        return new ShowTableCreationCommand(dbName, wild);
+    }
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index fac113564f857e..e98d9f6afd5ae2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -249,5 +249,6 @@ public enum PlanType {
     CREATE_ENCRYPTKEY_COMMAND,
     CREATE_WORKLOAD_GROUP_COMMAND,
     CREATE_FILE_COMMAND,
-    CREATE_ROUTINE_LOAD_COMMAND
+    CREATE_ROUTINE_LOAD_COMMAND,
+    SHOW_TABLE_CREATION_COMMAND
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCreationCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCreationCommand.java
new file mode 100644
index 00000000000000..8d2102a0b04182
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCreationCommand.java
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.ListComparator;
+import org.apache.doris.common.util.OrderByPair;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * show table id command
+ */
+public class ShowTableCreationCommand extends ShowCommand {
+    private static final ShowResultSetMetaData META_DATA =
+            ShowResultSetMetaData.builder()
+                .addColumn(new Column("Database", ScalarType.createVarchar(20)))
+                .addColumn(new Column("Table", ScalarType.createVarchar(20)))
+                .addColumn(new Column("Status", ScalarType.createVarchar(10)))
+                .addColumn(new Column("Create Time", ScalarType.createVarchar(20)))
+                .addColumn(new Column("Error Msg", ScalarType.createVarchar(100)))
+                .build();
+
+    private String dbName;
+    private String wild;
+
+    /**
+     * constructor
+     */
+    public ShowTableCreationCommand(String dbName, String wild) {
+        super(PlanType.SHOW_TABLE_CREATION_COMMAND);
+        this.dbName = dbName;
+        this.wild = wild;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        if (Strings.isNullOrEmpty(dbName)) {
+            dbName = ConnectContext.get().getDatabase();
+            if (Strings.isNullOrEmpty(dbName)) {
+                ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
+            }
+        }
+
+        /* Need to implement fetching records from iceberg table */
+        List<List<Comparable>> rowSet = Lists.newArrayList();
+        // sort function rows by fourth column (Create Time) asc
+        ListComparator<List<Comparable>> comparator = null;
+        OrderByPair orderByPair = new OrderByPair(3, false);
+        comparator = new ListComparator<>(orderByPair);
+        Collections.sort(rowSet, comparator);
+        List<List<String>> resultRowSet = Lists.newArrayList();
+
+        Set<String> keyNameSet = new HashSet<>();
+        for (List<Comparable> row : rowSet) {
+            List<String> resultRow = Lists.newArrayList();
+            for (Comparable column : row) {
+                resultRow.add(column.toString());
+            }
+            resultRowSet.add(resultRow);
+            keyNameSet.add(resultRow.get(0));
+        }
+
+        return new ShowResultSet(META_DATA, resultRowSet);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowTableCreationCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 05ee8548cfa93d..9131671f98abfd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -116,6 +116,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTabletsBelongCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTrashCommand;
@@ -610,4 +611,8 @@ default R visitAdminCheckTabletsCommand(AdminCheckTabletsCommand adminCheckTable
     default R visitShowDataSkewCommand(ShowDataSkewCommand showDataSkewCommand, C context) {
         return visitCommand(showDataSkewCommand, context);
     }
+
+    default R visitShowTableCreationCommand(ShowTableCreationCommand showTableCreationCommand, C context) {
+        return visitCommand(showTableCreationCommand, context);
+    }
 }
diff --git a/regression-test/suites/nereids_p0/show/test_nereids_show_table_creation.groovy b/regression-test/suites/nereids_p0/show/test_nereids_show_table_creation.groovy
new file mode 100644
index 00000000000000..ca29bd82ed640e
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_nereids_show_table_creation.groovy
@@ -0,0 +1,70 @@
+package show
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_nereids_show_table_creation") {
+    def table = "test_nereids_show_table_creation"
+    // create table and insert data
+    String dbName = 'test_nereids_show_table_creation'
+    try_sql """drop database if exists ${dbName}"""
+    sql """create database ${dbName}"""
+    sql """ drop table if exists ${dbName}.${table} force"""
+
+    sql """
+    create table ${dbName}.${table} (
+        `id` int(11),
+        `name` varchar(128),
+        `da` date
+    )
+    engine=olap
+    duplicate key(id)
+    partition by range(da)(
+        PARTITION p3 VALUES LESS THAN ('2023-01-01'),
+        PARTITION p4 VALUES LESS THAN ('2024-01-01'),
+        PARTITION p5 VALUES LESS THAN ('2025-01-01')
+    )
+    distributed by hash(id) buckets 2
+    properties(
+        "replication_num"="1",
+        "light_schema_change"="true"
+    );
+    """
+
+    checkNereidsExecute("SHOW TABLE CREATION FROM ${dbName}")
+    checkNereidsExecute("SHOW TABLE CREATION IN ${dbName}")
+    checkNereidsExecute("SHOW TABLE CREATION FROM ${dbName} like '%${table}%'")
+    checkNereidsExecute("SHOW TABLE CREATION like '%${table}%'")
+    checkNereidsExecute("SHOW TABLE CREATION")
+
+    def res = sql """SHOW TABLE CREATION FROM ${dbName}"""
+    assertTrue(res.size() == 0)
+
+    res = sql """SHOW TABLE CREATION IN ${dbName}"""
+    assertTrue(res.size() == 0)
+
+    res = sql """SHOW TABLE CREATION FROM ${dbName} like '%${table}%'"""
+    assertTrue(res.size() == 0)
+
+    res = sql """SHOW TABLE CREATION like '%${table}%'"""
+    assertTrue(res.size() == 0)
+
+    res = sql """SHOW TABLE CREATION"""
+    assertTrue(res.size() == 0)
+
+}
+

From 5aee0cc8d122f30499f39c060ff20747ce362cbf Mon Sep 17 00:00:00 2001
From: "Mingyu Chen (Rayner)" <morningman@163.com>
Date: Mon, 9 Dec 2024 16:55:19 -0800
Subject: [PATCH 244/399] [fix](planner) query should be cancelled if limit
 reached (#44338)

### What problem does this PR solve?

Problem Summary:
When there is a `limit` cluse in SQL, if FE has obtained data with more
than the `limit` number of rows,
it should send a cancel command to BE to cancel the query to prevent BE
from reading more data.
However, this function has problems in the current code and does not
work.
Especially in external table query, this may result in lots of
unnecessary network io read.

1. `isBlockQuery`

In the old optimizer, if a query statement contains a `sort` or `agg`
node,
    `isBlockQuery` will be marked as true, otherwise it will be false.
    In the new optimizer, this value is always true.

    Regardless of the old or new optimizer, this logic is wrong.
But only when `isBlockQuery = false` will the reach limit logic be
triggered.

2. Calling problem of reach limit logic

The reach limit logic judgment will only be performed when `eos = true`
in the rowBatch returned by BE.
    This is wrong.
Because for `limit N` queries, each BE's own `limit` is N. But for FE,
as long as the total number of rows
returned by all BEs exceeds N, the reach limit logic can be triggered.
    So it should not be processed only when `eos = true`.

The PR mainly changes:

1. Remove `isBlockQuery`

`isBlockQuery` is only used in the reach limit logic. And it is not
needed. Remove it completely.

2. Modify the judgment position of reach limit.

    When the number of rows obtained by FE is greater than the limit,
    it will check the reach limit logic.

3. fix wrong `limitRows` in `QueryProcessor`

    the limitRows should be got from the first fragment, not last.

4. In scanner scheduler on BE side, if scanner has limit, ignore the
scan bytes threshold per round.

### Release note

[fix](planner) query should be cancelled if limit reached
---
 be/src/vec/exec/scan/scanner_scheduler.cpp    | 13 +++-
 be/src/vec/exec/scan/vscanner.h               |  2 +
 .../apache/doris/nereids/NereidsPlanner.java  |  5 --
 .../apache/doris/planner/OriginalPlanner.java | 15 -----
 .../org/apache/doris/planner/Planner.java     |  6 --
 .../java/org/apache/doris/qe/Coordinator.java | 25 ++++----
 .../apache/doris/qe/CoordinatorContext.java   |  6 +-
 .../java/org/apache/doris/qe/LimitUtils.java  | 54 +++++++++++++++++
 .../runtime/MultiFragmentsPipelineTask.java   |  5 +-
 .../doris/qe/runtime/QueryProcessor.java      | 25 ++++----
 .../org/apache/doris/qe/LimitUtilsTest.java   | 59 +++++++++++++++++++
 11 files changed, 152 insertions(+), 63 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/qe/LimitUtils.java
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/LimitUtilsTest.java

diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp
index 7c5aa8db0a77af..f419f58037aab6 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -268,7 +268,7 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
             }
 
             size_t raw_bytes_threshold = config::doris_scanner_row_bytes;
-            size_t raw_bytes_read = 0; bool first_read = true;
+            size_t raw_bytes_read = 0; bool first_read = true; int64_t limit = scanner->limit();
             while (!eos && raw_bytes_read < raw_bytes_threshold) {
                 if (UNLIKELY(ctx->done())) {
                     eos = true;
@@ -316,6 +316,17 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
                     ctx->inc_block_usage(free_block->allocated_bytes());
                     scan_task->cached_blocks.emplace_back(std::move(free_block), free_block_bytes);
                 }
+                if (limit > 0 && limit < ctx->batch_size()) {
+                    // If this scanner has limit, and less than batch size,
+                    // return immediately and no need to wait raw_bytes_threshold.
+                    // This can save time that each scanner may only return a small number of rows,
+                    // but rows are enough from all scanners.
+                    // If not break, the query like "select * from tbl where id=1 limit 10"
+                    // may scan a lot data when the "id=1"'s filter ratio is high.
+                    // If limit is larger than batch size, this rule is skipped,
+                    // to avoid user specify a large limit and causing too much small blocks.
+                    break;
+                }
             } // end for while
 
             if (UNLIKELY(!status.ok())) {
diff --git a/be/src/vec/exec/scan/vscanner.h b/be/src/vec/exec/scan/vscanner.h
index 6c4f3294ce1bcc..bb68055e1f07a3 100644
--- a/be/src/vec/exec/scan/vscanner.h
+++ b/be/src/vec/exec/scan/vscanner.h
@@ -156,6 +156,8 @@ class VScanner {
         _query_statistics = query_statistics;
     }
 
+    int64_t limit() const { return _limit; }
+
 protected:
     void _discard_conjuncts() {
         for (auto& conjunct : _conjuncts) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 58af5cd3e92199..16fe1353facfb6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -722,11 +722,6 @@ public String getExplainString(ExplainOptions explainOptions) {
         return plan;
     }
 
-    @Override
-    public boolean isBlockQuery() {
-        return true;
-    }
-
     @Override
     public DescriptorTable getDescTable() {
         return descTable;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
index 503cb181766113..feb8c45787a013 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
@@ -79,10 +79,6 @@ public OriginalPlanner(Analyzer analyzer) {
         this.analyzer = analyzer;
     }
 
-    public boolean isBlockQuery() {
-        return isBlockQuery;
-    }
-
     public PlannerContext getPlannerContext() {
         return plannerContext;
     }
@@ -276,17 +272,6 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue
 
         if (queryStmt instanceof SelectStmt) {
             SelectStmt selectStmt = (SelectStmt) queryStmt;
-            if (queryStmt.getSortInfo() != null || selectStmt.getAggInfo() != null) {
-                isBlockQuery = true;
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("this is block query");
-                }
-            } else {
-                isBlockQuery = false;
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("this isn't block query");
-                }
-            }
             if (selectStmt.isTwoPhaseReadOptEnabled()) {
                 // Optimize query like `SELECT ... FROM <tbl> WHERE ... ORDER BY ... LIMIT ...`
                 if (singleNodePlan instanceof SortNode
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
index 5617ad57e8f66e..cfcd27af8fa5d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
@@ -44,8 +44,6 @@ public abstract class Planner {
 
     protected ArrayList<PlanFragment> fragments = Lists.newArrayList();
 
-    protected boolean isBlockQuery = false;
-
     protected TQueryOptions queryOptions;
 
     public abstract List<ScanNode> getScanNodes();
@@ -116,10 +114,6 @@ public List<PlanFragment> getFragments() {
         return fragments;
     }
 
-    public boolean isBlockQuery() {
-        return isBlockQuery;
-    }
-
     public TQueryOptions getQueryOptions() {
         return queryOptions;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 262b5836689ab0..4905050e6e8cfe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -235,8 +235,6 @@ public class Coordinator implements CoordInterface {
     // same as backend_exec_states_.size() after Exec()
     private final Set<TUniqueId> instanceIds = Sets.newHashSet();
 
-    private final boolean isBlockQuery;
-
     private int numReceivedRows = 0;
 
     private List<String> deltaUrls;
@@ -332,7 +330,6 @@ public Coordinator(ConnectContext context, Analyzer analyzer, Planner planner,
     // Used for query/insert/test
     public Coordinator(ConnectContext context, Analyzer analyzer, Planner planner) {
         this.context = context;
-        this.isBlockQuery = planner.isBlockQuery();
         this.queryId = context.queryId();
         this.fragments = planner.getFragments();
         this.scanNodes = planner.getScanNodes();
@@ -375,7 +372,6 @@ public Coordinator(ConnectContext context, Analyzer analyzer, Planner planner) {
     // Constructor of Coordinator is too complicated.
     public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, List<PlanFragment> fragments,
             List<ScanNode> scanNodes, String timezone, boolean loadZeroTolerance, boolean enableProfile) {
-        this.isBlockQuery = true;
         this.jobId = jobId;
         this.queryId = queryId;
         this.descTable = descTable.toThrift();
@@ -1202,23 +1198,22 @@ public RowBatch getNext() throws Exception {
             numReceivedRows += resultBatch.getBatch().getRowsSize();
         }
 
+        // if reached limit rows, cancel this query immediately
+        // to avoid BE from reading more data.
+        // ATTN: if change here, also need to change the same logic in QueryProcessor.getNext();
+        Long limitRows = fragments.get(0).getPlanRoot().getLimit();
+        boolean reachedLimit = LimitUtils.cancelIfReachLimit(
+                resultBatch, limitRows, numReceivedRows, this::cancelInternal);
+
         if (resultBatch.isEos()) {
             receivers.remove(receiver);
             if (receivers.isEmpty()) {
                 returnedAllResults = true;
-            } else {
+            } else if (!reachedLimit) {
+                // if reachedLimit is true, which means this query has been cancelled.
+                // so no need to set eos to false again.
                 resultBatch.setEos(false);
             }
-
-            // if this query is a block query do not cancel.
-            Long numLimitRows = fragments.get(0).getPlanRoot().getLimit();
-            boolean hasLimit = numLimitRows > 0;
-            if (!isBlockQuery && instanceIds.size() > 1 && hasLimit && numReceivedRows >= numLimitRows) {
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("no block query, return num >= limit rows, need cancel");
-                }
-                cancelInternal(new Status(TStatusCode.LIMIT_REACH, "query reach limit"));
-            }
         }
 
         if (!returnedAllResults) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java
index aed0fd9c98ce71..343456c296a79b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java
@@ -84,7 +84,6 @@ public class CoordinatorContext {
     // these are some constant parameters
     public final NereidsCoordinator coordinator;
     public final List<PlanFragment> fragments;
-    public final boolean isBlockQuery;
     public final DataSink dataSink;
     public final ExecutionProfile executionProfile;
     public final ConnectContext connectContext;
@@ -120,7 +119,6 @@ public class CoordinatorContext {
     private CoordinatorContext(
             NereidsCoordinator coordinator,
             ConnectContext connectContext,
-            boolean isBlockQuery,
             List<PipelineDistributedPlan> distributedPlans,
             List<PlanFragment> fragments,
             List<RuntimeFilter> runtimeFilters,
@@ -131,7 +129,6 @@ private CoordinatorContext(
             TQueryOptions queryOptions,
             TDescriptorTable descriptorTable) {
         this.connectContext = connectContext;
-        this.isBlockQuery = isBlockQuery;
         this.fragments = fragments;
         this.distributedPlans = distributedPlans;
         this.topDistributedPlan = distributedPlans.get(distributedPlans.size() - 1);
@@ -161,7 +158,6 @@ private CoordinatorContext(
             TDescriptorTable descriptorTable,
             ExecutionProfile executionProfile) {
         this.coordinator = coordinator;
-        this.isBlockQuery = true;
         this.fragments = fragments;
         this.distributedPlans = distributedPlans;
         this.topDistributedPlan = distributedPlans.get(distributedPlans.size() - 1);
@@ -290,7 +286,7 @@ public static CoordinatorContext buildForSql(NereidsPlanner planner, NereidsCoor
                         .collect(Collectors.toList())
         );
         return new CoordinatorContext(
-                coordinator, connectContext, planner.isBlockQuery(),
+                coordinator, connectContext,
                 planner.getDistributedPlans().valueList(),
                 planner.getFragments(), planner.getRuntimeFilters(), planner.getTopnFilters(),
                 planner.getScanNodes(), executionProfile, queryGlobals, queryOptions, descriptorTable
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/LimitUtils.java b/fe/fe-core/src/main/java/org/apache/doris/qe/LimitUtils.java
new file mode 100644
index 00000000000000..cbbe5c71a0ff06
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/LimitUtils.java
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.qe;
+
+import org.apache.doris.common.Status;
+import org.apache.doris.thrift.TStatusCode;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.function.Consumer;
+
+/**
+ * This is a utility class for limit related operations.
+ * Because current there are 2 places need to check limit rows, so put the logic here for unification.
+ *  - Coordinator.getNext();
+ *  - QueryProcessor.getNext();
+ */
+public class LimitUtils {
+    private static final Logger LOG = LogManager.getLogger(LimitUtils.class);
+    private static final Status LIMIT_REACH_STATUS = new Status(TStatusCode.LIMIT_REACH, "query reach limit");
+
+    // if reached limit rows, cancel this query immediately
+    // to avoid BE from reading more data.
+    public static boolean cancelIfReachLimit(RowBatch resultBatch, long limitRows, long numReceivedRows,
+            Consumer<Status> cancelFunc) {
+        boolean reachedLimit = false;
+        if (limitRows > 0 && numReceivedRows >= limitRows) {
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("reach limit rows: {}, received rows: {}, cancel query", limitRows, numReceivedRows);
+            }
+            cancelFunc.accept(LIMIT_REACH_STATUS);
+            // set this
+            resultBatch.setEos(true);
+            reachedLimit = true;
+        }
+        return reachedLimit;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java
index a1a91aab5176b8..92b2a00597b2e7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java
@@ -130,9 +130,10 @@ public void onSuccess(InternalService.PCancelPlanFragmentResult result) {
                                 LOG.warn("Failed to cancel query {} backend: {}, reason: {}",
                                         DebugUtil.printId(queryId), backend, status.toString());
                             }
+                        } else {
+                            LOG.warn("Failed to cancel query {} backend: {} reason: {}",
+                                    DebugUtil.printId(queryId), backend, "without status");
                         }
-                        LOG.warn("Failed to cancel query {} backend: {} reason: {}",
-                                DebugUtil.printId(queryId), backend, "without status");
                     }
 
                     public void onFailure(Throwable t) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java
index a5a5100faece1a..6eb19d250ede4c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java
@@ -28,12 +28,12 @@
 import org.apache.doris.qe.AbstractJobProcessor;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.CoordinatorContext;
+import org.apache.doris.qe.LimitUtils;
 import org.apache.doris.qe.ResultReceiver;
 import org.apache.doris.qe.RowBatch;
 import org.apache.doris.rpc.RpcException;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TReportExecStatusParams;
-import org.apache.doris.thrift.TStatusCode;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
@@ -64,7 +64,7 @@ public QueryProcessor(CoordinatorContext coordinatorContext, List<ResultReceiver
                 Objects.requireNonNull(runningReceivers, "runningReceivers can not be null")
         );
 
-        this.limitRows = coordinatorContext.fragments.get(coordinatorContext.fragments.size() - 1)
+        this.limitRows = coordinatorContext.fragments.get(0)
                 .getPlanRoot()
                 .getLimit();
     }
@@ -145,22 +145,19 @@ public RowBatch getNext() throws UserException, TException, RpcException {
             numReceivedRows += resultBatch.getBatch().getRowsSize();
         }
 
+        // if reached limit rows, cancel this query immediately
+        // to avoid BE from reading more data.
+        // ATTN: if change here, also need to change the same logic in Coordinator.getNext();
+        boolean reachedLimit = LimitUtils.cancelIfReachLimit(
+                resultBatch, limitRows, numReceivedRows, coordinatorContext::cancelSchedule);
+
         if (resultBatch.isEos()) {
             runningReceivers.remove(receiver);
-            if (!runningReceivers.isEmpty()) {
+            // if reachedLimit is true, which means this query has been cancelled.
+            // so no need to set eos to false again.
+            if (!runningReceivers.isEmpty() && !reachedLimit) {
                 resultBatch.setEos(false);
             }
-
-            // if this query is a block query do not cancel.
-            boolean hasLimit = limitRows > 0;
-            if (!coordinatorContext.isBlockQuery
-                    && coordinatorContext.instanceNum.get() > 1
-                    && hasLimit && numReceivedRows >= limitRows) {
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("no block query, return num >= limit rows, need cancel");
-                }
-                coordinatorContext.cancelSchedule(new Status(TStatusCode.LIMIT_REACH, "query reach limit"));
-            }
         }
 
         if (!runningReceivers.isEmpty()) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/LimitUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/LimitUtilsTest.java
new file mode 100644
index 00000000000000..012fbad18a5ddb
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/LimitUtilsTest.java
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.qe;
+
+
+import org.apache.doris.common.Status;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.function.Consumer;
+
+public class LimitUtilsTest {
+
+    private static int res = 0;
+
+    @Test
+    public void testUpperBound() {
+        Consumer<Status> cancelFunc = batch -> res = 666;
+        RowBatch rowBatch = new RowBatch();
+        rowBatch.setEos(false);
+        // - no limit
+        Assert.assertFalse(LimitUtils.cancelIfReachLimit(rowBatch, 0, 10, cancelFunc));
+        Assert.assertFalse(rowBatch.isEos());
+        Assert.assertEquals(0, res);
+
+        // - not reach limit
+        Assert.assertFalse(LimitUtils.cancelIfReachLimit(rowBatch, 10, 1, cancelFunc));
+        Assert.assertFalse(rowBatch.isEos());
+        Assert.assertEquals(0, res);
+
+        // - reach limit
+        Assert.assertTrue(LimitUtils.cancelIfReachLimit(rowBatch, 10, 10, cancelFunc));
+        Assert.assertTrue(rowBatch.isEos());
+        Assert.assertEquals(666, res);
+
+        // - reach limit
+        res = 0;
+        rowBatch.setEos(false);
+        Assert.assertTrue(LimitUtils.cancelIfReachLimit(rowBatch, 10, 100, cancelFunc));
+        Assert.assertTrue(rowBatch.isEos());
+        Assert.assertEquals(666, res);
+    }
+}

From ed112527f9ba41cefbf5593aa94dc0bd9fbfa2ea Mon Sep 17 00:00:00 2001
From: Rijesh Kunhi Parambattu <147430310+rijeshkp@users.noreply.github.com>
Date: Tue, 10 Dec 2024 07:17:25 +0530
Subject: [PATCH 245/399] [Feat](nereids) support ShowTabletStorageFormat
 command in nereids (#43295)

Issue Number: close #42783
---
 .../org/apache/doris/nereids/DorisParser.g4   |   4 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  13 ++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../ShowTabletStorageFormatCommand.java       | 129 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   6 +
 .../test_database_management_auth.groovy      |   2 +-
 ..._nereids_show_tablet_storage_format.groovy |  25 ++++
 7 files changed, 177 insertions(+), 3 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java
 create mode 100644 regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 3bc884408c55ce..1b01cad4293569 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -270,6 +270,7 @@ supportedShowStatement
     | SHOW DATA SKEW FROM baseTableRef                                              #showDataSkew
     | SHOW TABLE CREATION ((FROM | IN) database=multipartIdentifier)?
         (LIKE STRING_LITERAL)?                                                      #showTableCreation
+    | SHOW TABLET STORAGE FORMAT VERBOSE?                                           #showTabletStorageFormat
     ;
 
 supportedLoadStatement
@@ -369,7 +370,6 @@ unsupportedShowStatement
     | SHOW (CLUSTERS | (COMPUTE GROUPS))                                            #showClusters
     | SHOW CONVERT_LSC ((FROM | IN) database=multipartIdentifier)?                  #showConvertLsc
     | SHOW REPLICA STATUS FROM baseTableRef wildWhere?                              #showReplicaStatus
-    | SHOW TABLET STORAGE FORMAT VERBOSE?                                           #showTabletStorageFormat
     | SHOW COPY ((FROM | IN) database=multipartIdentifier)?
         whereClause? sortClause? limitClause?                                       #showCopy
     | SHOW WARM UP JOB wildWhere?                                                   #showWarmUpJob
@@ -495,6 +495,7 @@ supportedAdminStatement
     | ADMIN SHOW REPLICA STATUS FROM baseTableRef (WHERE STATUS EQ|NEQ STRING_LITERAL)?   #adminShowReplicaStatus
     | ADMIN COMPACT TABLE baseTableRef (WHERE TYPE EQ STRING_LITERAL)?              #adminCompactTable
     | ADMIN CHECK tabletList properties=propertyClause?                             #adminCheckTablets
+    | ADMIN SHOW TABLET STORAGE FORMAT VERBOSE?                                     #adminShowTabletStorageFormat
     ;
 
 supportedRecoverStatement
@@ -520,7 +521,6 @@ unsupportedAdminStatement
         (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?                              #adminCleanTrash
     | ADMIN SET TABLE name=multipartIdentifier
         PARTITION VERSION properties=propertyClause?                                #adminSetPartitionVersion
-    | ADMIN SHOW TABLET STORAGE FORMAT VERBOSE?                                     #adminShowTabletStorageFormat
     | ADMIN COPY TABLET tabletId=INTEGER_VALUE properties=propertyClause?           #adminCopyTablet
     | ADMIN SET TABLE name=multipartIdentifier STATUS properties=propertyClause?    #adminSetTableStatus
     ;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 8c964e997069a5..95dbbb504712f8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -55,6 +55,7 @@
 import org.apache.doris.nereids.DorisParser.AdminDiagnoseTabletContext;
 import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext;
 import org.apache.doris.nereids.DorisParser.AdminShowReplicaStatusContext;
+import org.apache.doris.nereids.DorisParser.AdminShowTabletStorageFormatContext;
 import org.apache.doris.nereids.DorisParser.AggClauseContext;
 import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext;
 import org.apache.doris.nereids.DorisParser.AliasQueryContext;
@@ -273,6 +274,7 @@
 import org.apache.doris.nereids.DorisParser.ShowStorageEnginesContext;
 import org.apache.doris.nereids.DorisParser.ShowTableCreationContext;
 import org.apache.doris.nereids.DorisParser.ShowTableIdContext;
+import org.apache.doris.nereids.DorisParser.ShowTabletStorageFormatContext;
 import org.apache.doris.nereids.DorisParser.ShowTabletsBelongContext;
 import org.apache.doris.nereids.DorisParser.ShowTrashContext;
 import org.apache.doris.nereids.DorisParser.ShowTriggersContext;
@@ -577,6 +579,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowTabletStorageFormatCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTabletsBelongCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTriggersCommand;
@@ -5001,5 +5004,15 @@ public LogicalPlan visitShowTableCreation(ShowTableCreationContext ctx) {
         }
         return new ShowTableCreationCommand(dbName, wild);
     }
+
+    @Override
+    public LogicalPlan visitAdminShowTabletStorageFormat(AdminShowTabletStorageFormatContext ctx) {
+        return new ShowTabletStorageFormatCommand(ctx.VERBOSE() != null);
+    }
+
+    @Override
+    public LogicalPlan visitShowTabletStorageFormat(ShowTabletStorageFormatContext ctx) {
+        return new ShowTabletStorageFormatCommand(ctx.VERBOSE() != null);
+    }
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index e98d9f6afd5ae2..91ec3591003500 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -235,6 +235,7 @@ public enum PlanType {
     SHOW_STORAGE_ENGINES_COMMAND,
     SHOW_TABLE_ID_COMMAND,
     SHOW_TRASH_COMMAND,
+    SHOW_TABLET_STORAGE_FORMAT_COMMAND,
     SHOW_TRIGGERS_COMMAND,
     SHOW_VARIABLES_COMMAND,
     SHOW_AUTHORS_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java
new file mode 100644
index 00000000000000..aa535e0f39dd15
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletStorageFormatCommand.java
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.system.Backend;
+import org.apache.doris.task.AgentClient;
+import org.apache.doris.thrift.TCheckStorageFormatResult;
+
+import com.google.common.collect.Lists;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * show tablet storage format command
+ */
+public class ShowTabletStorageFormatCommand extends ShowCommand {
+    public static final Logger LOG = LogManager.getLogger(ShowTabletStorageFormatCommand.class);
+    private final boolean verbose;
+
+    /**
+     * constructor
+     */
+    public ShowTabletStorageFormatCommand(boolean verbose) {
+        super(PlanType.SHOW_TABLET_STORAGE_FORMAT_COMMAND);
+        this.verbose = verbose;
+    }
+
+    /**
+     * get Meta for show
+     */
+    public ShowResultSetMetaData getMetaData() {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        if (verbose) {
+            builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("TabletId", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("StorageFormat", ScalarType.createVarchar(30)));
+        } else {
+            builder.addColumn(new Column("BackendId", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("V1Count", ScalarType.createVarchar(30)))
+                    .addColumn(new Column("V2Count", ScalarType.createVarchar(30)));
+        }
+        return builder.build();
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR,
+                    PrivPredicate.ADMIN.getPrivs().toString());
+        }
+
+        List<List<String>> resultRowSet = Lists.newArrayList();
+        for (Backend be : Env.getCurrentSystemInfo().getAllBackendsByAllCluster().values()) {
+            if (be.isQueryAvailable() && be.isLoadAvailable()) {
+                AgentClient client = new AgentClient(be.getHost(), be.getBePort());
+                TCheckStorageFormatResult result = client.checkStorageFormat();
+                if (result == null) {
+                    throw new AnalysisException("get tablet data from backend: " + be.getId() + "error.");
+                }
+                if (verbose) {
+                    for (long tabletId : result.getV1Tablets()) {
+                        List<String> row = new ArrayList<>();
+                        row.add(String.valueOf(be.getId()));
+                        row.add(String.valueOf(tabletId));
+                        row.add("V1");
+                        resultRowSet.add(row);
+                    }
+                    for (long tabletId : result.getV2Tablets()) {
+                        List<String> row = new ArrayList<>();
+                        row.add(String.valueOf(be.getId()));
+                        row.add(String.valueOf(tabletId));
+                        row.add("V2");
+                        resultRowSet.add(row);
+                    }
+                } else {
+                    List<String> row = new ArrayList<>();
+                    row.add(String.valueOf(be.getId()));
+                    row.add(String.valueOf(result.getV1Tablets().size()));
+                    row.add(String.valueOf(result.getV2Tablets().size()));
+                    resultRowSet.add(row);
+                }
+            }
+        }
+        return new ShowResultSet(getMetaData(), resultRowSet);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowTabletStorageFormatCommand(this, context);
+    }
+
+    @Override
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
+        LOG.info("show tablet storage format not supported in cloud mode");
+        throw new DdlException("Unsupported operation");
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 9131671f98abfd..5da741f303fed4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -118,6 +118,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowTabletStorageFormatCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTabletsBelongCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTriggersCommand;
@@ -615,4 +616,9 @@ default R visitShowDataSkewCommand(ShowDataSkewCommand showDataSkewCommand, C co
     default R visitShowTableCreationCommand(ShowTableCreationCommand showTableCreationCommand, C context) {
         return visitCommand(showTableCreationCommand, context);
     }
+
+    default R visitShowTabletStorageFormatCommand(ShowTabletStorageFormatCommand showTabletStorageFormatCommand,
+                                                  C context) {
+        return visitCommand(showTabletStorageFormatCommand, context);
+    }
 }
diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy
index 79114c18e789b5..b481cf8de75c00 100644
--- a/regression-test/suites/auth_call/test_database_management_auth.groovy
+++ b/regression-test/suites/auth_call/test_database_management_auth.groovy
@@ -107,7 +107,7 @@ suite("test_database_management_auth","p0,auth_call") {
         }
         test {
             sql """show tablet storage format verbose;"""
-            exception "denied"
+            exception "${error_in_cloud}"
         }
         test {
             sql """ADMIN CLEAN TRASH;"""
diff --git a/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy b/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy
new file mode 100644
index 00000000000000..b4b78bd1c28729
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_nereids_show_tablet_storage_format") {
+    checkNereidsExecute("""show tablet storage format;""")
+    checkNereidsExecute("""show tablet storage format verbose;""")
+    checkNereidsExecute("""admin show tablet storage format;""")
+    checkNereidsExecute("""admin show tablet storage format verbose;""")
+}
+

From 18dc92a40d33b7fe19e0c04dcfc675f4da0bcfcf Mon Sep 17 00:00:00 2001
From: "Mingyu Chen (Rayner)" <morningman@163.com>
Date: Mon, 9 Dec 2024 19:46:13 -0800
Subject: [PATCH 246/399] [fix](catalog) opt the count pushdown rule for
 iceberg/paimon/hive scan node (#44038)

### What problem does this PR solve?

1. Opt the parallelism when doing count push down optimization

Count push down optimization is used to optimize queries such as `select
count(*) from table`.
In this scenario, we can directly obtain the number of rows through the
row count statistics
    of the external table, or the metadata of the Parquet/ORC file,
without reading the actual file content, thereby speeding up such
queries.

Currently, we support count push down optimization for Hive, Iceberg,
and Paimon tables.
    There are two ways to obtain the number of rows:

    1. Obtain directly from statistics

For Iceberg tables, we can obtain the number of rows directly from
statistics.
However, due to the historical issues of Iceberg, if there is
position/equality delete in the table,
        this method cannot be used to prevent incorrect row count.
In this case, it will degenerate to obtaining from the metadata of the
file.

    2. Obtain from the metadata of the file

For Hive, Paimon, and some of Iceberg tables, the number of rows can be
obtained directly
        from the metadata of the Parquet/ORC file.
For Text format tables, efficiency can also be improved by only
performing row separation, without column separation.

In the task splitting logic, for Count push-down optimization, the
number of split tasks should comprehensively
consider the file format, number of files, parallelism, number of BE
nodes, and the Local Shuffle:

1. Count push-down optimization should avoid Local Shuffle, so the
number of split tasks should be greater than or equal to `parallelism *
number of BE nodes`.

2. Fix the incorrect logic of Count push-down optimization

In the previous code, for Iceberg and Paimon tables, Count push-down
optimization did not take effect because we did not push
CountPushDown information to FileFormatReader inside TableForamtReader.
This PR fixes this problem.

3. Store SessionVaraible variables in FileQueryScanNode.

SessionVaraible is a variable in ConnectionContext. And
ConnectionContext is a ThreadLocal variable.
In FileQueryScanNode, SessionVaraible may be accessed in other threads
in some cases,
    so ThreadLocal variables may not be obtained.
Therefore, the SessionVaraible reference is stored in FileQueryScanNode
to prevent illegal access.

4. Independent FileSplitter class.

The FileSplitter class is a tool class that allows users to split
`Split` according to different strategies.
This PR does not modify the splitting strategy, but only extracts this
part of the logic separately,
    to be able to perform logic optimization later.
---
 .../vec/exec/format/table/iceberg_reader.cpp  |  18 +--
 be/src/vec/exec/format/table/iceberg_reader.h |   4 +-
 .../vec/exec/format/table/paimon_reader.cpp   |   3 +
 be/src/vec/exec/scan/vfile_scanner.cpp        |   4 +
 .../analysis/TableValuedFunctionRef.java      |   5 +-
 .../doris/datasource/FileQueryScanNode.java   |  27 ++++-
 .../apache/doris/datasource/FileScanNode.java |  79 -------------
 .../apache/doris/datasource/FileSplit.java    |   5 +-
 .../apache/doris/datasource/FileSplitter.java | 104 ++++++++++++++++++
 .../doris/datasource/SplitAssignment.java     |   2 +-
 .../apache/doris/datasource/SplitCreator.java |   2 +-
 .../doris/datasource/SplitGenerator.java      |   5 +-
 .../datasource/hive/source/HiveScanNode.java  |  52 ++++++---
 .../datasource/hive/source/HiveSplit.java     |   6 +-
 .../datasource/hudi/source/HudiScanNode.java  |   8 +-
 .../iceberg/source/IcebergScanNode.java       |  42 ++++---
 .../iceberg/source/IcebergSplit.java          |  11 +-
 .../lakesoul/source/LakeSoulScanNode.java     |   7 +-
 .../maxcompute/source/MaxComputeScanNode.java |  28 ++---
 .../paimon/source/PaimonScanNode.java         |  65 ++++++-----
 .../datasource/paimon/source/PaimonSplit.java |   7 +-
 .../source/TrinoConnectorScanNode.java        |   9 +-
 .../datasource/tvf/source/TVFScanNode.java    |  22 +++-
 .../translator/PhysicalPlanTranslator.java    |  20 ++--
 .../doris/planner/SingleNodePlanner.java      |  24 ++--
 .../DataGenTableValuedFunction.java           |   3 +-
 .../ExternalFileTableValuedFunction.java      |   5 +-
 .../GroupCommitTableValuedFunction.java       |   3 +-
 .../JdbcQueryTableValueFunction.java          |   3 +-
 .../MetadataTableValuedFunction.java          |   3 +-
 .../QueryTableValueFunction.java              |   3 +-
 .../tablefunction/TableValuedFunctionIf.java  |   3 +-
 .../paimon/test_paimon_catalog.out            |  80 ++++++++++++++
 .../test_iceberg_optimize_count.groovy        |   2 +-
 .../paimon/test_paimon_catalog.groovy         |  14 +++
 35 files changed, 450 insertions(+), 228 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplitter.java

diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp
index 8f130ca6002d5d..837269b0bb355d 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.cpp
+++ b/be/src/vec/exec/format/table/iceberg_reader.cpp
@@ -96,25 +96,25 @@ IcebergTableReader::IcebergTableReader(std::unique_ptr<GenericReader> file_forma
     _iceberg_profile.delete_rows_sort_time =
             ADD_CHILD_TIMER(_profile, "DeleteRowsSortTime", iceberg_profile);
     if (range.table_format_params.iceberg_params.__isset.row_count) {
-        _remaining_push_down_count = range.table_format_params.iceberg_params.row_count;
+        _remaining_table_level_row_count = range.table_format_params.iceberg_params.row_count;
     } else {
-        _remaining_push_down_count = -1;
+        _remaining_table_level_row_count = -1;
     }
 }
 
 Status IcebergTableReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
     // already get rows from be
-    if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_push_down_count > 0) {
-        auto rows =
-                std::min(_remaining_push_down_count, (int64_t)_state->query_options().batch_size);
-        _remaining_push_down_count -= rows;
+    if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count > 0) {
+        auto rows = std::min(_remaining_table_level_row_count,
+                             (int64_t)_state->query_options().batch_size);
+        _remaining_table_level_row_count -= rows;
         auto mutate_columns = block->mutate_columns();
         for (auto& col : mutate_columns) {
             col->resize(rows);
         }
         block->set_columns(std::move(mutate_columns));
         *read_rows = rows;
-        if (_remaining_push_down_count == 0) {
+        if (_remaining_table_level_row_count == 0) {
             *eof = true;
         }
 
@@ -164,7 +164,7 @@ Status IcebergTableReader::get_columns(
 
 Status IcebergTableReader::init_row_filters(const TFileRangeDesc& range, io::IOContext* io_ctx) {
     // We get the count value by doris's be, so we don't need to read the delete file
-    if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_push_down_count > 0) {
+    if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count > 0) {
         return Status::OK();
     }
 
@@ -187,9 +187,11 @@ Status IcebergTableReader::init_row_filters(const TFileRangeDesc& range, io::IOC
     if (position_delete_files.size() > 0) {
         RETURN_IF_ERROR(
                 _position_delete_base(table_desc.original_file_path, position_delete_files));
+        _file_format_reader->set_push_down_agg_type(TPushAggOp::NONE);
     }
     if (equality_delete_files.size() > 0) {
         RETURN_IF_ERROR(_equality_delete_base(equality_delete_files));
+        _file_format_reader->set_push_down_agg_type(TPushAggOp::NONE);
     }
 
     COUNTER_UPDATE(_iceberg_profile.num_delete_files, table_desc.delete_files.size());
diff --git a/be/src/vec/exec/format/table/iceberg_reader.h b/be/src/vec/exec/format/table/iceberg_reader.h
index 2e240f465b6a2c..ee7dcdd68d24fa 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.h
+++ b/be/src/vec/exec/format/table/iceberg_reader.h
@@ -167,7 +167,9 @@ class IcebergTableReader : public TableFormatReader {
     bool _has_schema_change = false;
     bool _has_iceberg_schema = false;
 
-    int64_t _remaining_push_down_count;
+    // the table level row count for optimizing query like:
+    // select count(*) from table;
+    int64_t _remaining_table_level_row_count;
     Fileformat _file_format = Fileformat::NONE;
 
     const int64_t MIN_SUPPORT_DELETE_FILES_VERSION = 2;
diff --git a/be/src/vec/exec/format/table/paimon_reader.cpp b/be/src/vec/exec/format/table/paimon_reader.cpp
index 263fdc8014bc36..055d6179b2c422 100644
--- a/be/src/vec/exec/format/table/paimon_reader.cpp
+++ b/be/src/vec/exec/format/table/paimon_reader.cpp
@@ -40,6 +40,9 @@ Status PaimonReader::init_row_filters(const TFileRangeDesc& range, io::IOContext
         return Status::OK();
     }
 
+    // set push down agg type to NONE because we can not do count push down opt
+    // if there are delete files.
+    _file_format_reader->set_push_down_agg_type(TPushAggOp::NONE);
     const auto& deletion_file = table_desc.deletion_file;
     io::FileSystemProperties properties = {
             .system_type = _params.file_type,
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp
index 246a2edb082c92..76639e4bed4a28 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -822,6 +822,9 @@ Status VFileScanner::_get_next_reader() {
                     _should_enable_file_meta_cache() ? ExecEnv::GetInstance()->file_meta_cache()
                                                      : nullptr,
                     _state->query_options().enable_parquet_lazy_mat);
+            // ATTN: the push down agg type may be set back to NONE,
+            // see IcebergTableReader::init_row_filters for example.
+            parquet_reader->set_push_down_agg_type(_get_push_down_agg_type());
             {
                 SCOPED_TIMER(_open_reader_timer);
                 RETURN_IF_ERROR(parquet_reader->open());
@@ -887,6 +890,7 @@ Status VFileScanner::_get_next_reader() {
                     _profile, _state, *_params, range, _state->query_options().batch_size,
                     _state->timezone(), _io_ctx.get(), _state->query_options().enable_orc_lazy_mat,
                     unsupported_pushdown_types);
+            orc_reader->set_push_down_agg_type(_get_push_down_agg_type());
             if (push_down_predicates) {
                 RETURN_IF_ERROR(_process_late_arrival_conjuncts());
             }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
index b1e7c7c89e91f5..9eacb8a0422695 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableValuedFunctionRef.java
@@ -26,6 +26,7 @@
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.planner.ScanNode;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.tablefunction.BackendsTableValuedFunction;
 import org.apache.doris.tablefunction.LocalTableValuedFunction;
 import org.apache.doris.tablefunction.TableValuedFunctionIf;
@@ -119,8 +120,8 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
         analyzeJoin(analyzer);
     }
 
-    public ScanNode getScanNode(PlanNodeId id) {
-        return tableFunction.getScanNode(id, desc);
+    public ScanNode getScanNode(PlanNodeId id, SessionVariable sv) {
+        return tableFunction.getScanNode(id, desc, sv);
     }
 
     public TableValuedFunctionIf getTableFunction() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index 2672cc016eb528..16b19ac77b2bb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -39,6 +39,7 @@
 import org.apache.doris.datasource.hive.source.HiveSplit;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.system.Backend;
@@ -94,6 +95,9 @@ public abstract class FileQueryScanNode extends FileScanNode {
     protected String brokerName;
 
     protected TableSnapshot tableSnapshot;
+    // Save the reference of session variable, so that we don't need to get it from connection context.
+    // connection context is a thread local variable, it is not available is running in other thread.
+    protected SessionVariable sessionVariable;
 
     /**
      * External file scan node for Query hms table
@@ -102,8 +106,10 @@ public abstract class FileQueryScanNode extends FileScanNode {
      * These scan nodes do not have corresponding catalog/database/table info, so no need to do priv check
      */
     public FileQueryScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName,
-                             StatisticalType statisticalType, boolean needCheckColumnPriv) {
+            StatisticalType statisticalType, boolean needCheckColumnPriv,
+            SessionVariable sv) {
         super(id, desc, planNodeName, statisticalType, needCheckColumnPriv);
+        this.sessionVariable = sv;
     }
 
     @Override
@@ -112,7 +118,6 @@ public void init(Analyzer analyzer) throws UserException {
             ConnectContext.get().getExecutor().getSummaryProfile().setInitScanNodeStartTime();
         }
         super.init(analyzer);
-        initFileSplitSize();
         doInitialize();
         if (ConnectContext.get().getExecutor() != null) {
             ConnectContext.get().getExecutor().getSummaryProfile().setInitScanNodeFinishTime();
@@ -314,6 +319,7 @@ public void createScanRangeLocations() throws UserException {
             params.setProperties(locationProperties);
         }
 
+        int numBackends = backendPolicy.numBackends();
         List<String> pathPartitionKeys = getPathPartitionKeys();
         if (isBatchMode()) {
             // File splits are generated lazily, and fetched by backends while scanning.
@@ -356,7 +362,7 @@ public void createScanRangeLocations() throws UserException {
                 scanBackendIds.add(backend.getId());
             }
         } else {
-            List<Split> inputSplits = getSplits();
+            List<Split> inputSplits = getSplits(numBackends);
             if (ConnectContext.get().getExecutor() != null) {
                 ConnectContext.get().getExecutor().getSummaryProfile().setGetSplitsFinishTime();
             }
@@ -605,4 +611,19 @@ public TableSnapshot getQueryTableSnapshot() {
         }
         return this.tableSnapshot;
     }
+
+    /**
+     * The real file split size is determined by:
+     * 1. If user specify the split size in session variable `file_split_size`, use user specified value.
+     * 2. Otherwise, use the max value of DEFAULT_SPLIT_SIZE and block size.
+     * @param blockSize, got from file system, eg, hdfs
+     * @return the real file split size
+     */
+    protected long getRealFileSplitSize(long blockSize) {
+        long realSplitSize = sessionVariable.getFileSplitSize();
+        if (realSplitSize <= 0) {
+            realSplitSize = Math.max(DEFAULT_SPLIT_SIZE, blockSize);
+        }
+        return realSplitSize;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index 29fdb2b09acfcd..b7d34312313308 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -25,15 +25,10 @@
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.UserException;
-import org.apache.doris.common.util.LocationPath;
-import org.apache.doris.common.util.Util;
 import org.apache.doris.planner.PlanNodeId;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.thrift.TExplainLevel;
 import org.apache.doris.thrift.TExpr;
-import org.apache.doris.thrift.TFileCompressType;
 import org.apache.doris.thrift.TFileRangeDesc;
 import org.apache.doris.thrift.TFileScanNode;
 import org.apache.doris.thrift.TFileScanRangeParams;
@@ -46,11 +41,9 @@
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
-import org.apache.hadoop.fs.BlockLocation;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -68,8 +61,6 @@ public abstract class FileScanNode extends ExternalScanNode {
     // For explain
     protected long totalFileSize = 0;
     protected long totalPartitionNum = 0;
-    protected long fileSplitSize;
-    protected boolean isSplitSizeSetBySession = false;
 
     public FileScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType,
             boolean needCheckColumnPriv) {
@@ -77,19 +68,6 @@ public FileScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, St
         this.needCheckColumnPriv = needCheckColumnPriv;
     }
 
-    @Override
-    public void init() throws UserException {
-        initFileSplitSize();
-    }
-
-    protected void initFileSplitSize() {
-        this.fileSplitSize = ConnectContext.get().getSessionVariable().getFileSplitSize();
-        this.isSplitSizeSetBySession = this.fileSplitSize > 0;
-        if (this.fileSplitSize <= 0) {
-            this.fileSplitSize = DEFAULT_SPLIT_SIZE;
-        }
-    }
-
     @Override
     protected void toThrift(TPlanNode planNode) {
         planNode.setPushDownAggTypeOpt(pushDownAggNoGroupingOp);
@@ -256,61 +234,4 @@ protected void setDefaultValueExprs(TableIf tbl,
             }
         }
     }
-
-    protected List<Split> splitFile(LocationPath path, long blockSize, BlockLocation[] blockLocations, long length,
-            long modificationTime, boolean splittable, List<String> partitionValues, SplitCreator splitCreator)
-            throws IOException {
-        if (blockLocations == null) {
-            blockLocations = new BlockLocation[0];
-        }
-        List<Split> result = Lists.newArrayList();
-        TFileCompressType compressType = Util.inferFileCompressTypeByPath(path.get());
-        if (!splittable || compressType != TFileCompressType.PLAIN) {
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("Path {} is not splittable.", path);
-            }
-            String[] hosts = blockLocations.length == 0 ? null : blockLocations[0].getHosts();
-            result.add(splitCreator.create(path, 0, length, length, modificationTime, hosts, partitionValues));
-            return result;
-        }
-        // if file split size is set by session variable, use session variable.
-        // Otherwise, use max(file split size, block size)
-        if (!isSplitSizeSetBySession) {
-            fileSplitSize = Math.max(fileSplitSize, blockSize);
-        }
-        long bytesRemaining;
-        for (bytesRemaining = length; (double) bytesRemaining / (double) fileSplitSize > 1.1D;
-                bytesRemaining -= fileSplitSize) {
-            int location = getBlockIndex(blockLocations, length - bytesRemaining);
-            String[] hosts = location == -1 ? null : blockLocations[location].getHosts();
-            result.add(splitCreator.create(path, length - bytesRemaining, fileSplitSize,
-                    length, modificationTime, hosts, partitionValues));
-        }
-        if (bytesRemaining != 0L) {
-            int location = getBlockIndex(blockLocations, length - bytesRemaining);
-            String[] hosts = location == -1 ? null : blockLocations[location].getHosts();
-            result.add(splitCreator.create(path, length - bytesRemaining, bytesRemaining,
-                    length, modificationTime, hosts, partitionValues));
-        }
-
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("Path {} includes {} splits.", path, result.size());
-        }
-        return result;
-    }
-
-    protected int getBlockIndex(BlockLocation[] blkLocations, long offset) {
-        if (blkLocations == null || blkLocations.length == 0) {
-            return -1;
-        }
-        for (int i = 0; i < blkLocations.length; ++i) {
-            if (blkLocations[i].getOffset() <= offset
-                    && offset < blkLocations[i].getOffset() + blkLocations[i].getLength()) {
-                return i;
-            }
-        }
-        BlockLocation last = blkLocations[blkLocations.length - 1];
-        long fileLength = last.getOffset() + last.getLength() - 1L;
-        throw new IllegalArgumentException(String.format("Offset %d is outside of file (0..%d)", offset, fileLength));
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplit.java
index 1ebb390e90438f..37e66c7056fbd7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplit.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplit.java
@@ -87,9 +87,12 @@ public static class FileSplitCreator implements SplitCreator {
 
         @Override
         public Split create(LocationPath path, long start, long length, long fileLength,
+                long fileSplitSize,
                 long modificationTime, String[] hosts,
                 List<String> partitionValues) {
-            return new FileSplit(path, start, length, fileLength, modificationTime, hosts, partitionValues);
+            FileSplit split = new FileSplit(path, start, length, fileLength, modificationTime, hosts, partitionValues);
+            split.setTargetSplitSize(fileSplitSize);
+            return split;
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplitter.java
new file mode 100644
index 00000000000000..b923c87d3ac96d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplitter.java
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource;
+
+import org.apache.doris.common.util.LocationPath;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.spi.Split;
+import org.apache.doris.thrift.TFileCompressType;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+import java.util.List;
+
+public class FileSplitter {
+    private static final Logger LOG = LogManager.getLogger(FileSplitter.class);
+
+    // If the number of files is larger than parallel instances * num of backends,
+    // we don't need to split the file.
+    // Otherwise, split the file to avoid local shuffle.
+    public static boolean needSplitForCountPushdown(int parallelism, int numBackends, long totalFileNum) {
+        return totalFileNum < parallelism * numBackends;
+    }
+
+    public static List<Split> splitFile(
+            LocationPath path,
+            long fileSplitSize,
+            BlockLocation[] blockLocations,
+            long length,
+            long modificationTime,
+            boolean splittable,
+            List<String> partitionValues,
+            SplitCreator splitCreator)
+            throws IOException {
+        if (blockLocations == null) {
+            blockLocations = new BlockLocation[0];
+        }
+        List<Split> result = Lists.newArrayList();
+        TFileCompressType compressType = Util.inferFileCompressTypeByPath(path.get());
+        if (!splittable || compressType != TFileCompressType.PLAIN) {
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Path {} is not splittable.", path);
+            }
+            String[] hosts = blockLocations.length == 0 ? null : blockLocations[0].getHosts();
+            result.add(splitCreator.create(path, 0, length, length, fileSplitSize,
+                    modificationTime, hosts, partitionValues));
+            return result;
+        }
+        long bytesRemaining;
+        for (bytesRemaining = length; (double) bytesRemaining / (double) fileSplitSize > 1.1D;
+                bytesRemaining -= fileSplitSize) {
+            int location = getBlockIndex(blockLocations, length - bytesRemaining);
+            String[] hosts = location == -1 ? null : blockLocations[location].getHosts();
+            result.add(splitCreator.create(path, length - bytesRemaining, fileSplitSize,
+                    length, fileSplitSize, modificationTime, hosts, partitionValues));
+        }
+        if (bytesRemaining != 0L) {
+            int location = getBlockIndex(blockLocations, length - bytesRemaining);
+            String[] hosts = location == -1 ? null : blockLocations[location].getHosts();
+            result.add(splitCreator.create(path, length - bytesRemaining, bytesRemaining,
+                    length, fileSplitSize, modificationTime, hosts, partitionValues));
+        }
+
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("Path {} includes {} splits.", path, result.size());
+        }
+        return result;
+    }
+
+    private static int getBlockIndex(BlockLocation[] blkLocations, long offset) {
+        if (blkLocations == null || blkLocations.length == 0) {
+            return -1;
+        }
+        for (int i = 0; i < blkLocations.length; ++i) {
+            if (blkLocations[i].getOffset() <= offset
+                    && offset < blkLocations[i].getOffset() + blkLocations[i].getLength()) {
+                return i;
+            }
+        }
+        BlockLocation last = blkLocations[blkLocations.length - 1];
+        long fileLength = last.getOffset() + last.getLength() - 1L;
+        throw new IllegalArgumentException(String.format("Offset %d is outside of file (0..%d)", offset, fileLength));
+    }
+
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java
index 928854b91d1810..a26abc7fc5e037 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java
@@ -69,7 +69,7 @@ public SplitAssignment(
     }
 
     public void init() throws UserException {
-        splitGenerator.startSplit();
+        splitGenerator.startSplit(backendPolicy.numBackends());
         synchronized (assignLock) {
             while (sampleSplit == null && waitFirstSplit()) {
                 try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitCreator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitCreator.java
index 4df30459db7021..6df84d2f0f5ee9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitCreator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitCreator.java
@@ -23,6 +23,6 @@
 import java.util.List;
 
 public interface SplitCreator {
-    Split create(LocationPath path, long start, long length, long fileLength,
+    Split create(LocationPath path, long start, long length, long fileLength, long fileSplitSize,
             long modificationTime, String[] hosts, List<String> partitionValues);
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java
index c4a373bc85b3b1..34ff3911445bfe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java
@@ -30,8 +30,9 @@
 public interface SplitGenerator {
     /**
      * Get all file splits if the producer doesn't support batch mode.
+     * @param numBackends the number of backends, this is useful when determine the number of splits.
      */
-    default List<Split> getSplits() throws UserException {
+    default List<Split> getSplits(int numBackends) throws UserException {
         // todo: remove this interface if batch mode is stable
         throw new NotImplementedException("Not implement");
     }
@@ -51,7 +52,7 @@ default int numApproximateSplits() {
         return -1;
     }
 
-    default void startSplit() {
+    default void startSplit(int numBackends) {
     }
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index 99d3cd1cd21622..35b21c368ea9f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -32,6 +32,7 @@
 import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.FileQueryScanNode;
 import org.apache.doris.datasource.FileSplit;
+import org.apache.doris.datasource.FileSplitter;
 import org.apache.doris.datasource.hive.HMSExternalCatalog;
 import org.apache.doris.datasource.hive.HMSExternalTable;
 import org.apache.doris.datasource.hive.HiveMetaStoreCache;
@@ -44,12 +45,14 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.thrift.TFileAttributes;
 import org.apache.doris.thrift.TFileCompressType;
 import org.apache.doris.thrift.TFileFormatType;
 import org.apache.doris.thrift.TFileTextScanRangeParams;
+import org.apache.doris.thrift.TPushAggOp;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
@@ -98,15 +101,13 @@ public class HiveScanNode extends FileQueryScanNode {
      * eg: s3 tvf
      * These scan nodes do not have corresponding catalog/database/table info, so no need to do priv check
      */
-    public HiveScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
-        super(id, desc, "HIVE_SCAN_NODE", StatisticalType.HIVE_SCAN_NODE, needCheckColumnPriv);
-        hmsTable = (HMSExternalTable) desc.getTable();
-        brokerName = hmsTable.getCatalog().bindBrokerName();
+    public HiveScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv, SessionVariable sv) {
+        this(id, desc, "HIVE_SCAN_NODE", StatisticalType.HIVE_SCAN_NODE, needCheckColumnPriv, sv);
     }
 
     public HiveScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName,
-                        StatisticalType statisticalType, boolean needCheckColumnPriv) {
-        super(id, desc, planNodeName, statisticalType, needCheckColumnPriv);
+            StatisticalType statisticalType, boolean needCheckColumnPriv, SessionVariable sv) {
+        super(id, desc, planNodeName, statisticalType, needCheckColumnPriv, sv);
         hmsTable = (HMSExternalTable) desc.getTable();
         brokerName = hmsTable.getCatalog().bindBrokerName();
     }
@@ -163,7 +164,7 @@ protected List<HivePartition> getPartitions() throws AnalysisException {
     }
 
     @Override
-    public List<Split> getSplits() throws UserException {
+    public List<Split> getSplits(int numBackends) throws UserException {
         long start = System.currentTimeMillis();
         try {
             if (!partitionInit) {
@@ -174,7 +175,7 @@ public List<Split> getSplits() throws UserException {
                     .getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog());
             String bindBrokerName = hmsTable.getCatalog().bindBrokerName();
             List<Split> allFiles = Lists.newArrayList();
-            getFileSplitByPartitions(cache, prunedPartitions, allFiles, bindBrokerName);
+            getFileSplitByPartitions(cache, prunedPartitions, allFiles, bindBrokerName, numBackends);
             if (ConnectContext.get().getExecutor() != null) {
                 ConnectContext.get().getExecutor().getSummaryProfile().setGetPartitionFilesFinishTime();
             }
@@ -193,7 +194,7 @@ public List<Split> getSplits() throws UserException {
     }
 
     @Override
-    public void startSplit() {
+    public void startSplit(int numBackends) {
         if (prunedPartitions.isEmpty()) {
             splitAssignment.finishSchedule();
             return;
@@ -214,12 +215,12 @@ public void startSplit() {
                         try {
                             List<Split> allFiles = Lists.newArrayList();
                             getFileSplitByPartitions(
-                                    cache, Collections.singletonList(partition), allFiles, bindBrokerName);
+                                    cache, Collections.singletonList(partition), allFiles, bindBrokerName, numBackends);
                             if (allFiles.size() > numSplitsPerPartition.get()) {
                                 numSplitsPerPartition.set(allFiles.size());
                             }
                             splitAssignment.addToQueue(allFiles);
-                        } catch (IOException e) {
+                        } catch (Exception e) {
                             batchException.set(new UserException(e.getMessage(), e));
                         } finally {
                             splittersOnFlight.release();
@@ -263,7 +264,7 @@ public int numApproximateSplits() {
     }
 
     private void getFileSplitByPartitions(HiveMetaStoreCache cache, List<HivePartition> partitions,
-                                          List<Split> allFiles, String bindBrokerName) throws IOException {
+            List<Split> allFiles, String bindBrokerName, int numBackends) throws IOException, UserException {
         List<FileCacheValue> fileCaches;
         if (hiveTransaction != null) {
             fileCaches = getFileSplitByTransaction(cache, partitions, bindBrokerName);
@@ -276,11 +277,34 @@ private void getFileSplitByPartitions(HiveMetaStoreCache cache, List<HivePartiti
             splitAllFiles(allFiles, hiveFileStatuses);
             return;
         }
+
+        /**
+         * If the push down aggregation operator is COUNT,
+         * we don't need to split the file because for parquet/orc format, only metadata is read.
+         * If we split the file, we will read metadata of a file multiple times, which is not efficient.
+         *
+         * - Hive Transactional Table may need merge on read, so do not apply this optimization.
+         * - If the file format is not parquet/orc, eg, text, we need to split the file to increase the parallelism.
+         */
+        boolean needSplit = true;
+        if (getPushDownAggNoGroupingOp() == TPushAggOp.COUNT
+                && hiveTransaction != null) {
+            int totalFileNum = 0;
+            for (FileCacheValue fileCacheValue : fileCaches) {
+                if (fileCacheValue.getFiles() != null) {
+                    totalFileNum += fileCacheValue.getFiles().size();
+                }
+            }
+            int parallelNum = sessionVariable.getParallelExecInstanceNum();
+            needSplit = FileSplitter.needSplitForCountPushdown(parallelNum, numBackends, totalFileNum);
+        }
         for (HiveMetaStoreCache.FileCacheValue fileCacheValue : fileCaches) {
             if (fileCacheValue.getFiles() != null) {
                 boolean isSplittable = fileCacheValue.isSplittable();
                 for (HiveMetaStoreCache.HiveFileStatus status : fileCacheValue.getFiles()) {
-                    allFiles.addAll(splitFile(status.getPath(), status.getBlockSize(),
+                    allFiles.addAll(FileSplitter.splitFile(status.getPath(),
+                            // set block size to Long.MAX_VALUE to avoid splitting the file.
+                            getRealFileSplitSize(needSplit ? status.getBlockSize() : Long.MAX_VALUE),
                             status.getBlockLocations(), status.getLength(), status.getModificationTime(),
                             isSplittable, fileCacheValue.getPartitionValues(),
                             new HiveSplitCreator(fileCacheValue.getAcidInfo())));
@@ -292,7 +316,7 @@ private void getFileSplitByPartitions(HiveMetaStoreCache cache, List<HivePartiti
     private void splitAllFiles(List<Split> allFiles,
                                List<HiveMetaStoreCache.HiveFileStatus> hiveFileStatuses) throws IOException {
         for (HiveMetaStoreCache.HiveFileStatus status : hiveFileStatuses) {
-            allFiles.addAll(splitFile(status.getPath(), status.getBlockSize(),
+            allFiles.addAll(FileSplitter.splitFile(status.getPath(), getRealFileSplitSize(status.getBlockSize()),
                     status.getBlockLocations(), status.getLength(), status.getModificationTime(),
                     status.isSplittable(), status.getPartitionValues(),
                     new HiveSplitCreator(status.getAcidInfo())));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveSplit.java
index 5dd63e734c9c91..58bfb32e617e34 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveSplit.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveSplit.java
@@ -54,9 +54,13 @@ public HiveSplitCreator(AcidInfo acidInfo) {
 
         @Override
         public Split create(LocationPath path, long start, long length, long fileLength,
+                long fileSplitSize,
                 long modificationTime, String[] hosts,
                 List<String> partitionValues) {
-            return new HiveSplit(path, start, length, fileLength, modificationTime, hosts, partitionValues, acidInfo);
+            HiveSplit split =  new HiveSplit(path, start, length, fileLength, modificationTime,
+                    hosts, partitionValues, acidInfo);
+            split.setTargetSplitSize(fileSplitSize);
+            return split;
         }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
index b2cad8ab710178..e1dfaa40aefce1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
@@ -125,8 +125,8 @@ public class HudiScanNode extends HiveScanNode {
      */
     public HudiScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv,
             Optional<TableScanParams> scanParams, Optional<IncrementalRelation> incrementalRelation,
-            SessionVariable sessionVariable) {
-        super(id, desc, "HUDI_SCAN_NODE", StatisticalType.HUDI_SCAN_NODE, needCheckColumnPriv);
+            SessionVariable sv) {
+        super(id, desc, "HUDI_SCAN_NODE", StatisticalType.HUDI_SCAN_NODE, needCheckColumnPriv, sv);
         isCowTable = hmsTable.isHoodieCowTable();
         if (LOG.isDebugEnabled()) {
             if (isCowTable) {
@@ -390,7 +390,7 @@ private void getPartitionsSplits(List<HivePartition> partitions, List<Split> spl
     }
 
     @Override
-    public List<Split> getSplits() throws UserException {
+    public List<Split> getSplits(int numBackends) throws UserException {
         if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) {
             return getIncrementalSplits();
         }
@@ -406,7 +406,7 @@ public List<Split> getSplits() throws UserException {
     }
 
     @Override
-    public void startSplit() {
+    public void startSplit(int numBackends) {
         if (prunedPartitions.isEmpty()) {
             splitAssignment.finishSchedule();
             return;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index f7b58158d1a72c..c78140b9d3cd99 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -36,7 +36,7 @@
 import org.apache.doris.datasource.iceberg.IcebergExternalTable;
 import org.apache.doris.datasource.iceberg.IcebergUtils;
 import org.apache.doris.planner.PlanNodeId;
-import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.thrift.TExplainLevel;
@@ -87,7 +87,13 @@ public class IcebergScanNode extends FileQueryScanNode {
     private IcebergSource source;
     private Table icebergTable;
     private List<String> pushdownIcebergPredicates = Lists.newArrayList();
-    private boolean pushDownCount = false;
+    // If tableLevelPushDownCount is true, means we can do count push down opt at table level.
+    // which means all splits have no position/equality delete files,
+    // so for query like "select count(*) from ice_tbl", we can get count from snapshot row count info directly.
+    // If tableLevelPushDownCount is false, means we can't do count push down opt at table level,
+    // But for part of splits which have no position/equality delete files, we can still do count push down opt.
+    // And for split level count push down opt, the flag is set in each split.
+    private boolean tableLevelPushDownCount = false;
     private static final long COUNT_WITH_PARALLEL_SPLITS = 10000;
 
     /**
@@ -96,8 +102,8 @@ public class IcebergScanNode extends FileQueryScanNode {
      * eg: s3 tvf
      * These scan nodes do not have corresponding catalog/database/table info, so no need to do priv check
      */
-    public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
-        super(id, desc, "ICEBERG_SCAN_NODE", StatisticalType.ICEBERG_SCAN_NODE, needCheckColumnPriv);
+    public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv, SessionVariable sv) {
+        super(id, desc, "ICEBERG_SCAN_NODE", StatisticalType.ICEBERG_SCAN_NODE, needCheckColumnPriv, sv);
 
         ExternalTable table = (ExternalTable) desc.getTable();
         if (table instanceof HMSExternalTable) {
@@ -140,8 +146,8 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli
         int formatVersion = icebergSplit.getFormatVersion();
         fileDesc.setFormatVersion(formatVersion);
         fileDesc.setOriginalFilePath(icebergSplit.getOriginalPath());
-        if (pushDownCount) {
-            fileDesc.setRowCount(icebergSplit.getRowCount());
+        if (tableLevelPushDownCount) {
+            fileDesc.setRowCount(icebergSplit.getTableLevelRowCount());
         }
         if (formatVersion < MIN_DELETE_FILE_SUPPORT_VERSION) {
             fileDesc.setContent(FileContent.DATA.id());
@@ -177,11 +183,12 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli
     }
 
     @Override
-    public List<Split> getSplits() throws UserException {
-        return HiveMetaStoreClientHelper.ugiDoAs(source.getCatalog().getConfiguration(), this::doGetSplits);
+    public List<Split> getSplits(int numBackends) throws UserException {
+        return HiveMetaStoreClientHelper.ugiDoAs(source.getCatalog().getConfiguration(),
+                () -> doGetSplits(numBackends));
     }
 
-    private List<Split> doGetSplits() throws UserException {
+    private List<Split> doGetSplits(int numBackends) throws UserException {
         TableScan scan = icebergTable.newScan();
 
         // set snapshot
@@ -209,9 +216,10 @@ private List<Split> doGetSplits() throws UserException {
         HashSet<String> partitionPathSet = new HashSet<>();
         boolean isPartitionedTable = icebergTable.spec().isPartitioned();
 
-        CloseableIterable<FileScanTask> fileScanTasks = TableScanUtil.splitFiles(scan.planFiles(), fileSplitSize);
+        long realFileSplitSize = getRealFileSplitSize(0);
+        CloseableIterable<FileScanTask> fileScanTasks = TableScanUtil.splitFiles(scan.planFiles(), realFileSplitSize);
         try (CloseableIterable<CombinedScanTask> combinedScanTasks =
-                TableScanUtil.planTasks(fileScanTasks, fileSplitSize, 1, 0)) {
+                TableScanUtil.planTasks(fileScanTasks, realFileSplitSize, 1, 0)) {
             combinedScanTasks.forEach(taskGrp -> taskGrp.files().forEach(splitTask -> {
                 List<String> partitionValues = new ArrayList<>();
                 if (isPartitionedTable) {
@@ -250,6 +258,7 @@ private List<Split> doGetSplits() throws UserException {
                         source.getCatalog().getProperties(),
                         partitionValues,
                         originalPath);
+                split.setTargetSplitSize(realFileSplitSize);
                 if (formatVersion >= MIN_DELETE_FILE_SUPPORT_VERSION) {
                     split.setDeleteFileFilters(getDeleteFileFilters(splitTask));
                 }
@@ -268,11 +277,11 @@ private List<Split> doGetSplits() throws UserException {
             }
             long countFromSnapshot = getCountFromSnapshot();
             if (countFromSnapshot >= 0) {
-                pushDownCount = true;
+                tableLevelPushDownCount = true;
                 List<Split> pushDownCountSplits;
                 if (countFromSnapshot > COUNT_WITH_PARALLEL_SPLITS) {
-                    int parallelNum = ConnectContext.get().getSessionVariable().getParallelExecInstanceNum();
-                    pushDownCountSplits = splits.subList(0, Math.min(splits.size(), parallelNum));
+                    int minSplits = sessionVariable.getParallelExecInstanceNum() * numBackends;
+                    pushDownCountSplits = splits.subList(0, Math.min(splits.size(), minSplits));
                 } else {
                     pushDownCountSplits = Collections.singletonList(splits.get(0));
                 }
@@ -282,7 +291,6 @@ private List<Split> doGetSplits() throws UserException {
         }
 
         selectedPartitionNum = partitionPathSet.size();
-        splits.forEach(s -> s.setTargetSplitSize(fileSplitSize));
         return splits;
     }
 
@@ -422,8 +430,8 @@ private void assignCountToSplits(List<Split> splits, long totalCount) {
         int size = splits.size();
         long countPerSplit = totalCount / size;
         for (int i = 0; i < size - 1; i++) {
-            ((IcebergSplit) splits.get(i)).setRowCount(countPerSplit);
+            ((IcebergSplit) splits.get(i)).setTableLevelRowCount(countPerSplit);
         }
-        ((IcebergSplit) splits.get(size - 1)).setRowCount(countPerSplit + totalCount % size);
+        ((IcebergSplit) splits.get(size - 1)).setTableLevelRowCount(countPerSplit + totalCount % size);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java
index 580d3cf1bb23f3..0520612935a778 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java
@@ -37,7 +37,8 @@ public class IcebergSplit extends FileSplit {
     private Integer formatVersion;
     private List<IcebergDeleteFileFilter> deleteFileFilters;
     private Map<String, String> config;
-    private long rowCount = -1;
+    // tableLevelRowCount will be set only table-level count push down opt is available.
+    private long tableLevelRowCount = -1;
 
     // File path will be changed if the file is modified, so there's no need to get modification time.
     public IcebergSplit(LocationPath file, long start, long length, long fileLength, String[] hosts,
@@ -50,14 +51,6 @@ public IcebergSplit(LocationPath file, long start, long length, long fileLength,
         this.selfSplitWeight = length;
     }
 
-    public long getRowCount() {
-        return rowCount;
-    }
-
-    public void setRowCount(long rowCount) {
-        this.rowCount = rowCount;
-    }
-
     public void setDeleteFileFilters(List<IcebergDeleteFileFilter> deleteFileFilters) {
         this.deleteFileFilters = deleteFileFilters;
         this.selfSplitWeight += deleteFileFilters.stream().mapToLong(IcebergDeleteFileFilter::getFilesize).sum();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java
index 21bf1e794560cc..f668661944272b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java
@@ -28,6 +28,7 @@
 import org.apache.doris.datasource.property.constants.OssProperties;
 import org.apache.doris.datasource.property.constants.S3Properties;
 import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.thrift.TFileFormatType;
@@ -80,8 +81,8 @@ public class LakeSoulScanNode extends FileQueryScanNode {
 
     String readType;
 
-    public LakeSoulScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
-        super(id, desc, "planNodeName", StatisticalType.LAKESOUL_SCAN_NODE, needCheckColumnPriv);
+    public LakeSoulScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv, SessionVariable sv) {
+        super(id, desc, "planNodeName", StatisticalType.LAKESOUL_SCAN_NODE, needCheckColumnPriv, sv);
     }
 
     @Override
@@ -209,7 +210,7 @@ private void setLakeSoulParams(TFileRangeDesc rangeDesc, LakeSoulSplit lakeSoulS
         rangeDesc.setTableFormatParams(tableFormatFileDesc);
     }
 
-    public List<Split> getSplits() throws UserException {
+    public List<Split> getSplits(int numBackends) throws UserException {
         if (LOG.isDebugEnabled()) {
             LOG.debug("getSplits with columnFilters={}", columnFilters);
             LOG.debug("getSplits with columnNameToRange={}", columnNameToRange);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index 9fa22a0fffaab8..e4bb8b5e9dcc53 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -44,7 +44,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
 import org.apache.doris.nereids.util.DateUtils;
 import org.apache.doris.planner.PlanNodeId;
-import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.thrift.TFileFormatType;
@@ -107,21 +107,23 @@ public class MaxComputeScanNode extends FileQueryScanNode {
 
     // For new planner
     public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc,
-            SelectedPartitions selectedPartitions, boolean needCheckColumnPriv) {
+            SelectedPartitions selectedPartitions, boolean needCheckColumnPriv,
+            SessionVariable sv) {
         this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE,
-                selectedPartitions, needCheckColumnPriv);
+                selectedPartitions, needCheckColumnPriv, sv);
     }
 
     // For old planner
-    public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
+    public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv,
+            SessionVariable sv) {
         this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE,
-                SelectedPartitions.NOT_PRUNED, needCheckColumnPriv);
+                SelectedPartitions.NOT_PRUNED, needCheckColumnPriv, sv);
     }
 
     private MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName,
             StatisticalType statisticalType, SelectedPartitions selectedPartitions,
-            boolean needCheckColumnPriv) {
-        super(id, desc, planNodeName, statisticalType, needCheckColumnPriv);
+            boolean needCheckColumnPriv, SessionVariable sv) {
+        super(id, desc, planNodeName, statisticalType, needCheckColumnPriv, sv);
         table = (MaxComputeExternalTable) desc.getTable();
         this.selectedPartitions = selectedPartitions;
     }
@@ -214,7 +216,7 @@ public boolean isBatchMode() {
             return false;
         }
 
-        int numPartitions = ConnectContext.get().getSessionVariable().getNumPartitionsInBatchMode();
+        int numPartitions = sessionVariable.getNumPartitionsInBatchMode();
         return numPartitions > 0
                 && selectedPartitions != SelectedPartitions.NOT_PRUNED
                 && selectedPartitions.selectedPartitions.size() >= numPartitions;
@@ -226,7 +228,7 @@ public int numApproximateSplits() {
     }
 
     @Override
-    public void startSplit() {
+    public void startSplit(int numBackends) {
         this.totalPartitionNum = selectedPartitions.totalPartitionNum;
         this.selectedPartitionNum = selectedPartitions.selectedPartitions.size();
 
@@ -241,8 +243,7 @@ public void startSplit() {
                 (key, value) -> requiredPartitionSpecs.add(new PartitionSpec(key))
         );
 
-
-        int batchNumPartitions = ConnectContext.get().getSessionVariable().getNumPartitionsInBatchMode();
+        int batchNumPartitions = sessionVariable.getNumPartitionsInBatchMode();
 
         Executor scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor();
         AtomicReference<UserException> batchException = new AtomicReference<>(null);
@@ -546,7 +547,7 @@ protected Map<String, String> getLocationProperties() throws UserException {
         return new HashMap<>();
     }
 
-    List<Split> getSplitByTableSession(TableBatchReadSession tableBatchReadSession) throws java.io.IOException {
+    private List<Split> getSplitByTableSession(TableBatchReadSession tableBatchReadSession) throws IOException {
         List<Split> result = new ArrayList<>();
         String scanSessionSerialize =  serializeSession(tableBatchReadSession);
         InputSplitAssigner assigner = tableBatchReadSession.getInputSplitAssigner();
@@ -595,9 +596,8 @@ List<Split> getSplitByTableSession(TableBatchReadSession tableBatchReadSession)
         return result;
     }
 
-
     @Override
-    public List<Split> getSplits() throws UserException {
+    public List<Split> getSplits(int numBackends) throws UserException {
         List<Split> result = new ArrayList<>();
         com.aliyun.odps.Table odpsTable = table.getOdpsTable();
         if (desc.getSlots().isEmpty() || odpsTable.getFileNum() <= 0) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index 5009ec3c9049fb..28efbc58f51b84 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -25,6 +25,7 @@
 import org.apache.doris.common.util.FileFormatUtils;
 import org.apache.doris.common.util.LocationPath;
 import org.apache.doris.datasource.FileQueryScanNode;
+import org.apache.doris.datasource.FileSplitter;
 import org.apache.doris.datasource.paimon.PaimonExternalCatalog;
 import org.apache.doris.datasource.paimon.PaimonExternalTable;
 import org.apache.doris.planner.PlanNodeId;
@@ -36,6 +37,7 @@
 import org.apache.doris.thrift.TFileRangeDesc;
 import org.apache.doris.thrift.TPaimonDeletionFileDesc;
 import org.apache.doris.thrift.TPaimonFileDesc;
+import org.apache.doris.thrift.TPushAggOp;
 import org.apache.doris.thrift.TTableFormatFileDesc;
 
 import com.google.common.base.Preconditions;
@@ -101,15 +103,16 @@ public String toString() {
     private int rawFileSplitNum = 0;
     private int paimonSplitNum = 0;
     private List<SplitStat> splitStats = new ArrayList<>();
-    private SessionVariable sessionVariable;
     private String serializedTable;
 
+    private boolean pushDownCount = false;
+    private static final long COUNT_WITH_PARALLEL_SPLITS = 10000;
+
     public PaimonScanNode(PlanNodeId id,
             TupleDescriptor desc,
             boolean needCheckColumnPriv,
-            SessionVariable sessionVariable) {
-        super(id, desc, "PAIMON_SCAN_NODE", StatisticalType.PAIMON_SCAN_NODE, needCheckColumnPriv);
-        this.sessionVariable = sessionVariable;
+            SessionVariable sv) {
+        super(id, desc, "PAIMON_SCAN_NODE", StatisticalType.PAIMON_SCAN_NODE, needCheckColumnPriv, sv);
     }
 
     @Override
@@ -199,7 +202,7 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit)
     }
 
     @Override
-    public List<Split> getSplits() throws UserException {
+    public List<Split> getSplits(int numBackends) throws UserException {
         boolean forceJniScanner = sessionVariable.isForceJniScanner();
         SessionVariable.IgnoreSplitType ignoreSplitType = SessionVariable.IgnoreSplitType
                 .valueOf(sessionVariable.getIgnoreSplitType());
@@ -211,6 +214,8 @@ public List<Split> getSplits() throws UserException {
         List<org.apache.paimon.table.source.Split> paimonSplits = readBuilder.withFilter(predicates)
                 .withProjection(projected)
                 .newScan().plan().splits();
+
+        boolean applyCountPushdown = getPushDownAggNoGroupingOp() == TPushAggOp.COUNT;
         // Just for counting the number of selected partitions for this paimon table
         Set<BinaryRow> selectedPartitionValues = Sets.newHashSet();
         for (org.apache.paimon.table.source.Split split : paimonSplits) {
@@ -238,9 +243,9 @@ public List<Split> getSplits() throws UserException {
                             LocationPath locationPath = new LocationPath(file.path(),
                                     source.getCatalog().getProperties());
                             try {
-                                List<Split> dorisSplits = splitFile(
+                                List<Split> dorisSplits = FileSplitter.splitFile(
                                         locationPath,
-                                        0,
+                                        getRealFileSplitSize(0),
                                         null,
                                         file.length(),
                                         -1,
@@ -261,25 +266,7 @@ public List<Split> getSplits() throws UserException {
                             }
                         }
                     } else {
-                        for (RawFile file : rawFiles) {
-                            LocationPath locationPath = new LocationPath(file.path(),
-                                    source.getCatalog().getProperties());
-                            try {
-                                splits.addAll(
-                                        splitFile(
-                                                locationPath,
-                                                0,
-                                                null,
-                                                file.length(),
-                                                -1,
-                                                true,
-                                                null,
-                                                PaimonSplit.PaimonSplitCreator.DEFAULT));
-                                ++rawFileSplitNum;
-                            } catch (IOException e) {
-                                throw new UserException("Paimon error to split file: " + e.getMessage(), e);
-                            }
-                        }
+                        createRawFileSplits(rawFiles, splits, applyCountPushdown ? Long.MAX_VALUE : 0);
                     }
                 } else {
                     if (ignoreSplitType == SessionVariable.IgnoreSplitType.IGNORE_JNI) {
@@ -297,14 +284,34 @@ public List<Split> getSplits() throws UserException {
             }
             splitStats.add(splitStat);
         }
+
         this.selectedPartitionNum = selectedPartitionValues.size();
         // TODO: get total partition number
-        // We should set fileSplitSize at the end because fileSplitSize may be modified
-        // in splitFile.
-        splits.forEach(s -> s.setTargetSplitSize(fileSplitSize));
         return splits;
     }
 
+    private void createRawFileSplits(List<RawFile> rawFiles, List<Split> splits, long blockSize) throws UserException {
+        for (RawFile file : rawFiles) {
+            LocationPath locationPath = new LocationPath(file.path(),
+                    source.getCatalog().getProperties());
+            try {
+                splits.addAll(
+                        FileSplitter.splitFile(
+                                locationPath,
+                                getRealFileSplitSize(blockSize),
+                                null,
+                                file.length(),
+                                -1,
+                                true,
+                                null,
+                                PaimonSplit.PaimonSplitCreator.DEFAULT));
+                ++rawFileSplitNum;
+            } catch (IOException e) {
+                throw new UserException("Paimon error to split file: " + e.getMessage(), e);
+            }
+        }
+    }
+
     private String getFileFormat(String path) {
         return FileFormatUtils.getFileFormatBySuffix(path).orElse(source.getFileFormatFromTableProperties());
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java
index 3ab38c7db28e9e..988f043ad0e7d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java
@@ -38,7 +38,6 @@ public class PaimonSplit extends FileSplit {
     private TableFormatType tableFormatType;
     private Optional<DeletionFile> optDeletionFile;
 
-
     public PaimonSplit(Split split) {
         super(DUMMY_PATH, 0, 0, 0, 0, null, null);
         this.split = split;
@@ -100,10 +99,14 @@ public org.apache.doris.spi.Split create(LocationPath path,
                 long start,
                 long length,
                 long fileLength,
+                long fileSplitSize,
                 long modificationTime,
                 String[] hosts,
                 List<String> partitionValues) {
-            return new PaimonSplit(path, start, length, fileLength, modificationTime, hosts, partitionValues);
+            PaimonSplit split =  new PaimonSplit(path, start, length, fileLength,
+                    modificationTime, hosts, partitionValues);
+            split.setTargetSplitSize(fileSplitSize);
+            return split;
         }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
index 6f660993d6350a..50c1d5752a1fe3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
@@ -28,6 +28,7 @@
 import org.apache.doris.datasource.TableFormatType;
 import org.apache.doris.datasource.trinoconnector.TrinoConnectorPluginLoader;
 import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.thrift.TFileAttributes;
@@ -97,8 +98,10 @@ public class TrinoConnectorScanNode extends FileQueryScanNode {
     private ConnectorMetadata connectorMetadata;
     private Constraint constraint;
 
-    public TrinoConnectorScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
-        super(id, desc, "TRINO_CONNECTOR_SCAN_NODE", StatisticalType.TRINO_CONNECTOR_SCAN_NODE, needCheckColumnPriv);
+    public TrinoConnectorScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv,
+            SessionVariable sv) {
+        super(id, desc, "TRINO_CONNECTOR_SCAN_NODE", StatisticalType.TRINO_CONNECTOR_SCAN_NODE, needCheckColumnPriv,
+                sv);
     }
 
     @Override
@@ -129,7 +132,7 @@ protected void convertPredicate() {
     }
 
     @Override
-    public List<Split> getSplits() throws UserException {
+    public List<Split> getSplits(int numBackends) throws UserException {
         // 1. Get necessary objects
         Connector connector = source.getConnector();
         connectorMetadata = source.getConnectorMetadata();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
index b0f0406c215c34..5e65093036517e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
@@ -29,7 +29,9 @@
 import org.apache.doris.datasource.FileQueryScanNode;
 import org.apache.doris.datasource.FileSplit;
 import org.apache.doris.datasource.FileSplit.FileSplitCreator;
+import org.apache.doris.datasource.FileSplitter;
 import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.system.Backend;
@@ -40,6 +42,7 @@
 import org.apache.doris.thrift.TFileCompressType;
 import org.apache.doris.thrift.TFileFormatType;
 import org.apache.doris.thrift.TFileType;
+import org.apache.doris.thrift.TPushAggOp;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -63,8 +66,8 @@ public class TVFScanNode extends FileQueryScanNode {
      * eg: s3 tvf
      * These scan nodes do not have corresponding catalog/database/table info, so no need to do priv check
      */
-    public TVFScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
-        super(id, desc, "TVF_SCAN_NODE", StatisticalType.TVF_SCAN_NODE, needCheckColumnPriv);
+    public TVFScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv, SessionVariable sv) {
+        super(id, desc, "TVF_SCAN_NODE", StatisticalType.TVF_SCAN_NODE, needCheckColumnPriv, sv);
         table = (FunctionGenTable) this.desc.getTable();
         tableValuedFunction = (ExternalFileTableValuedFunction) table.getTvf();
     }
@@ -126,16 +129,27 @@ public TableIf getTargetTable() {
     }
 
     @Override
-    public List<Split> getSplits() throws UserException {
+    public List<Split> getSplits(int numBackends) throws UserException {
         List<Split> splits = Lists.newArrayList();
         if (tableValuedFunction.getTFileType() == TFileType.FILE_STREAM) {
             return splits;
         }
+
         List<TBrokerFileStatus> fileStatuses = tableValuedFunction.getFileStatuses();
+
+        // Push down count optimization.
+        boolean needSplit = true;
+        if (getPushDownAggNoGroupingOp() == TPushAggOp.COUNT) {
+            int parallelNum = sessionVariable.getParallelExecInstanceNum();
+            int totalFileNum = fileStatuses.size();
+            needSplit = FileSplitter.needSplitForCountPushdown(parallelNum, numBackends, totalFileNum);
+        }
+
         for (TBrokerFileStatus fileStatus : fileStatuses) {
             Map<String, String> prop = Maps.newHashMap();
             try {
-                splits.addAll(splitFile(new LocationPath(fileStatus.getPath(), prop), fileStatus.getBlockSize(),
+                splits.addAll(FileSplitter.splitFile(new LocationPath(fileStatus.getPath(), prop),
+                        getRealFileSplitSize(needSplit ? fileStatus.getBlockSize() : Long.MAX_VALUE),
                         null, fileStatus.getSize(),
                         fileStatus.getModificationTime(), fileStatus.isSplitable, null,
                         FileSplitCreator.DEFAULT));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index bea5eec432b2ab..d66ff3ebb7649d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -196,6 +196,7 @@
 import org.apache.doris.planner.TableFunctionNode;
 import org.apache.doris.planner.UnionNode;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.statistics.StatisticConstants;
 import org.apache.doris.tablefunction.TableValuedFunctionIf;
 import org.apache.doris.thrift.TFetchOption;
@@ -552,15 +553,16 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla
         ExternalTable table = fileScan.getTable();
         TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context);
 
+        SessionVariable sv = ConnectContext.get().getSessionVariable();
         // TODO(cmy): determine the needCheckColumnPriv param
         ScanNode scanNode;
         if (table instanceof HMSExternalTable) {
             switch (((HMSExternalTable) table).getDlaType()) {
                 case ICEBERG:
-                    scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
+                    scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv);
                     break;
                 case HIVE:
-                    scanNode = new HiveScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
+                    scanNode = new HiveScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv);
                     HiveScanNode hiveScanNode = (HiveScanNode) scanNode;
                     hiveScanNode.setSelectedPartitions(fileScan.getSelectedPartitions());
                     if (fileScan.getTableSample().isPresent()) {
@@ -572,17 +574,16 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla
                     throw new RuntimeException("do not support DLA type " + ((HMSExternalTable) table).getDlaType());
             }
         } else if (table instanceof IcebergExternalTable) {
-            scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
+            scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv);
         } else if (table instanceof PaimonExternalTable) {
-            scanNode = new PaimonScanNode(context.nextPlanNodeId(), tupleDescriptor, false,
-                ConnectContext.get().getSessionVariable());
+            scanNode = new PaimonScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv);
         } else if (table instanceof TrinoConnectorExternalTable) {
-            scanNode = new TrinoConnectorScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
+            scanNode = new TrinoConnectorScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv);
         } else if (table instanceof MaxComputeExternalTable) {
             scanNode = new MaxComputeScanNode(context.nextPlanNodeId(), tupleDescriptor,
-                    fileScan.getSelectedPartitions(), false);
+                    fileScan.getSelectedPartitions(), false, sv);
         } else if (table instanceof LakeSoulExternalTable) {
-            scanNode = new LakeSoulScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
+            scanNode = new LakeSoulScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv);
         } else {
             throw new RuntimeException("do not support table type " + table.getType());
         }
@@ -908,7 +909,8 @@ public PlanFragment visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, Pl
         TupleDescriptor tupleDescriptor = generateTupleDesc(slots, tvfRelation.getFunction().getTable(), context);
 
         TableValuedFunctionIf catalogFunction = tvfRelation.getFunction().getCatalogFunction();
-        ScanNode scanNode = catalogFunction.getScanNode(context.nextPlanNodeId(), tupleDescriptor);
+        SessionVariable sv = ConnectContext.get().getSessionVariable();
+        ScanNode scanNode = catalogFunction.getScanNode(context.nextPlanNodeId(), tupleDescriptor, sv);
         scanNode.setNereidsId(tvfRelation.getId());
         Utils.execWithUncheckedException(scanNode::init);
         context.getRuntimeTranslator().ifPresent(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index ae5d562bd5bd30..f29ff4dba32b9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -75,6 +75,7 @@
 import org.apache.doris.datasource.paimon.source.PaimonScanNode;
 import org.apache.doris.datasource.trinoconnector.source.TrinoConnectorScanNode;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException;
 import org.apache.doris.thrift.TPushAggOp;
 
@@ -1916,8 +1917,8 @@ private boolean canMigrateConjuncts(InlineViewRef inlineViewRef) {
      */
     private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt selectStmt)
             throws UserException {
-        ScanNode scanNode = null;
-
+        SessionVariable sv = ConnectContext.get().getSessionVariable();
+        ScanNode scanNode;
         switch (tblRef.getTable().getType()) {
             case OLAP:
             case MATERIALIZED_VIEW:
@@ -1955,7 +1956,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s
                 scanNode = new JdbcScanNode(ctx.getNextNodeId(), tblRef.getDesc(), false);
                 break;
             case TABLE_VALUED_FUNCTION:
-                scanNode = ((TableValuedFunctionRef) tblRef).getScanNode(ctx.getNextNodeId());
+                scanNode = ((TableValuedFunctionRef) tblRef).getScanNode(ctx.getNextNodeId(), sv);
                 break;
             case HMS_EXTERNAL_TABLE:
                 TableIf table = tblRef.getDesc().getTable();
@@ -1968,13 +1969,13 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s
                                     + "please set enable_nereids_planner = true to enable new optimizer");
                         }
                         scanNode = new HudiScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true,
-                                Optional.empty(), Optional.empty(), ConnectContext.get().getSessionVariable());
+                                Optional.empty(), Optional.empty(), sv);
                         break;
                     case ICEBERG:
-                        scanNode = new IcebergScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
+                        scanNode = new IcebergScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true, sv);
                         break;
                     case HIVE:
-                        scanNode = new HiveScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
+                        scanNode = new HiveScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true, sv);
                         ((HiveScanNode) scanNode).setTableSample(tblRef.getTableSample());
                         break;
                     default:
@@ -1982,17 +1983,16 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s
                 }
                 break;
             case ICEBERG_EXTERNAL_TABLE:
-                scanNode = new IcebergScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
+                scanNode = new IcebergScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true, sv);
                 break;
             case PAIMON_EXTERNAL_TABLE:
-                scanNode = new PaimonScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true,
-                    ConnectContext.get().getSessionVariable());
+                scanNode = new PaimonScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true, sv);
                 break;
             case TRINO_CONNECTOR_EXTERNAL_TABLE:
-                scanNode = new TrinoConnectorScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
+                scanNode = new TrinoConnectorScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true, sv);
                 break;
             case MAX_COMPUTE_EXTERNAL_TABLE:
-                scanNode = new MaxComputeScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
+                scanNode = new MaxComputeScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true, sv);
                 break;
             case ES_EXTERNAL_TABLE:
                 scanNode = new EsScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
@@ -2001,7 +2001,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s
                 scanNode = new JdbcScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
                 break;
             case LAKESOUl_EXTERNAL_TABLE:
-                scanNode = new LakeSoulScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);
+                scanNode = new LakeSoulScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true, sv);
                 break;
             case TEST_EXTERNAL_TABLE:
                 scanNode = new TestExternalTableScanNode(ctx.getNextNodeId(), tblRef.getDesc());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/DataGenTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/DataGenTableValuedFunction.java
index 629b410e676590..66f344f03be0a3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/DataGenTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/DataGenTableValuedFunction.java
@@ -22,6 +22,7 @@
 import org.apache.doris.planner.DataGenScanNode;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.planner.ScanNode;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.thrift.TDataGenFunctionName;
 
 import java.util.List;
@@ -32,7 +33,7 @@ public abstract class DataGenTableValuedFunction extends TableValuedFunctionIf {
     public abstract TDataGenFunctionName getDataGenFunctionName();
 
     @Override
-    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc) {
+    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv) {
         return new DataGenScanNode(id, desc, this);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index e137d5d200cc84..6f45a1cc0eb03d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -52,6 +52,7 @@
 import org.apache.doris.proto.Types.PTypeDesc;
 import org.apache.doris.proto.Types.PTypeNode;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.rpc.BackendServiceProxy;
 import org.apache.doris.rpc.RpcException;
 import org.apache.doris.system.Backend;
@@ -301,8 +302,8 @@ public TFileAttributes getFileAttributes() {
     }
 
     @Override
-    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc) {
-        return new TVFScanNode(id, desc, false);
+    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv) {
+        return new TVFScanNode(id, desc, false, sv);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java
index 3bd262f467d6a3..324e17d4f24490 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java
@@ -30,6 +30,7 @@
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.planner.ScanNode;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.thrift.TFileType;
 
 import java.util.ArrayList;
@@ -83,7 +84,7 @@ public List<Column> getTableColumns() throws AnalysisException {
     }
 
     @Override
-    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc) {
+    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv) {
         return new GroupCommitScanNode(id, desc, tableId);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/JdbcQueryTableValueFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/JdbcQueryTableValueFunction.java
index b884dab38824b2..a9847c7eadd329 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/JdbcQueryTableValueFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/JdbcQueryTableValueFunction.java
@@ -26,6 +26,7 @@
 import org.apache.doris.datasource.jdbc.source.JdbcScanNode;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.planner.ScanNode;
+import org.apache.doris.qe.SessionVariable;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -47,7 +48,7 @@ public List<Column> getTableColumns() throws AnalysisException {
     }
 
     @Override
-    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc) {
+    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv) {
         JdbcExternalCatalog catalog = (JdbcExternalCatalog) catalogIf;
         JdbcTable jdbcTable = new JdbcTable(1, desc.getTable().getName(), desc.getTable().getFullSchema(),
                 TableType.JDBC);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
index a7e25bc7f82445..7bd28f363e7523 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
@@ -22,6 +22,7 @@
 import org.apache.doris.datasource.tvf.source.MetadataScanNode;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.planner.ScanNode;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.thrift.TMetaScanRange;
 import org.apache.doris.thrift.TMetadataTableRequestParams;
 import org.apache.doris.thrift.TMetadataType;
@@ -60,7 +61,7 @@ public static Integer getColumnIndexFromColumnName(TMetadataType type, String co
     public abstract TMetaScanRange getMetaScanRange();
 
     @Override
-    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc) {
+    public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv) {
         return new MetadataScanNode(id, desc, this);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/QueryTableValueFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/QueryTableValueFunction.java
index cb0f5100229060..07a125836b75fd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/QueryTableValueFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/QueryTableValueFunction.java
@@ -28,6 +28,7 @@
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.planner.ScanNode;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -87,5 +88,5 @@ public String getTableName() {
     public abstract List<Column> getTableColumns() throws AnalysisException;
 
     @Override
-    public abstract ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc);
+    public abstract ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv);
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
index d4faa46019541c..eb323a7667224a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
@@ -25,6 +25,7 @@
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.planner.ScanNode;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 
 import java.util.List;
 import java.util.Map;
@@ -88,7 +89,7 @@ public static TableValuedFunctionIf getTableFunction(String funcName, Map<String
 
     public abstract List<Column> getTableColumns() throws AnalysisException;
 
-    public abstract ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc);
+    public abstract ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv);
 
     public void checkAuth(ConnectContext ctx) {
 
diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
index f3b44964915230..a394836625d751 100644
--- a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
@@ -578,6 +578,26 @@ bbb
 
 -- !c109 --
 
+-- !c110 --
+3
+
+-- !c111 --
+3
+
+-- !c112 --
+2
+
+-- !c113 --
+2
+
+-- !c114 --
+3	3_1
+4	4_1
+
+-- !c115 --
+3	3_1
+4	4_1
+
 -- !all --
 1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
 10	20	30	40	50	60	70	80	90.1	100.1	110.10	2020-03-02	130str	140varchar	b	false	bbbb	2023-08-14T08:32:52.821
@@ -1157,6 +1177,26 @@ bbb
 
 -- !c109 --
 
+-- !c110 --
+3
+
+-- !c111 --
+3
+
+-- !c112 --
+2
+
+-- !c113 --
+2
+
+-- !c114 --
+3	3_1
+4	4_1
+
+-- !c115 --
+3	3_1
+4	4_1
+
 -- !all --
 1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
 10	20	30	40	50	60	70	80	90.1	100.1	110.10	2020-03-02	130str	140varchar	b	false	bbbb	2023-08-14T08:32:52.821
@@ -1736,6 +1776,26 @@ bbb
 
 -- !c109 --
 
+-- !c110 --
+3
+
+-- !c111 --
+3
+
+-- !c112 --
+2
+
+-- !c113 --
+2
+
+-- !c114 --
+3	3_1
+4	4_1
+
+-- !c115 --
+3	3_1
+4	4_1
+
 -- !all --
 1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
 10	20	30	40	50	60	70	80	90.1	100.1	110.10	2020-03-02	130str	140varchar	b	false	bbbb	2023-08-14T08:32:52.821
@@ -2315,3 +2375,23 @@ bbb
 
 -- !c109 --
 
+-- !c110 --
+3
+
+-- !c111 --
+3
+
+-- !c112 --
+2
+
+-- !c113 --
+2
+
+-- !c114 --
+3	3_1
+4	4_1
+
+-- !c115 --
+3	3_1
+4	4_1
+
diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy
index 306af3b2cb2852..7a9e90a61fe2cb 100644
--- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy
+++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy
@@ -110,7 +110,7 @@ suite("test_iceberg_optimize_count", "p0,external,doris,external_docker,external
 
     } finally {
         sql """ set enable_count_push_down_for_external_table=true; """
-        sql """drop catalog if exists ${catalog_name}"""
+        // sql """drop catalog if exists ${catalog_name}"""
     }
 }
 
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index 41afb02e0f932d..9668cbb0950c5d 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -181,6 +181,13 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_
             def c108= """ select id from tb_with_upper_case where id = 1 """
             def c109= """ select id from tb_with_upper_case where id < 1 """
 
+            def c110 = """select count(*) from deletion_vector_orc;"""
+            def c111 = """select count(*) from deletion_vector_parquet;"""
+            def c112 = """select count(*) from deletion_vector_orc where id > 2;"""
+            def c113 = """select count(*) from deletion_vector_parquet where id > 2;"""
+            def c114 = """select * from deletion_vector_orc where id > 2;"""
+            def c115 = """select * from deletion_vector_parquet where id > 2;"""
+
             String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
             String catalog_name = "ctl_test_paimon_catalog"
             String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
@@ -289,6 +296,13 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_
                 qt_c107 c107
                 qt_c108 c108
                 qt_c109 c109
+
+                qt_c110 c110
+                qt_c111 c111
+                qt_c112 c112
+                qt_c113 c113
+                qt_c114 c114
+                qt_c115 c115
             }
 
             test_cases("false", "false")

From 3d7163ea2cf0eaa13147f89c9c350141a2f24f9e Mon Sep 17 00:00:00 2001
From: zclllhhjj <zhaochangle@selectdb.com>
Date: Tue, 10 Dec 2024 12:05:16 +0800
Subject: [PATCH 247/399] [Enhancement](thread) FE fetch_data use brpc thread
 locally in BE  (#44928)

1. do `fetch_data` in brpc thread locally
2. add more gauge for fragment async thread
3. fix wrong log info for FifoThreadPool
---
 be/src/runtime/buffer_control_block.cpp      |  6 ++--
 be/src/runtime/buffer_control_block.h        | 10 +++---
 be/src/runtime/fragment_mgr.cpp              | 21 +++++-------
 be/src/runtime/fragment_mgr.h                |  3 +-
 be/src/service/internal_service.cpp          | 14 +++-----
 be/src/util/doris_metrics.h                  |  1 +
 be/src/util/threadpool.h                     | 35 +++++++++++---------
 be/src/util/work_thread_pool.hpp             | 14 ++++----
 be/src/vec/sink/writer/async_result_writer.h |  6 ++--
 9 files changed, 53 insertions(+), 57 deletions(-)

diff --git a/be/src/runtime/buffer_control_block.cpp b/be/src/runtime/buffer_control_block.cpp
index 8c1ae79955f317..6f4427746f8103 100644
--- a/be/src/runtime/buffer_control_block.cpp
+++ b/be/src/runtime/buffer_control_block.cpp
@@ -30,12 +30,10 @@
 #include <utility>
 #include <vector>
 
-#include "arrow/record_batch.h"
 #include "arrow/type_fwd.h"
 #include "pipeline/dependency.h"
 #include "runtime/thread_context.h"
 #include "util/runtime_profile.h"
-#include "util/string_util.h"
 #include "util/thrift_util.h"
 #include "vec/core/block.h"
 
@@ -149,8 +147,8 @@ void GetArrowResultBatchCtx::on_data(
     delete this;
 }
 
-BufferControlBlock::BufferControlBlock(const TUniqueId& id, int buffer_size, RuntimeState* state)
-        : _fragment_id(id),
+BufferControlBlock::BufferControlBlock(TUniqueId id, int buffer_size, RuntimeState* state)
+        : _fragment_id(std::move(id)),
           _is_close(false),
           _is_cancelled(false),
           _buffer_limit(buffer_size),
diff --git a/be/src/runtime/buffer_control_block.h b/be/src/runtime/buffer_control_block.h
index 249e1ba7652df7..9060007232e3bd 100644
--- a/be/src/runtime/buffer_control_block.h
+++ b/be/src/runtime/buffer_control_block.h
@@ -21,10 +21,10 @@
 #include <cctz/time_zone.h>
 #include <gen_cpp/PaloInternalService_types.h>
 #include <gen_cpp/Types_types.h>
-#include <stdint.h>
 
 #include <atomic>
 #include <condition_variable>
+#include <cstdint>
 #include <deque>
 #include <list>
 #include <memory>
@@ -34,7 +34,6 @@
 #include "common/status.h"
 #include "runtime/query_statistics.h"
 #include "runtime/runtime_state.h"
-#include "util/hash_util.hpp"
 
 namespace google::protobuf {
 class Closure;
@@ -98,13 +97,15 @@ struct GetArrowResultBatchCtx {
 // buffer used for result customer and producer
 class BufferControlBlock {
 public:
-    BufferControlBlock(const TUniqueId& id, int buffer_size, RuntimeState* state);
+    BufferControlBlock(TUniqueId id, int buffer_size, RuntimeState* state);
     ~BufferControlBlock();
 
     Status init();
+    // try to consume _waiting_rpc or make data waiting in _fe_result_batch_queue. try to combine block to reduce rpc first.
     Status add_batch(RuntimeState* state, std::unique_ptr<TFetchDataResult>& result);
     Status add_arrow_batch(RuntimeState* state, std::shared_ptr<vectorized::Block>& result);
 
+    // if there's Block waiting in _fe_result_batch_queue, send it(by on_data). otherwise make a rpc wait in _waiting_rpc.
     void get_batch(GetResultBatchCtx* ctx);
     // for ArrowFlightBatchLocalReader
     Status get_arrow_batch(std::shared_ptr<vectorized::Block>* result,
@@ -150,7 +151,7 @@ class BufferControlBlock {
     const int _buffer_limit;
     int64_t _packet_num;
 
-    // blocking queue for batch
+    // Producer. blocking queue for result batch waiting to sent to FE by _waiting_rpc.
     FeResultQueue _fe_result_batch_queue;
     ArrowFlightResultQueue _arrow_flight_result_batch_queue;
     // for arrow flight
@@ -163,6 +164,7 @@ class BufferControlBlock {
     // TODO, waiting for data will block pipeline, so use a request pool to save requests waiting for data.
     std::condition_variable _arrow_data_arrival;
 
+    // Consumer. RPCs which FE waiting for result. when _fe_result_batch_queue filled, the rpc could be sent.
     std::deque<GetResultBatchCtx*> _waiting_rpc;
     std::deque<GetArrowResultBatchCtx*> _waiting_arrow_result_batch_rpc;
 
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index f96e4152500808..b1bc42491b5ce8 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -34,17 +34,16 @@
 #include <gen_cpp/Types_types.h>
 #include <gen_cpp/internal_service.pb.h>
 #include <pthread.h>
-#include <stddef.h>
 #include <sys/time.h>
 #include <thrift/TApplicationException.h>
 #include <thrift/Thrift.h>
 #include <thrift/protocol/TDebugProtocol.h>
 #include <thrift/transport/TTransportException.h>
-#include <time.h>
 #include <unistd.h>
 
 #include <algorithm>
-#include <atomic>
+#include <cstddef>
+#include <ctime>
 
 #include "common/status.h"
 // IWYU pragma: no_include <bits/chrono.h>
@@ -58,19 +57,16 @@
 #include <unordered_set>
 #include <utility>
 
-#include "cloud/config.h"
 #include "common/config.h"
 #include "common/logging.h"
 #include "common/object_pool.h"
 #include "common/utils.h"
-#include "gutil/strings/substitute.h"
 #include "io/fs/stream_load_pipe.h"
 #include "pipeline/pipeline_fragment_context.h"
 #include "runtime/client_cache.h"
 #include "runtime/descriptors.h"
 #include "runtime/exec_env.h"
 #include "runtime/frontend_info.h"
-#include "runtime/memory/mem_tracker_limiter.h"
 #include "runtime/primitive_type.h"
 #include "runtime/query_context.h"
 #include "runtime/runtime_filter_mgr.h"
@@ -89,24 +85,20 @@
 #include "util/debug_points.h"
 #include "util/debug_util.h"
 #include "util/doris_metrics.h"
-#include "util/hash_util.hpp"
-#include "util/mem_info.h"
 #include "util/network_util.h"
-#include "util/pretty_printer.h"
 #include "util/runtime_profile.h"
 #include "util/thread.h"
 #include "util/threadpool.h"
 #include "util/thrift_util.h"
 #include "util/uid_util.h"
-#include "util/url_coding.h"
 #include "vec/runtime/shared_hash_table_controller.h"
-#include "vec/runtime/vdatetime_value.h"
 
 namespace doris {
 
 DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(fragment_instance_count, MetricUnit::NOUNIT);
 DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(timeout_canceled_fragment_count, MetricUnit::NOUNIT);
 DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(fragment_thread_pool_queue_size, MetricUnit::NOUNIT);
+DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(fragment_thread_pool_num_active_threads, MetricUnit::NOUNIT);
 bvar::LatencyRecorder g_fragmentmgr_prepare_latency("doris_FragmentMgr", "prepare");
 
 bvar::Adder<uint64_t> g_fragment_executing_count("fragment_executing_count");
@@ -184,7 +176,7 @@ static Status _do_fetch_running_queries_rpc(const FrontendInfo& fe_info,
     }
 
     // Avoid logic error in frontend.
-    if (rpc_result.__isset.status == false || rpc_result.status.status_code != TStatusCode::OK) {
+    if (!rpc_result.__isset.status || rpc_result.status.status_code != TStatusCode::OK) {
         LOG_WARNING("Failed to fetch running queries from {}, reason: {}",
                     PrintThriftNetworkAddress(fe_info.info.coordinator_address),
                     doris::to_string(rpc_result.status.status_code));
@@ -193,7 +185,7 @@ static Status _do_fetch_running_queries_rpc(const FrontendInfo& fe_info,
                                      doris::to_string(rpc_result.status.status_code));
     }
 
-    if (rpc_result.__isset.running_queries == false) {
+    if (!rpc_result.__isset.running_queries) {
         return Status::InternalError("Failed to fetch running queries from {}, reason: {}",
                                      PrintThriftNetworkAddress(fe_info.info.coordinator_address),
                                      "running_queries is not set");
@@ -254,6 +246,8 @@ FragmentMgr::FragmentMgr(ExecEnv* exec_env)
 
     REGISTER_HOOK_METRIC(fragment_thread_pool_queue_size,
                          [this]() { return _thread_pool->get_queue_size(); });
+    REGISTER_HOOK_METRIC(fragment_thread_pool_num_active_threads,
+                         [this]() { return _thread_pool->num_active_threads(); });
     CHECK(s.ok()) << s.to_string();
 }
 
@@ -262,6 +256,7 @@ FragmentMgr::~FragmentMgr() = default;
 void FragmentMgr::stop() {
     DEREGISTER_HOOK_METRIC(fragment_instance_count);
     DEREGISTER_HOOK_METRIC(fragment_thread_pool_queue_size);
+    DEREGISTER_HOOK_METRIC(fragment_thread_pool_num_active_threads);
     _stop_background_threads_latch.count_down();
     if (_cancel_thread) {
         _cancel_thread->join();
diff --git a/be/src/runtime/fragment_mgr.h b/be/src/runtime/fragment_mgr.h
index 63d666788d0a5f..e85fb07cba68da 100644
--- a/be/src/runtime/fragment_mgr.h
+++ b/be/src/runtime/fragment_mgr.h
@@ -21,9 +21,8 @@
 #include <gen_cpp/QueryPlanExtra_types.h>
 #include <gen_cpp/Types_types.h>
 #include <gen_cpp/types.pb.h>
-#include <stdint.h>
 
-#include <condition_variable>
+#include <cstdint>
 #include <functional>
 #include <iosfwd>
 #include <memory>
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index 439f3f17faf00f..fb0b2f090bc045 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -665,15 +665,11 @@ void PInternalService::cancel_plan_fragment(google::protobuf::RpcController* /*c
 void PInternalService::fetch_data(google::protobuf::RpcController* controller,
                                   const PFetchDataRequest* request, PFetchDataResult* result,
                                   google::protobuf::Closure* done) {
-    bool ret = _heavy_work_pool.try_offer([this, controller, request, result, done]() {
-        brpc::Controller* cntl = static_cast<brpc::Controller*>(controller);
-        GetResultBatchCtx* ctx = new GetResultBatchCtx(cntl, result, done);
-        _exec_env->result_mgr()->fetch_data(request->finst_id(), ctx);
-    });
-    if (!ret) {
-        offer_failed(result, done, _heavy_work_pool);
-        return;
-    }
+    // fetch_data is a light operation which will put a request rather than wait inplace when there's no data ready.
+    // when there's data ready, use brpc to send. there's queue in brpc service. won't take it too long.
+    auto* cntl = static_cast<brpc::Controller*>(controller);
+    auto* ctx = new GetResultBatchCtx(cntl, result, done);
+    _exec_env->result_mgr()->fetch_data(request->finst_id(), ctx);
 }
 
 void PInternalService::fetch_arrow_data(google::protobuf::RpcController* controller,
diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h
index 69516773debdbc..31b907eec9ed6c 100644
--- a/be/src/util/doris_metrics.h
+++ b/be/src/util/doris_metrics.h
@@ -202,6 +202,7 @@ class DorisMetrics {
     UIntGauge* send_batch_thread_pool_thread_num = nullptr;
     UIntGauge* send_batch_thread_pool_queue_size = nullptr;
     UIntGauge* fragment_thread_pool_queue_size = nullptr;
+    UIntGauge* fragment_thread_pool_num_active_threads = nullptr;
 
     // Upload metrics
     UIntGauge* upload_total_byte = nullptr;
diff --git a/be/src/util/threadpool.h b/be/src/util/threadpool.h
index 9bd4a7246fb0b1..f822c307aa6b8e 100644
--- a/be/src/util/threadpool.h
+++ b/be/src/util/threadpool.h
@@ -20,12 +20,11 @@
 
 #pragma once
 
-#include <limits.h>
-#include <stddef.h>
-
 #include <boost/intrusive/detail/algo_type.hpp>
 #include <boost/intrusive/list.hpp>
 #include <boost/intrusive/list_hook.hpp>
+#include <climits>
+#include <cstddef>
 // IWYU pragma: no_include <bits/chrono.h>
 #include <chrono> // IWYU pragma: keep
 #include <condition_variable>
@@ -50,7 +49,7 @@ class ThreadPoolToken;
 class Runnable {
 public:
     virtual void run() = 0;
-    virtual ~Runnable() {}
+    virtual ~Runnable() = default;
 };
 
 // ThreadPool takes a lot of arguments. We provide sane defaults with a builder.
@@ -127,6 +126,9 @@ class ThreadPoolBuilder {
         return Status::OK();
     }
 
+    ThreadPoolBuilder(const ThreadPoolBuilder&) = delete;
+    void operator=(const ThreadPoolBuilder&) = delete;
+
 private:
     friend class ThreadPool;
     const std::string _name;
@@ -136,9 +138,6 @@ class ThreadPoolBuilder {
     std::weak_ptr<CgroupCpuCtl> _cgroup_cpu_ctl;
     std::chrono::milliseconds _idle_timeout;
 
-    ThreadPoolBuilder(const ThreadPoolBuilder&) = delete;
-    void operator=(const ThreadPoolBuilder&) = delete;
-
     template <typename T>
     static constexpr bool always_false_v = false;
 };
@@ -256,13 +255,22 @@ class ThreadPool {
         return _total_queued_tasks;
     }
 
-    std::vector<int> debug_info() {
+    std::vector<int> debug_info() const {
         std::lock_guard<std::mutex> l(_lock);
         std::vector<int> arr = {_num_threads, static_cast<int>(_threads.size()), _min_threads,
                                 _max_threads};
         return arr;
     }
 
+    std::string get_info() const {
+        std::lock_guard<std::mutex> l(_lock);
+        return fmt::format("ThreadPool(name={}, threads(active/pending)=({}/{}), queued_task={})",
+                           _name, _active_threads, _num_threads_pending_start, _total_queued_tasks);
+    }
+
+    ThreadPool(const ThreadPool&) = delete;
+    void operator=(const ThreadPool&) = delete;
+
 private:
     friend class ThreadPoolBuilder;
     friend class ThreadPoolToken;
@@ -372,7 +380,7 @@ class ThreadPool {
     //
     // Protected by _lock.
     struct IdleThread : public boost::intrusive::list_base_hook<> {
-        explicit IdleThread() {}
+        explicit IdleThread() = default;
 
         // Condition variable for "queue is not empty". Waiters wake up when a new
         // task is queued.
@@ -384,9 +392,6 @@ class ThreadPool {
 
     // ExecutionMode::CONCURRENT token used by the pool for tokenless submission.
     std::unique_ptr<ThreadPoolToken> _tokenless;
-
-    ThreadPool(const ThreadPool&) = delete;
-    void operator=(const ThreadPool&) = delete;
 };
 
 // Entry point for token-based task submission and blocking for a particular
@@ -434,6 +439,9 @@ class ThreadPoolToken {
         return _entries.size();
     }
 
+    ThreadPoolToken(const ThreadPoolToken&) = delete;
+    void operator=(const ThreadPoolToken&) = delete;
+
 private:
     // All possible token states. Legal state transitions:
     //   IDLE      -> RUNNING: task is submitted via token
@@ -516,9 +524,6 @@ class ThreadPoolToken {
     int _num_submitted_tasks;
     // Number of tasks which has not been submitted to the thread pool's queue.
     int _num_unsubmitted_tasks;
-
-    ThreadPoolToken(const ThreadPoolToken&) = delete;
-    void operator=(const ThreadPoolToken&) = delete;
 };
 
 } // namespace doris
diff --git a/be/src/util/work_thread_pool.hpp b/be/src/util/work_thread_pool.hpp
index 00430ff75148fc..1da8a08f90d234 100644
--- a/be/src/util/work_thread_pool.hpp
+++ b/be/src/util/work_thread_pool.hpp
@@ -18,7 +18,6 @@
 #pragma once
 
 #include <mutex>
-#include <thread>
 
 #include "util/blocking_priority_queue.hpp"
 #include "util/blocking_queue.hpp"
@@ -126,12 +125,13 @@ class WorkThreadPool {
     }
 
     std::string get_info() const {
-        return fmt::format(
-                "PriorityThreadPool(name={}, queue_size={}/{}, active_thread={}/{}, "
-                "total_get_wait_time={}, total_put_wait_time={})",
-                _name, get_queue_size(), _work_queue.get_capacity(), _active_threads,
-                _threads.size(), _work_queue.total_get_wait_time(),
-                _work_queue.total_put_wait_time());
+        return (Priority ? "PriorityThreadPool" : "FifoThreadPool") +
+               fmt::format(
+                       "(name={}, queue_size={}/{}, active_thread={}/{}, "
+                       "total_get_wait_time={}, total_put_wait_time={})",
+                       _name, get_queue_size(), _work_queue.get_capacity(), _active_threads,
+                       _threads.size(), _work_queue.total_get_wait_time(),
+                       _work_queue.total_put_wait_time());
     }
 
 protected:
diff --git a/be/src/vec/sink/writer/async_result_writer.h b/be/src/vec/sink/writer/async_result_writer.h
index 513f2aa7984ca5..2a90dd2dbd04e9 100644
--- a/be/src/vec/sink/writer/async_result_writer.h
+++ b/be/src/vec/sink/writer/async_result_writer.h
@@ -19,7 +19,7 @@
 #include <concurrentqueue.h>
 
 #include <condition_variable>
-#include <queue>
+#include <queue> // IWYU pragma: keep
 
 #include "runtime/result_writer.h"
 #include "vec/exprs/vexpr_fwd.h"
@@ -49,7 +49,7 @@ class Block;
  *  pipeline execution engine performance.
  *
  *  The Sub class of AsyncResultWriter need to impl two virtual function
- *     * Status open() the first time IO work like: create file/ connect networking
+ *     * Status open() the first time IO work like: create file/ connect network
  *     * Status write() do the real IO work for block 
  */
 class AsyncResultWriter : public ResultWriter {
@@ -64,7 +64,7 @@ class AsyncResultWriter : public ResultWriter {
 
     virtual Status open(RuntimeState* state, RuntimeProfile* profile) = 0;
 
-    // sink the block date to date queue, it is async
+    // sink the block data to data queue, it is async
     Status sink(Block* block, bool eos);
 
     // Add the IO thread task process block() to thread pool to dispose the IO

From 552315cb9e3e9b9f542c7a9e187a57fcd2e4bfbf Mon Sep 17 00:00:00 2001
From: zclllhhjj <zhaochangle@selectdb.com>
Date: Tue, 10 Dec 2024 12:05:56 +0800
Subject: [PATCH 248/399] [chore](BE) remove some useless class (#45049)

### What problem does this PR solve?
remove some useless class
---
 be/src/olap/hll.cpp | 39 ---------------------------------
 be/src/olap/hll.h   | 53 ---------------------------------------------
 be/src/util/slice.h | 21 ------------------
 3 files changed, 113 deletions(-)

diff --git a/be/src/olap/hll.cpp b/be/src/olap/hll.cpp
index 80be40f5643977..2b5d213c952589 100644
--- a/be/src/olap/hll.cpp
+++ b/be/src/olap/hll.cpp
@@ -367,43 +367,4 @@ int64_t HyperLogLog::estimate_cardinality() const {
     return (int64_t)(estimate + 0.5);
 }
 
-void HllSetResolver::parse() {
-    // skip LengthValueType
-    char* pdata = _buf_ref;
-    _set_type = (HllDataType)pdata[0];
-    char* sparse_data = nullptr;
-    switch (_set_type) {
-    case HLL_DATA_EXPLICIT:
-        // first byte : type
-        // second～five byte : hash values's number
-        // five byte later : hash value
-        _explicit_num = (ExplicitLengthValueType)(pdata[sizeof(SetTypeValueType)]);
-        _explicit_value =
-                (uint64_t*)(pdata + sizeof(SetTypeValueType) + sizeof(ExplicitLengthValueType));
-        break;
-    case HLL_DATA_SPARSE:
-        // first byte : type
-        // second ～（2^HLL_COLUMN_PRECISION)/8 byte : bitmap mark which is not zero
-        // 2^HLL_COLUMN_PRECISION)/8 ＋ 1以后value
-        _sparse_count = (SparseLengthValueType*)(pdata + sizeof(SetTypeValueType));
-        sparse_data = pdata + sizeof(SetTypeValueType) + sizeof(SparseLengthValueType);
-        for (int i = 0; i < *_sparse_count; i++) {
-            auto* index = (SparseIndexType*)sparse_data;
-            sparse_data += sizeof(SparseIndexType);
-            auto* value = (SparseValueType*)sparse_data;
-            _sparse_map[*index] = *value;
-            sparse_data += sizeof(SetTypeValueType);
-        }
-        break;
-    case HLL_DATA_FULL:
-        // first byte : type
-        // second byte later : hll register value
-        _full_value_position = pdata + sizeof(SetTypeValueType);
-        break;
-    default:
-        // HLL_DATA_EMPTY
-        break;
-    }
-}
-
 } // namespace doris
diff --git a/be/src/olap/hll.h b/be/src/olap/hll.h
index 56cbd97110dd97..1d01223c2573ad 100644
--- a/be/src/olap/hll.h
+++ b/be/src/olap/hll.h
@@ -303,57 +303,4 @@ class HyperLogLog {
     uint8_t* _registers = nullptr;
 };
 
-// todo(kks): remove this when dpp_sink class was removed
-class HllSetResolver {
-public:
-    HllSetResolver() = default;
-
-    ~HllSetResolver() = default;
-
-    using SetTypeValueType = uint8_t;
-    using ExplicitLengthValueType = uint8_t;
-    using SparseLengthValueType = int32_t;
-    using SparseIndexType = uint16_t;
-    using SparseValueType = uint8_t;
-
-    // only save pointer
-    void init(char* buf, int len) {
-        this->_buf_ref = buf;
-        this->_buf_len = len;
-    }
-
-    // hll set type
-    HllDataType get_hll_data_type() { return _set_type; }
-
-    // explicit value num
-    int get_explicit_count() const { return (int)_explicit_num; }
-
-    // get explicit index value 64bit
-    uint64_t get_explicit_value(int index) {
-        if (index >= _explicit_num) {
-            return -1;
-        }
-        return _explicit_value[index];
-    }
-
-    // get full register value
-    char* get_full_value() { return _full_value_position; }
-
-    // get (index, value) map
-    std::map<SparseIndexType, SparseValueType>& get_sparse_map() { return _sparse_map; }
-
-    // parse set , call after copy() or init()
-    void parse();
-
-private:
-    char* _buf_ref = nullptr; // set
-    int _buf_len {};          // set len
-    HllDataType _set_type {}; //set type
-    char* _full_value_position = nullptr;
-    uint64_t* _explicit_value = nullptr;
-    ExplicitLengthValueType _explicit_num {};
-    std::map<SparseIndexType, SparseValueType> _sparse_map;
-    SparseLengthValueType* _sparse_count;
-};
-
 } // namespace doris
diff --git a/be/src/util/slice.h b/be/src/util/slice.h
index fd6bcf0adfb510..b70a82e17ce74d 100644
--- a/be/src/util/slice.h
+++ b/be/src/util/slice.h
@@ -302,27 +302,6 @@ inline int Slice::compare(const Slice& b) const {
     return r;
 }
 
-/// @brief STL map whose keys are Slices.
-///
-/// An example of usage:
-/// @code
-///   typedef SliceMap<int>::type MySliceMap;
-///
-///   MySliceMap my_map;
-///   my_map.insert(MySliceMap::value_type(a, 1));
-///   my_map.insert(MySliceMap::value_type(b, 2));
-///   my_map.insert(MySliceMap::value_type(c, 3));
-///
-///   for (const MySliceMap::value_type& pair : my_map) {
-///     ...
-///   }
-/// @endcode
-template <typename T>
-struct SliceMap {
-    /// A handy typedef for the slice map with appropriate comparison operator.
-    typedef std::map<Slice, T, Slice::Comparator> type;
-};
-
 // A move-only type which manage the lifecycle of externally allocated data.
 // Unlike std::unique_ptr<uint8_t[]>, OwnedSlice remembers the size of data so that clients can access
 // the underlying buffer as a Slice.

From 602ad0761fefb021561ce5d2a67b9cee86244f20 Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Tue, 10 Dec 2024 12:58:02 +0800
Subject: [PATCH 249/399] [improvement](statistics)Support get oracle jdbc row
 count. (#45214)

### What problem does this PR solve?

Support get oracle jdbc row count from oracle statistics if exists.

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [x] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [x] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [x] Confirm the release note
- [x] Confirm test cases
- [x] Confirm document
- [x] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 .../datasource/jdbc/JdbcExternalTable.java    |  5 ++++
 .../jdbc/test_jdbc_row_count.groovy           | 30 +++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
index 9e188a711b0297..772c366850b9a4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
@@ -63,6 +63,9 @@ public class JdbcExternalTable extends ExternalTable {
     public static final String SQLSERVER_ROW_COUNT_SQL = "SELECT sum(rows) as rows FROM sys.partitions "
             + "WHERE object_id = (SELECT object_id('${dbName}.${tblName}')) AND index_id IN (0, 1)";
 
+    public static final String ORACLE_ROW_COUNT_SQL = "SELECT NUM_ROWS as \\\"rows\\\" FROM ALL_TABLES WHERE "
+            + "OWNER = '${dbName}' and TABLE_NAME = '${tblName}'";
+
     public static final String FETCH_ROW_COUNT_TEMPLATE = "SELECT * FROM QUERY"
             + "(\"catalog\"=\"${ctlName}\", \"query\"=\"${sql}\");";
 
@@ -144,6 +147,8 @@ public long fetchRowCount() {
                 params.put("sql", SQLSERVER_ROW_COUNT_SQL);
                 return getRowCount(params);
             case JdbcResource.ORACLE:
+                params.put("sql", ORACLE_ROW_COUNT_SQL);
+                return getRowCount(params);
             default:
                 break;
         }
diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy
index 5cb1f6267e32cb..c8220ce13dd0b1 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy
@@ -102,6 +102,36 @@ suite("test_jdbc_row_count", "p0,external,mysql,external_docker,external_docker_
         }
         assertEquals("3", result[0][2])
         sql """drop catalog ${catalog_name}"""
+
+        // Test oracle
+        catalog_name = "test_oracle_jdbc_row_count";
+        String oracle_port = context.config.otherConfigs.get("oracle_11_port");
+        String SID = "XE";
+        driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/ojdbc8.jar"
+        sql """drop catalog if exists ${catalog_name} """
+        sql """
+            create catalog if not exists ${catalog_name} properties(
+                    "type"="jdbc",
+                    "user"="doris_test",
+                    "password"="123456",
+                    "jdbc_url" = "jdbc:oracle:thin:@${externalEnvIp}:${oracle_port}:${SID}",
+                    "driver_url" = "${driver_url}",
+                    "driver_class" = "oracle.jdbc.driver.OracleDriver"
+            );
+        """
+        sql """use ${catalog_name}.DORIS_TEST"""
+        result = sql """show table stats STUDENT"""
+        Thread.sleep(1000)
+        for (int i = 0; i < 30; i++) {
+            result = sql """show table stats STUDENT""";
+            if (result[0][2] != "-1") {
+                break;
+            }
+            logger.info("Table row count not ready yet. Wait 1 second.")
+            Thread.sleep(1000)
+        }
+        assertTrue("4".equals(result[0][2]) || "-1".equals(result[0][2]))
+        sql """drop catalog ${catalog_name}"""
     }
 }
 

From 8a144548a3f8dcada0c3d9e31f60513b84da1bb2 Mon Sep 17 00:00:00 2001
From: zxealous <zhouchangyue@baidu.com>
Date: Tue, 10 Dec 2024 13:52:26 +0800
Subject: [PATCH 250/399] [fix](nereids) SqlParser support load data into
 temporary partition (#45025)

Problem Summary:
Currently, use DorisParser.g4 parse sql, and not support specified
temporary partition while load data now.

LOAD LABEL db.label ( DATA INFILE("hdfs://hdfs:9000/user/partition/partition_type") INTO TABLE `tb` TEMPORARY PARTITION (partition_g)) WITH BROKER "ahdfs" ( "username" = "xxx",  "password" = "") PROPERTIES( "max_filter_ratio"="0.5" )

ERROR 1105 (HY000): errCode = 2, detailMessage =
mismatched input 'TEMPORARY' expecting {')', ','}(line 1, pos xxx)

This pr support specified temporary partition while load data, because
load operation will backup to old optimizer, so only support specified
temporary partition while load data can successfully load data into the
temporary partition.

TODO:
1.nereids need support load data to temporary partition.
2.nereids need support export data from temporary partition.
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../test_broker_load_with_partition.out       |  8 +++
 .../test_broker_load_with_partition.groovy    | 54 +++++++++++++++++++
 3 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 1b01cad4293569..fdad6baccdc056 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -929,7 +929,7 @@ identityOrFunction
 dataDesc
     : ((WITH)? mergeType)? DATA INFILE LEFT_PAREN filePaths+=STRING_LITERAL (COMMA filePath+=STRING_LITERAL)* RIGHT_PAREN
         INTO TABLE targetTableName=identifier
-        (PARTITION partition=identifierList)?
+        (partitionSpec)?
         (COLUMNS TERMINATED BY comma=STRING_LITERAL)?
         (LINES TERMINATED BY separator=STRING_LITERAL)?
         (FORMAT AS format=identifierOrText)?
diff --git a/regression-test/data/load_p0/broker_load/test_broker_load_with_partition.out b/regression-test/data/load_p0/broker_load/test_broker_load_with_partition.out
index 5015d98b3c9bfe..e62e67a8f74909 100644
--- a/regression-test/data/load_p0/broker_load/test_broker_load_with_partition.out
+++ b/regression-test/data/load_p0/broker_load/test_broker_load_with_partition.out
@@ -8,3 +8,11 @@
 11001	2023-09-01	1	2	10
 11001	2023-09-01	2	1	10
 
+-- !select --
+11001	2023-10-01	1	3	10
+11001	2023-10-01	2	2	10
+11001	2023-10-01	2	3	10
+11001	2023-09-01	1	1	10
+11001	2023-09-01	1	2	10
+11001	2023-09-01	2	1	10
+
diff --git a/regression-test/suites/load_p0/broker_load/test_broker_load_with_partition.groovy b/regression-test/suites/load_p0/broker_load/test_broker_load_with_partition.groovy
index 032b48baf6025e..45f0cc50be7eed 100644
--- a/regression-test/suites/load_p0/broker_load/test_broker_load_with_partition.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_broker_load_with_partition.groovy
@@ -54,6 +54,13 @@ suite("test_broker_load_with_partition", "load_p0") {
         assertTrue(result2.size() == 1)
         assertTrue(result2[0].size() == 1)
         assertTrue(result2[0][0] == 1, "Insert should update 1 rows")
+
+        def result3 = sql """
+            ALTER TABLE ${testTable} ADD TEMPORARY PARTITION partition_t VALUES LESS THAN ("2023-11-01");
+            """
+        assertTrue(result3.size() == 1)
+        assertTrue(result3[0].size() == 1)
+        assertTrue(result3[0][0] == 0, "Alter table should update 0 rows")
     }
 
     def load_from_hdfs_partition = {testTablex, label, hdfsFilePath, format, brokerName, hdfsUser, hdfsPasswd ->
@@ -78,6 +85,28 @@ suite("test_broker_load_with_partition", "load_p0") {
         assertTrue(result1[0][0] == 0, "Query OK, 0 rows affected")
     }
 
+    def load_from_hdfs_tmp_partition = {testTablex, label, hdfsFilePath, format, brokerName, hdfsUser, hdfsPasswd ->
+        def result1= sql """
+                        LOAD LABEL ${label} (
+                            DATA INFILE("${hdfsFilePath}")
+                            INTO TABLE ${testTablex}
+                            TEMPORARY PARTITION (`partition_t`)
+                            COLUMNS TERMINATED BY ","
+                            FORMAT as "${format}"
+                        )
+                        with BROKER "${brokerName}" (
+                        "username"="${hdfsUser}",
+                        "password"="${hdfsPasswd}")
+                        PROPERTIES  (
+                        "timeout"="1200",
+                        "max_filter_ratio"="0.1");
+                        """
+
+        assertTrue(result1.size() == 1)
+        assertTrue(result1[0].size() == 1)
+        assertTrue(result1[0][0] == 0, "Query OK, 0 rows affected")
+    }
+
     def check_load_result = {checklabel, testTablex ->
         max_try_milli_secs = 10000
         while(max_try_milli_secs) {
@@ -97,6 +126,25 @@ suite("test_broker_load_with_partition", "load_p0") {
         }
     }
 
+    def check_load_tmp_partition_result = {checklabel, testTablex ->
+        max_try_milli_secs = 10000
+        while(max_try_milli_secs) {
+            result = sql "show load where label = '${checklabel}'"
+            log.info("result: ${result}")
+            if(result[0][2] == "FINISHED") {
+                //sql "sync"
+                qt_select "select * from ${testTablex} TEMPORARY PARTITION (`partition_t`) order by k1"
+                break
+            } else {
+                sleep(1000) // wait 1 second every time
+                max_try_milli_secs -= 1000
+                if(max_try_milli_secs <= 0) {
+                    assertEquals(1, 2)
+                }
+            }
+        }
+    }
+
     // if 'enableHdfs' in regression-conf.groovy has been set to true,
     // the test will run these case as below.
     if (enableHdfs()) {
@@ -115,6 +163,12 @@ suite("test_broker_load_with_partition", "load_p0") {
                                 brokerName, hdfsUser, hdfsPasswd)
 
             check_load_result.call(test_load_label, testTable)
+
+            def test_tmp_partition_load_label = UUID.randomUUID().toString().replaceAll("-", "")
+            load_from_hdfs_tmp_partition.call(testTable, test_tmp_partition_load_label,
+                                hdfs_csv_file_path, "csv", brokerName, hdfsUser, hdfsPasswd)
+
+            check_load_tmp_partition_result.call(test_tmp_partition_load_label, testTable)
         } finally {
             try_sql("DROP TABLE IF EXISTS ${testTable}")
         }

From 972c8dfe5ae48bfbe29b9fd0b6ed163a87778fd2 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@selectdb.com>
Date: Tue, 10 Dec 2024 15:24:39 +0800
Subject: [PATCH 251/399] [Refactor]refactor log workload group when query
 arrives be (#45034)

---
 be/src/runtime/fragment_mgr.cpp               | 33 +++++++++++--------
 be/src/runtime/load_channel.cpp               |  2 +-
 be/src/runtime/query_context.cpp              |  3 +-
 be/src/runtime/query_context.h                |  2 +-
 .../workload_group/workload_group_manager.cpp |  6 ++--
 .../workload_group/workload_group_manager.h   |  2 +-
 6 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index b1bc42491b5ce8..ce18071fda0d07 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -648,12 +648,25 @@ Status FragmentMgr::_get_or_create_query_ctx(const TPipelineFragmentParams& para
             }
 
             if (!query_ctx) {
+                WorkloadGroupPtr workload_group_ptr = nullptr;
+                std::string wg_info_str = "Workload Group not set";
+                if (params.__isset.workload_groups && !params.workload_groups.empty()) {
+                    uint64_t wg_id = params.workload_groups[0].id;
+                    workload_group_ptr = _exec_env->workload_group_mgr()->get_group(wg_id);
+                    if (workload_group_ptr != nullptr) {
+                        wg_info_str = workload_group_ptr->debug_string();
+                    } else {
+                        wg_info_str = "set wg but not find it in be";
+                    }
+                }
+
                 // First time a fragment of a query arrived. print logs.
                 LOG(INFO) << "query_id: " << print_id(query_id) << ", coord_addr: " << params.coord
                           << ", total fragment num on current host: " << params.fragment_num_on_host
                           << ", fe process uuid: " << params.query_options.fe_process_uuid
                           << ", query type: " << params.query_options.query_type
-                          << ", report audit fe:" << params.current_connect_fe;
+                          << ", report audit fe:" << params.current_connect_fe
+                          << ", use wg:" << wg_info_str;
 
                 // This may be a first fragment request of the query.
                 // Create the query fragments context.
@@ -678,19 +691,11 @@ Status FragmentMgr::_get_or_create_query_ctx(const TPipelineFragmentParams& para
 
                 _set_scan_concurrency(params, query_ctx.get());
 
-                if (params.__isset.workload_groups && !params.workload_groups.empty()) {
-                    uint64_t tg_id = params.workload_groups[0].id;
-                    WorkloadGroupPtr workload_group_ptr =
-                            _exec_env->workload_group_mgr()->get_task_group_by_id(tg_id);
-                    if (workload_group_ptr != nullptr) {
-                        RETURN_IF_ERROR(workload_group_ptr->add_query(query_id, query_ctx));
-                        RETURN_IF_ERROR(query_ctx->set_workload_group(workload_group_ptr));
-                        _exec_env->runtime_query_statistics_mgr()->set_workload_group_id(
-                                print_id(query_id), tg_id);
-                    } else {
-                        LOG(WARNING) << "Query/load id: " << print_id(query_ctx->query_id())
-                                     << "can't find its workload group " << tg_id;
-                    }
+                if (workload_group_ptr != nullptr) {
+                    RETURN_IF_ERROR(workload_group_ptr->add_query(query_id, query_ctx));
+                    query_ctx->set_workload_group(workload_group_ptr);
+                    _exec_env->runtime_query_statistics_mgr()->set_workload_group_id(
+                            print_id(query_id), workload_group_ptr->id());
                 }
                 // There is some logic in query ctx's dctor, we could not check if exists and delete the
                 // temp query ctx now. For example, the query id maybe removed from workload group's queryset.
diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp
index dd426f1ab81d3e..6dfd5d46eb6480 100644
--- a/be/src/runtime/load_channel.cpp
+++ b/be/src/runtime/load_channel.cpp
@@ -59,7 +59,7 @@ LoadChannel::LoadChannel(const UniqueId& load_id, int64_t timeout_s, bool is_hig
                 fmt::format("(FromLoadChannel)Load#Id={}", _load_id.to_string()));
         if (wg_id > 0) {
             WorkloadGroupPtr workload_group_ptr =
-                    ExecEnv::GetInstance()->workload_group_mgr()->get_task_group_by_id(wg_id);
+                    ExecEnv::GetInstance()->workload_group_mgr()->get_group(wg_id);
             if (workload_group_ptr) {
                 wg_ptr = workload_group_ptr;
                 wg_ptr->add_mem_tracker_limiter(mem_tracker);
diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp
index 161964420e9ad6..c777c8100ef213 100644
--- a/be/src/runtime/query_context.cpp
+++ b/be/src/runtime/query_context.cpp
@@ -323,14 +323,13 @@ ThreadPool* QueryContext::get_memtable_flush_pool() {
     }
 }
 
-Status QueryContext::set_workload_group(WorkloadGroupPtr& tg) {
+void QueryContext::set_workload_group(WorkloadGroupPtr& tg) {
     _workload_group = tg;
     // Should add query first, then the workload group will not be deleted.
     // see task_group_manager::delete_workload_group_by_ids
     _workload_group->add_mem_tracker_limiter(query_mem_tracker);
     _workload_group->get_query_scheduler(&_task_scheduler, &_scan_task_scheduler,
                                          &_memtable_flush_pool, &_remote_scan_task_scheduler);
-    return Status::OK();
 }
 
 void QueryContext::add_fragment_profile(
diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h
index d557245bf2339d..35431d54394268 100644
--- a/be/src/runtime/query_context.h
+++ b/be/src/runtime/query_context.h
@@ -138,7 +138,7 @@ class QueryContext {
         }
     }
 
-    Status set_workload_group(WorkloadGroupPtr& tg);
+    void set_workload_group(WorkloadGroupPtr& tg);
 
     int execution_timeout() const {
         return _query_options.__isset.execution_timeout ? _query_options.execution_timeout
diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp b/be/src/runtime/workload_group/workload_group_manager.cpp
index 4d32fc8700eaa5..e2c4b8c7f5e6c6 100644
--- a/be/src/runtime/workload_group/workload_group_manager.cpp
+++ b/be/src/runtime/workload_group/workload_group_manager.cpp
@@ -86,10 +86,10 @@ void WorkloadGroupMgr::get_related_workload_groups(
     }
 }
 
-WorkloadGroupPtr WorkloadGroupMgr::get_task_group_by_id(uint64_t tg_id) {
+WorkloadGroupPtr WorkloadGroupMgr::get_group(uint64_t wg_id) {
     std::shared_lock<std::shared_mutex> r_lock(_group_mutex);
-    if (_workload_groups.find(tg_id) != _workload_groups.end()) {
-        return _workload_groups.at(tg_id);
+    if (_workload_groups.find(wg_id) != _workload_groups.end()) {
+        return _workload_groups.at(wg_id);
     }
     return nullptr;
 }
diff --git a/be/src/runtime/workload_group/workload_group_manager.h b/be/src/runtime/workload_group/workload_group_manager.h
index 18a0687b373325..c0eb0dfc0b237e 100644
--- a/be/src/runtime/workload_group/workload_group_manager.h
+++ b/be/src/runtime/workload_group/workload_group_manager.h
@@ -55,7 +55,7 @@ class WorkloadGroupMgr {
 
     void delete_workload_group_by_ids(std::set<uint64_t> id_set);
 
-    WorkloadGroupPtr get_task_group_by_id(uint64_t tg_id);
+    WorkloadGroupPtr get_group(uint64_t wg_id);
 
     void do_sweep();
 

From 7dbae6e7f9148a8f38742536eecaf298441d5a02 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@selectdb.com>
Date: Tue, 10 Dec 2024 15:25:00 +0800
Subject: [PATCH 252/399] [Improment]Add workload group resource usage metrics
 (#44870)

---
 be/src/common/config.cpp                      |   2 +
 be/src/common/config.h                        |   1 +
 be/src/common/daemon.cpp                      |  13 ++
 be/src/common/daemon.h                        |   1 +
 be/src/pipeline/pipeline_task.cpp             |   2 +-
 be/src/runtime/query_context.h                |   4 +-
 be/src/runtime/thread_context.h               |  72 +++++-----
 .../runtime/workload_group/workload_group.cpp |  36 +++--
 .../runtime/workload_group/workload_group.h   |  27 ++--
 .../workload_group/workload_group_manager.cpp |  20 ++-
 .../workload_group/workload_group_manager.h   |   2 +
 .../workload_group/workload_group_metrics.cpp | 128 ++++++++++++++++++
 .../workload_group/workload_group_metrics.h   |  89 ++++++++++++
 .../workload_management/io_throttle.cpp       |  11 --
 .../runtime/workload_management/io_throttle.h |  10 +-
 be/src/util/system_metrics.cpp                |  21 +++
 be/src/util/system_metrics.h                  |   2 +
 be/src/vec/exec/scan/vscanner.cpp             |   2 +-
 .../vec/sink/writer/async_result_writer.cpp   |   2 +-
 19 files changed, 353 insertions(+), 92 deletions(-)
 create mode 100644 be/src/runtime/workload_group/workload_group_metrics.cpp
 create mode 100644 be/src/runtime/workload_group/workload_group_metrics.h

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 1528d9091c2f43..68b082c08734a6 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1216,6 +1216,8 @@ DEFINE_mString(doris_cgroup_cpu_path, "");
 DEFINE_mBool(enable_be_proc_monitor, "false");
 DEFINE_mInt32(be_proc_monitor_interval_ms, "10000");
 
+DEFINE_Int32(workload_group_metrics_interval_ms, "5000");
+
 DEFINE_mBool(enable_workload_group_memory_gc, "true");
 
 DEFINE_Bool(ignore_always_true_predicate_for_segment, "true");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 3bdbeb95edb65d..c0b2e19b49a6be 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1295,6 +1295,7 @@ DECLARE_mBool(exit_on_exception);
 DECLARE_mString(doris_cgroup_cpu_path);
 DECLARE_mBool(enable_be_proc_monitor);
 DECLARE_mInt32(be_proc_monitor_interval_ms);
+DECLARE_Int32(workload_group_metrics_interval_ms);
 
 DECLARE_mBool(enable_workload_group_memory_gc);
 
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index d3d55f10dde5fb..73035ecf3957eb 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -522,6 +522,13 @@ void Daemon::be_proc_monitor_thread() {
     }
 }
 
+void Daemon::calculate_workload_group_metrics_thread() {
+    while (!_stop_background_threads_latch.wait_for(
+            std::chrono::milliseconds(config::workload_group_metrics_interval_ms))) {
+        ExecEnv::GetInstance()->workload_group_mgr()->refresh_workload_group_metrics();
+    }
+}
+
 void Daemon::start() {
     Status st;
     st = Thread::create(
@@ -570,6 +577,12 @@ void Daemon::start() {
                 &_threads.emplace_back());
     }
     CHECK(st.ok()) << st;
+
+    st = Thread::create(
+            "Daemon", "workload_group_metrics",
+            [this]() { this->calculate_workload_group_metrics_thread(); },
+            &_threads.emplace_back());
+    CHECK(st.ok()) << st;
 }
 
 void Daemon::stop() {
diff --git a/be/src/common/daemon.h b/be/src/common/daemon.h
index fe723877dcd027..bd635f5a4b1920 100644
--- a/be/src/common/daemon.h
+++ b/be/src/common/daemon.h
@@ -47,6 +47,7 @@ class Daemon {
     void cache_prune_stale_thread();
     void report_runtime_query_statistics_thread();
     void be_proc_monitor_thread();
+    void calculate_workload_group_metrics_thread();
 
     CountDownLatch _stop_background_threads_latch;
     std::vector<scoped_refptr<Thread>> _threads;
diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp
index 6f9e59c8291966..6814881ac7a300 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -315,7 +315,7 @@ Status PipelineTask::execute(bool* eos) {
         if (cpu_qs) {
             cpu_qs->add_cpu_nanos(delta_cpu_time);
         }
-        query_context()->update_wg_cpu_adder(delta_cpu_time);
+        query_context()->update_cpu_time(delta_cpu_time);
     }};
     if (_wait_to_start()) {
         return Status::OK();
diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h
index 35431d54394268..621c5ebca90cad 100644
--- a/be/src/runtime/query_context.h
+++ b/be/src/runtime/query_context.h
@@ -241,9 +241,9 @@ class QueryContext {
     // only for file scan node
     std::map<int, TFileScanRangeParams> file_scan_range_params_map;
 
-    void update_wg_cpu_adder(int64_t delta_cpu_time) {
+    void update_cpu_time(int64_t delta_cpu_time) {
         if (_workload_group != nullptr) {
-            _workload_group->update_cpu_adder(delta_cpu_time);
+            _workload_group->update_cpu_time(delta_cpu_time);
         }
     }
 
diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h
index 72e1532e58d4f1..e0a44af69c1d66 100644
--- a/be/src/runtime/thread_context.h
+++ b/be/src/runtime/thread_context.h
@@ -119,38 +119,40 @@
         __VA_ARGS__;                                                       \
     } while (0)
 
-#define LIMIT_LOCAL_SCAN_IO(data_dir, bytes_read)                     \
-    std::shared_ptr<IOThrottle> iot = nullptr;                        \
-    auto* t_ctx = doris::thread_context(true);                        \
-    if (t_ctx) {                                                      \
-        iot = t_ctx->get_local_scan_io_throttle(data_dir);            \
-    }                                                                 \
-    if (iot) {                                                        \
-        iot->acquire(-1);                                             \
-    }                                                                 \
-    Defer defer {                                                     \
-        [&]() {                                                       \
-            if (iot) {                                                \
-                iot->update_next_io_time(*bytes_read);                \
-                t_ctx->update_total_local_scan_io_adder(*bytes_read); \
-            }                                                         \
-        }                                                             \
+#define LIMIT_LOCAL_SCAN_IO(data_dir, bytes_read)                   \
+    std::shared_ptr<IOThrottle> iot = nullptr;                      \
+    auto* t_ctx = doris::thread_context(true);                      \
+    if (t_ctx) {                                                    \
+        iot = t_ctx->get_local_scan_io_throttle(data_dir);          \
+    }                                                               \
+    if (iot) {                                                      \
+        iot->acquire(-1);                                           \
+    }                                                               \
+    Defer defer {                                                   \
+        [&]() {                                                     \
+            if (iot) {                                              \
+                iot->update_next_io_time(*bytes_read);              \
+                t_ctx->update_local_scan_io(data_dir, *bytes_read); \
+            }                                                       \
+        }                                                           \
     }
 
-#define LIMIT_REMOTE_SCAN_IO(bytes_read)               \
-    std::shared_ptr<IOThrottle> iot = nullptr;         \
-    if (auto* t_ctx = doris::thread_context(true)) {   \
-        iot = t_ctx->get_remote_scan_io_throttle();    \
-    }                                                  \
-    if (iot) {                                         \
-        iot->acquire(-1);                              \
-    }                                                  \
-    Defer defer {                                      \
-        [&]() {                                        \
-            if (iot) {                                 \
-                iot->update_next_io_time(*bytes_read); \
-            }                                          \
-        }                                              \
+#define LIMIT_REMOTE_SCAN_IO(bytes_read)                   \
+    std::shared_ptr<IOThrottle> iot = nullptr;             \
+    auto* t_ctx = doris::thread_context(true);             \
+    if (t_ctx) {                                           \
+        iot = t_ctx->get_remote_scan_io_throttle();        \
+    }                                                      \
+    if (iot) {                                             \
+        iot->acquire(-1);                                  \
+    }                                                      \
+    Defer defer {                                          \
+        [&]() {                                            \
+            if (iot) {                                     \
+                iot->update_next_io_time(*bytes_read);     \
+                t_ctx->update_remote_scan_io(*bytes_read); \
+            }                                              \
+        }                                                  \
     }
 
 namespace doris {
@@ -282,9 +284,15 @@ class ThreadContext {
         return nullptr;
     }
 
-    void update_total_local_scan_io_adder(size_t bytes_read) {
+    void update_local_scan_io(std::string path, size_t bytes_read) {
+        if (std::shared_ptr<WorkloadGroup> wg_ptr = _wg_wptr.lock()) {
+            wg_ptr->update_local_scan_io(path, bytes_read);
+        }
+    }
+
+    void update_remote_scan_io(size_t bytes_read) {
         if (std::shared_ptr<WorkloadGroup> wg_ptr = _wg_wptr.lock()) {
-            wg_ptr->update_total_local_scan_io_adder(bytes_read);
+            wg_ptr->update_remote_scan_io(bytes_read);
         }
     }
 
diff --git a/be/src/runtime/workload_group/workload_group.cpp b/be/src/runtime/workload_group/workload_group.cpp
index f9405de12737dc..6b9388af30a7f7 100644
--- a/be/src/runtime/workload_group/workload_group.cpp
+++ b/be/src/runtime/workload_group/workload_group.cpp
@@ -35,6 +35,7 @@
 #include "runtime/exec_env.h"
 #include "runtime/memory/global_memory_arbitrator.h"
 #include "runtime/memory/mem_tracker_limiter.h"
+#include "runtime/workload_group/workload_group_metrics.h"
 #include "runtime/workload_management/io_throttle.h"
 #include "util/mem_info.h"
 #include "util/parse_util.h"
@@ -71,18 +72,11 @@ WorkloadGroup::WorkloadGroup(const WorkloadGroupInfo& tg_info, bool need_create_
           _need_create_query_thread_pool(need_create_query_thread_pool) {
     std::vector<DataDirInfo>& data_dir_list = io::BeConfDataDirReader::be_config_data_dir_list;
     for (const auto& data_dir : data_dir_list) {
-        _scan_io_throttle_map[data_dir.path] =
-                std::make_shared<IOThrottle>(_name, data_dir.bvar_name + "_read_bytes");
-    }
-    _remote_scan_io_throttle = std::make_shared<IOThrottle>(_name, "remote_read_bytes");
-    _mem_used_status = std::make_unique<bvar::Status<int64_t>>(_name, "memory_used", 0);
-    _cpu_usage_adder = std::make_unique<bvar::Adder<uint64_t>>(_name, "cpu_usage_adder");
-    _cpu_usage_per_second = std::make_unique<bvar::PerSecond<bvar::Adder<uint64_t>>>(
-            _name, "cpu_usage", _cpu_usage_adder.get(), 10);
-    _total_local_scan_io_adder =
-            std::make_unique<bvar::Adder<size_t>>(_name, "total_local_read_bytes");
-    _total_local_scan_io_per_second = std::make_unique<bvar::PerSecond<bvar::Adder<size_t>>>(
-            _name, "total_local_read_bytes_per_second", _total_local_scan_io_adder.get(), 1);
+        _scan_io_throttle_map[data_dir.path] = std::make_shared<IOThrottle>(data_dir.bvar_name);
+    }
+    _remote_scan_io_throttle = std::make_shared<IOThrottle>();
+
+    _wg_metrics = std::make_shared<WorkloadGroupMetrics>(this);
 }
 
 std::string WorkloadGroup::debug_string() const {
@@ -169,11 +163,11 @@ int64_t WorkloadGroup::make_memory_tracker_snapshots(
     }
     // refresh total memory used.
     _total_mem_used = used_memory;
+    _wg_metrics->update_memory_used_bytes(used_memory);
     // reserve memory is recorded in the query mem tracker
     // and _total_mem_used already contains all the current reserve memory.
     // so after refreshing _total_mem_used, reset _wg_refresh_interval_memory_growth.
     _wg_refresh_interval_memory_growth.store(0.0);
-    _mem_used_status->set_value(used_memory);
     return used_memory;
 }
 
@@ -658,16 +652,20 @@ std::shared_ptr<IOThrottle> WorkloadGroup::get_remote_scan_io_throttle() {
     return _remote_scan_io_throttle;
 }
 
-void WorkloadGroup::update_cpu_adder(int64_t delta_cpu_time) {
-    (*_cpu_usage_adder) << (uint64_t)delta_cpu_time;
+void WorkloadGroup::update_cpu_time(int64_t delta_cpu_time) {
+    _wg_metrics->update_cpu_time_nanos(delta_cpu_time);
+}
+
+void WorkloadGroup::update_local_scan_io(std::string path, size_t scan_bytes) {
+    _wg_metrics->update_local_scan_io_bytes(path, (uint64_t)scan_bytes);
 }
 
-void WorkloadGroup::update_total_local_scan_io_adder(size_t scan_bytes) {
-    (*_total_local_scan_io_adder) << scan_bytes;
+void WorkloadGroup::update_remote_scan_io(size_t scan_bytes) {
+    _wg_metrics->update_remote_scan_io_bytes((uint64_t)scan_bytes);
 }
 
-int64_t WorkloadGroup::get_remote_scan_bytes_per_second() {
-    return _remote_scan_io_throttle->get_bvar_io_per_second();
+int64_t WorkloadGroup::get_mem_used() {
+    return _total_mem_used;
 }
 
 void WorkloadGroup::try_stop_schedulers() {
diff --git a/be/src/runtime/workload_group/workload_group.h b/be/src/runtime/workload_group/workload_group.h
index fb89ed8101ad49..35a8802e4c449a 100644
--- a/be/src/runtime/workload_group/workload_group.h
+++ b/be/src/runtime/workload_group/workload_group.h
@@ -17,7 +17,6 @@
 
 #pragma once
 
-#include <bvar/bvar.h>
 #include <gen_cpp/BackendService_types.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -54,6 +53,8 @@ class TaskScheduler;
 class WorkloadGroup;
 struct WorkloadGroupInfo;
 struct TrackerLimiterGroup;
+class WorkloadGroupMetrics;
+
 class WorkloadGroup : public std::enable_shared_from_this<WorkloadGroup> {
 public:
     explicit WorkloadGroup(const WorkloadGroupInfo& tg_info);
@@ -189,16 +190,13 @@ class WorkloadGroup : public std::enable_shared_from_this<WorkloadGroup> {
 
     void upsert_scan_io_throttle(WorkloadGroupInfo* tg_info);
 
-    void update_cpu_adder(int64_t delta_cpu_time);
+    void update_cpu_time(int64_t delta_cpu_time);
 
-    void update_total_local_scan_io_adder(size_t scan_bytes);
+    void update_local_scan_io(std::string path, size_t scan_bytes);
 
-    int64_t get_mem_used() { return _mem_used_status->get_value(); }
-    uint64_t get_cpu_usage() { return _cpu_usage_per_second->get_value(); }
-    int64_t get_local_scan_bytes_per_second() {
-        return _total_local_scan_io_per_second->get_value();
-    }
-    int64_t get_remote_scan_bytes_per_second();
+    void update_remote_scan_io(size_t scan_bytes);
+
+    int64_t get_mem_used();
 
     ThreadPool* get_memtable_flush_pool_ptr() {
         // no lock here because this is called by memtable flush,
@@ -209,6 +207,10 @@ class WorkloadGroup : public std::enable_shared_from_this<WorkloadGroup> {
 
     std::weak_ptr<CgroupCpuCtl> get_cgroup_cpu_ctl_wptr();
 
+    std::shared_ptr<WorkloadGroupMetrics> get_metrics() { return _wg_metrics; }
+
+    friend class WorkloadGroupMetrics;
+
 private:
     void create_cgroup_cpu_ctl_no_lock();
     void upsert_cgroup_cpu_ctl_no_lock(WorkloadGroupInfo* wg_info);
@@ -260,12 +262,7 @@ class WorkloadGroup : public std::enable_shared_from_this<WorkloadGroup> {
     // for some background workload, it doesn't need to create query thread pool
     const bool _need_create_query_thread_pool;
 
-    // bvar metric
-    std::unique_ptr<bvar::Status<int64_t>> _mem_used_status;
-    std::unique_ptr<bvar::Adder<uint64_t>> _cpu_usage_adder;
-    std::unique_ptr<bvar::PerSecond<bvar::Adder<uint64_t>>> _cpu_usage_per_second;
-    std::unique_ptr<bvar::Adder<size_t>> _total_local_scan_io_adder;
-    std::unique_ptr<bvar::PerSecond<bvar::Adder<size_t>>> _total_local_scan_io_per_second;
+    std::shared_ptr<WorkloadGroupMetrics> _wg_metrics {nullptr};
 };
 
 using WorkloadGroupPtr = std::shared_ptr<WorkloadGroup>;
diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp b/be/src/runtime/workload_group/workload_group_manager.cpp
index e2c4b8c7f5e6c6..1e01a7ce1bafb1 100644
--- a/be/src/runtime/workload_group/workload_group_manager.cpp
+++ b/be/src/runtime/workload_group/workload_group_manager.cpp
@@ -26,6 +26,7 @@
 #include "pipeline/task_scheduler.h"
 #include "runtime/memory/mem_tracker_limiter.h"
 #include "runtime/workload_group/workload_group.h"
+#include "runtime/workload_group/workload_group_metrics.h"
 #include "util/mem_info.h"
 #include "util/threadpool.h"
 #include "util/time.h"
@@ -287,16 +288,25 @@ void WorkloadGroupMgr::get_wg_resource_usage(vectorized::Block* block) {
     for (const auto& [id, wg] : _workload_groups) {
         SchemaScannerHelper::insert_int64_value(0, be_id, block);
         SchemaScannerHelper::insert_int64_value(1, wg->id(), block);
-        SchemaScannerHelper::insert_int64_value(2, wg->get_mem_used(), block);
+        SchemaScannerHelper::insert_int64_value(2, wg->get_metrics()->get_memory_used(), block);
 
-        double cpu_usage_p =
-                (double)wg->get_cpu_usage() / (double)total_cpu_time_ns_per_second * 100;
+        double cpu_usage_p = (double)wg->get_metrics()->get_cpu_time_nanos_per_second() /
+                             (double)total_cpu_time_ns_per_second * 100;
         cpu_usage_p = std::round(cpu_usage_p * 100.0) / 100.0;
 
         SchemaScannerHelper::insert_double_value(3, cpu_usage_p, block);
 
-        SchemaScannerHelper::insert_int64_value(4, wg->get_local_scan_bytes_per_second(), block);
-        SchemaScannerHelper::insert_int64_value(5, wg->get_remote_scan_bytes_per_second(), block);
+        SchemaScannerHelper::insert_int64_value(
+                4, wg->get_metrics()->get_local_scan_bytes_per_second(), block);
+        SchemaScannerHelper::insert_int64_value(
+                5, wg->get_metrics()->get_remote_scan_bytes_per_second(), block);
+    }
+}
+
+void WorkloadGroupMgr::refresh_workload_group_metrics() {
+    std::shared_lock<std::shared_mutex> r_lock(_group_mutex);
+    for (const auto& [id, wg] : _workload_groups) {
+        wg->get_metrics()->refresh_metrics();
     }
 }
 
diff --git a/be/src/runtime/workload_group/workload_group_manager.h b/be/src/runtime/workload_group/workload_group_manager.h
index c0eb0dfc0b237e..5d75a4558ef4f8 100644
--- a/be/src/runtime/workload_group/workload_group_manager.h
+++ b/be/src/runtime/workload_group/workload_group_manager.h
@@ -76,6 +76,8 @@ class WorkloadGroupMgr {
         return _workload_groups[INTERNAL_WORKLOAD_GROUP_ID];
     }
 
+    void refresh_workload_group_metrics();
+
 private:
     std::shared_mutex _group_mutex;
     std::unordered_map<uint64_t, WorkloadGroupPtr> _workload_groups;
diff --git a/be/src/runtime/workload_group/workload_group_metrics.cpp b/be/src/runtime/workload_group/workload_group_metrics.cpp
new file mode 100644
index 00000000000000..18ff7aa2f4f185
--- /dev/null
+++ b/be/src/runtime/workload_group/workload_group_metrics.cpp
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/workload_group/workload_group_metrics.h"
+
+#include "runtime/workload_group/workload_group.h"
+#include "runtime/workload_management/io_throttle.h"
+#include "util/doris_metrics.h"
+#include "util/metrics.h"
+
+namespace doris {
+
+#include "common/compile_check_begin.h"
+
+WorkloadGroupMetrics::~WorkloadGroupMetrics() {
+    DorisMetrics::instance()->metric_registry()->deregister_entity(_entity);
+}
+
+WorkloadGroupMetrics::WorkloadGroupMetrics(WorkloadGroup* wg) {
+    _entity = DorisMetrics::instance()->metric_registry()->register_entity(
+            "workload_group." + wg->name(), {{"name", wg->name()}});
+
+    _cpu_time_metric = std::make_unique<doris::MetricPrototype>(
+            doris::MetricType::COUNTER, doris::MetricUnit::SECONDS, "workload_group_cpu_time_sec");
+    _cpu_time_counter =
+            (IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(_cpu_time_metric.get()));
+
+    _mem_used_bytes_metric = std::make_unique<doris::MetricPrototype>(
+            doris::MetricType::COUNTER, doris::MetricUnit::BYTES, "workload_group_mem_used_bytes");
+    _mem_used_bytes_counter = (IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(
+            _mem_used_bytes_metric.get()));
+
+    _local_scan_bytes_metric = std::make_unique<doris::MetricPrototype>(
+            doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
+            "workload_group_local_scan_bytes");
+    _local_scan_bytes_counter = (IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(
+            _local_scan_bytes_metric.get()));
+
+    _remote_scan_bytes_metric = std::make_unique<doris::MetricPrototype>(
+            doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
+            "workload_group_remote_scan_bytes");
+    _remote_scan_bytes_counter = (IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(
+            _remote_scan_bytes_metric.get()));
+
+    for (const auto& [key, io_throttle] : wg->_scan_io_throttle_map) {
+        std::unique_ptr<doris::MetricPrototype> metric = std::make_unique<doris::MetricPrototype>(
+                doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
+                "workload_group_local_scan_bytes_" + io_throttle->metric_name());
+        _local_scan_bytes_counter_map[key] =
+                (IntAtomicCounter*)(_entity->register_metric<IntAtomicCounter>(metric.get()));
+        _local_scan_bytes_metric_map[key] = std::move(metric);
+    }
+}
+
+void WorkloadGroupMetrics::update_cpu_time_nanos(uint64_t delta_cpu_time) {
+    _cpu_time_nanos += delta_cpu_time;
+}
+
+void WorkloadGroupMetrics::update_memory_used_bytes(int64_t memory_used) {
+    _memory_used = memory_used;
+}
+
+void WorkloadGroupMetrics::update_local_scan_io_bytes(std::string path, uint64_t delta_io_bytes) {
+    _local_scan_bytes_counter->increment(delta_io_bytes);
+    _local_scan_bytes_counter_map[path]->increment((int64_t)delta_io_bytes);
+}
+
+void WorkloadGroupMetrics::update_remote_scan_io_bytes(uint64_t delta_io_bytes) {
+    _remote_scan_bytes_counter->increment(delta_io_bytes);
+}
+
+void WorkloadGroupMetrics::refresh_metrics() {
+    int interval_second = config::workload_group_metrics_interval_ms / 1000;
+
+    // cpu
+    uint64_t _current_cpu_time_nanos = _cpu_time_nanos.load();
+    uint64_t _cpu_time_sec = _current_cpu_time_nanos / (1000L * 1000L * 1000L);
+    _cpu_time_counter->set_value(_cpu_time_sec);
+    _per_sec_cpu_time_nanos = (_current_cpu_time_nanos - _last_cpu_time_nanos) / interval_second;
+    _last_cpu_time_nanos = _current_cpu_time_nanos;
+
+    // memory
+    _mem_used_bytes_counter->set_value(_memory_used);
+
+    // local scan
+    int64_t current_local_scan_bytes = _local_scan_bytes_counter->value();
+    _per_sec_local_scan_bytes =
+            (current_local_scan_bytes - _last_local_scan_bytes) / interval_second;
+    _last_local_scan_bytes = current_local_scan_bytes;
+
+    // remote scan
+    int64_t current_remote_scan_bytes = _remote_scan_bytes_counter->value();
+    _per_sec_remote_scan_bytes =
+            (current_remote_scan_bytes - _last_remote_scan_bytes) / interval_second;
+    _last_remote_scan_bytes = current_remote_scan_bytes;
+}
+
+uint64_t WorkloadGroupMetrics::get_cpu_time_nanos_per_second() {
+    return _per_sec_cpu_time_nanos.load();
+}
+
+int64_t WorkloadGroupMetrics::get_local_scan_bytes_per_second() {
+    return _per_sec_local_scan_bytes.load();
+}
+
+int64_t WorkloadGroupMetrics::get_remote_scan_bytes_per_second() {
+    return _last_remote_scan_bytes.load();
+}
+
+int64_t WorkloadGroupMetrics::get_memory_used() {
+    return _mem_used_bytes_counter->value();
+}
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_group/workload_group_metrics.h b/be/src/runtime/workload_group/workload_group_metrics.h
new file mode 100644
index 00000000000000..e68715df249dee
--- /dev/null
+++ b/be/src/runtime/workload_group/workload_group_metrics.h
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <atomic>
+#include <map>
+#include <memory>
+#include <string>
+
+namespace doris {
+
+class WorkloadGroup;
+
+template <typename T>
+class AtomicCounter;
+using IntAtomicCounter = AtomicCounter<int64_t>;
+class MetricEntity;
+struct MetricPrototype;
+
+class WorkloadGroupMetrics {
+public:
+    WorkloadGroupMetrics(WorkloadGroup* wg);
+
+    ~WorkloadGroupMetrics();
+
+    void update_cpu_time_nanos(uint64_t delta_cpu_time);
+
+    void update_memory_used_bytes(int64_t memory_used);
+
+    void update_local_scan_io_bytes(std::string path, uint64_t delta_io_bytes);
+
+    void update_remote_scan_io_bytes(uint64_t delta_io_bytes);
+
+    void refresh_metrics();
+
+    uint64_t get_cpu_time_nanos_per_second();
+
+    int64_t get_local_scan_bytes_per_second();
+
+    int64_t get_remote_scan_bytes_per_second();
+
+    int64_t get_memory_used();
+
+private:
+    std::unique_ptr<doris::MetricPrototype> _cpu_time_metric {nullptr};
+    std::unique_ptr<doris::MetricPrototype> _mem_used_bytes_metric {nullptr};
+    std::unique_ptr<doris::MetricPrototype> _local_scan_bytes_metric {nullptr};
+    std::unique_ptr<doris::MetricPrototype> _remote_scan_bytes_metric {nullptr};
+    // NOTE: _local_scan_bytes_metric is sum of all disk's IO
+    // _local_disk_io_metric is every disk's IO
+    std::map<std::string, std::unique_ptr<doris::MetricPrototype>> _local_scan_bytes_metric_map;
+
+    IntAtomicCounter* _cpu_time_counter {nullptr};                          // used for metric
+    IntAtomicCounter* _mem_used_bytes_counter {nullptr};                    // used for metric
+    IntAtomicCounter* _local_scan_bytes_counter {nullptr};                  // used for metric
+    IntAtomicCounter* _remote_scan_bytes_counter {nullptr};                 // used for metric
+    std::map<std::string, IntAtomicCounter*> _local_scan_bytes_counter_map; // used for metric
+
+    std::atomic<uint64_t> _cpu_time_nanos {0};
+    std::atomic<uint64_t> _last_cpu_time_nanos {0};
+    std::atomic<uint64_t> _per_sec_cpu_time_nanos {0}; // used for system table
+
+    std::atomic<uint64_t> _per_sec_local_scan_bytes {0};
+    std::atomic<uint64_t> _last_local_scan_bytes {0}; // used for system table
+
+    std::atomic<uint64_t> _per_sec_remote_scan_bytes {0};
+    std::atomic<uint64_t> _last_remote_scan_bytes {0}; // used for system table
+
+    std::atomic<uint64_t> _memory_used {0};
+
+    std::shared_ptr<MetricEntity> _entity {nullptr};
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/io_throttle.cpp b/be/src/runtime/workload_management/io_throttle.cpp
index dacfa29012f59f..118fc518072272 100644
--- a/be/src/runtime/workload_management/io_throttle.cpp
+++ b/be/src/runtime/workload_management/io_throttle.cpp
@@ -22,12 +22,6 @@
 
 namespace doris {
 
-IOThrottle::IOThrottle(std::string prefix, std::string name) {
-    _io_adder = std::make_unique<bvar::Adder<size_t>>(prefix, name);
-    _io_adder_per_second = std::make_unique<bvar::PerSecond<bvar::Adder<size_t>>>(
-            prefix, name + "_per_second", _io_adder.get(), 1);
-}
-
 bool IOThrottle::acquire(int64_t block_timeout_ms) {
     if (_io_bytes_per_second_limit < 0) {
         return true;
@@ -57,11 +51,6 @@ bool IOThrottle::try_acquire() {
 }
 
 void IOThrottle::update_next_io_time(int64_t io_bytes) {
-    Defer defer {[&]() {
-        if (io_bytes > 0) {
-            (*_io_adder) << io_bytes;
-        }
-    }};
     if (_io_bytes_per_second_limit <= 0 || io_bytes <= 0) {
         return;
     }
diff --git a/be/src/runtime/workload_management/io_throttle.h b/be/src/runtime/workload_management/io_throttle.h
index 4212527020e0e2..f688922fcd29f9 100644
--- a/be/src/runtime/workload_management/io_throttle.h
+++ b/be/src/runtime/workload_management/io_throttle.h
@@ -28,7 +28,9 @@ namespace doris {
 
 class IOThrottle {
 public:
-    IOThrottle(std::string prefix, std::string name);
+    IOThrottle() = default;
+
+    IOThrottle(std::string metric_name) : _metric_name(metric_name) {}
 
     ~IOThrottle() = default;
 
@@ -41,7 +43,7 @@ class IOThrottle {
 
     void set_io_bytes_per_second(int64_t read_bytes_per_second);
 
-    size_t get_bvar_io_per_second() { return _io_adder_per_second->get_value(); }
+    std::string metric_name() { return _metric_name; }
 
 private:
     std::mutex _mutex;
@@ -49,8 +51,6 @@ class IOThrottle {
     int64_t _next_io_time_micros {0};
     std::atomic<int64_t> _io_bytes_per_second_limit {-1};
 
-    // bvar monitor
-    std::unique_ptr<bvar::Adder<size_t>> _io_adder;
-    std::unique_ptr<bvar::PerSecond<bvar::Adder<size_t>>> _io_adder_per_second;
+    std::string _metric_name;
 };
 }; // namespace doris
\ No newline at end of file
diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp
index c1385b6244bf62..973f461d8defe7 100644
--- a/be/src/util/system_metrics.cpp
+++ b/be/src/util/system_metrics.cpp
@@ -38,6 +38,16 @@
 
 namespace doris {
 
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(host_cpu_num, MetricUnit::NOUNIT);
+struct CpuNumberMetrics {
+    CpuNumberMetrics(MetricEntity* ent) : entity(ent) {
+        INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, host_cpu_num);
+    }
+
+    IntAtomicCounter* host_cpu_num {nullptr};
+    MetricEntity* entity = nullptr;
+};
+
 #define DEFINE_CPU_COUNTER_METRIC(metric)                                            \
     DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(cpu_##metric, MetricUnit::PERCENT, "", cpu, \
                                          Labels({{"mode", #metric}}));
@@ -386,11 +396,22 @@ void SystemMetrics::update() {
 
 void SystemMetrics::_install_cpu_metrics() {
     get_cpu_name();
+
+    int cpu_num = 0;
     for (auto cpu_name : _cpu_names) {
+        // NOTE: cpu_name comes from /proc/stat which named 'cpu' is not a real cpu name, it should be skipped.
+        if (cpu_name != "cpu") {
+            cpu_num++;
+        }
         auto cpu_entity = _registry->register_entity(cpu_name, {{"device", cpu_name}});
         CpuMetrics* metrics = new CpuMetrics(cpu_entity.get());
         _cpu_metrics.emplace(cpu_name, metrics);
     }
+
+    auto cpu_num_entity = _registry->register_entity("doris_be_host_cpu_num");
+    _cpu_num_metrics = std::make_unique<CpuNumberMetrics>(cpu_num_entity.get());
+
+    _cpu_num_metrics->host_cpu_num->set_value(cpu_num);
 }
 
 #ifdef BE_TEST
diff --git a/be/src/util/system_metrics.h b/be/src/util/system_metrics.h
index c72ba3693012fb..29ce8c9c02b359 100644
--- a/be/src/util/system_metrics.h
+++ b/be/src/util/system_metrics.h
@@ -31,6 +31,7 @@
 namespace doris {
 
 struct CpuMetrics;
+struct CpuNumberMetrics;
 struct MemoryMetrics;
 struct DiskMetrics;
 struct NetworkMetrics;
@@ -99,6 +100,7 @@ class SystemMetrics {
     static const char* _s_hook_name;
 
     std::map<std::string, CpuMetrics*> _cpu_metrics;
+    std::unique_ptr<CpuNumberMetrics> _cpu_num_metrics;
     std::unique_ptr<MemoryMetrics> _memory_metrics;
     std::map<std::string, DiskMetrics*> _disk_metrics;
     std::map<std::string, NetworkMetrics*> _network_metrics;
diff --git a/be/src/vec/exec/scan/vscanner.cpp b/be/src/vec/exec/scan/vscanner.cpp
index 97bf563db1fa58..0087a19d92f54a 100644
--- a/be/src/vec/exec/scan/vscanner.cpp
+++ b/be/src/vec/exec/scan/vscanner.cpp
@@ -262,7 +262,7 @@ void VScanner::update_scan_cpu_timer() {
     _scan_cpu_timer += cpu_time;
     _query_statistics->add_cpu_nanos(cpu_time);
     if (_state && _state->get_query_ctx()) {
-        _state->get_query_ctx()->update_wg_cpu_adder(cpu_time);
+        _state->get_query_ctx()->update_cpu_time(cpu_time);
     }
 }
 
diff --git a/be/src/vec/sink/writer/async_result_writer.cpp b/be/src/vec/sink/writer/async_result_writer.cpp
index 65210a53ec3b55..ed4f71677f2afe 100644
--- a/be/src/vec/sink/writer/async_result_writer.cpp
+++ b/be/src/vec/sink/writer/async_result_writer.cpp
@@ -127,7 +127,7 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* profi
             cpu_time_stop_watch.start();
             Defer defer {[&]() {
                 if (state && state->get_query_ctx()) {
-                    state->get_query_ctx()->update_wg_cpu_adder(cpu_time_stop_watch.elapsed_time());
+                    state->get_query_ctx()->update_cpu_time(cpu_time_stop_watch.elapsed_time());
                 }
             }};
             if (!_eos && _data_queue.empty() && _writer_status.ok()) {

From 8ad6b283403fd8b3407dc91af25c0f7906109b81 Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Tue, 10 Dec 2024 16:21:59 +0800
Subject: [PATCH 253/399] [fix](nereids) not rewrite first_value when
 first_value ignore nulls (#45065)

this pr fix 2 things:
1.when first_value/last_value second parameter is true, which means
ignore nulls, fe should not do some rewrite.
2.adding check that first_value/last_value second parameter must be true
or false.

Related PR: #44996 #27623
---
 .../rules/analysis/WindowFunctionChecker.java | 11 ++++++
 .../functions/window/FirstOrLastValue.java    | 15 ++++++++
 .../nereids_syntax_p0/window_function.groovy  | 35 +++++++++++++++++++
 3 files changed, 61 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/WindowFunctionChecker.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/WindowFunctionChecker.java
index 036b6be8b20201..d6904ae074c7da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/WindowFunctionChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/WindowFunctionChecker.java
@@ -37,6 +37,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.window.PercentRank;
 import org.apache.doris.nereids.trees.expressions.functions.window.Rank;
 import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
 import org.apache.doris.nereids.util.TypeCoercionUtils;
@@ -315,6 +316,10 @@ public Lead visitLead(Lead lead, Void ctx) {
      */
     @Override
     public FirstOrLastValue visitFirstValue(FirstValue firstValue, Void ctx) {
+        FirstOrLastValue.checkSecondParameter(firstValue);
+        if (2 == firstValue.arity() && firstValue.child(1).equals(BooleanLiteral.TRUE)) {
+            return firstValue;
+        }
         Optional<WindowFrame> windowFrame = windowExpression.getWindowFrame();
         if (windowFrame.isPresent()) {
             WindowFrame wf = windowFrame.get();
@@ -339,6 +344,12 @@ public FirstOrLastValue visitFirstValue(FirstValue firstValue, Void ctx) {
         return firstValue;
     }
 
+    @Override
+    public FirstOrLastValue visitLastValue(LastValue lastValue, Void ctx) {
+        FirstOrLastValue.checkSecondParameter(lastValue);
+        return lastValue;
+    }
+
     /**
      * required WindowFrame: (RANGE, UNBOUNDED PRECEDING, CURRENT ROW)
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/FirstOrLastValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/FirstOrLastValue.java
index bb0a68eef6adbc..c21c420f703d44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/FirstOrLastValue.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/FirstOrLastValue.java
@@ -18,9 +18,11 @@
 package org.apache.doris.nereids.trees.expressions.functions.window;
 
 import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.types.coercion.AnyDataType;
 
@@ -63,4 +65,17 @@ public FirstOrLastValue reverse() {
     public List<FunctionSignature> getSignatures() {
         return SIGNATURES;
     }
+
+    /**check the second parameter must be true or false*/
+    public static void checkSecondParameter(FirstOrLastValue firstOrLastValue) {
+        if (1 == firstOrLastValue.arity()) {
+            return;
+        }
+        if (!BooleanLiteral.TRUE.equals(firstOrLastValue.child(1))
+                && !BooleanLiteral.FALSE.equals(firstOrLastValue.child(1))) {
+            throw new AnalysisException("The second parameter of " + firstOrLastValue.getName()
+                    + " must be a constant or a constant expression, and the result of "
+                    + "the calculated constant or constant expression must be true or false.");
+        }
+    }
 }
diff --git a/regression-test/suites/nereids_syntax_p0/window_function.groovy b/regression-test/suites/nereids_syntax_p0/window_function.groovy
index 31cb425fcf7a44..f2ce708f4c372a 100644
--- a/regression-test/suites/nereids_syntax_p0/window_function.groovy
+++ b/regression-test/suites/nereids_syntax_p0/window_function.groovy
@@ -279,4 +279,39 @@ suite("window_function") {
                 )t
             )a where rn=1
     """
+
+    // test first value second param is not constant
+    test {
+        sql "select first_value(c1,c1) over() from window_test"
+        exception "The second parameter of first_value must be a constant or a constant expression, and the result of the calculated constant or constant expression must be true or false."
+    }
+
+    test {
+        sql "select last_value(c1,c1) over() from window_test"
+        exception "The second parameter of last_value must be a constant or a constant expression, and the result of the calculated constant or constant expression must be true or false."
+    }
+
+    test {
+        sql "select first_value(c1,cast('abc' as boolean)) over() from window_test"
+        exception "The second parameter of first_value must be a constant or a constant expression, and the result of the calculated constant or constant expression must be true or false."
+    }
+    test {
+        sql "select first_value(c1,'') over() from window_test"
+        exception "The second parameter of first_value must be a constant or a constant expression, and the result of the calculated constant or constant expression must be true or false."
+    }
+    test {
+        sql "select last_value(c1,'345_a') over() from window_test"
+        exception "The second parameter of last_value must be a constant or a constant expression, and the result of the calculated constant or constant expression must be true or false."
+    }
+    sql "select last_value(c1,cast('67' as boolean)) over() from window_test"
+    sql "select first_value(c1,cast(56 as boolean)) over() from window_test"
+    sql "select last_value(c1,cast(56 as boolean)) over() from window_test"
+    sql "select first_value(c1,cast('true' as boolean)) over() from window_test"
+    sql "select last_value(c1,cast('false' as boolean)) over() from window_test"
+    sql "select first_value(c1,cast('1' as boolean)) over() from window_test"
+    sql "select last_value(c1,cast('0' as boolean)) over() from window_test"
+    sql "select first_value(c1,true) over() from window_test"
+    sql "select last_value(c1,false) over() from window_test"
+    sql "select first_value(c1,1) over() from window_test"
+    sql "select last_value(c1,0) over() from window_test"
 }

From df90de94618865e4832648e967b279c68e1c8703 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Tue, 10 Dec 2024 16:28:13 +0800
Subject: [PATCH 254/399] [performance](load) increase max_broker_concurrency
 to 100 (#44929)

Increase default `max_broker_concurrency` to 100 to improve broker load
performance.
This option will affect the max number of scan / sink instances allowed
in a broker load.

```
parallel instance = min(max_broker_concurrency,
                        source file size / min_bytes_per_broker_scanner,
                        num backends * load_parallelism)
```

S3 load time of tpcds_1000g catalog_sales:
* before: 438s
* after: 225s
---
 fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 247c61ecd365ec..50d43bffa1b821 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -784,7 +784,7 @@ public class Config extends ConfigBase {
 
     @ConfField(mutable = true, masterOnly = true, description = {
             "单个 broker scanner 的最大并发数。", "Maximal concurrency of broker scanners."})
-    public static int max_broker_concurrency = 10;
+    public static int max_broker_concurrency = 100;
 
     // TODO(cmy): Disable by default because current checksum logic has some bugs.
     @ConfField(mutable = true, masterOnly = true, description = {

From 5e655d488c67cf36457a48dec4534eb9f2033d61 Mon Sep 17 00:00:00 2001
From: deardeng <dengxin@selectdb.com>
Date: Tue, 10 Dec 2024 16:33:37 +0800
Subject: [PATCH 255/399] =?UTF-8?q?[fix](docker=20case)=20Fix=20`test=5Fsq?=
 =?UTF-8?q?l=5Fmode=5Fnode=5Fmgr`=20and=20add=20cloud=20multi=20f=E2=80=A6?=
 =?UTF-8?q?=20(#44124)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ollower fuzzy test

### What problem does this PR solve?
1. fix start docker cluster slow, double cycle use one break, can't
break out
2. Fix the test_sql_mode_node_mgr case that fails to meet testing
objectives, as the container exits after dropping the follower.
3. Add fuzzy add/drop follower/observer cases on the cloud.
4. Fix docker compose error `java.lang.Exception: doris compose output
is empty, err:`
5. Fix show frontends slow query when change fe nodes, due to `show
frontends` use is isJoin function call
getHostName(https://bugs.openjdk.org/browse/JDK-8143378#:~:text=getHostName()%3B%20takes%20about%205,millisecond%20on%20JDK%20update%2051.)
6. and Fix
```
2024-11-28 15:50:05,272 WARN (cloud cluster check|302) [CloudClusterChecker.checkCloudBackends():518] diff cluster has exception, Cannot invoke "org.apache.doris.system.Backend.isDecommissioned()" because "be" is null
java.lang.NullPointerException: Cannot invoke "org.apache.doris.system.Backend.isDecommissioned()" because "be" is null
        at org.apache.doris.cloud.catalog.CloudClusterChecker.updateStatus(CloudClusterChecker.java:175) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.cloud.catalog.CloudClusterChecker.checkDiffNode(CloudClusterChecker.java:263) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.cloud.catalog.CloudClusterChecker.checkCloudBackends(CloudClusterChecker.java:513) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.cloud.catalog.CloudClusterChecker.runAfterCatalogReady(CloudClusterChecker.java:320) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.common.util.MasterDaemon.runOneCycle(MasterDaemon.java:58) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.common.util.Daemon.run(Daemon.java:119) ~[doris-fe.jar:1.2-SNAPSHOT]
```
---
 docker/runtime/doris-compose/cluster.py       |  12 +-
 docker/runtime/doris-compose/command.py       |  32 +++--
 docker/runtime/doris-compose/database.py      |  58 +++++---
 docker/runtime/doris-compose/doris-compose.py |  25 ++--
 docker/runtime/doris-compose/requirements.txt |   3 +-
 .../runtime/doris-compose/resource/init_fe.sh |   7 -
 docker/runtime/doris-compose/utils.py         |  55 ++++----
 .../cloud/catalog/CloudClusterChecker.java    |   4 +
 .../apache/doris/cloud/catalog/CloudEnv.java  |   3 +-
 .../doris/common/proc/FrontendsProcNode.java  |  25 +++-
 .../regression/suite/SuiteCluster.groovy      |  21 +--
 .../node_mgr/test_sql_mode_node_mgr.groovy    | 127 ++++++++----------
 12 files changed, 206 insertions(+), 166 deletions(-)

diff --git a/docker/runtime/doris-compose/cluster.py b/docker/runtime/doris-compose/cluster.py
index ba834167bd1c63..f4522181d4b18e 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -23,6 +23,7 @@
 import os
 import os.path
 import utils
+import time
 
 DOCKER_DORIS_PATH = "/opt/apache-doris"
 LOCAL_DORIS_PATH = os.getenv("LOCAL_DORIS_PATH", "/tmp/doris")
@@ -139,11 +140,15 @@ def gen_subnet_prefix16():
 
 def get_master_fe_endpoint(cluster_name):
     master_fe_ip_file = get_cluster_path(cluster_name) + "/status/master_fe_ip"
-    if os.path.exists(master_fe_ip_file):
-        with open(master_fe_ip_file, "r") as f:
-            return "{}:{}".format(f.read().strip(), FE_QUERY_PORT)
+    max_retries = 10
+    for attempt in range(max_retries):
+        if os.path.exists(master_fe_ip_file):
+            with open(master_fe_ip_file, "r") as f:
+                return "{}:{}".format(f.read().strip(), FE_QUERY_PORT)
+        time.sleep(1)
     try:
         cluster = Cluster.load(cluster_name)
+        LOG.info("master file not exist, master ip get from node 1")
         return "{}:{}".format(
             cluster.get_node(Node.TYPE_FE, 1).get_ip(), FE_QUERY_PORT)
     except:
@@ -468,6 +473,7 @@ def get_add_init_config(self):
             for key in ("JAVA_OPTS", "JAVA_OPTS_FOR_JDK_17"):
                 value = parser["dummy_section"].get(key)
                 if value:
+                    value = value.strip().strip('"')
                     cfg.append(
                         f"{key} = \"{value} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:{FE_JAVA_DBG_PORT}\""
                     )
diff --git a/docker/runtime/doris-compose/command.py b/docker/runtime/doris-compose/command.py
index 7a2f3f3c195f18..638c1c465d75b1 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -183,7 +183,7 @@ def _support_boolean_action(self):
         return sys.version_info.major == 3 and sys.version_info.minor >= 9
 
     def _print_table(self, header, datas):
-        if utils.is_enable_log():
+        if utils.is_log_stdout():
             table = prettytable.PrettyTable(
                 [utils.render_green(field) for field in header])
             for row in datas:
@@ -598,13 +598,6 @@ def do_add_node(node_type, add_num, add_ids):
                                           related_nodes,
                                           output_real_time=output_real_time)
 
-        ls_cmd = "python docker/runtime/doris-compose/doris-compose.py ls " + cluster.name
-        LOG.info("Inspect command: " + utils.render_green(ls_cmd) + "\n")
-        LOG.info(
-            "Master fe query address: " +
-            utils.render_green(CLUSTER.get_master_fe_endpoint(cluster.name)) +
-            "\n")
-
         if not args.start:
             LOG.info(
                 utils.render_green(
@@ -618,14 +611,18 @@ def do_add_node(node_type, add_num, add_ids):
             LOG.info("Waiting for FE master to be elected...")
             expire_ts = time.time() + 30
             while expire_ts > time.time():
+                ready = False
                 db_mgr = database.get_db_mgr(args.NAME, False)
                 for id in add_fe_ids:
                     fe_state = db_mgr.get_fe(id)
                     if fe_state is not None and fe_state.alive:
+                        ready = True
                         break
-                    LOG.info("there is no fe ready")
-                time.sleep(5)
-
+                if ready:
+                    break
+                LOG.info("there is no fe ready")
+                time.sleep(1)
+            LOG.info("after Waiting for FE master to be elected...")
             if cluster.is_cloud and args.sql_mode_node_mgr:
                 db_mgr = database.get_db_mgr(args.NAME, False)
                 master_fe_endpoint = CLUSTER.get_master_fe_endpoint(
@@ -635,7 +632,9 @@ def do_add_node(node_type, add_num, add_ids):
                     fe_endpoint = f"{fe.get_ip()}:{CLUSTER.FE_EDITLOG_PORT}"
                     if fe_endpoint != master_fe_endpoint:
                         try:
-                            db_mgr.add_fe(fe_endpoint)
+                            db_mgr.add_fe(
+                                fe_endpoint, "FOLLOWER"
+                                if cluster.fe_follower else "OBSERVER")
                             LOG.info(f"Added FE {fe_endpoint} successfully.")
                         except Exception as e:
                             LOG.error(
@@ -661,6 +660,12 @@ def do_add_node(node_type, add_num, add_ids):
                     "Up cluster {} succ, related node num {}".format(
                         args.NAME, related_node_num)))
 
+        ls_cmd = "python docker/runtime/doris-compose/doris-compose.py ls " + cluster.name
+        LOG.info("Inspect command: " + utils.render_green(ls_cmd) + "\n")
+        LOG.info(
+            "Master fe query address: " +
+            utils.render_green(CLUSTER.get_master_fe_endpoint(cluster.name)) +
+            "\n")
         return {
             "fe": {
                 "add_list": add_fe_ids,
@@ -1066,8 +1071,7 @@ def parse_cluster_compose_file(cluster_name):
                 if services is None:
                     return COMPOSE_BAD, {}
                 return COMPOSE_GOOD, {
-                    service:
-                    ComposeService(
+                    service: ComposeService(
                         service,
                         list(service_conf["networks"].values())[0]
                         ["ipv4_address"], service_conf["image"])
diff --git a/docker/runtime/doris-compose/database.py b/docker/runtime/doris-compose/database.py
index 46cdd961c9f888..370f1d5ee2afa7 100644
--- a/docker/runtime/doris-compose/database.py
+++ b/docker/runtime/doris-compose/database.py
@@ -27,12 +27,13 @@
 
 class FEState(object):
 
-    def __init__(self, id, is_master, alive, last_heartbeat, err_msg):
+    def __init__(self, id, is_master, alive, last_heartbeat, err_msg, role):
         self.id = id
         self.is_master = is_master
         self.alive = alive
         self.last_heartbeat = last_heartbeat
         self.err_msg = err_msg
+        self.role = role
 
 
 class BEState(object):
@@ -66,11 +67,11 @@ def load_states(self):
         self._load_fe_states()
         self._load_be_states()
 
-    def add_fe(self, fe_endpoint):
+    def add_fe(self, fe_endpoint, role):
         try:
-            sql = f"ALTER SYSTEM ADD FOLLOWER '{fe_endpoint}'"
+            sql = f"ALTER SYSTEM ADD {role} '{fe_endpoint}'"
             self._exec_query(sql)
-            LOG.info(f"Added FE {fe_endpoint} via SQL successfully.")
+            LOG.info(f"Added {role} FE {fe_endpoint} via SQL successfully.")
         except Exception as e:
             LOG.error(f"Failed to add FE {fe_endpoint} via SQL: {str(e)}")
             raise
@@ -78,8 +79,9 @@ def add_fe(self, fe_endpoint):
     def drop_fe(self, fe_endpoint):
         id = CLUSTER.Node.get_id_from_ip(fe_endpoint[:fe_endpoint.find(":")])
         try:
-            self._exec_query(
-                "ALTER SYSTEM DROP FOLLOWER '{}'".format(fe_endpoint))
+            role = self.get_fe(id).role if self.get_fe(id) else "FOLLOWER"
+            self._exec_query("ALTER SYSTEM DROP {} '{}'".format(
+                role, fe_endpoint))
             LOG.info("Drop fe {} with id {} from db succ.".format(
                 fe_endpoint, id))
         except Exception as e:
@@ -152,7 +154,7 @@ def decommission_be(self, be_endpoint):
                 .format(be_endpoint, be.alive, be.decommissioned, be.tablet_num, old_tablet_num,
                         int(time.time() - start_ts)))
 
-            time.sleep(5)
+            time.sleep(1)
 
     def create_default_storage_vault(self, cloud_store_config):
         try:
@@ -194,7 +196,7 @@ def _load_fe_states(self):
             id = CLUSTER.Node.get_id_from_ip(ip)
             last_heartbeat = utils.escape_null(record["LastHeartbeat"])
             err_msg = record["ErrMsg"]
-            fe = FEState(id, is_master, alive, last_heartbeat, err_msg)
+            fe = FEState(id, is_master, alive, last_heartbeat, err_msg, role)
             fe_states[id] = fe
             if is_master and alive:
                 alive_master_fe_ip = ip
@@ -223,13 +225,23 @@ def _load_be_states(self):
         self.be_states = be_states
 
     # return rows, and each row is a record map
-    def _exec_query(self, sql):
+    def _exec_query(self, sql, retries=3):
         self._prepare_conn()
-        with self.conn.cursor() as cursor:
-            cursor.execute(sql)
-            fields = [field_md[0] for field_md in cursor.description
-                      ] if cursor.description else []
-            return [dict(zip(fields, row)) for row in cursor.fetchall()]
+        for attempt in range(retries):
+            try:
+                with self.conn.cursor() as cursor:
+                    cursor.execute(sql)
+                    fields = [field_md[0] for field_md in cursor.description
+                              ] if cursor.description else []
+                    return [dict(zip(fields, row)) for row in cursor.fetchall()]
+            except Exception as e:
+                LOG.warn(f"Error occurred: {e}")
+                if "timed out" in str(e).lower() and attempt < retries - 1:
+                    LOG.warn(f"Query timed out. Retrying {attempt + 1}/{retries}...")
+                    self._reset_conn()
+                else:
+                    raise e
+        raise Exception("Max retries exceeded")
 
     def _prepare_conn(self):
         if self.conn:
@@ -257,19 +269,23 @@ def get_db_mgr(cluster_name, required_load_succ=True):
     if not master_fe_ip:
         return db_mgr
 
-    has_alive_fe = False
+    alive_fe = None
+    cluster = CLUSTER.Cluster.load(cluster_name)
     containers = utils.get_doris_containers(cluster_name).get(cluster_name, [])
     for container in containers:
         if utils.is_container_running(container):
-            _, node_type, _ = utils.parse_service_name(container.name)
+            _, node_type, id = utils.parse_service_name(container.name)
             if node_type == CLUSTER.Node.TYPE_FE:
-                has_alive_fe = True
-                break
-
-    if not has_alive_fe:
+                node = cluster.get_node(node_type, id)
+                if not alive_fe:
+                    alive_fe = node
+                if node.get_ip() == master_fe_ip:
+                    alive_fe = node
+                    break
+    if not alive_fe:
         return db_mgr
 
-    db_mgr.master_fe_ip = master_fe_ip
+    db_mgr.master_fe_ip = alive_fe.get_ip()
     try:
         db_mgr.load_states()
     except Exception as e:
diff --git a/docker/runtime/doris-compose/doris-compose.py b/docker/runtime/doris-compose/doris-compose.py
index a2d3a517553f71..cf3692d53215e0 100644
--- a/docker/runtime/doris-compose/doris-compose.py
+++ b/docker/runtime/doris-compose/doris-compose.py
@@ -16,7 +16,9 @@
 # under the License.
 
 import argparse
+import cluster as CLUSTER
 import command
+import os.path
 import sys
 import traceback
 import utils
@@ -31,12 +33,12 @@ def parse_args():
     return ap.parse_args(), ap.format_help()
 
 
-def run(args, disable_log, help):
+def run(args, disable_log_stdout, help):
     for cmd in command.ALL_COMMANDS:
         if args.command == cmd.name:
             timer = utils.Timer()
             result = cmd.run(args)
-            if cmd.print_use_time() and not disable_log:
+            if cmd.print_use_time() and not disable_log_stdout:
                 timer.show()
             return result
     print(help)
@@ -48,19 +50,26 @@ def run(args, disable_log, help):
     verbose = getattr(args, "verbose", False)
     if verbose:
         utils.set_log_verbose()
-    disable_log = getattr(args, "output_json", False)
-    if disable_log:
-        utils.set_enable_log(False)
+    disable_log_stdout = getattr(args, "output_json", False)
+    if disable_log_stdout:
+        log_file_name = ""
+        cluster_name = getattr(args, "NAME", "")
+        if cluster_name:
+            if type(cluster_name) == type([]):
+                cluster_name = cluster_name[0]
+            log_file_name = os.path.join(
+                CLUSTER.get_cluster_path(cluster_name), "doris-compose.log")
+        utils.set_log_to(log_file_name, False)
 
     code = None
     try:
-        data = run(args, disable_log, help)
-        if disable_log:
+        data = run(args, disable_log_stdout, help)
+        if disable_log_stdout:
             print(utils.pretty_json({"code": 0, "data": data}), flush=True)
         code = 0
     except:
         err = traceback.format_exc()
-        if disable_log:
+        if disable_log_stdout:
             print(utils.pretty_json({"code": 1, "err": err}), flush=True)
         else:
             print(err, flush=True)
diff --git a/docker/runtime/doris-compose/requirements.txt b/docker/runtime/doris-compose/requirements.txt
index 2f962ed68d8bf8..1f32223a02e1f4 100644
--- a/docker/runtime/doris-compose/requirements.txt
+++ b/docker/runtime/doris-compose/requirements.txt
@@ -22,5 +22,6 @@ jsonpickle
 prettytable
 pymysql
 python-dateutil
-#pyyaml==5.4.1
+# if mac install pyyaml failed, change pyyaml version
+#pyyaml==5.3.1
 requests<=2.31.0
diff --git a/docker/runtime/doris-compose/resource/init_fe.sh b/docker/runtime/doris-compose/resource/init_fe.sh
index b69ac3a209e409..a58723db1d7b2a 100755
--- a/docker/runtime/doris-compose/resource/init_fe.sh
+++ b/docker/runtime/doris-compose/resource/init_fe.sh
@@ -102,9 +102,6 @@ start_cloud_fe() {
         fe_daemon &
         run_fe
 
-        if [ "$MY_ID" == "1" ]; then
-            echo $MY_IP >$MASTER_FE_IP_FILE
-        fi
         return
     fi
 
@@ -168,10 +165,6 @@ start_cloud_fe() {
 
     fe_daemon &
     run_fe
-
-    if [ "$MY_ID" == "1" ]; then
-        echo $MY_IP >$MASTER_FE_IP_FILE
-    fi
 }
 
 stop_frontend() {
diff --git a/docker/runtime/doris-compose/utils.py b/docker/runtime/doris-compose/utils.py
index 4332ae6cf48d03..dcb821ddffdd00 100644
--- a/docker/runtime/doris-compose/utils.py
+++ b/docker/runtime/doris-compose/utils.py
@@ -23,6 +23,7 @@
 import pwd
 import socket
 import subprocess
+import sys
 import time
 import yaml
 
@@ -30,7 +31,7 @@
 
 LOG = None
 
-ENABLE_LOG = True
+ENALBE_LOG_STDOUT = True
 
 
 class Timer(object):
@@ -48,39 +49,41 @@ def cancel(self):
         self.canceled = True
 
 
-def set_enable_log(enabled):
-    global ENABLE_LOG
-    ENABLE_LOG = enabled
-    get_logger().disabled = not enabled
-
-
-def is_enable_log():
-    return ENABLE_LOG
+def is_log_stdout():
+    return ENALBE_LOG_STDOUT
 
 
 def set_log_verbose():
     get_logger().setLevel(logging.DEBUG)
 
 
-def get_logger(name=None):
-    global LOG
-    if LOG != None:
-        return LOG
-
-    logger = logging.getLogger(name)
-    if not logger.hasHandlers():
+def set_log_to(log_file_name, is_to_stdout):
+    logger = get_logger()
+    for ch in logger.handlers:
+        logger.removeHandler(ch)
+    if log_file_name:
+        os.makedirs(os.path.dirname(log_file_name), exist_ok=True)
+        logger.addHandler(logging.FileHandler(log_file_name))
+    global ENALBE_LOG_STDOUT
+    ENALBE_LOG_STDOUT = is_to_stdout
+    if is_to_stdout:
+        logger.addHandler(logging.StreamHandler(sys.stdout))
+    for ch in logger.handlers:
         formatter = logging.Formatter(
             '%(asctime)s - %(filename)s - %(lineno)dL - %(levelname)s - %(message)s'
         )
-        ch = logging.StreamHandler()
         ch.setLevel(logging.DEBUG)
         ch.setFormatter(formatter)
-        logger.addHandler(ch)
-        logger.setLevel(logging.INFO)
 
-    LOG = logger
 
-    return logger
+def get_logger(name="doris-compose"):
+    global LOG
+    if LOG is None:
+        LOG = logging.getLogger(name)
+        LOG.setLevel(logging.INFO)
+        set_log_to(None, True)
+
+    return LOG
 
 
 get_logger()
@@ -196,15 +199,17 @@ def exec_shell_command(command, ignore_errors=False, output_real_time=False):
     if output_real_time:
         while p.poll() is None:
             s = p.stdout.readline().decode('utf-8')
-            if ENABLE_LOG and s.rstrip():
-                print(s.rstrip())
+            if s.rstrip():
+                for line in s.strip().splitlines():
+                    LOG.info("(docker) " + line)
             out += s
         exitcode = p.wait()
     else:
         out = p.communicate()[0].decode('utf-8')
         exitcode = p.returncode
-        if ENABLE_LOG and out:
-            print(out)
+        if out:
+            for line in out.splitlines():
+                LOG.info("(docker) " + line)
     if not ignore_errors:
         assert exitcode == 0, out
     return exitcode, out
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java
index e27339c2aacc14..9468c8acecd032 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java
@@ -170,6 +170,10 @@ private void updateStatus(List<Backend> currentBes, List<Cloud.NodeInfoPB> expec
             String endpoint = addr + ":" + node.getHeartbeatPort();
             Cloud.NodeStatusPB status = node.getStatus();
             Backend be = currentMap.get(endpoint);
+            if (be == null) {
+                LOG.warn("cant get valid be {} from fe mem, ignore it checker will add this be at next", endpoint);
+                continue;
+            }
 
             if (status == Cloud.NodeStatusPB.NODE_STATUS_DECOMMISSIONING) {
                 if (!be.isDecommissioned()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
index f4c6005a0d828a..89338c228fc0b6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
@@ -415,7 +415,8 @@ public void dropFrontend(FrontendNodeType role, String host, int port) throws Dd
 
         Frontend frontend = checkFeExist(host, port);
         if (frontend == null) {
-            throw new DdlException("Frontend does not exist.");
+            throw new DdlException("frontend does not exist[" + NetUtils
+                .getHostPortInAccessibleFormat(host, port) + "]");
         }
 
         if (frontend.getRole() != role) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java
index eb75bc9312aab4..ede8cb56258636 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java
@@ -126,7 +126,9 @@ public static void getFrontendsInfo(Env env, List<List<String>> infos) {
             selfNode = ConnectContext.get().getCurrentConnectedFEIp();
         }
 
-        for (Frontend fe : env.getFrontends(null /* all */)) {
+        List<Frontend> envFes = env.getFrontends(null /* all */);
+        LOG.info("bdbje fes {}, env fes {}", allFe, envFes);
+        for (Frontend fe : envFes) {
             List<String> info = new ArrayList<String>();
             info.add(fe.getNodeName());
             info.add(fe.getHost());
@@ -211,11 +213,6 @@ private static boolean isJoin(List<InetSocketAddress> allFeHosts, Frontend fe) {
             if (fe.getEditLogPort() != addr.getPort()) {
                 continue;
             }
-            if (!Strings.isNullOrEmpty(addr.getHostName())) {
-                if (addr.getHostName().equals(fe.getHost())) {
-                    return true;
-                }
-            }
             // if hostname of InetSocketAddress is ip, addr.getHostName() may be not equal to fe.getIp()
             // so we need to compare fe.getIp() with address.getHostAddress()
             InetAddress address = addr.getAddress();
@@ -227,6 +224,22 @@ private static boolean isJoin(List<InetSocketAddress> allFeHosts, Frontend fe) {
                 return true;
             }
         }
+
+        // Avoid calling getHostName multiple times, don't remove it
+        for (InetSocketAddress addr : allFeHosts) {
+            // Avoid calling getHostName multiple times, don't remove it
+            if (fe.getEditLogPort() != addr.getPort()) {
+                continue;
+            }
+            // https://bugs.openjdk.org/browse/JDK-8143378#:~:text=getHostName()%3B%20takes%20about%205,millisecond%20on%20JDK%20update%2051
+            // getHostName sometime has bug, take 5s
+            String host = addr.getHostName();
+            if (!Strings.isNullOrEmpty(host)) {
+                if (host.equals(fe.getHost())) {
+                    return true;
+                }
+            }
+        }
         return false;
     }
 }
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
index 856b0e76956395..e77658793fe5b2 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
@@ -23,12 +23,14 @@ import org.apache.doris.regression.util.JdbcUtils
 import org.apache.doris.regression.util.NodeType
 
 import com.google.common.collect.Maps
+import org.awaitility.Awaitility
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
 import groovy.json.JsonSlurper
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
+import static java.util.concurrent.TimeUnit.SECONDS
 import java.util.stream.Collectors
 import java.sql.Connection
 
@@ -333,7 +335,7 @@ class SuiteCluster {
 
         sqlModeNodeMgr = options.sqlModeNodeMgr
 
-        runCmd(cmd.join(' '), -1)
+        runCmd(cmd.join(' '), 180)
 
         // wait be report disk
         Thread.sleep(5000)
@@ -483,6 +485,9 @@ class SuiteCluster {
             if (followerMode) {
                 sb.append('--fe-follower' + ' ')
             }
+            if (sqlModeNodeMgr) {
+                sb.append('--sql-mode-node-mgr' + ' ')
+            }
         }
         if (beNum > 0) {
             sb.append('--add-be-num ' + beNum + ' ')
@@ -492,7 +497,7 @@ class SuiteCluster {
         }
         sb.append('--wait-timeout 60')
 
-        def data = (Map<String, Map<String, Object>>) runCmd(sb.toString(), -1)
+        def data = (Map<String, Map<String, Object>>) runCmd(sb.toString(), 180)
         def newFrontends = (List<Integer>) data.get('fe').get('add_list')
         def newBackends = (List<Integer>) data.get('be').get('add_list')
 
@@ -636,17 +641,15 @@ class SuiteCluster {
     }
 
     private Object runCmd(String cmd, int timeoutSecond = 60) throws Exception {
-        def fullCmd = String.format('python -W ignore %s %s --output-json', config.dorisComposePath, cmd)
+        def fullCmd = String.format('python -W ignore %s %s -v --output-json', config.dorisComposePath, cmd)
         logger.info('Run doris compose cmd: {}', fullCmd)
         def proc = fullCmd.execute()
         def outBuf = new StringBuilder()
         def errBuf = new StringBuilder()
-        proc.consumeProcessOutput(outBuf, errBuf)
-        if (timeoutSecond > 0) {
-            proc.waitForOrKill(timeoutSecond * 1000)
-        } else {
-            proc.waitFor()
-        }
+        Awaitility.await().atMost(timeoutSecond, SECONDS).until({
+            proc.waitForProcessOutput(outBuf, errBuf)
+            return true
+        })
         if (proc.exitValue() != 0) {
             throw new Exception(String.format('Exit value: %s != 0, stdout: %s, stderr: %s',
                                               proc.exitValue(), outBuf.toString(), errBuf.toString()))
diff --git a/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy b/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy
index 7405cb864d889d..70372f68ab865f 100644
--- a/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy
+++ b/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy
@@ -38,6 +38,7 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') {
         options.sqlModeNodeMgr = true
         options.waitTimeout = 0
         options.feNum = 3
+        options.useFollowersMode = true
         options.feConfigs += ["resource_not_ready_sleep_seconds=1",
                 "heartbeat_interval_second=1",]
     }
@@ -121,6 +122,9 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') {
                 
                 // Check FE number
                 def frontendResult = sql_return_maparray """SHOW FRONTENDS;"""
+                // Check that all frontends are alive
+                def aliveCount = frontendResult.count { it['Alive'] == 'true' }
+                assert aliveCount == expectedFeNum, "Expected all $expectedFeNum frontends to be alive, but only ${aliveCount} are alive"
                 assert frontendResult.size() == expectedFeNum, "Expected ${expectedFeNum} frontends, but got ${frontendResult.size()}"
                 logger.info("FE number check passed: ${frontendResult.size()} FEs found")
 
@@ -272,28 +276,23 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') {
             def feEditLogPort = feToDropMap['EditLogPort']
             def feRole = feToDropMap['Role']
 
-            logger.info("Dropping non-master frontend: {}:{}", feHost, feEditLogPort)
+            def dropFeInx = cluster.getFrontends().find { it.host == feHost }.index 
+            logger.info("Dropping non-master frontend: {}:{}, index: {}", feHost, feEditLogPort, dropFeInx)
 
             // Drop the selected non-master frontend
             sql """ ALTER SYSTEM DROP ${feRole} "${feHost}:${feEditLogPort}"; """
-
+            // After drop feHost container will exit
+            cluster.dropFrontends(true, dropFeInx)
+            sleep(3 * 1000)
+            logger.info("Dropping frontend index: {}, remove it from docker compose", dropFeInx)
             // Wait for the frontend to be fully dropped
-            maxWaitSeconds = 300
-            waited = 0
-            while (waited < maxWaitSeconds) {
+
+            dockerAwaitUntil(300) {
                 reconnectFe()
                 def currentFrontends = sql_return_maparray("SHOW FRONTENDS")
-                if (currentFrontends.size() == frontends.size() - 1) {
-                    logger.info("Non-master frontend successfully dropped")
-                    break
-                }
-                sleep(10000)
-                waited += 10
+                currentFrontends.size() == frontends.size() - 1
             }
 
-            if (waited >= maxWaitSeconds) {
-                throw new Exception("Timeout waiting for non-master frontend to be dropped")
-            }
 
             checkClusterStatus(2, 3, 4)
 
@@ -309,86 +308,72 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') {
             }
             
             assert droppedFE != null, "Could not find the dropped frontend"
-
-            feHost = droppedFE['Host']
-            feEditLogPort = droppedFE['EditLogPort']
-
-            logger.info("Adding back frontend: {}:{}", feHost, feEditLogPort)
-
-            // Add the frontend back
-            sql """ ALTER SYSTEM ADD FOLLOWER "${feHost}:${feEditLogPort}"; """
+            
+            // Up a new follower fe and add to docker compose
+            // ATTN: in addFrontend, sql node mode, will execute `ALTER SYSTEM ADD FOLLOWER "${feHost}:${feEditLogPort}";`
+            boolean fuzzyUpFollower = (getRandomBoolean() == "true") ? true : false
+            logger.info("Want up a new role [{}] frontend", fuzzyUpFollower ? "FOLLOWER" : "OBSERVER")
+            def addList = cluster.addFrontend(1, fuzzyUpFollower)
+            logger.info("Up a new frontend, addList: {}", addList)
+
+            def addFE = cluster.getFeByIndex(addList[0])
+            feHost = addFE['Host']
+            feEditLogPort = addFE['EditLogPort']
+            def showFes = sql """SHOW FRONTENDS"""
+            logger.info("Adding back frontend: {}", showFes)
 
             // Wait for the frontend to be fully added back
-            maxWaitSeconds = 300
-            waited = 0
-            while (waited < maxWaitSeconds) {
+            dockerAwaitUntil(300, 5) {
                 def updatedFrontends = sql_return_maparray("SHOW FRONTENDS")
-                if (updatedFrontends.size() == frontends.size()) {
-                    logger.info("Frontend successfully added back")
-                    break
-                }
-                sleep(10000)
-                waited += 10
+                updatedFrontends.size() == frontends.size()
             }
-
-            if (waited >= maxWaitSeconds) {
-                throw new Exception("Timeout waiting for frontend to be added back")
-            }
-
-            // Verify cluster status after adding the frontend back
+           
             checkClusterStatus(3, 3, 5)
 
             logger.info("Frontend successfully added back and cluster status verified")
 
             // CASE 6. Drop frontend and add back again
             logger.info("Dropping frontend and adding back again")
-
             // Get the frontend to be dropped
-            def frontendToDrop = frontends.find { it['Host'] == feHost && it['EditLogPort'] == feEditLogPort }
+            currentFrontends = sql_return_maparray("SHOW FRONTENDS")
+
+            int obServerCount = currentFrontends.count { it['Role'] == 'OBSERVER' } 
+            String fuzzyDropRole
+            if (obServerCount != 0) {
+                fuzzyDropRole = (getRandomBoolean() == "true") ? "FOLLOWER" : "OBSERVER"
+            } else {
+                fuzzyDropRole = "FOLLOWER"
+            }
+
+            def frontendToDrop = currentFrontends.find {it['IsMaster'] == "false" && it['Role'] == fuzzyDropRole}
+            logger.info("Find drop again frontend: {}, drop role [{}]", frontendToDrop, fuzzyDropRole)
             assert frontendToDrop != null, "Could not find the frontend to drop"
 
+            def role = frontendToDrop.Role
             // Drop the frontend
-            sql """ ALTER SYSTEM DROP FOLLOWER "${feHost}:${feEditLogPort}"; """
-            sleep(30000)
+            sql """ ALTER SYSTEM DROP $role "${frontendToDrop.Host}:${frontendToDrop.EditLogPort}"; """
+            dropFeInx = cluster.getFrontends().find { it.host == frontendToDrop.Host }.index 
+            // After drop frontendToDrop.Host container will exit
+            cluster.dropFrontends(true, dropFeInx)
+            logger.info("Dropping again frontend index: {}, remove it from docker compose", dropFeInx)
+            sleep(3 * 1000)
             reconnectFe()
 
             // Wait for the frontend to be fully dropped
-            maxWaitSeconds = 300
-            waited = 0
-            while (waited < maxWaitSeconds) {
+            dockerAwaitUntil(300, 5) {
                 def updatedFrontends = sql_return_maparray("SHOW FRONTENDS")
-                if (!updatedFrontends.any { it['Host'] == feHost && it['EditLogPort'] == feEditLogPort }) {
-                    logger.info("Frontend successfully dropped")
-                    break
-                }
-                sleep(10000)
-                waited += 10
-            }
-
-            if (waited >= maxWaitSeconds) {
-                throw new Exception("Timeout waiting for frontend to be dropped")
+                !updatedFrontends.any { it['Host'] == frontendToDrop.Host && it['EditLogPort'] == frontendToDrop.EditLogPort }
             }
 
-            // Add the frontend back
-            sql """ ALTER SYSTEM ADD FOLLOWER "${feHost}:${feEditLogPort}"; """
+            // Up a new follower fe and add to docker compose
+            // ATTN: in addFrontend, sql node mode, will execute `ALTER SYSTEM ADD FOLLOWER "${feHost}:${feEditLogPort}";`
+            addList = cluster.addFrontend(1, true)
+            logger.info("Up a new frontend, addList: {}", addList)
 
-            // Wait for the frontend to be fully added back
-            maxWaitSeconds = 300
-            waited = 0
-            while (waited < maxWaitSeconds) {
+            dockerAwaitUntil(300, 5) {
                 def updatedFrontends = sql_return_maparray("SHOW FRONTENDS")
-                if (updatedFrontends.any { it['Host'] == feHost && it['EditLogPort'] == feEditLogPort }) {
-                    logger.info("Frontend successfully added back")
-                    break
-                }
-                sleep(10000)
-                waited += 10
+                updatedFrontends.size() == 3
             }
-
-            if (waited >= maxWaitSeconds) {
-                throw new Exception("Timeout waiting for frontend to be added back")
-            }
-
             // Verify cluster status after adding the frontend back
             checkClusterStatus(3, 3, 6)
 

From 82d799ad724d430b0f440466cd90eb883d46acd0 Mon Sep 17 00:00:00 2001
From: abmdocrt <lianyukang@selectdb.com>
Date: Tue, 10 Dec 2024 16:39:12 +0800
Subject: [PATCH 256/399] [Enhancement](compaction)Optimize compaction task
 permit allocation (#45197)

The current implementation of compaction task submission reserves
permits before task execution, which can lead to inefficient resource
utilization. Tasks waiting in the thread pool queue may hold permits,
potentially blocking other tasks from being executed.

## Solution

Change total_permits_for_compaction_score to 1,000,000, which will
effectively remove the limit on total permits. The original purpose of
total permits was to control the memory of compaction tasks, but
currently, memory is controlled by individual compaction tasks, so total
permits are no longer serving any purpose. If no memory issues arise in
the next two versions after making this change, we will remove the
permits mechanism.
---
 be/src/common/config.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 68b082c08734a6..c3d00e23c98e5e 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -455,7 +455,7 @@ DEFINE_mInt32(cumulative_compaction_max_deltas_factor, "10");
 DEFINE_mInt32(multi_get_max_threads, "10");
 
 // The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction.
-DEFINE_mInt64(total_permits_for_compaction_score, "10000");
+DEFINE_mInt64(total_permits_for_compaction_score, "1000000");
 
 // sleep interval in ms after generated compaction tasks
 DEFINE_mInt32(generate_compaction_tasks_interval_ms, "100");

From cebf6481da4513e6ee1bb88f1d5c01fcea37e545 Mon Sep 17 00:00:00 2001
From: huanghaibin <huanghaibin@selectdb.com>
Date: Tue, 10 Dec 2024 17:05:46 +0800
Subject: [PATCH 257/399] [improve](mow) merge and remove old version of delete
 bitmap when cumulative compaction is done on local mode (#41636)

pr https://github.com/apache/doris/pull/40204 remove delete bitmap on
cloud mode, this pr is used for local mode
---
 be/src/cloud/cloud_cumulative_compaction.cpp  |  28 +---
 .../action/delete_bitmap_action.cpp}          |  36 ++---
 .../action/delete_bitmap_action.h}            |  13 +-
 be/src/olap/compaction.cpp                    |  64 +++++++++
 be/src/olap/compaction.h                      |   8 ++
 be/src/olap/rowset/beta_rowset_writer.cpp     |   3 +-
 be/src/olap/tablet.cpp                        |  10 ++
 be/src/olap/tablet_meta.cpp                   |  18 ++-
 be/src/service/http_service.cpp               |  21 ++-
 ...on_remove_old_version_delete_bitmap.groovy | 134 +++++++++---------
 10 files changed, 212 insertions(+), 123 deletions(-)
 rename be/src/{cloud/cloud_delete_bitmap_action.cpp => http/action/delete_bitmap_action.cpp} (83%)
 rename be/src/{cloud/cloud_delete_bitmap_action.h => http/action/delete_bitmap_action.h} (79%)

diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp
index f63054563aa18a..6b4dbe360da651 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -383,28 +383,10 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() {
     }
     std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator);
     if (!pre_rowsets.empty()) {
-        auto pre_max_version = _output_rowset->version().second;
-        DeleteBitmapPtr new_delete_bitmap =
-                std::make_shared<DeleteBitmap>(_tablet->tablet_meta()->tablet_id());
         std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>
                 to_remove_vec;
-        for (auto& rowset : pre_rowsets) {
-            if (rowset->rowset_meta()->total_disk_size() == 0) {
-                continue;
-            }
-            for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) {
-                rowset->rowset_id().to_string();
-                DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0};
-                DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version};
-                auto d = _tablet->tablet_meta()->delete_bitmap().get_agg(
-                        {rowset->rowset_id(), seg_id, pre_max_version});
-                to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end));
-                if (d->isEmpty()) {
-                    continue;
-                }
-                new_delete_bitmap->set(end, *d);
-            }
-        }
+        DeleteBitmapPtr new_delete_bitmap = nullptr;
+        agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap);
         if (!new_delete_bitmap->empty()) {
             // store agg delete bitmap
             DBUG_EXECUTE_IF("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed",
@@ -424,9 +406,9 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() {
             }
             _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(),
                                                                         to_remove_vec);
-            DBUG_EXECUTE_IF(
-                    "CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets",
-                    { static_cast<CloudTablet*>(_tablet.get())->delete_expired_stale_rowsets(); });
+            DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", {
+                static_cast<CloudTablet*>(_tablet.get())->delete_expired_stale_rowsets();
+            });
         }
     }
     return Status::OK();
diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp b/be/src/http/action/delete_bitmap_action.cpp
similarity index 83%
rename from be/src/cloud/cloud_delete_bitmap_action.cpp
rename to be/src/http/action/delete_bitmap_action.cpp
index 3d834bfe7b373c..2fa0a73c2f338c 100644
--- a/be/src/cloud/cloud_delete_bitmap_action.cpp
+++ b/be/src/http/action/delete_bitmap_action.cpp
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "cloud_delete_bitmap_action.h"
+#include "delete_bitmap_action.h"
 
 #include <rapidjson/document.h>
 #include <rapidjson/encodings.h>
@@ -34,8 +34,10 @@
 #include <utility>
 
 #include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_storage_engine.h"
 #include "cloud/cloud_tablet.h"
 #include "cloud/cloud_tablet_mgr.h"
+#include "cloud/config.h"
 #include "common/logging.h"
 #include "common/status.h"
 #include "gutil/strings/substitute.h"
@@ -44,7 +46,6 @@
 #include "http/http_request.h"
 #include "http/http_status.h"
 #include "olap/olap_define.h"
-#include "olap/storage_engine.h"
 #include "olap/tablet_manager.h"
 #include "util/doris_metrics.h"
 #include "util/stopwatch.hpp"
@@ -59,10 +60,9 @@ constexpr std::string_view HEADER_JSON = "application/json";
 
 } // namespace
 
-CloudDeleteBitmapAction::CloudDeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env,
-                                                 CloudStorageEngine& engine,
-                                                 TPrivilegeHier::type hier,
-                                                 TPrivilegeType::type ptype)
+DeleteBitmapAction::DeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env,
+                                       BaseStorageEngine& engine, TPrivilegeHier::type hier,
+                                       TPrivilegeType::type ptype)
         : HttpHandlerWithAuth(exec_env, hier, ptype),
           _engine(engine),
           _delete_bitmap_action_type(ctype) {}
@@ -80,8 +80,8 @@ static Status _check_param(HttpRequest* req, uint64_t* tablet_id) {
     return Status::OK();
 }
 
-Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req,
-                                                                       std::string* json_result) {
+Status DeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req,
+                                                                  std::string* json_result) {
     uint64_t tablet_id = 0;
     // check & retrieve tablet_id from req if it contains
     RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed");
@@ -89,11 +89,15 @@ Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpReque
         return Status::InternalError("check param failed: missing tablet_id");
     }
 
-    CloudTabletSPtr tablet = DORIS_TRY(_engine.tablet_mgr().get_tablet(tablet_id));
+    BaseTabletSPtr tablet = nullptr;
+    if (config::is_cloud_mode()) {
+        tablet = DORIS_TRY(_engine.to_cloud().tablet_mgr().get_tablet(tablet_id));
+    } else {
+        tablet = _engine.to_local().tablet_manager()->get_tablet(tablet_id);
+    }
     if (tablet == nullptr) {
         return Status::NotFound("Tablet not found. tablet_id={}", tablet_id);
     }
-
     auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
     auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality();
     auto size = tablet->tablet_meta()->delete_bitmap().get_size();
@@ -115,8 +119,8 @@ Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpReque
     return Status::OK();
 }
 
-Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req,
-                                                                    std::string* json_result) {
+Status DeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req,
+                                                               std::string* json_result) {
     uint64_t tablet_id = 0;
     // check & retrieve tablet_id from req if it contains
     RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed");
@@ -124,14 +128,14 @@ Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest*
         return Status::InternalError("check param failed: missing tablet_id");
     }
     TabletMetaSharedPtr tablet_meta;
-    auto st = _engine.meta_mgr().get_tablet_meta(tablet_id, &tablet_meta);
+    auto st = _engine.to_cloud().meta_mgr().get_tablet_meta(tablet_id, &tablet_meta);
     if (!st.ok()) {
         LOG(WARNING) << "failed to get_tablet_meta tablet=" << tablet_id
                      << ", st=" << st.to_string();
         return st;
     }
-    auto tablet = std::make_shared<CloudTablet>(_engine, std::move(tablet_meta));
-    st = _engine.meta_mgr().sync_tablet_rowsets(tablet.get(), false, true, true);
+    auto tablet = std::make_shared<CloudTablet>(_engine.to_cloud(), std::move(tablet_meta));
+    st = _engine.to_cloud().meta_mgr().sync_tablet_rowsets(tablet.get(), false, true, true);
     if (!st.ok()) {
         LOG(WARNING) << "failed to sync tablet=" << tablet_id << ", st=" << st;
         return st;
@@ -157,7 +161,7 @@ Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest*
     return Status::OK();
 }
 
-void CloudDeleteBitmapAction::handle(HttpRequest* req) {
+void DeleteBitmapAction::handle(HttpRequest* req) {
     req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data());
     if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_LOCAL) {
         std::string json_result;
diff --git a/be/src/cloud/cloud_delete_bitmap_action.h b/be/src/http/action/delete_bitmap_action.h
similarity index 79%
rename from be/src/cloud/cloud_delete_bitmap_action.h
rename to be/src/http/action/delete_bitmap_action.h
index ce507ee9991757..284e8dbcf5705b 100644
--- a/be/src/cloud/cloud_delete_bitmap_action.h
+++ b/be/src/http/action/delete_bitmap_action.h
@@ -21,9 +21,9 @@
 
 #include <string>
 
-#include "cloud/cloud_storage_engine.h"
 #include "common/status.h"
 #include "http/http_handler_with_auth.h"
+#include "olap/storage_engine.h"
 #include "olap/tablet.h"
 
 namespace doris {
@@ -35,13 +35,12 @@ class ExecEnv;
 enum class DeleteBitmapActionType { COUNT_LOCAL = 1, COUNT_MS = 2 };
 
 /// This action is used for viewing the delete bitmap status
-class CloudDeleteBitmapAction : public HttpHandlerWithAuth {
+class DeleteBitmapAction : public HttpHandlerWithAuth {
 public:
-    CloudDeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env,
-                            CloudStorageEngine& engine, TPrivilegeHier::type hier,
-                            TPrivilegeType::type ptype);
+    DeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, BaseStorageEngine& engine,
+                       TPrivilegeHier::type hier, TPrivilegeType::type ptype);
 
-    ~CloudDeleteBitmapAction() override = default;
+    ~DeleteBitmapAction() override = default;
 
     void handle(HttpRequest* req) override;
 
@@ -50,7 +49,7 @@ class CloudDeleteBitmapAction : public HttpHandlerWithAuth {
     Status _handle_show_ms_delete_bitmap_count(HttpRequest* req, std::string* json_result);
 
 private:
-    CloudStorageEngine& _engine;
+    BaseStorageEngine& _engine;
     DeleteBitmapActionType _delete_bitmap_action_type;
 };
 #include "common/compile_check_end.h"
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 8c45c20f799427..7b2bb5d4ee3cfb 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -929,6 +929,33 @@ Status CloudCompactionMixin::update_delete_bitmap() {
     return Status::OK();
 }
 
+void Compaction::agg_and_remove_old_version_delete_bitmap(
+        std::vector<RowsetSharedPtr>& pre_rowsets,
+        std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>&
+                to_remove_vec,
+        DeleteBitmapPtr& new_delete_bitmap) {
+    // agg previously rowset old version delete bitmap
+    auto pre_max_version = _output_rowset->version().second;
+    new_delete_bitmap = std::make_shared<DeleteBitmap>(_tablet->tablet_meta()->tablet_id());
+    for (auto& rowset : pre_rowsets) {
+        if (rowset->rowset_meta()->total_disk_size() == 0) {
+            continue;
+        }
+        for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) {
+            rowset->rowset_id().to_string();
+            DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0};
+            DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version};
+            auto d = _tablet->tablet_meta()->delete_bitmap().get_agg(
+                    {rowset->rowset_id(), seg_id, pre_max_version});
+            to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end));
+            if (d->isEmpty()) {
+                continue;
+            }
+            new_delete_bitmap->set(end, *d);
+        }
+    }
+}
+
 Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx) {
     // only do index compaction for dup_keys and unique_keys with mow enabled
     if (config::inverted_index_compaction_enable &&
@@ -1103,6 +1130,13 @@ Status CompactionMixin::modify_rowsets() {
         tablet()->delete_expired_stale_rowset();
     }
 
+    if (config::enable_delete_bitmap_merge_on_compaction &&
+        compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION &&
+        _tablet->keys_type() == KeysType::UNIQUE_KEYS &&
+        _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) {
+        process_old_version_delete_bitmap();
+    }
+
     int64_t cur_max_version = 0;
     {
         std::shared_lock rlock(_tablet->get_header_lock());
@@ -1121,6 +1155,36 @@ Status CompactionMixin::modify_rowsets() {
     return Status::OK();
 }
 
+void CompactionMixin::process_old_version_delete_bitmap() {
+    std::vector<RowsetSharedPtr> pre_rowsets {};
+    for (const auto& it : tablet()->rowset_map()) {
+        if (it.first.second < _input_rowsets.front()->start_version()) {
+            pre_rowsets.emplace_back(it.second);
+        }
+    }
+    std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator);
+    if (!pre_rowsets.empty()) {
+        std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>
+                to_remove_vec;
+        DeleteBitmapPtr new_delete_bitmap = nullptr;
+        agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap);
+        if (!new_delete_bitmap->empty()) {
+            // store agg delete bitmap
+            Version version(_input_rowsets.front()->start_version(),
+                            _input_rowsets.back()->end_version());
+            for (auto it = new_delete_bitmap->delete_bitmap.begin();
+                 it != new_delete_bitmap->delete_bitmap.end(); it++) {
+                _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second);
+            }
+            _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(),
+                                                                        to_remove_vec);
+            DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", {
+                static_cast<Tablet*>(_tablet.get())->delete_expired_stale_rowset();
+            });
+        }
+    }
+}
+
 bool CompactionMixin::_check_if_includes_input_rowsets(
         const RowsetIdUnorderedSet& commit_rowset_ids_set) const {
     std::vector<RowsetId> commit_rowset_ids {};
diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h
index 7f92a6c5f4d7dc..ccabf7dadb4733 100644
--- a/be/src/olap/compaction.h
+++ b/be/src/olap/compaction.h
@@ -86,6 +86,12 @@ class Compaction {
 
     virtual Status update_delete_bitmap() = 0;
 
+    void agg_and_remove_old_version_delete_bitmap(
+            std::vector<RowsetSharedPtr>& pre_rowsets,
+            std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>&
+                    to_remove_vec,
+            DeleteBitmapPtr& new_delete_bitmap);
+
     // the root tracker for this compaction
     std::shared_ptr<MemTrackerLimiter> _mem_tracker;
 
@@ -162,6 +168,8 @@ class CompactionMixin : public Compaction {
 
     Status do_compact_ordered_rowsets();
 
+    void process_old_version_delete_bitmap();
+
     bool _check_if_includes_input_rowsets(const RowsetIdUnorderedSet& commit_rowset_ids_set) const;
 
     PendingRowsetGuard _pending_rs_guard;
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
index ab5bc48db80b00..dc155efe0165bc 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -337,7 +337,8 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
     LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " << _context.tablet->tablet_id()
               << ", rowset_ids: " << _context.mow_context->rowset_ids.size()
               << ", cur max_version: " << _context.mow_context->max_version
-              << ", transaction_id: " << _context.mow_context->txn_id
+              << ", transaction_id: " << _context.mow_context->txn_id << ", delete_bitmap_count: "
+              << _context.tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count()
               << ", cost: " << watch.get_elapse_time_us() << "(us), total rows: " << total_rows;
     return Status::OK();
 }
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 379fb6eec3cb43..644ca9133eb885 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -817,10 +817,13 @@ void Tablet::delete_expired_stale_rowset() {
         auto old_meta_size = _tablet_meta->all_stale_rs_metas().size();
 
         // do delete operation
+        std::vector<std::string> version_to_delete;
         auto to_delete_iter = stale_version_path_map.begin();
         while (to_delete_iter != stale_version_path_map.end()) {
             std::vector<TimestampedVersionSharedPtr>& to_delete_version =
                     to_delete_iter->second->timestamped_versions();
+            int64_t start_version = -1;
+            int64_t end_version = -1;
             for (auto& timestampedVersion : to_delete_version) {
                 auto it = _stale_rs_version_map.find(timestampedVersion->version());
                 if (it != _stale_rs_version_map.end()) {
@@ -841,10 +844,17 @@ void Tablet::delete_expired_stale_rowset() {
                                  << timestampedVersion->version().second
                                  << "] not find in stale rs version map";
                 }
+                if (start_version < 0) {
+                    start_version = timestampedVersion->version().first;
+                }
+                end_version = timestampedVersion->version().second;
                 _delete_stale_rowset_by_version(timestampedVersion->version());
             }
+            Version version(start_version, end_version);
+            version_to_delete.emplace_back(version.to_string());
             to_delete_iter++;
         }
+        _tablet_meta->delete_bitmap().remove_stale_delete_bitmap_from_queue(version_to_delete);
 
         bool reconstructed = _reconstruct_version_tracker_if_necessary();
 
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 3247f34656fb5d..43b0d5d8bd0ae0 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -1091,7 +1091,9 @@ uint64_t DeleteBitmap::cardinality() const {
     std::shared_lock l(lock);
     uint64_t res = 0;
     for (auto entry : delete_bitmap) {
-        res += entry.second.cardinality();
+        if (std::get<1>(entry.first) != DeleteBitmap::INVALID_SEGMENT_ID) {
+            res += entry.second.cardinality();
+        }
     }
     return res;
 }
@@ -1100,7 +1102,9 @@ uint64_t DeleteBitmap::get_size() const {
     std::shared_lock l(lock);
     uint64_t charge = 0;
     for (auto& [k, v] : delete_bitmap) {
-        charge += v.getSizeInBytes();
+        if (std::get<1>(k) != DeleteBitmap::INVALID_SEGMENT_ID) {
+            charge += v.getSizeInBytes();
+        }
     }
     return charge;
 }
@@ -1219,7 +1223,7 @@ void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vector<std::
             _stale_delete_bitmap.erase(version_str);
         }
     }
-    if (tablet_id == -1 || to_delete.empty()) {
+    if (tablet_id == -1 || to_delete.empty() || !config::is_cloud_mode()) {
         return;
     }
     CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud();
@@ -1232,7 +1236,13 @@ void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vector<std::
 
 uint64_t DeleteBitmap::get_delete_bitmap_count() {
     std::shared_lock l(lock);
-    return delete_bitmap.size();
+    uint64_t count = 0;
+    for (auto it = delete_bitmap.begin(); it != delete_bitmap.end(); it++) {
+        if (std::get<1>(it->first) != DeleteBitmap::INVALID_SEGMENT_ID) {
+            count++;
+        }
+    }
+    return count;
 }
 
 // We cannot just copy the underlying memory to construct a string
diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp
index c4b01a641987e6..eb0824170b30cc 100644
--- a/be/src/service/http_service.cpp
+++ b/be/src/service/http_service.cpp
@@ -25,7 +25,6 @@
 #include <vector>
 
 #include "cloud/cloud_compaction_action.h"
-#include "cloud/cloud_delete_bitmap_action.h"
 #include "cloud/config.h"
 #include "cloud/injection_point_action.h"
 #include "common/config.h"
@@ -43,6 +42,7 @@
 #include "http/action/compaction_score_action.h"
 #include "http/action/config_action.h"
 #include "http/action/debug_point_action.h"
+#include "http/action/delete_bitmap_action.h"
 #include "http/action/download_action.h"
 #include "http/action/download_binlog_action.h"
 #include "http/action/file_cache_action.h"
@@ -389,6 +389,13 @@ void HttpService::register_local_handler(StorageEngine& engine) {
 
     _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status",
                                       run_status_compaction_action);
+
+    DeleteBitmapAction* count_delete_bitmap_action =
+            _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine,
+                                             TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
+    _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_local",
+                                      count_delete_bitmap_action);
+
     CheckTabletSegmentAction* check_tablet_segment_action = _pool.add(new CheckTabletSegmentAction(
             _env, engine, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
     _ev_http_server->register_handler(HttpMethod::POST, "/api/check_tablet_segment_lost",
@@ -437,14 +444,14 @@ void HttpService::register_cloud_handler(CloudStorageEngine& engine) {
                                       TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
     _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status",
                                       run_status_compaction_action);
-    CloudDeleteBitmapAction* count_local_delete_bitmap_action =
-            _pool.add(new CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine,
-                                                  TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
+    DeleteBitmapAction* count_local_delete_bitmap_action =
+            _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine,
+                                             TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
     _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_local",
                                       count_local_delete_bitmap_action);
-    CloudDeleteBitmapAction* count_ms_delete_bitmap_action =
-            _pool.add(new CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_MS, _env, engine,
-                                                  TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
+    DeleteBitmapAction* count_ms_delete_bitmap_action =
+            _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_MS, _env, engine,
+                                             TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
     _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_ms",
                                       count_ms_delete_bitmap_action);
 #ifdef ENABLE_INJECTION_POINT
diff --git a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy
index a36cb4579ca487..e14bdbbb2c6505 100644
--- a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy
+++ b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy
@@ -18,9 +18,6 @@
 import org.codehaus.groovy.runtime.IOGroovyMethods
 
 suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
-    if (!isCloudMode()) {
-        return
-    }
     def backendId_to_backendIP = [:]
     def backendId_to_backendHttpPort = [:]
     def backendId_to_params = [string: [:]]
@@ -163,7 +160,7 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
     def timeout = 10000
     sql """ DROP TABLE IF EXISTS ${testTable}"""
     def testTableDDL = """
-        create table ${testTable} 
+        create table ${testTable}
             (
             `plan_id` bigint(20) NOT NULL,
             `target_id` int(20) NOT NULL,
@@ -189,7 +186,7 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
     set_be_param("tablet_rowset_stale_sweep_time_sec", "0")
 
     try {
-        GetDebugPoint().enableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets")
+        GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets")
         // 1. test normal
         sql "sync"
         sql """ INSERT INTO ${testTable} VALUES (0,0,'1'),(1,1,'1'); """
@@ -219,18 +216,20 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
 
             // before compaction, delete_bitmap_count is (rowsets num - 1)
             local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
-            logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
-            assertTrue(local_delete_bitmap_count == 7)
-            assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count)
-
             local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
-            ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+            logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
             logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality)
-            logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
+            assertTrue(local_delete_bitmap_count == 7)
             assertTrue(local_delete_bitmap_cardinality == 7)
-            assertTrue(local_delete_bitmap_cardinality == ms_delete_bitmap_cardinality)
+
+            if (isCloudMode()) {
+                ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+                ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+                logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
+                logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
+                assertTrue(ms_delete_bitmap_count == 7)
+                assertTrue(ms_delete_bitmap_cardinality == 7)
+            }
 
 
             assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id],
@@ -265,18 +264,20 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
             // before compaction, local delete_bitmap_count is (total rowsets num - 1), ms delete_bitmap_count is new rowset num
             String trigger_backend_id = tablet.BackendId
             local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
-            logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
-            assertTrue(local_delete_bitmap_count == 12)
-            assertTrue(ms_delete_bitmap_count == 5)
-
             local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
-            ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+            logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
             logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality)
-            logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
+            assertTrue(local_delete_bitmap_count == 12)
             assertTrue(local_delete_bitmap_cardinality == 17)
-            assertTrue(ms_delete_bitmap_cardinality == 10)
+
+            if (isCloudMode()) {
+                ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+                ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+                logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
+                logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
+                assertTrue(ms_delete_bitmap_count == 5)
+                assertTrue(ms_delete_bitmap_cardinality == 10)
+            }
 
             getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
             assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id],
@@ -285,20 +286,21 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
             getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
 
             Thread.sleep(1000)
-            // after compaction, delete_bitmap_count is 1
+            // after compaction, delete_bitmap_count is 1, cardinality is 2, check it
             local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
-            logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
-            assertTrue(local_delete_bitmap_count == 1)
-            assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count)
-
             local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
-            ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+            logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
             logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality)
-            logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
+            assertTrue(local_delete_bitmap_count == 1)
             assertTrue(local_delete_bitmap_cardinality == 2)
-            assertTrue(ms_delete_bitmap_cardinality == 2)
+            if (isCloudMode()) {
+                ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+                ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+                logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
+                logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
+                assertTrue(ms_delete_bitmap_count == 1)
+                assertTrue(ms_delete_bitmap_cardinality == 2)
+            }
         }
 
         qt_sql "select * from ${testTable} order by plan_id"
@@ -319,37 +321,39 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
 
         qt_sql "select * from ${testTable} order by plan_id"
         GetDebugPoint().enableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed")
-        for (def tablet in tablets) {
-            String tablet_id = tablet.TabletId
-            def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """
-            logger.info("tablet: " + tablet_info)
-            String trigger_backend_id = tablet.BackendId
-
-            local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
-            logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
-            assertTrue(local_delete_bitmap_count == 6)
-            assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count)
-
-            local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
-            ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
-            logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality)
-            logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
-            assertTrue(local_delete_bitmap_cardinality == 12)
-            assertTrue(ms_delete_bitmap_cardinality == 12)
-
-            getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
-            assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id],
-                    "cumulative", tablet_id).contains("Success"));
-            waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id)
-            getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
-
-            // update fail, local delete_bitmap_count will not change
-            Thread.sleep(1000)
-            local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
-            logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
-            assertTrue(local_delete_bitmap_count == 6)
+        if (isCloudMode()) {
+            for (def tablet in tablets) {
+                String tablet_id = tablet.TabletId
+                def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """
+                logger.info("tablet: " + tablet_info)
+                String trigger_backend_id = tablet.BackendId
+
+                local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+                ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+                logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
+                logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
+                assertTrue(local_delete_bitmap_count == 6)
+                assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count)
+
+                local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+                ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+                logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality)
+                logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
+                assertTrue(local_delete_bitmap_cardinality == 12)
+                assertTrue(ms_delete_bitmap_cardinality == 12)
+
+                getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
+                assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id],
+                        "cumulative", tablet_id).contains("Success"));
+                waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id)
+                getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
+
+                // update fail, local delete_bitmap_count will not change
+                Thread.sleep(1000)
+                local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+                logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
+                assertTrue(local_delete_bitmap_count == 6)
+            }
         }
 
         qt_sql "select * from ${testTable} order by plan_id"
@@ -372,7 +376,7 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
     } finally {
         reset_be_param("compaction_promotion_version_count")
         reset_be_param("tablet_rowset_stale_sweep_time_sec")
-        GetDebugPoint().disableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets")
+        GetDebugPoint().disableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets")
         GetDebugPoint().disableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed")
     }
 

From 54fbbef432daece146c0984f104970dcae8a688e Mon Sep 17 00:00:00 2001
From: lihangyu <lihangyu@selectdb.com>
Date: Tue, 10 Dec 2024 19:20:08 +0800
Subject: [PATCH 258/399] [chore](regression-test) move some variant cases to
 directory (#45169)

1. remove redunant variant_github_events_new_p2/sql
2. mv some cases
---
 .../{ => sql}/affinityByIssuesAndPRs1.out     |  0
 .../{ => sql}/affinityByIssuesAndPRs2.out     |  0
 .../{ => sql}/authorsWithTheMostPushes.out    |  0
 .../{ => sql}/countingStar1.out               |  0
 .../{ => sql}/countingStar2.out               |  0
 .../{ => sql}/countingStar3.out               |  0
 .../distributionOfRepositoriesByStarCount.out |  0
 .../{ => sql}/githubRoulette.out              |  0
 ...tOfTopRepositoriesChangedOverTheYears1.out |  0
 ...tOfTopRepositoriesChangedOverTheYears2.out |  0
 ...tOfTopRepositoriesChangedOverTheYears3.out |  0
 ...tOfTopRepositoriesChangedOverTheYears4.out |  0
 ...tOfTopRepositoriesChangedOverTheYears5.out |  0
 ...tOfTopRepositoriesChangedOverTheYears6.out |  0
 ...asTheTotalNumberOfStarsChangedOverTime.out |  0
 .../{ => sql}/issuesWithTheMostComments1.out  |  0
 .../{ => sql}/issuesWithTheMostComments2.out  |  0
 .../{ => sql}/issuesWithTheMostComments3.out  |  0
 .../{ => sql}/issuesWithTheMostComments4.out  |  0
 .../{ => sql}/issuesWithTheMostComments5.out  |  0
 .../{ => sql}/issuesWithTheMostComments6.out  |  0
 .../{ => sql}/issuesWithTheMostComments7.out  |  0
 .../{ => sql}/load.out                        |  0
 .../{ => sql}/mostForkedRepositories.out      |  0
 .../{ => sql}/mostPopularCommentsOnGithub.out |  0
 ...organizationsByTheNumberOfRepositories.out |  0
 .../organizationsByTheNumberOfStars.out       |  0
 .../proportionsBetweenStarsAndForks1.out      |  0
 .../proportionsBetweenStarsAndForks2.out      |  0
 .../proportionsBetweenStarsAndForks3.out      |  0
 .../proportionsBetweenStarsAndForks4.out      |  0
 .../proportionsBetweenStarsAndForks5.out      |  0
 .../repositoriesByAmountOfModifiedCode.out    |  0
 .../repositoriesByTheNumberOfPushes.out       |  0
 ...toriesWithClickhouse_related_comments1.out |  0
 ...toriesWithClickhouse_related_comments2.out |  0
 ...epositoriesWithDoris_related_comments1.out |  0
 ...epositoriesWithDoris_related_comments2.out |  0
 .../repositoriesWithTheHighestGrowthYoY.out   |  0
 ...ositoriesWithTheMaximumAmountOfIssues1.out |  0
 ...ositoriesWithTheMaximumAmountOfIssues2.out |  0
 ...ositoriesWithTheMaximumAmountOfIssues3.out |  0
 ...ositoriesWithTheMaximumAmountOfIssues4.out |  0
 ...iesWithTheMaximumAmountOfPullRequests1.out |  0
 ...iesWithTheMaximumAmountOfPullRequests2.out |  0
 ...hTheMaximumNumberOfAcceptedInvitations.out |  0
 ...iesWithTheMostPeopleWhoHavePushAccess1.out |  0
 ...epositoriesWithTheMostStarsOverOneDay1.out |  0
 ...itoriesWithTheMostSteadyGrowthOverTime.out |  0
 ...positoriesWithTheWorstStagnation_order.out |  0
 .../{ => sql}/repositoryAffinityList1.out     |  0
 .../{ => sql}/repositoryAffinityList2.out     |  0
 .../{ => sql}/starsFromHeavyGithubUsers1.out  |  0
 .../{ => sql}/starsFromHeavyGithubUsers2.out  |  0
 .../{ => sql}/theLongestRepositoryNames1.out  |  0
 .../{ => sql}/theLongestRepositoryNames2.out  |  0
 .../{ => sql}/theMostToughCodeReviews.out     |  0
 .../theTotalNumberOfRepositoriesOnGithub.out  |  0
 .../theTotalNumberOfUsersOnGithub1.out        |  0
 .../theTotalNumberOfUsersOnGithub2.out        |  0
 .../theTotalNumberOfUsersOnGithub3.out        |  0
 .../theTotalNumberOfUsersOnGithub4.out        |  0
 .../{ => sql}/topRepositoriesByStars.out      |  0
 .../whatIsTheBestDayOfTheWeekToCatchAStar.out |  0
 .../whoAreAllThosePeopleGivingStars1.out      |  0
 .../whoAreAllThosePeopleGivingStars2.out      |  0
 .../whoAreAllThosePeopleGivingStars3.out      |  0
 .../sql/affinityByIssuesAndPRs1.sql           | 14 ---------
 .../sql/affinityByIssuesAndPRs2.sql           | 14 ---------
 .../sql/authorsWithTheMostPushes.sql          |  9 ------
 .../sql/countingStar1.sql                     |  1 -
 .../sql/countingStar2.sql                     |  1 -
 .../sql/countingStar3.sql                     |  1 -
 .../distributionOfRepositoriesByStarCount.sql | 14 ---------
 .../sql/githubRoulette.sql                    |  1 -
 ...tOfTopRepositoriesChangedOverTheYears1.sql |  1 -
 ...tOfTopRepositoriesChangedOverTheYears2.sql |  1 -
 ...tOfTopRepositoriesChangedOverTheYears3.sql |  1 -
 ...tOfTopRepositoriesChangedOverTheYears4.sql |  1 -
 ...tOfTopRepositoriesChangedOverTheYears5.sql |  1 -
 ...tOfTopRepositoriesChangedOverTheYears6.sql |  1 -
 ...tOfTopRepositoriesChangedOverTheYears7.sql | 30 -------------------
 ...asTheTotalNumberOfStarsChangedOverTime.sql |  2 --
 .../sql/issuesWithTheMostComments1.sql        |  1 -
 .../sql/issuesWithTheMostComments2.sql        |  1 -
 .../sql/issuesWithTheMostComments3.sql        | 17 -----------
 .../sql/issuesWithTheMostComments4.sql        |  9 ------
 .../sql/issuesWithTheMostComments5.sql        |  9 ------
 .../sql/issuesWithTheMostComments6.sql        | 11 -------
 .../sql/issuesWithTheMostComments7.sql        |  9 ------
 .../sql/issuesWithTheMostComments8.sql        | 13 --------
 .../sql/mostForkedRepositories.sql            |  1 -
 .../sql/mostPopularCommentsOnGithub.sql       |  1 -
 ...organizationsByTheNumberOfRepositories.sql | 14 ---------
 .../sql/organizationsByTheNumberOfStars.sql   |  8 -----
 .../sql/organizationsByTheSizeOfCommunity.sql | 23 --------------
 .../sql/proportionsBetweenStarsAndForks1.sql  | 17 -----------
 .../sql/proportionsBetweenStarsAndForks2.sql  | 18 -----------
 .../sql/proportionsBetweenStarsAndForks3.sql  | 18 -----------
 .../sql/proportionsBetweenStarsAndForks4.sql  | 13 --------
 .../sql/proportionsBetweenStarsAndForks5.sql  | 21 -------------
 .../sql/repositoriesByTheNumberOfPushes.sql   | 17 -----------
 ...toriesWithClickhouse_related_comments1.sql |  1 -
 ...toriesWithClickhouse_related_comments2.sql | 17 -----------
 ...epositoriesWithDoris_related_comments1.sql |  1 -
 ...epositoriesWithDoris_related_comments2.sql | 17 -----------
 .../repositoriesWithTheHighestGrowthYoY.sql   | 20 -------------
 ...ositoriesWithTheMaximumAmountOfIssues1.sql |  1 -
 ...ositoriesWithTheMaximumAmountOfIssues2.sql | 18 -----------
 ...ositoriesWithTheMaximumAmountOfIssues3.sql | 19 ------------
 ...ositoriesWithTheMaximumAmountOfIssues4.sql | 18 -----------
 ...iesWithTheMaximumAmountOfPullRequests1.sql |  1 -
 ...iesWithTheMaximumAmountOfPullRequests2.sql |  1 -
 ...hTheMaximumNumberOfAcceptedInvitations.sql | 17 -----------
 ...iesWithTheMostPeopleWhoHavePushAccess1.sql | 13 --------
 ...iesWithTheMostPeopleWhoHavePushAccess2.sql | 13 --------
 ...iesWithTheMostPeopleWhoHavePushAccess3.sql | 16 ----------
 ...epositoriesWithTheMostStarsOverOneDay1.sql | 25 ----------------
 ...epositoriesWithTheMostStarsOverOneDay2.sql | 25 ----------------
 ...epositoriesWithTheMostStarsOverOneDay3.sql |  1 -
 ...itoriesWithTheMostSteadyGrowthOverTime.sql | 20 -------------
 ...positoriesWithTheWorstStagnation_order.sql | 20 -------------
 .../sql/repositoryAffinityList1.sql           | 13 --------
 .../sql/repositoryAffinityList2.sql           | 23 --------------
 .../sql/starsFromHeavyGithubUsers1.sql        | 13 --------
 .../sql/starsFromHeavyGithubUsers2.sql        | 15 ----------
 .../sql/test_distinct_streaming_agg.sql       |  5 ----
 .../sql/theLongestRepositoryNames1.sql        |  1 -
 .../sql/theLongestRepositoryNames2.sql        |  1 -
 .../sql/theMostToughCodeReviews.sql           | 10 -------
 .../theTotalNumberOfRepositoriesOnGithub.sql  |  1 -
 .../sql/theTotalNumberOfUsersOnGithub1.sql    |  1 -
 .../sql/theTotalNumberOfUsersOnGithub2.sql    |  1 -
 .../sql/theTotalNumberOfUsersOnGithub3.sql    |  1 -
 .../sql/theTotalNumberOfUsersOnGithub4.sql    |  1 -
 .../sql/topLabels1.sql                        |  9 ------
 .../sql/topLabels2.sql                        |  9 ------
 .../sql/topLabels3.sql                        | 14 ---------
 .../sql/topRepositoriesByStars.sql            |  1 -
 .../whatIsTheBestDayOfTheWeekToCatchAStar.sql |  1 -
 .../sql/whoAreAllThosePeopleGivingStars1.sql  |  1 -
 .../sql/whoAreAllThosePeopleGivingStars2.sql  |  1 -
 .../sql/whoAreAllThosePeopleGivingStars3.sql  | 13 --------
 .../repositoriesByAmountOfModifiedCode.sql    | 12 --------
 .../{ => sql}/affinityByIssuesAndPRs1.sql     |  0
 .../{ => sql}/affinityByIssuesAndPRs2.sql     |  0
 .../{ => sql}/authorsWithTheMostPushes.sql    |  0
 .../{ => sql}/countingStar1.sql               |  0
 .../{ => sql}/countingStar2.sql               |  0
 .../{ => sql}/countingStar3.sql               |  0
 .../distributionOfRepositoriesByStarCount.sql |  0
 .../{ => sql}/githubRoulette.sql              |  0
 ...tOfTopRepositoriesChangedOverTheYears1.sql |  0
 ...tOfTopRepositoriesChangedOverTheYears2.sql |  0
 ...tOfTopRepositoriesChangedOverTheYears3.sql |  0
 ...tOfTopRepositoriesChangedOverTheYears4.sql |  0
 ...tOfTopRepositoriesChangedOverTheYears5.sql |  0
 ...tOfTopRepositoriesChangedOverTheYears6.sql |  0
 ...tOfTopRepositoriesChangedOverTheYears7.sql |  0
 ...asTheTotalNumberOfStarsChangedOverTime.sql |  0
 .../{ => sql}/issuesWithTheMostComments1.sql  |  0
 .../{ => sql}/issuesWithTheMostComments2.sql  |  0
 .../{ => sql}/issuesWithTheMostComments3.sql  |  0
 .../{ => sql}/issuesWithTheMostComments4.sql  |  0
 .../{ => sql}/issuesWithTheMostComments5.sql  |  0
 .../{ => sql}/issuesWithTheMostComments6.sql  |  0
 .../{ => sql}/issuesWithTheMostComments7.sql  |  0
 .../{ => sql}/issuesWithTheMostComments8.sql  |  0
 .../{ => sql}/mostForkedRepositories.sql      |  0
 .../{ => sql}/mostPopularCommentsOnGithub.sql |  0
 ...organizationsByTheNumberOfRepositories.sql |  0
 .../organizationsByTheNumberOfStars.sql       |  0
 .../organizationsByTheSizeOfCommunity.sql     |  0
 .../proportionsBetweenStarsAndForks1.sql      |  0
 .../proportionsBetweenStarsAndForks2.sql      |  0
 .../proportionsBetweenStarsAndForks3.sql      |  0
 .../proportionsBetweenStarsAndForks4.sql      |  0
 .../proportionsBetweenStarsAndForks5.sql      |  0
 .../repositoriesByAmountOfModifiedCode.sql    |  0
 .../repositoriesByTheNumberOfPushes.sql       |  0
 ...toriesWithClickhouse_related_comments1.sql |  0
 ...toriesWithClickhouse_related_comments2.sql |  0
 ...epositoriesWithDoris_related_comments1.sql |  0
 ...epositoriesWithDoris_related_comments2.sql |  0
 .../repositoriesWithTheHighestGrowthYoY.sql   |  0
 ...ositoriesWithTheMaximumAmountOfIssues1.sql |  0
 ...ositoriesWithTheMaximumAmountOfIssues2.sql |  0
 ...ositoriesWithTheMaximumAmountOfIssues3.sql |  0
 ...ositoriesWithTheMaximumAmountOfIssues4.sql |  0
 ...iesWithTheMaximumAmountOfPullRequests1.sql |  0
 ...iesWithTheMaximumAmountOfPullRequests2.sql |  0
 ...hTheMaximumNumberOfAcceptedInvitations.sql |  0
 ...iesWithTheMostPeopleWhoHavePushAccess1.sql |  0
 ...iesWithTheMostPeopleWhoHavePushAccess2.sql |  0
 ...iesWithTheMostPeopleWhoHavePushAccess3.sql |  0
 ...epositoriesWithTheMostStarsOverOneDay1.sql |  0
 ...epositoriesWithTheMostStarsOverOneDay2.sql |  0
 ...epositoriesWithTheMostStarsOverOneDay3.sql |  0
 ...itoriesWithTheMostSteadyGrowthOverTime.sql |  0
 ...positoriesWithTheWorstStagnation_order.sql |  0
 .../{ => sql}/repositoryAffinityList1.sql     |  0
 .../{ => sql}/repositoryAffinityList2.sql     |  0
 .../{ => sql}/starsFromHeavyGithubUsers1.sql  |  0
 .../{ => sql}/starsFromHeavyGithubUsers2.sql  |  0
 .../{ => sql}/theLongestRepositoryNames1.sql  |  0
 .../{ => sql}/theLongestRepositoryNames2.sql  |  0
 .../{ => sql}/theMostToughCodeReviews.sql     |  0
 .../theTotalNumberOfRepositoriesOnGithub.sql  |  0
 .../theTotalNumberOfUsersOnGithub1.sql        |  0
 .../theTotalNumberOfUsersOnGithub2.sql        |  0
 .../theTotalNumberOfUsersOnGithub3.sql        |  0
 .../theTotalNumberOfUsersOnGithub4.sql        |  0
 .../{ => sql}/topLabels1.sql                  |  0
 .../{ => sql}/topLabels2.sql                  |  0
 .../{ => sql}/topLabels3.sql                  |  0
 .../{ => sql}/topRepositoriesByStars.sql      |  0
 .../whatIsTheBestDayOfTheWeekToCatchAStar.sql |  0
 .../whoAreAllThosePeopleGivingStars1.sql      |  0
 .../whoAreAllThosePeopleGivingStars2.sql      |  0
 .../whoAreAllThosePeopleGivingStars3.sql      |  0
 220 files changed, 725 deletions(-)
 rename regression-test/data/variant_github_events_p2/{ => sql}/affinityByIssuesAndPRs1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/affinityByIssuesAndPRs2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/authorsWithTheMostPushes.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/countingStar1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/countingStar2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/countingStar3.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/distributionOfRepositoriesByStarCount.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/githubRoulette.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears3.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears4.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears5.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears6.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/howHasTheTotalNumberOfStarsChangedOverTime.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/issuesWithTheMostComments1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/issuesWithTheMostComments2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/issuesWithTheMostComments3.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/issuesWithTheMostComments4.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/issuesWithTheMostComments5.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/issuesWithTheMostComments6.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/issuesWithTheMostComments7.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/load.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/mostForkedRepositories.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/mostPopularCommentsOnGithub.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/organizationsByTheNumberOfRepositories.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/organizationsByTheNumberOfStars.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks3.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks4.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks5.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesByAmountOfModifiedCode.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesByTheNumberOfPushes.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithClickhouse_related_comments1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithClickhouse_related_comments2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithDoris_related_comments1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithDoris_related_comments2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheHighestGrowthYoY.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfIssues1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfIssues2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfIssues3.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfIssues4.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfPullRequests1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfPullRequests2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMostPeopleWhoHavePushAccess1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMostStarsOverOneDay1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheMostSteadyGrowthOverTime.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoriesWithTheWorstStagnation_order.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoryAffinityList1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/repositoryAffinityList2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/starsFromHeavyGithubUsers1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/starsFromHeavyGithubUsers2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/theLongestRepositoryNames1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/theLongestRepositoryNames2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/theMostToughCodeReviews.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/theTotalNumberOfRepositoriesOnGithub.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/theTotalNumberOfUsersOnGithub1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/theTotalNumberOfUsersOnGithub2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/theTotalNumberOfUsersOnGithub3.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/theTotalNumberOfUsersOnGithub4.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/topRepositoriesByStars.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/whatIsTheBestDayOfTheWeekToCatchAStar.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/whoAreAllThosePeopleGivingStars1.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/whoAreAllThosePeopleGivingStars2.out (100%)
 rename regression-test/data/variant_github_events_p2/{ => sql}/whoAreAllThosePeopleGivingStars3.out (100%)
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/affinityByIssuesAndPRs1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/affinityByIssuesAndPRs2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/authorsWithTheMostPushes.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/countingStar1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/countingStar2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/countingStar3.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/distributionOfRepositoriesByStarCount.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/githubRoulette.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments3.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments4.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments5.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments6.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments7.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments8.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/mostForkedRepositories.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/mostPopularCommentsOnGithub.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheNumberOfRepositories.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheNumberOfStars.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheSizeOfCommunity.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks3.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks4.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks5.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesByTheNumberOfPushes.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheHighestGrowthYoY.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheWorstStagnation_order.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoryAffinityList1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/repositoryAffinityList2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/test_distinct_streaming_agg.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/theLongestRepositoryNames1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/theLongestRepositoryNames2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/theMostToughCodeReviews.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub3.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub4.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/topLabels1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/topLabels2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/topLabels3.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/topRepositoriesByStars.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars1.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars2.sql
 delete mode 100644 regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars3.sql
 delete mode 100644 regression-test/suites/variant_github_events_p2/repositoriesByAmountOfModifiedCode.sql
 rename regression-test/suites/variant_github_events_p2/{ => sql}/affinityByIssuesAndPRs1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/affinityByIssuesAndPRs2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/authorsWithTheMostPushes.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/countingStar1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/countingStar2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/countingStar3.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/distributionOfRepositoriesByStarCount.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/githubRoulette.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/howHasTheTotalNumberOfStarsChangedOverTime.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/issuesWithTheMostComments1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/issuesWithTheMostComments2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/issuesWithTheMostComments3.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/issuesWithTheMostComments4.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/issuesWithTheMostComments5.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/issuesWithTheMostComments6.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/issuesWithTheMostComments7.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/issuesWithTheMostComments8.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/mostForkedRepositories.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/mostPopularCommentsOnGithub.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/organizationsByTheNumberOfRepositories.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/organizationsByTheNumberOfStars.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/organizationsByTheSizeOfCommunity.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks3.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks4.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/proportionsBetweenStarsAndForks5.sql (100%)
 rename regression-test/suites/{variant_github_events_new_p2 => variant_github_events_p2}/sql/repositoriesByAmountOfModifiedCode.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesByTheNumberOfPushes.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithClickhouse_related_comments1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithClickhouse_related_comments2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithDoris_related_comments1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithDoris_related_comments2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheHighestGrowthYoY.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfIssues1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfIssues2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfIssues3.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfIssues4.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfPullRequests1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumAmountOfPullRequests2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMostStarsOverOneDay1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMostStarsOverOneDay2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMostStarsOverOneDay3.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheMostSteadyGrowthOverTime.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoriesWithTheWorstStagnation_order.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoryAffinityList1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/repositoryAffinityList2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/starsFromHeavyGithubUsers1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/starsFromHeavyGithubUsers2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/theLongestRepositoryNames1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/theLongestRepositoryNames2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/theMostToughCodeReviews.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/theTotalNumberOfRepositoriesOnGithub.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/theTotalNumberOfUsersOnGithub1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/theTotalNumberOfUsersOnGithub2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/theTotalNumberOfUsersOnGithub3.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/theTotalNumberOfUsersOnGithub4.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/topLabels1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/topLabels2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/topLabels3.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/topRepositoriesByStars.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/whatIsTheBestDayOfTheWeekToCatchAStar.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/whoAreAllThosePeopleGivingStars1.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/whoAreAllThosePeopleGivingStars2.sql (100%)
 rename regression-test/suites/variant_github_events_p2/{ => sql}/whoAreAllThosePeopleGivingStars3.sql (100%)

diff --git a/regression-test/data/variant_github_events_p2/affinityByIssuesAndPRs1.out b/regression-test/data/variant_github_events_p2/sql/affinityByIssuesAndPRs1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/affinityByIssuesAndPRs1.out
rename to regression-test/data/variant_github_events_p2/sql/affinityByIssuesAndPRs1.out
diff --git a/regression-test/data/variant_github_events_p2/affinityByIssuesAndPRs2.out b/regression-test/data/variant_github_events_p2/sql/affinityByIssuesAndPRs2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/affinityByIssuesAndPRs2.out
rename to regression-test/data/variant_github_events_p2/sql/affinityByIssuesAndPRs2.out
diff --git a/regression-test/data/variant_github_events_p2/authorsWithTheMostPushes.out b/regression-test/data/variant_github_events_p2/sql/authorsWithTheMostPushes.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/authorsWithTheMostPushes.out
rename to regression-test/data/variant_github_events_p2/sql/authorsWithTheMostPushes.out
diff --git a/regression-test/data/variant_github_events_p2/countingStar1.out b/regression-test/data/variant_github_events_p2/sql/countingStar1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/countingStar1.out
rename to regression-test/data/variant_github_events_p2/sql/countingStar1.out
diff --git a/regression-test/data/variant_github_events_p2/countingStar2.out b/regression-test/data/variant_github_events_p2/sql/countingStar2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/countingStar2.out
rename to regression-test/data/variant_github_events_p2/sql/countingStar2.out
diff --git a/regression-test/data/variant_github_events_p2/countingStar3.out b/regression-test/data/variant_github_events_p2/sql/countingStar3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/countingStar3.out
rename to regression-test/data/variant_github_events_p2/sql/countingStar3.out
diff --git a/regression-test/data/variant_github_events_p2/distributionOfRepositoriesByStarCount.out b/regression-test/data/variant_github_events_p2/sql/distributionOfRepositoriesByStarCount.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/distributionOfRepositoriesByStarCount.out
rename to regression-test/data/variant_github_events_p2/sql/distributionOfRepositoriesByStarCount.out
diff --git a/regression-test/data/variant_github_events_p2/githubRoulette.out b/regression-test/data/variant_github_events_p2/sql/githubRoulette.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/githubRoulette.out
rename to regression-test/data/variant_github_events_p2/sql/githubRoulette.out
diff --git a/regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears1.out b/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears1.out
rename to regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out
diff --git a/regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears2.out b/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears2.out
rename to regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out
diff --git a/regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears3.out b/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears3.out
rename to regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out
diff --git a/regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears4.out b/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears4.out
rename to regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out
diff --git a/regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears5.out b/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears5.out
rename to regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out
diff --git a/regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears6.out b/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears6.out
rename to regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out
diff --git a/regression-test/data/variant_github_events_p2/howHasTheTotalNumberOfStarsChangedOverTime.out b/regression-test/data/variant_github_events_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/howHasTheTotalNumberOfStarsChangedOverTime.out
rename to regression-test/data/variant_github_events_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out
diff --git a/regression-test/data/variant_github_events_p2/issuesWithTheMostComments1.out b/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/issuesWithTheMostComments1.out
rename to regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments1.out
diff --git a/regression-test/data/variant_github_events_p2/issuesWithTheMostComments2.out b/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/issuesWithTheMostComments2.out
rename to regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments2.out
diff --git a/regression-test/data/variant_github_events_p2/issuesWithTheMostComments3.out b/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/issuesWithTheMostComments3.out
rename to regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments3.out
diff --git a/regression-test/data/variant_github_events_p2/issuesWithTheMostComments4.out b/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/issuesWithTheMostComments4.out
rename to regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments4.out
diff --git a/regression-test/data/variant_github_events_p2/issuesWithTheMostComments5.out b/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments5.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/issuesWithTheMostComments5.out
rename to regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments5.out
diff --git a/regression-test/data/variant_github_events_p2/issuesWithTheMostComments6.out b/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments6.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/issuesWithTheMostComments6.out
rename to regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments6.out
diff --git a/regression-test/data/variant_github_events_p2/issuesWithTheMostComments7.out b/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments7.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/issuesWithTheMostComments7.out
rename to regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments7.out
diff --git a/regression-test/data/variant_github_events_p2/load.out b/regression-test/data/variant_github_events_p2/sql/load.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/load.out
rename to regression-test/data/variant_github_events_p2/sql/load.out
diff --git a/regression-test/data/variant_github_events_p2/mostForkedRepositories.out b/regression-test/data/variant_github_events_p2/sql/mostForkedRepositories.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/mostForkedRepositories.out
rename to regression-test/data/variant_github_events_p2/sql/mostForkedRepositories.out
diff --git a/regression-test/data/variant_github_events_p2/mostPopularCommentsOnGithub.out b/regression-test/data/variant_github_events_p2/sql/mostPopularCommentsOnGithub.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/mostPopularCommentsOnGithub.out
rename to regression-test/data/variant_github_events_p2/sql/mostPopularCommentsOnGithub.out
diff --git a/regression-test/data/variant_github_events_p2/organizationsByTheNumberOfRepositories.out b/regression-test/data/variant_github_events_p2/sql/organizationsByTheNumberOfRepositories.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/organizationsByTheNumberOfRepositories.out
rename to regression-test/data/variant_github_events_p2/sql/organizationsByTheNumberOfRepositories.out
diff --git a/regression-test/data/variant_github_events_p2/organizationsByTheNumberOfStars.out b/regression-test/data/variant_github_events_p2/sql/organizationsByTheNumberOfStars.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/organizationsByTheNumberOfStars.out
rename to regression-test/data/variant_github_events_p2/sql/organizationsByTheNumberOfStars.out
diff --git a/regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks1.out b/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks1.out
rename to regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks1.out
diff --git a/regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks2.out b/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks2.out
rename to regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks2.out
diff --git a/regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks3.out b/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks3.out
rename to regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks3.out
diff --git a/regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks4.out b/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks4.out
rename to regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks4.out
diff --git a/regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks5.out b/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks5.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/proportionsBetweenStarsAndForks5.out
rename to regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks5.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesByAmountOfModifiedCode.out b/regression-test/data/variant_github_events_p2/sql/repositoriesByAmountOfModifiedCode.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesByAmountOfModifiedCode.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesByAmountOfModifiedCode.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesByTheNumberOfPushes.out b/regression-test/data/variant_github_events_p2/sql/repositoriesByTheNumberOfPushes.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesByTheNumberOfPushes.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesByTheNumberOfPushes.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithClickhouse_related_comments1.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithClickhouse_related_comments1.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments1.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithClickhouse_related_comments2.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithClickhouse_related_comments2.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments2.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithDoris_related_comments1.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithDoris_related_comments1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithDoris_related_comments1.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithDoris_related_comments1.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithDoris_related_comments2.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithDoris_related_comments2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithDoris_related_comments2.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithDoris_related_comments2.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheHighestGrowthYoY.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheHighestGrowthYoY.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheHighestGrowthYoY.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheHighestGrowthYoY.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues1.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues1.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues2.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues2.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues3.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues3.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues4.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues4.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfPullRequests1.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfPullRequests1.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfPullRequests2.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumAmountOfPullRequests2.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMostPeopleWhoHavePushAccess1.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMostPeopleWhoHavePushAccess1.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMostStarsOverOneDay1.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMostStarsOverOneDay1.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheMostSteadyGrowthOverTime.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheMostSteadyGrowthOverTime.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out
diff --git a/regression-test/data/variant_github_events_p2/repositoriesWithTheWorstStagnation_order.out b/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheWorstStagnation_order.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoriesWithTheWorstStagnation_order.out
rename to regression-test/data/variant_github_events_p2/sql/repositoriesWithTheWorstStagnation_order.out
diff --git a/regression-test/data/variant_github_events_p2/repositoryAffinityList1.out b/regression-test/data/variant_github_events_p2/sql/repositoryAffinityList1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoryAffinityList1.out
rename to regression-test/data/variant_github_events_p2/sql/repositoryAffinityList1.out
diff --git a/regression-test/data/variant_github_events_p2/repositoryAffinityList2.out b/regression-test/data/variant_github_events_p2/sql/repositoryAffinityList2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/repositoryAffinityList2.out
rename to regression-test/data/variant_github_events_p2/sql/repositoryAffinityList2.out
diff --git a/regression-test/data/variant_github_events_p2/starsFromHeavyGithubUsers1.out b/regression-test/data/variant_github_events_p2/sql/starsFromHeavyGithubUsers1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/starsFromHeavyGithubUsers1.out
rename to regression-test/data/variant_github_events_p2/sql/starsFromHeavyGithubUsers1.out
diff --git a/regression-test/data/variant_github_events_p2/starsFromHeavyGithubUsers2.out b/regression-test/data/variant_github_events_p2/sql/starsFromHeavyGithubUsers2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/starsFromHeavyGithubUsers2.out
rename to regression-test/data/variant_github_events_p2/sql/starsFromHeavyGithubUsers2.out
diff --git a/regression-test/data/variant_github_events_p2/theLongestRepositoryNames1.out b/regression-test/data/variant_github_events_p2/sql/theLongestRepositoryNames1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/theLongestRepositoryNames1.out
rename to regression-test/data/variant_github_events_p2/sql/theLongestRepositoryNames1.out
diff --git a/regression-test/data/variant_github_events_p2/theLongestRepositoryNames2.out b/regression-test/data/variant_github_events_p2/sql/theLongestRepositoryNames2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/theLongestRepositoryNames2.out
rename to regression-test/data/variant_github_events_p2/sql/theLongestRepositoryNames2.out
diff --git a/regression-test/data/variant_github_events_p2/theMostToughCodeReviews.out b/regression-test/data/variant_github_events_p2/sql/theMostToughCodeReviews.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/theMostToughCodeReviews.out
rename to regression-test/data/variant_github_events_p2/sql/theMostToughCodeReviews.out
diff --git a/regression-test/data/variant_github_events_p2/theTotalNumberOfRepositoriesOnGithub.out b/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfRepositoriesOnGithub.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/theTotalNumberOfRepositoriesOnGithub.out
rename to regression-test/data/variant_github_events_p2/sql/theTotalNumberOfRepositoriesOnGithub.out
diff --git a/regression-test/data/variant_github_events_p2/theTotalNumberOfUsersOnGithub1.out b/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/theTotalNumberOfUsersOnGithub1.out
rename to regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub1.out
diff --git a/regression-test/data/variant_github_events_p2/theTotalNumberOfUsersOnGithub2.out b/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/theTotalNumberOfUsersOnGithub2.out
rename to regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub2.out
diff --git a/regression-test/data/variant_github_events_p2/theTotalNumberOfUsersOnGithub3.out b/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/theTotalNumberOfUsersOnGithub3.out
rename to regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub3.out
diff --git a/regression-test/data/variant_github_events_p2/theTotalNumberOfUsersOnGithub4.out b/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/theTotalNumberOfUsersOnGithub4.out
rename to regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub4.out
diff --git a/regression-test/data/variant_github_events_p2/topRepositoriesByStars.out b/regression-test/data/variant_github_events_p2/sql/topRepositoriesByStars.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/topRepositoriesByStars.out
rename to regression-test/data/variant_github_events_p2/sql/topRepositoriesByStars.out
diff --git a/regression-test/data/variant_github_events_p2/whatIsTheBestDayOfTheWeekToCatchAStar.out b/regression-test/data/variant_github_events_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/whatIsTheBestDayOfTheWeekToCatchAStar.out
rename to regression-test/data/variant_github_events_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out
diff --git a/regression-test/data/variant_github_events_p2/whoAreAllThosePeopleGivingStars1.out b/regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/whoAreAllThosePeopleGivingStars1.out
rename to regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars1.out
diff --git a/regression-test/data/variant_github_events_p2/whoAreAllThosePeopleGivingStars2.out b/regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/whoAreAllThosePeopleGivingStars2.out
rename to regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars2.out
diff --git a/regression-test/data/variant_github_events_p2/whoAreAllThosePeopleGivingStars3.out b/regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/whoAreAllThosePeopleGivingStars3.out
rename to regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars3.out
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/affinityByIssuesAndPRs1.sql b/regression-test/suites/variant_github_events_new_p2/sql/affinityByIssuesAndPRs1.sql
deleted file mode 100644
index 043ff99f6a0ec4..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/affinityByIssuesAndPRs1.sql
+++ /dev/null
@@ -1,14 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    count() AS prs,
-    count(distinct cast(v["actor"]["login"] as string)) AS authors
-FROM github_events
-WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["actor"]["login"] as string) IN
-(
-    SELECT cast(v["actor"]["login"] as string)
-    FROM github_events
-    WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string)= 'opened') AND (cast(v["repo"]["name"] as string) IN ('rspec/rspec-core', 'golden-warning/giraffedraft-server', 'apache/spark'))
-)) AND (lower(cast(v["repo"]["name"] as string)) NOT LIKE '%clickhouse%')
-GROUP BY repo_name
-ORDER BY authors DESC, prs DESC, repo_name DESC
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/affinityByIssuesAndPRs2.sql b/regression-test/suites/variant_github_events_new_p2/sql/affinityByIssuesAndPRs2.sql
deleted file mode 100644
index b3cf37124fc802..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/affinityByIssuesAndPRs2.sql
+++ /dev/null
@@ -1,14 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    count() AS prs,
-    count(distinct cast(v["actor"]["login"] as string)) AS authors
-FROM github_events
-WHERE (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["actor"]["login"] as string) IN
-(
-    SELECT cast(v["actor"]["login"] as string)
-    FROM github_events
-    WHERE (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["repo"]["name"] as string) IN ('No-CQRT/GooGuns', 'ivolunteerph/ivolunteerph', 'Tribler/tribler'))
-)) AND (lower(cast(v["repo"]["name"] as string)) NOT LIKE '%clickhouse%')
-GROUP BY repo_name 
-ORDER BY authors DESC, prs DESC, repo_name ASC
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/authorsWithTheMostPushes.sql b/regression-test/suites/variant_github_events_new_p2/sql/authorsWithTheMostPushes.sql
deleted file mode 100644
index 637dba7f6206e8..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/authorsWithTheMostPushes.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-      cast(v["actor"]["login"] as string) as actor_login,
-      count() AS c,
-      count(distinct cast(v["repo"]["name"] as string)) AS repos
-  FROM github_events
-  WHERE cast(v["type"] as string) = 'PushEvent'
-  GROUP BY actor_login
-  ORDER BY c DESC, 1, 3
-  LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/countingStar1.sql b/regression-test/suites/variant_github_events_new_p2/sql/countingStar1.sql
deleted file mode 100644
index cae7d585bf0784..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/countingStar1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent'
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/countingStar2.sql b/regression-test/suites/variant_github_events_new_p2/sql/countingStar2.sql
deleted file mode 100644
index 15bc6efe16961b..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/countingStar2.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["payload"]["action"] as string) as payload_action, count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY payload_action
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/countingStar3.sql b/regression-test/suites/variant_github_events_new_p2/sql/countingStar3.sql
deleted file mode 100644
index 172967acbfdd24..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/countingStar3.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND cast(v["repo"]["name"] as string) IN ('apache/spark', 'GunZi200/Memory-Colour', 'isohuntto/openbay', 'wasabeef/awesome-android-ui') GROUP BY cast(v["payload"]["action"] as string)  
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/distributionOfRepositoriesByStarCount.sql b/regression-test/suites/variant_github_events_new_p2/sql/distributionOfRepositoriesByStarCount.sql
deleted file mode 100644
index 5dbec453df93ef..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/distributionOfRepositoriesByStarCount.sql
+++ /dev/null
@@ -1,14 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    pow(10, floor(log10(c))) AS stars,
-    count(distinct repo)
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo,
-        count() AS c
-    FROM github_events
-    WHERE cast(v["type"] as string) = 'WatchEvent'
-    GROUP BY repo
-) t
-GROUP BY stars
-ORDER BY stars ASC
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/githubRoulette.sql b/regression-test/suites/variant_github_events_new_p2/sql/githubRoulette.sql
deleted file mode 100644
index 20cd59b11ecccc..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/githubRoulette.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' ORDER BY cast(v["created_at"] as datetime),  cast(v["repo"]["name"] as string) LIMIT 50
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql b/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql
deleted file mode 100644
index e3f1053a6be1c2..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY repo_name ORDER BY stars DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql b/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql
deleted file mode 100644
index e3f1053a6be1c2..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY repo_name ORDER BY stars DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql b/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql
deleted file mode 100644
index f68bdcfbf0e57a..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY repo_name ORDER BY stars, repo_name DESC LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql b/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql
deleted file mode 100644
index e3f1053a6be1c2..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY repo_name ORDER BY stars DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql b/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql
deleted file mode 100644
index e3f1053a6be1c2..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY repo_name ORDER BY stars DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql b/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql
deleted file mode 100644
index e3f1053a6be1c2..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY repo_name ORDER BY stars DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql b/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql
deleted file mode 100644
index 8f7e3f7b36fbfa..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql
+++ /dev/null
@@ -1,30 +0,0 @@
--- FIXME: UNSTABLE
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
---     repo,
---     year,
---     cnt
--- FROM
--- (
---     SELECT
---         row_number() OVER (PARTITION BY year ORDER BY cnt DESC) AS r,
---         repo,
---         year,
---         cnt
---     FROM
---     (
---         SELECT
---         lower(cast(v["repo"]["name"] as string)) AS repo,
---         year(cast(v["created_at"] as datetime)) AS year,
---         count() AS cnt
---         FROM github_events
---         WHERE (cast(v["type"] as string) = 'WatchEvent') AND (year(cast(v["created_at"] as datetime)) >= 2015)
---         GROUP BY
---             repo,
---             year
---     ) t
--- ) t2
--- WHERE r <= 10
--- ORDER BY
---     year ASC,
---     cnt DESC, repo
--- 
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql b/regression-test/suites/variant_github_events_new_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql
deleted file mode 100644
index f0d95d00ead3fe..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ year(cast(v["created_at"] as datetime)) AS year, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY year ORDER BY year
-
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments1.sql b/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments1.sql
deleted file mode 100644
index 5c0e58e47ef961..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ count() FROM github_events WHERE cast(v["type"] as string) = 'IssueCommentEvent'
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments2.sql b/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments2.sql
deleted file mode 100644
index bb12557117b2c7..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments2.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() FROM github_events WHERE cast(v["type"] as string) = 'IssueCommentEvent' GROUP BY repo_name ORDER BY count() DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments3.sql b/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments3.sql
deleted file mode 100644
index 1ab1e2969bfdb4..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments3.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    comments,
-    issues,
-    round(comments / issues, 2) AS ratio
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        count() AS comments,
-        count(distinct cast(v["payload"]["issue"]["number"] as int)) AS issues
-    FROM github_events
-    WHERE cast(v["type"] as string) = 'IssueCommentEvent'
-    GROUP BY repo_name
-) t
-ORDER BY comments DESC, 1, 3, 4
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments4.sql b/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments4.sql
deleted file mode 100644
index 62bba924a9926c..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments4.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    cast(v["payload"]["issue"]["number"] as int)  as number,
-    count() AS comments
-FROM github_events
-WHERE cast(v["type"] as string) = 'IssueCommentEvent' AND (cast(v["payload"]["action"] as string) = 'created')
-GROUP BY repo_name, number 
-ORDER BY comments DESC, number ASC, 1
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments5.sql b/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments5.sql
deleted file mode 100644
index fb8f3866328549..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments5.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    cast(v["payload"]["issue"]["number"] as int)  as number,
-    count() AS comments
-FROM github_events
-WHERE cast(v["type"] as string) = 'IssueCommentEvent' AND (cast(v["payload"]["action"] as string) = 'created') AND (cast(v["payload"]["issue"]["number"] as int)  > 10)
-GROUP BY repo_name, number
-ORDER BY comments DESC, repo_name, number
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments6.sql b/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments6.sql
deleted file mode 100644
index 3782b46cd001c0..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments6.sql
+++ /dev/null
@@ -1,11 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    cast(v["payload"]["issue"]["number"] as int)  as number,
-    count() AS comments,
-    count(distinct cast(v["actor"]["login"] as string)) AS authors
-FROM github_events
-WHERE cast(v["type"] as string) = 'IssueCommentEvent' AND (cast(v["payload"]["action"] as string) = 'created') AND (cast(v["payload"]["issue"]["number"] as int) > 10)
-GROUP BY repo_name, number
-HAVING authors >= 4
-ORDER BY comments DESC, repo_name
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments7.sql b/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments7.sql
deleted file mode 100644
index b429dcc8bfe4f7..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments7.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    count() AS comments,
-    count(distinct cast(v["actor"]["login"] as string)) AS authors
-FROM github_events
-WHERE cast(v["type"] as string) = 'CommitCommentEvent'
-GROUP BY repo_name
-ORDER BY count() DESC, 1, 3
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments8.sql b/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments8.sql
deleted file mode 100644
index 762a7dec440a63..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/issuesWithTheMostComments8.sql
+++ /dev/null
@@ -1,13 +0,0 @@
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
---     concat('https://github.com/', cast(v["repo"]["name"] as string), '/commit/', cast(v["payload"]["commit_id"] as string)) URL,
---     cast(v["payload"]["commit_id"] as string) AS commit_id,
---     count() AS comments,
---     count(distinct cast(v["actor"]["login"] as string)) AS authors
--- FROM github_events
--- WHERE (cast(v["type"] as string) = 'CommitCommentEvent') AND commit_id != ""
--- GROUP BY
---     cast(v["repo"]["name"] as string),
---     commit_id 
--- HAVING authors >= 10
--- ORDER BY count() DESC, URL, authors
--- LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/mostForkedRepositories.sql b/regression-test/suites/variant_github_events_new_p2/sql/mostForkedRepositories.sql
deleted file mode 100644
index f0ccb6e2c3d8fd..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/mostForkedRepositories.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() AS forks FROM github_events WHERE cast(v["type"] as string) = 'ForkEvent' GROUP BY repo_name ORDER BY forks DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/mostPopularCommentsOnGithub.sql b/regression-test/suites/variant_github_events_new_p2/sql/mostPopularCommentsOnGithub.sql
deleted file mode 100644
index 8b7094ac153b9f..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/mostPopularCommentsOnGithub.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["payload"]["comment"]["body"] as string) as comment_body, count() FROM github_events WHERE cast(v["payload"]["comment"]["body"] as string) != "" AND length(cast(v["payload"]["comment"]["body"] as string)) < 100 GROUP BY comment_body  ORDER BY count(), 1 DESC LIMIT 50
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheNumberOfRepositories.sql b/regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheNumberOfRepositories.sql
deleted file mode 100644
index f6d653413c3dc5..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheNumberOfRepositories.sql
+++ /dev/null
@@ -1,14 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    lower(split_part(repo_name, '/', 1)) AS org,
-    count(distinct repo_name) AS repos
-FROM
-(
-    SELECT cast(v["repo"]["name"] as string) as repo_name
-    FROM github_events
-    WHERE cast(v["type"] as string) = 'WatchEvent'
-    GROUP BY repo_name 
-    HAVING count() >= 10
-) t
-GROUP BY org
-ORDER BY repos DESC, org ASC
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheNumberOfStars.sql b/regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheNumberOfStars.sql
deleted file mode 100644
index 04b55d39e51291..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheNumberOfStars.sql
+++ /dev/null
@@ -1,8 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    lower(split_part(cast(v["repo"]["name"] as string), '/', 1)) AS org,
-    count() AS stars
-FROM github_events
-WHERE cast(v["type"] as string) = 'WatchEvent'
-GROUP BY org
-ORDER BY stars DESC, 1
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheSizeOfCommunity.sql b/regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheSizeOfCommunity.sql
deleted file mode 100644
index e0352afc55fa0f..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/organizationsByTheSizeOfCommunity.sql
+++ /dev/null
@@ -1,23 +0,0 @@
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
---     lower(split_part(cast(v["repo"]["name"] as string), '/', 1)) AS org,
---     count(distinct cast(v["actor"]["login"] as string)) AS authors,
---     count(distinct pr_author) AS pr_authors,
---     count(distinct issue_author) AS issue_authors,
---     count(distinct comment_author) AS comment_authors,
---     count(distinct review_author) AS review_authors,
---     count(distinct push_author) AS push_authors
--- FROM
--- (
--- SELECT
---     cast(v["repo"]["name"] as string),
---     cast(v["actor"]["login"] as string),
---     CASE WHEN cast(v["type"] as string) = 'PullRequestEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END pr_author,
---     CASE WHEN cast(v["type"] as string) = 'IssuesEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END issue_author,
---     CASE WHEN cast(v["type"] as string) = 'IssueCommentEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END comment_author,
---     CASE WHEN cast(v["type"] as string) = 'PullRequestReviewCommentEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END review_author,
---     CASE WHEN cast(v["type"] as string) = 'PushEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END push_author
--- FROM github_events
--- WHERE cast(v["type"] as string) IN ('PullRequestEvent', 'IssuesEvent', 'IssueCommentEvent', 'PullRequestReviewCommentEvent', 'PushEvent')
--- ) t
--- GROUP BY org
--- ORDER BY authors DESC
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks1.sql b/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks1.sql
deleted file mode 100644
index 9e3f2875ef2e10..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks1.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(fork) AS forks,
-    sum(star) AS stars,
-    round(sum(star) / sum(fork), 3) AS ratio
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star
-    FROM github_events
-    WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent')
-) t
-GROUP BY repo_name
-ORDER BY forks DESC, 1, 3, 4
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks2.sql b/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks2.sql
deleted file mode 100644
index 2e1cb1b9c1f4e5..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks2.sql
+++ /dev/null
@@ -1,18 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(fork) AS forks,
-    sum(star) AS stars,
-    round(sum(star) / sum(fork), 3) AS ratio
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star
-    FROM github_events
-    WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent')
-) t
-GROUP BY  repo_name 
-HAVING (stars > 20) AND (forks >= 10)
-ORDER BY ratio DESC, repo_name 
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks3.sql b/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks3.sql
deleted file mode 100644
index 6f91ab15eb9823..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks3.sql
+++ /dev/null
@@ -1,18 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(fork) AS forks,
-    sum(star) AS stars,
-    round(sum(fork) / sum(star), 2) AS ratio
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star
-    FROM github_events
-    WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent')
-) t
-GROUP BY repo_name
-HAVING (stars > 4) AND (forks > 4)
-ORDER BY ratio DESC
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks4.sql b/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks4.sql
deleted file mode 100644
index 8be41f0949737f..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks4.sql
+++ /dev/null
@@ -1,13 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    sum(fork) AS forks,
-    sum(star) AS stars,
-    round(sum(star) / sum(fork), 2) AS ratio
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string),
-        CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star
-    FROM github_events
-    WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent')
-) t
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks5.sql b/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks5.sql
deleted file mode 100644
index 26bfb2ac6302ee..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks5.sql
+++ /dev/null
@@ -1,21 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    sum(forks) AS forks,
-    sum(stars) AS stars,
-    round(sum(stars) / sum(forks), 2) AS ratio
-FROM
-(
-    SELECT
-        sum(fork) AS forks,
-        sum(star) AS stars
-    FROM
-    (
-        SELECT
-            cast(v["repo"]["name"] as string) as repo_name,
-            CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork,
-            CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star
-        FROM github_events
-        WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent')
-    ) t
-    GROUP BY repo_name
-    HAVING stars > 10
-) t2
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesByTheNumberOfPushes.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesByTheNumberOfPushes.sql
deleted file mode 100644
index 6db36e257c5d45..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesByTheNumberOfPushes.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    count() AS pushes,
-    count(distinct cast(v["actor"]["login"] as string)) AS authors
-FROM github_events
-WHERE (cast(v["type"] as string) = 'PushEvent') AND (cast(v["repo"]["name"] as string) IN
-(
-    SELECT cast(v["repo"]["name"] as string) as repo_name
-    FROM github_events
-    WHERE cast(v["type"] as string) = 'WatchEvent'
-    GROUP BY repo_name
-    ORDER BY count() DESC
-    LIMIT 10000
-))
-GROUP BY repo_name
-ORDER BY count() DESC, repo_name
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments1.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments1.sql
deleted file mode 100644
index 254888d0e11901..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() FROM github_events WHERE lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%apache%' GROUP BY repo_name ORDER BY count() DESC, repo_name ASC LIMIT 50
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments2.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments2.sql
deleted file mode 100644
index 90ef187ba3520c..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments2.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(num_star) AS num_stars,
-    sum(num_comment) AS num_comments
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS num_star,
-        CASE WHEN lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%apache%' THEN 1 ELSE 0 END AS num_comment
-    FROM github_events
-    WHERE (lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%apache%') OR (cast(v["type"] as string) = 'WatchEvent')
-) t
-GROUP BY repo_name 
-HAVING num_comments > 0
-ORDER BY num_stars DESC,num_comments DESC, repo_name ASC
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments1.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments1.sql
deleted file mode 100644
index 90a73db9ca3897..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() FROM github_events WHERE lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%spark%' GROUP BY repo_name ORDER BY count() DESC, repo_name ASC LIMIT 50
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments2.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments2.sql
deleted file mode 100644
index 28c44705455d23..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments2.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(num_star) AS num_stars,
-    sum(num_comment) AS num_comments
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS num_star,
-        CASE WHEN lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%spark%' THEN 1 ELSE 0 END AS num_comment
-    FROM github_events
-    WHERE (lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%spark%') OR (cast(v["type"] as string) = 'WatchEvent')
-) t
-GROUP BY repo_name
-HAVING num_comments > 0
-ORDER BY num_stars DESC,num_comments DESC,repo_name ASC
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheHighestGrowthYoY.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheHighestGrowthYoY.sql
deleted file mode 100644
index 47d3ad8417dc4f..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheHighestGrowthYoY.sql
+++ /dev/null
@@ -1,20 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(created_at_2022) AS stars2022,
-    sum(created_at_2015) AS stars2015,
-    round(sum(created_at_2022) / sum(created_at_2015), 3) AS yoy,
-    min(created_at) AS first_seen
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE year(cast(v["created_at"] as datetime)) WHEN 2022 THEN 1 ELSE 0 END AS created_at_2022,
-        CASE year(cast(v["created_at"] as datetime)) WHEN 2015 THEN 1 ELSE 0 END AS created_at_2015,
-        cast(v["created_at"] as datetime) as created_at
-    FROM github_events
-    WHERE cast(v["type"] as string) = 'WatchEvent'
-) t
-GROUP BY  repo_name 
-HAVING (min(created_at) <= '2023-01-01 00:00:00') AND (stars2022 >= 1) and (stars2015 >= 1)
-ORDER BY yoy DESC, repo_name 
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql
deleted file mode 100644
index d93b20c783fe11..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() AS c, count(distinct cast(v["actor"]["login"] as string)) AS u FROM github_events WHERE cast(v["type"] as string) = 'IssuesEvent' AND cast(v["payload"]["action"] as string) = 'opened' GROUP BY repo_name ORDER BY c DESC, repo_name LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql
deleted file mode 100644
index 5703e50af8f5d9..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql
+++ /dev/null
@@ -1,18 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(issue_created) AS c,
-    count(distinct actor_login) AS u,
-    sum(star) AS stars
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN 1 ELSE 0 END AS issue_created,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star,
-        CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS actor_login 
-    FROM github_events
-    WHERE cast(v["type"] as string) IN ('IssuesEvent', 'WatchEvent')
-) t
-GROUP BY repo_name
-ORDER BY c DESC, repo_name
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql
deleted file mode 100644
index 32014cb4c195bc..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql
+++ /dev/null
@@ -1,19 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(issue_created) AS c,
-    count(distinct actor_login) AS u,
-    sum(star) AS stars
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN 1 ELSE 0 END AS issue_created,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star,
-        CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS actor_login 
-    FROM github_events
-    WHERE cast(v["type"] as string) IN ('IssuesEvent', 'WatchEvent')
-) t
-GROUP BY repo_name
-HAVING stars >= 10
-ORDER BY c, u, stars DESC, repo_name
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql
deleted file mode 100644
index 484d8b50ea8ccd..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql
+++ /dev/null
@@ -1,18 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(issue_created) AS c,
-    count(distinct actor_login) AS u,
-    sum(star) AS stars
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN 1 ELSE 0 END AS issue_created,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star,
-        CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS actor_login 
-    FROM github_events
-    WHERE cast(v["type"] as string) IN ('IssuesEvent', 'WatchEvent')
-) t
-GROUP BY repo_name
-ORDER BY u, c, stars DESC, 1
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql
deleted file mode 100644
index 688d42ec6a94b4..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count(), count(distinct cast(v["actor"]["login"] as string)) FROM github_events WHERE cast(v["type"] as string) = 'PullRequestEvent' AND cast(v["payload"]["action"] as string) = 'opened' GROUP BY repo_name ORDER BY 2,1,3 DESC LIMIT 50
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql
deleted file mode 100644
index e4b3b388e24159..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count(), count(distinct cast(v["actor"]["login"] as string)) AS u FROM github_events WHERE cast(v["type"] as string) = 'PullRequestEvent' AND cast(v["payload"]["action"] as string) = 'opened' GROUP BY repo_name ORDER BY u DESC, 2 DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql
deleted file mode 100644
index f6672875578f47..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(invitation) AS invitations,
-    sum(star) AS stars
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE WHEN cast(v["type"] as string) = 'MemberEvent' THEN 1 ELSE 0 END AS invitation,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star
-    FROM github_events
-    WHERE cast(v["type"] as string) IN ('MemberEvent', 'WatchEvent')
-) t
-GROUP BY repo_name
-HAVING stars >= 2
-ORDER BY invitations DESC, stars DESC, repo_name
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql
deleted file mode 100644
index d31a0f38a54a13..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql
+++ /dev/null
@@ -1,13 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    count(distinct actor_login) AS u,
-    sum(star) AS stars
-FROM
-(
-    SELECT
-        lower(cast(v["repo"]["name"] as string)) as repo_name,
-        CASE WHEN cast(v["type"] as string) = 'PushEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END AS actor_login,
-        CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star
-    FROM github_events WHERE cast(v["type"] as string) IN ('PushEvent', 'WatchEvent') AND cast(v["repo"]["name"] as string) != '/'
-) t
-GROUP BY repo_name ORDER BY u, stars, repo_name DESC LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql
deleted file mode 100644
index 5849b3527d3a55..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql
+++ /dev/null
@@ -1,13 +0,0 @@
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
---     cast(v["repo"]["name"] as string),
---     count(distinct cast(v["actor"]["login"] as string)) AS u,
---     sum(star) AS stars
--- FROM
--- (
---     SELECT
---         cast(v["repo"]["name"] as string),
---         CASE WHEN cast(v["type"] as string) = 'PushEvent' AND (ref LIKE '%/master' OR ref LIKE '%/main') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS cast(v["actor"]["login"] as string),
---         CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star
---     FROM github_events WHERE cast(v["type"] as string) IN ('PushEvent', 'WatchEvent') AND cast(v["repo"]["name"] as string) != '/'
--- ) t
--- GROUP BY cast(v["repo"]["name"] as string) ORDER BY u DESC LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql
deleted file mode 100644
index b3b2133eb2991c..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql
+++ /dev/null
@@ -1,16 +0,0 @@
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
---     cast(v["repo"]["name"] as string),
---     count(distinct cast(v["actor"]["login"] as string)) AS u,
---     sum(star) AS stars
--- FROM
--- (
---     SELECT
---         cast(v["repo"]["name"] as string),
---         CASE WHEN cast(v["type"] as string) = 'PushEvent' AND (ref LIKE '%/master' OR ref LIKE '%/main') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS cast(v["actor"]["login"] as string),
---         CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star
---     FROM github_events WHERE cast(v["type"] as string) IN ('PushEvent', 'WatchEvent') AND cast(v["repo"]["name"] as string) != '/'
--- ) t
--- GROUP BY cast(v["repo"]["name"] as string)
--- HAVING stars >= 100
--- ORDER BY u DESC
--- LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql
deleted file mode 100644
index dcf6eb23235a69..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql
+++ /dev/null
@@ -1,25 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    day,
-    stars
-FROM
-(
-    SELECT
-        row_number() OVER (PARTITION BY repo_name  ORDER BY stars DESC) AS rank,
-        repo_name,
-        day,
-        stars
-    FROM
-    (
-        SELECT
-            cast(v["repo"]["name"] as string) as repo_name,
-            to_date(cast(v["created_at"] as datetime)) AS day,
-            count() AS stars
-        FROM github_events
-        WHERE cast(v["type"] as string) = 'WatchEvent'
-        GROUP BY repo_name, day
-    ) t1
-) t2
-WHERE rank = 1
-ORDER BY stars DESC, 1
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql
deleted file mode 100644
index 66a7ab7cba55fd..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql
+++ /dev/null
@@ -1,25 +0,0 @@
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
---     cast(v["repo"]["name"] as string),
---     day,
---     stars
--- FROM
--- (
---     SELECT
---         row_number() OVER (PARTITION BY cast(v["repo"]["name"] as string) ORDER BY stars DESC) AS rank,
---         cast(v["repo"]["name"] as string),
---         day,
---         stars
---     FROM
---     (
---         SELECT
---             cast(v["repo"]["name"] as string),
---             to_date(cast(v["created_at"] as datetime)) AS day,
---             count() AS stars
---         FROM github_events
---         WHERE cast(v["type"] as string) = 'WatchEvent'
---         GROUP BY cast(v["repo"]["name"] as string), day
---     ) t1
--- ) t2
--- WHERE rank = 1
--- ORDER BY stars DESC
--- LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql
deleted file mode 100644
index 1bfa7869399474..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql
+++ /dev/null
@@ -1 +0,0 @@
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string), cast(v["created_at"] as datetime), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY cast(v["repo"]["name"] as string), cast(v["created_at"] as datetime) ORDER BY count() DESC LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql
deleted file mode 100644
index d1abdfd9dd4e0e..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql
+++ /dev/null
@@ -1,20 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    max(stars) AS daily_stars,
-    sum(stars) AS total_stars,
-    sum(stars) / max(stars) AS rate
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        to_date(cast(v["created_at"] as datetime)) AS day,
-        count() AS stars
-    FROM github_events
-    WHERE cast(v["type"] as string) = 'WatchEvent'
-    GROUP BY
-        repo_name,
-        day
-) t
-GROUP BY repo_name
-ORDER BY rate DESC, 1
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheWorstStagnation_order.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheWorstStagnation_order.sql
deleted file mode 100644
index a4c09020695e9d..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesWithTheWorstStagnation_order.sql
+++ /dev/null
@@ -1,20 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    repo_name,
-    sum(created_at_2022) AS stars2022,
-    sum(created_at_2015) AS stars2015,
-    round(sum(created_at_2022) / sum(created_at_2015), 3) AS yoy,
-    min(created_at) AS first_seen
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        CASE year(cast(v["created_at"] as datetime)) WHEN 2022 THEN 1 ELSE 0 END AS created_at_2022,
-        CASE year(cast(v["created_at"] as datetime)) WHEN 2015 THEN 1 ELSE 0 END AS created_at_2015,
-        cast(v["created_at"] as datetime) as created_at
-    FROM github_events
-    WHERE cast(v["type"] as string) = 'WatchEvent'
-) t
-GROUP BY repo_name
-HAVING (min(created_at) <= '2019-01-01 00:00:00') AND (max(created_at) >= '2020-06-01 00:00:00') AND (stars2015 >= 2)
-ORDER BY yoy, repo_name
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoryAffinityList1.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoryAffinityList1.sql
deleted file mode 100644
index f75b3d0d493adb..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoryAffinityList1.sql
+++ /dev/null
@@ -1,13 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-  cast(v["repo"]["name"] as string) as repo_name,
-  count() AS stars
-FROM github_events
-WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["actor"]["login"] as string) IN
-(
-    SELECT cast(v["actor"]["login"] as string)
-    FROM github_events
-    WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["repo"]["name"] as string) IN ('apache/spark', 'prakhar1989/awesome-courses'))
-)) AND (cast(v["repo"]["name"] as string) NOT IN ('ClickHouse/ClickHouse', 'yandex/ClickHouse'))
-GROUP BY repo_name
-ORDER BY stars DESC, repo_name 
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoryAffinityList2.sql b/regression-test/suites/variant_github_events_new_p2/sql/repositoryAffinityList2.sql
deleted file mode 100644
index 6f5ae4cbb33dc7..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/repositoryAffinityList2.sql
+++ /dev/null
@@ -1,23 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-  repo_name,
-  total_stars,
-  round(spark_stars / total_stars, 2) AS ratio
-FROM
-(
-    SELECT
-        cast(v["repo"]["name"] as string) as repo_name,
-        count(distinct cast(v["actor"]["login"] as string)) AS total_stars
-    FROM github_events
-    WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["repo"]["name"] as string) NOT IN ('apache/spark'))
-    GROUP BY repo_name
-    HAVING total_stars >= 10
-) t1
-JOIN
-(
-    SELECT
-        count(distinct cast(v["actor"]["login"] as string)) AS spark_stars
-    FROM github_events
-    WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["repo"]["name"] as string) IN ('apache/spark'))
-) t2
-ORDER BY ratio DESC, repo_name
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers1.sql b/regression-test/suites/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers1.sql
deleted file mode 100644
index 0cd0b5ade503a3..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers1.sql
+++ /dev/null
@@ -1,13 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    count()
-FROM github_events
-WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["actor"]["login"] as string) IN
-(
-    SELECT cast(v["actor"]["login"] as string)
-    FROM github_events
-    WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string) = 'opened')
-))
-GROUP BY repo_name
-ORDER BY count() DESC, repo_name
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers2.sql b/regression-test/suites/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers2.sql
deleted file mode 100644
index d230a3186fbe93..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers2.sql
+++ /dev/null
@@ -1,15 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    count()
-FROM github_events
-WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["actor"]["login"] as string) IN
-(
-    SELECT cast(v["actor"]["login"] as string) as actor_login
-    FROM github_events
-    WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string) = 'opened')
-    GROUP BY actor_login
-    HAVING count() >= 2
-))
-GROUP BY repo_name
-ORDER BY 1, count() DESC, 1
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/test_distinct_streaming_agg.sql b/regression-test/suites/variant_github_events_new_p2/sql/test_distinct_streaming_agg.sql
deleted file mode 100644
index 40854a2d53502f..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/test_distinct_streaming_agg.sql
+++ /dev/null
@@ -1,5 +0,0 @@
-SELECT
-    /*+SET_VAR(batch_size=50,experimental_enable_pipeline_x_engine=false,parallel_pipeline_task_num=1,disable_streaming_preaggregations=false) */
-    count(distinct v)
-FROM
-    github_events_2;
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/theLongestRepositoryNames1.sql b/regression-test/suites/variant_github_events_new_p2/sql/theLongestRepositoryNames1.sql
deleted file mode 100644
index 4644f27b54f784..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/theLongestRepositoryNames1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ count(), cast(v["repo"]["name"] as string) as repo_name FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY repo_name ORDER BY length(repo_name) DESC, repo_name LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/theLongestRepositoryNames2.sql b/regression-test/suites/variant_github_events_new_p2/sql/theLongestRepositoryNames2.sql
deleted file mode 100644
index ee34ddfaae05a1..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/theLongestRepositoryNames2.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND cast(v["repo"]["name"] as string) LIKE '%_/_%' GROUP BY repo_name ORDER BY length(repo_name) ASC, repo_name LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/theMostToughCodeReviews.sql b/regression-test/suites/variant_github_events_new_p2/sql/theMostToughCodeReviews.sql
deleted file mode 100644
index 29cf3d807c3760..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/theMostToughCodeReviews.sql
+++ /dev/null
@@ -1,10 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    concat('https://github.com/', cast(v["repo"]["name"] as string), '/pull/') AS URL,
-    count(distinct cast(v["actor"]["login"] as string)) AS authors
-FROM github_events
-WHERE (cast(v["type"] as string) = 'PullRequestReviewCommentEvent') AND (cast(v["payload"]["action"] as string) = 'created')
-GROUP BY
-    cast(v["repo"]["name"] as string),
-    cast(v["payload"]["issue"]["number"] as string) 
-ORDER BY authors DESC, URL ASC
-LIMIT 50
\ No newline at end of file
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql b/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql
deleted file mode 100644
index e10d25749b48bc..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ count(distinct cast(v["repo"]["name"] as string)) FROM github_events
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub1.sql b/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub1.sql
deleted file mode 100644
index fe5f1956c7948a..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ count(distinct cast(v["actor"]["login"] as string)) FROM github_events
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub2.sql b/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub2.sql
deleted file mode 100644
index 72dccfb12d697e..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub2.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ count(distinct cast(v["actor"]["login"] as string)) FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent'
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub3.sql b/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub3.sql
deleted file mode 100644
index ce96a89bd27105..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub3.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ count(distinct cast(v["actor"]["login"] as string)) FROM github_events WHERE cast(v["type"] as string) = 'PushEvent'
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub4.sql b/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub4.sql
deleted file mode 100644
index 53e611219f3dd0..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub4.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ count(distinct cast(v["actor"]["login"] as string)) FROM github_events WHERE cast(v["type"] as string) = 'PullRequestEvent' AND cast(v["payload"]["action"] as string) = 'opened'
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/topLabels1.sql b/regression-test/suites/variant_github_events_new_p2/sql/topLabels1.sql
deleted file mode 100644
index eb6dd56d26c556..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/topLabels1.sql
+++ /dev/null
@@ -1,9 +0,0 @@
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
---     label,
---     count() AS c
--- FROM github_events
--- LATERAL VIEW explode_split(labels, ',') t AS label
--- WHERE (cast(v["type"] as string) IN ('IssuesEvent', 'PullRequestEvent', 'IssueCommentEvent')) AND (action IN ('created', 'opened', 'labeled'))
--- GROUP BY label
--- ORDER BY c DESC
--- LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/topLabels2.sql b/regression-test/suites/variant_github_events_new_p2/sql/topLabels2.sql
deleted file mode 100644
index d04bded901d30c..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/topLabels2.sql
+++ /dev/null
@@ -1,9 +0,0 @@
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
---     label,
---     count() AS c
--- FROM github_events
--- LATERAL VIEW explode_split(labels, ',') t AS label
--- WHERE (cast(v["type"] as string) IN ('IssuesEvent', 'PullRequestEvent', 'IssueCommentEvent')) AND (action IN ('created', 'opened', 'labeled')) AND ((lower(label) LIKE '%bug%') OR (lower(label) LIKE '%feature%'))
--- GROUP BY label
--- ORDER BY c DESC
--- LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/topLabels3.sql b/regression-test/suites/variant_github_events_new_p2/sql/topLabels3.sql
deleted file mode 100644
index f35116aaa6d27f..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/topLabels3.sql
+++ /dev/null
@@ -1,14 +0,0 @@
--- SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
---     sum(bug) AS bugs,
---     sum(feature) AS feature,
---     sum(bug) / sum(feature) AS ratio
--- FROM
--- (
---     SELECT
---         CASE WHEN lower(label) LIKE '%bug%' THEN 1 ELSE 0 END AS bug,
---         CASE WHEN lower(label) LIKE '%feature%' THEN 1 ELSE 0 END AS feature
---     FROM github_events
---     LATERAL VIEW explode_split(labels, ',') t AS label
---     WHERE (cast(v["type"] as string) IN ('IssuesEvent', 'PullRequestEvent', 'IssueCommentEvent')) AND (action IN ('created', 'opened', 'labeled')) AND ((lower(label) LIKE '%bug%') OR (lower(label) LIKE '%feature%'))
--- ) t
--- LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/topRepositoriesByStars.sql b/regression-test/suites/variant_github_events_new_p2/sql/topRepositoriesByStars.sql
deleted file mode 100644
index 41eec01ee97809..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/topRepositoriesByStars.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["repo"]["name"] as string) as repo_name, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY repo_name ORDER BY stars DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql b/regression-test/suites/variant_github_events_new_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql
deleted file mode 100644
index c0d4576ce60088..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ dayofweek(cast(v["created_at"] as datetime)) AS day, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY day ORDER BY day
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars1.sql b/regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars1.sql
deleted file mode 100644
index 91f186645603d6..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars1.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["actor"]["login"] as string) as actor_login, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY actor_login ORDER BY stars DESC, 1 LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars2.sql b/regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars2.sql
deleted file mode 100644
index 30a9ea294ccf9e..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars2.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ cast(v["actor"]["login"] as string) as actor_login, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND cast(v["actor"]["login"] as string) = 'cliffordfajardo' GROUP BY actor_login ORDER BY stars DESC LIMIT 50
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars3.sql b/regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars3.sql
deleted file mode 100644
index 524737737b6bac..00000000000000
--- a/regression-test/suites/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars3.sql
+++ /dev/null
@@ -1,13 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    count() AS stars
-FROM github_events
-WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["repo"]["name"] as string) IN
-(
-    SELECT cast(v["repo"]["name"] as string)
-    FROM github_events
-    WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["actor"]["login"] as string) = 'cliffordfajardo')
-))
-GROUP BY repo_name 
-ORDER BY stars DESC, repo_name
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesByAmountOfModifiedCode.sql b/regression-test/suites/variant_github_events_p2/repositoriesByAmountOfModifiedCode.sql
deleted file mode 100644
index 6c1632769f3d9f..00000000000000
--- a/regression-test/suites/variant_github_events_p2/repositoriesByAmountOfModifiedCode.sql
+++ /dev/null
@@ -1,12 +0,0 @@
-SELECT /*+SET_VAR(enable_fallback_to_original_planner=false,disable_streaming_preaggregations=false) */
-    cast(v["repo"]["name"] as string) as repo_name,
-    count() AS prs,
-    count(distinct cast(v["actor"]["login"] as string)) AS authors,
-    sum(cast(v["payload"]["pull_request"]["additions"] as int)) AS adds,
-    sum(cast(v["payload"]["pull_request"]["deletions"] as int)) AS dels
-FROM github_events
-WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["payload"]["pull_request"]["additions"] as int) < 10000) AND (cast(v["payload"]["pull_request"]["deletions"] as int) < 10000)
-GROUP BY repo_name
-HAVING (adds / dels) < 10
-ORDER BY adds + dels DESC, 1
-LIMIT 50
diff --git a/regression-test/suites/variant_github_events_p2/affinityByIssuesAndPRs1.sql b/regression-test/suites/variant_github_events_p2/sql/affinityByIssuesAndPRs1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/affinityByIssuesAndPRs1.sql
rename to regression-test/suites/variant_github_events_p2/sql/affinityByIssuesAndPRs1.sql
diff --git a/regression-test/suites/variant_github_events_p2/affinityByIssuesAndPRs2.sql b/regression-test/suites/variant_github_events_p2/sql/affinityByIssuesAndPRs2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/affinityByIssuesAndPRs2.sql
rename to regression-test/suites/variant_github_events_p2/sql/affinityByIssuesAndPRs2.sql
diff --git a/regression-test/suites/variant_github_events_p2/authorsWithTheMostPushes.sql b/regression-test/suites/variant_github_events_p2/sql/authorsWithTheMostPushes.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/authorsWithTheMostPushes.sql
rename to regression-test/suites/variant_github_events_p2/sql/authorsWithTheMostPushes.sql
diff --git a/regression-test/suites/variant_github_events_p2/countingStar1.sql b/regression-test/suites/variant_github_events_p2/sql/countingStar1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/countingStar1.sql
rename to regression-test/suites/variant_github_events_p2/sql/countingStar1.sql
diff --git a/regression-test/suites/variant_github_events_p2/countingStar2.sql b/regression-test/suites/variant_github_events_p2/sql/countingStar2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/countingStar2.sql
rename to regression-test/suites/variant_github_events_p2/sql/countingStar2.sql
diff --git a/regression-test/suites/variant_github_events_p2/countingStar3.sql b/regression-test/suites/variant_github_events_p2/sql/countingStar3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/countingStar3.sql
rename to regression-test/suites/variant_github_events_p2/sql/countingStar3.sql
diff --git a/regression-test/suites/variant_github_events_p2/distributionOfRepositoriesByStarCount.sql b/regression-test/suites/variant_github_events_p2/sql/distributionOfRepositoriesByStarCount.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/distributionOfRepositoriesByStarCount.sql
rename to regression-test/suites/variant_github_events_p2/sql/distributionOfRepositoriesByStarCount.sql
diff --git a/regression-test/suites/variant_github_events_p2/githubRoulette.sql b/regression-test/suites/variant_github_events_p2/sql/githubRoulette.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/githubRoulette.sql
rename to regression-test/suites/variant_github_events_p2/sql/githubRoulette.sql
diff --git a/regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql b/regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql
rename to regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql
diff --git a/regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql b/regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql
rename to regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql
diff --git a/regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql b/regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql
rename to regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql
diff --git a/regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql b/regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql
rename to regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql
diff --git a/regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql b/regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql
rename to regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql
diff --git a/regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql b/regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql
rename to regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql
diff --git a/regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql b/regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql
rename to regression-test/suites/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql
diff --git a/regression-test/suites/variant_github_events_p2/howHasTheTotalNumberOfStarsChangedOverTime.sql b/regression-test/suites/variant_github_events_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/howHasTheTotalNumberOfStarsChangedOverTime.sql
rename to regression-test/suites/variant_github_events_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql
diff --git a/regression-test/suites/variant_github_events_p2/issuesWithTheMostComments1.sql b/regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/issuesWithTheMostComments1.sql
rename to regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments1.sql
diff --git a/regression-test/suites/variant_github_events_p2/issuesWithTheMostComments2.sql b/regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/issuesWithTheMostComments2.sql
rename to regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments2.sql
diff --git a/regression-test/suites/variant_github_events_p2/issuesWithTheMostComments3.sql b/regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/issuesWithTheMostComments3.sql
rename to regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments3.sql
diff --git a/regression-test/suites/variant_github_events_p2/issuesWithTheMostComments4.sql b/regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments4.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/issuesWithTheMostComments4.sql
rename to regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments4.sql
diff --git a/regression-test/suites/variant_github_events_p2/issuesWithTheMostComments5.sql b/regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments5.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/issuesWithTheMostComments5.sql
rename to regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments5.sql
diff --git a/regression-test/suites/variant_github_events_p2/issuesWithTheMostComments6.sql b/regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments6.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/issuesWithTheMostComments6.sql
rename to regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments6.sql
diff --git a/regression-test/suites/variant_github_events_p2/issuesWithTheMostComments7.sql b/regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments7.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/issuesWithTheMostComments7.sql
rename to regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments7.sql
diff --git a/regression-test/suites/variant_github_events_p2/issuesWithTheMostComments8.sql b/regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments8.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/issuesWithTheMostComments8.sql
rename to regression-test/suites/variant_github_events_p2/sql/issuesWithTheMostComments8.sql
diff --git a/regression-test/suites/variant_github_events_p2/mostForkedRepositories.sql b/regression-test/suites/variant_github_events_p2/sql/mostForkedRepositories.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/mostForkedRepositories.sql
rename to regression-test/suites/variant_github_events_p2/sql/mostForkedRepositories.sql
diff --git a/regression-test/suites/variant_github_events_p2/mostPopularCommentsOnGithub.sql b/regression-test/suites/variant_github_events_p2/sql/mostPopularCommentsOnGithub.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/mostPopularCommentsOnGithub.sql
rename to regression-test/suites/variant_github_events_p2/sql/mostPopularCommentsOnGithub.sql
diff --git a/regression-test/suites/variant_github_events_p2/organizationsByTheNumberOfRepositories.sql b/regression-test/suites/variant_github_events_p2/sql/organizationsByTheNumberOfRepositories.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/organizationsByTheNumberOfRepositories.sql
rename to regression-test/suites/variant_github_events_p2/sql/organizationsByTheNumberOfRepositories.sql
diff --git a/regression-test/suites/variant_github_events_p2/organizationsByTheNumberOfStars.sql b/regression-test/suites/variant_github_events_p2/sql/organizationsByTheNumberOfStars.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/organizationsByTheNumberOfStars.sql
rename to regression-test/suites/variant_github_events_p2/sql/organizationsByTheNumberOfStars.sql
diff --git a/regression-test/suites/variant_github_events_p2/organizationsByTheSizeOfCommunity.sql b/regression-test/suites/variant_github_events_p2/sql/organizationsByTheSizeOfCommunity.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/organizationsByTheSizeOfCommunity.sql
rename to regression-test/suites/variant_github_events_p2/sql/organizationsByTheSizeOfCommunity.sql
diff --git a/regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks1.sql b/regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks1.sql
rename to regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks1.sql
diff --git a/regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks2.sql b/regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks2.sql
rename to regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks2.sql
diff --git a/regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks3.sql b/regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks3.sql
rename to regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks3.sql
diff --git a/regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks4.sql b/regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks4.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks4.sql
rename to regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks4.sql
diff --git a/regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks5.sql b/regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks5.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/proportionsBetweenStarsAndForks5.sql
rename to regression-test/suites/variant_github_events_p2/sql/proportionsBetweenStarsAndForks5.sql
diff --git a/regression-test/suites/variant_github_events_new_p2/sql/repositoriesByAmountOfModifiedCode.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesByAmountOfModifiedCode.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_new_p2/sql/repositoriesByAmountOfModifiedCode.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesByAmountOfModifiedCode.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesByTheNumberOfPushes.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesByTheNumberOfPushes.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesByTheNumberOfPushes.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesByTheNumberOfPushes.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithClickhouse_related_comments1.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithClickhouse_related_comments1.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments1.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithClickhouse_related_comments2.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithClickhouse_related_comments2.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments2.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithDoris_related_comments1.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithDoris_related_comments1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithDoris_related_comments1.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithDoris_related_comments1.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithDoris_related_comments2.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithDoris_related_comments2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithDoris_related_comments2.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithDoris_related_comments2.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheHighestGrowthYoY.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheHighestGrowthYoY.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheHighestGrowthYoY.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheHighestGrowthYoY.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues1.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues1.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues2.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues2.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues3.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues3.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues4.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfIssues4.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfPullRequests1.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfPullRequests1.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfPullRequests2.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumAmountOfPullRequests2.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMostStarsOverOneDay1.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMostStarsOverOneDay1.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMostStarsOverOneDay2.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMostStarsOverOneDay2.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMostStarsOverOneDay3.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMostStarsOverOneDay3.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheMostSteadyGrowthOverTime.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheMostSteadyGrowthOverTime.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoriesWithTheWorstStagnation_order.sql b/regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheWorstStagnation_order.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoriesWithTheWorstStagnation_order.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoriesWithTheWorstStagnation_order.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoryAffinityList1.sql b/regression-test/suites/variant_github_events_p2/sql/repositoryAffinityList1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoryAffinityList1.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoryAffinityList1.sql
diff --git a/regression-test/suites/variant_github_events_p2/repositoryAffinityList2.sql b/regression-test/suites/variant_github_events_p2/sql/repositoryAffinityList2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/repositoryAffinityList2.sql
rename to regression-test/suites/variant_github_events_p2/sql/repositoryAffinityList2.sql
diff --git a/regression-test/suites/variant_github_events_p2/starsFromHeavyGithubUsers1.sql b/regression-test/suites/variant_github_events_p2/sql/starsFromHeavyGithubUsers1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/starsFromHeavyGithubUsers1.sql
rename to regression-test/suites/variant_github_events_p2/sql/starsFromHeavyGithubUsers1.sql
diff --git a/regression-test/suites/variant_github_events_p2/starsFromHeavyGithubUsers2.sql b/regression-test/suites/variant_github_events_p2/sql/starsFromHeavyGithubUsers2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/starsFromHeavyGithubUsers2.sql
rename to regression-test/suites/variant_github_events_p2/sql/starsFromHeavyGithubUsers2.sql
diff --git a/regression-test/suites/variant_github_events_p2/theLongestRepositoryNames1.sql b/regression-test/suites/variant_github_events_p2/sql/theLongestRepositoryNames1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/theLongestRepositoryNames1.sql
rename to regression-test/suites/variant_github_events_p2/sql/theLongestRepositoryNames1.sql
diff --git a/regression-test/suites/variant_github_events_p2/theLongestRepositoryNames2.sql b/regression-test/suites/variant_github_events_p2/sql/theLongestRepositoryNames2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/theLongestRepositoryNames2.sql
rename to regression-test/suites/variant_github_events_p2/sql/theLongestRepositoryNames2.sql
diff --git a/regression-test/suites/variant_github_events_p2/theMostToughCodeReviews.sql b/regression-test/suites/variant_github_events_p2/sql/theMostToughCodeReviews.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/theMostToughCodeReviews.sql
rename to regression-test/suites/variant_github_events_p2/sql/theMostToughCodeReviews.sql
diff --git a/regression-test/suites/variant_github_events_p2/theTotalNumberOfRepositoriesOnGithub.sql b/regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/theTotalNumberOfRepositoriesOnGithub.sql
rename to regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql
diff --git a/regression-test/suites/variant_github_events_p2/theTotalNumberOfUsersOnGithub1.sql b/regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/theTotalNumberOfUsersOnGithub1.sql
rename to regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub1.sql
diff --git a/regression-test/suites/variant_github_events_p2/theTotalNumberOfUsersOnGithub2.sql b/regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/theTotalNumberOfUsersOnGithub2.sql
rename to regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub2.sql
diff --git a/regression-test/suites/variant_github_events_p2/theTotalNumberOfUsersOnGithub3.sql b/regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/theTotalNumberOfUsersOnGithub3.sql
rename to regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub3.sql
diff --git a/regression-test/suites/variant_github_events_p2/theTotalNumberOfUsersOnGithub4.sql b/regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub4.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/theTotalNumberOfUsersOnGithub4.sql
rename to regression-test/suites/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub4.sql
diff --git a/regression-test/suites/variant_github_events_p2/topLabels1.sql b/regression-test/suites/variant_github_events_p2/sql/topLabels1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/topLabels1.sql
rename to regression-test/suites/variant_github_events_p2/sql/topLabels1.sql
diff --git a/regression-test/suites/variant_github_events_p2/topLabels2.sql b/regression-test/suites/variant_github_events_p2/sql/topLabels2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/topLabels2.sql
rename to regression-test/suites/variant_github_events_p2/sql/topLabels2.sql
diff --git a/regression-test/suites/variant_github_events_p2/topLabels3.sql b/regression-test/suites/variant_github_events_p2/sql/topLabels3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/topLabels3.sql
rename to regression-test/suites/variant_github_events_p2/sql/topLabels3.sql
diff --git a/regression-test/suites/variant_github_events_p2/topRepositoriesByStars.sql b/regression-test/suites/variant_github_events_p2/sql/topRepositoriesByStars.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/topRepositoriesByStars.sql
rename to regression-test/suites/variant_github_events_p2/sql/topRepositoriesByStars.sql
diff --git a/regression-test/suites/variant_github_events_p2/whatIsTheBestDayOfTheWeekToCatchAStar.sql b/regression-test/suites/variant_github_events_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/whatIsTheBestDayOfTheWeekToCatchAStar.sql
rename to regression-test/suites/variant_github_events_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql
diff --git a/regression-test/suites/variant_github_events_p2/whoAreAllThosePeopleGivingStars1.sql b/regression-test/suites/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars1.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/whoAreAllThosePeopleGivingStars1.sql
rename to regression-test/suites/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars1.sql
diff --git a/regression-test/suites/variant_github_events_p2/whoAreAllThosePeopleGivingStars2.sql b/regression-test/suites/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars2.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/whoAreAllThosePeopleGivingStars2.sql
rename to regression-test/suites/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars2.sql
diff --git a/regression-test/suites/variant_github_events_p2/whoAreAllThosePeopleGivingStars3.sql b/regression-test/suites/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars3.sql
similarity index 100%
rename from regression-test/suites/variant_github_events_p2/whoAreAllThosePeopleGivingStars3.sql
rename to regression-test/suites/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars3.sql

From 71f96e6e06fc8ac0fc30545514c1d4d95ef205af Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@selectdb.com>
Date: Tue, 10 Dec 2024 20:13:27 +0800
Subject: [PATCH 259/399] [Improvement]downgrade resource tag when there is not
 queryable replica (#44255)

---
 .../datasource/FederationBackendPolicy.java   |  3 +
 .../apache/doris/httpv2/rest/LoadAction.java  |  2 +
 .../org/apache/doris/mysql/MysqlProto.java    |  3 +-
 .../apache/doris/mysql/privilege/Auth.java    |  9 ++
 .../mysql/privilege/CommonUserProperties.java | 11 +++
 .../doris/mysql/privilege/UserProperty.java   | 22 +++++
 .../mysql/privilege/UserPropertyMgr.java      |  8 ++
 .../apache/doris/planner/OlapScanNode.java    | 47 ++++++++-
 .../org/apache/doris/qe/ConnectContext.java   |  8 +-
 .../org/apache/doris/qe/ConnectProcessor.java |  3 +-
 .../doris/system/BeSelectionPolicy.java       | 33 +++++--
 .../doris/planner/ResourceTagQueryTest.java   | 12 ++-
 .../doris/system/SystemInfoServiceTest.java   | 23 +++++
 .../skip_rg_test_table.csv                    |  2 +
 .../test_resource_tag.groovy                  | 99 +++++++++++++++++++
 15 files changed, 269 insertions(+), 16 deletions(-)
 create mode 100644 regression-test/data/workload_manager_p0/skip_rg_test_table.csv
 create mode 100644 regression-test/suites/workload_manager_p0/test_resource_tag.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FederationBackendPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FederationBackendPolicy.java
index 1e1787c1f64947..4a24645bf3ee03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FederationBackendPolicy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FederationBackendPolicy.java
@@ -155,6 +155,7 @@ public void init() throws UserException {
 
     public void init(List<String> preLocations) throws UserException {
         Set<Tag> tags = Sets.newHashSet();
+        boolean allowResourceTagDowngrade = false;
         if (ConnectContext.get() != null && ConnectContext.get().getCurrentUserIdentity() != null) {
             String qualifiedUser = ConnectContext.get().getCurrentUserIdentity().getQualifiedUser();
             // Some request from stream load(eg, mysql load) may not set user info in ConnectContext
@@ -164,6 +165,7 @@ public void init(List<String> preLocations) throws UserException {
                 if (tags == UserProperty.INVALID_RESOURCE_TAGS) {
                     throw new UserException("No valid resource tag for user: " + qualifiedUser);
                 }
+                allowResourceTagDowngrade = Env.getCurrentEnv().getAuth().isAllowResourceTagDowngrade(qualifiedUser);
             }
         } else {
             if (LOG.isDebugEnabled()) {
@@ -176,6 +178,7 @@ public void init(List<String> preLocations) throws UserException {
                 .needQueryAvailable()
                 .needLoadAvailable()
                 .addTags(tags)
+                .setAllowResourceTagDowngrade(allowResourceTagDowngrade)
                 .preferComputeNode(Config.prefer_compute_node_for_external_table)
                 .assignExpectBeNum(Config.min_backend_num_for_external_table)
                 .addPreLocations(preLocations)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
index 2f9efc1ed1b1bf..00348b2d83a61b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
@@ -425,8 +425,10 @@ private TNetworkAddress selectLocalRedirectBackend(boolean groupCommit, HttpServ
         BeSelectionPolicy policy = null;
         String qualifiedUser = ConnectContext.get().getQualifiedUser();
         Set<Tag> userTags = Env.getCurrentEnv().getAuth().getResourceTags(qualifiedUser);
+        boolean allowResourceTagDowngrade = Env.getCurrentEnv().getAuth().isAllowResourceTagDowngrade(qualifiedUser);
         policy = new BeSelectionPolicy.Builder()
                 .addTags(userTags)
+                .setAllowResourceTagDowngrade(allowResourceTagDowngrade)
                 .setEnableRoundRobin(true)
                 .needLoadAvailable().build();
         policy.nextRoundRobinIndex = getLastSelectedBackendIndexAndUpdate();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java
index a672a217a33795..c16cec5689afd4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java
@@ -266,7 +266,8 @@ public static boolean negotiate(ConnectContext context) throws IOException {
         }
 
         // set resource tag if has
-        context.setResourceTags(Env.getCurrentEnv().getAuth().getResourceTags(qualifiedUser));
+        context.setResourceTags(Env.getCurrentEnv().getAuth().getResourceTags(qualifiedUser),
+                Env.getCurrentEnv().getAuth().isAllowResourceTagDowngrade(qualifiedUser));
         return true;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
index a1fb57d01cafa4..9d6f52d5a51d7d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
@@ -1234,6 +1234,15 @@ public Set<Tag> getResourceTags(String qualifiedUser) {
         }
     }
 
+    public boolean isAllowResourceTagDowngrade(String qualifiedUser) {
+        readLock();
+        try {
+            return propertyMgr.isAllowResourceTagDowngrade(qualifiedUser);
+        } finally {
+            readUnlock();
+        }
+    }
+
     public long getExecMemLimit(String qualifiedUser) {
         readLock();
         try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java
index db838a91a56614..bd2c3d028231f4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java
@@ -69,6 +69,9 @@ public class CommonUserProperties implements Writable, GsonPostProcessable {
     @SerializedName(value = "wg", alternate = {"workloadGroup"})
     private String workloadGroup = WorkloadGroupMgr.DEFAULT_GROUP_NAME;
 
+    @SerializedName(value = "ard", alternate = {"AllowResourceTagDowngrade"})
+    private boolean allowResourceTagDowngrade = false;
+
     private String[] sqlBlockRulesSplit = {};
 
     long getMaxConn() {
@@ -164,6 +167,14 @@ public void setWorkloadGroup(String workloadGroup) {
         this.workloadGroup = workloadGroup;
     }
 
+    public void setAllowResourceTagDowngrade(boolean allowResourceTagDowngrade) {
+        this.allowResourceTagDowngrade = allowResourceTagDowngrade;
+    }
+
+    public boolean isAllowResourceTagDowngrade() {
+        return this.allowResourceTagDowngrade;
+    }
+
     @Deprecated
     public static CommonUserProperties read(DataInput in) throws IOException {
         String json = Text.readString(in);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java
index e4a76b23820b3b..08f64cc006e3d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java
@@ -86,6 +86,8 @@ public class UserProperty implements Writable {
 
     public static final String PROP_WORKLOAD_GROUP = "default_workload_group";
 
+    public static final String PROP_ALLOW_RESOURCE_TAG_DOWNGRADE = "allow_resource_tag_downgrade";
+
     public static final String DEFAULT_CLOUD_CLUSTER = "default_cloud_cluster";
     public static final String DEFAULT_COMPUTE_GROUP = "default_compute_group";
 
@@ -139,6 +141,8 @@ public class UserProperty implements Writable {
         ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_EXEC_MEM_LIMIT + "$", Pattern.CASE_INSENSITIVE));
         ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_USER_QUERY_TIMEOUT + "$", Pattern.CASE_INSENSITIVE));
         ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_USER_INSERT_TIMEOUT + "$", Pattern.CASE_INSENSITIVE));
+        ADVANCED_PROPERTIES.add(
+                Pattern.compile("^" + PROP_ALLOW_RESOURCE_TAG_DOWNGRADE + "$", Pattern.CASE_INSENSITIVE));
 
         COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_QUOTA + ".", Pattern.CASE_INSENSITIVE));
         COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_DEFAULT_LOAD_CLUSTER + "$", Pattern.CASE_INSENSITIVE));
@@ -201,6 +205,10 @@ public Set<Tag> getCopiedResourceTags() {
         return Sets.newHashSet(this.commonProperties.getResourceTags());
     }
 
+    public boolean isAllowResourceTagDowngrade() {
+        return this.commonProperties.isAllowResourceTagDowngrade();
+    }
+
     public long getExecMemLimit() {
         return commonProperties.getExecMemLimit();
     }
@@ -221,6 +229,7 @@ public void update(List<Pair<String, String>> properties, boolean isReplay) thro
         int queryTimeout = this.commonProperties.getQueryTimeout();
         int insertTimeout = this.commonProperties.getInsertTimeout();
         String workloadGroup = this.commonProperties.getWorkloadGroup();
+        boolean allowResourceTagDowngrade = this.commonProperties.isAllowResourceTagDowngrade();
 
         String newDefaultLoadCluster = defaultLoadCluster;
         String newDefaultCloudCluster = defaultCloudCluster;
@@ -358,6 +367,15 @@ public void update(List<Pair<String, String>> properties, boolean isReplay) thro
                     throw new DdlException("workload group " + value + " not exists");
                 }
                 workloadGroup = value;
+            } else if (keyArr[0].equalsIgnoreCase(PROP_ALLOW_RESOURCE_TAG_DOWNGRADE)) {
+                if (keyArr.length != 1) {
+                    throw new DdlException(PROP_ALLOW_RESOURCE_TAG_DOWNGRADE + " format error");
+                }
+                if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) {
+                    throw new DdlException(
+                            "allow_resource_tag_downgrade's value must be true or false");
+                }
+                allowResourceTagDowngrade = Boolean.parseBoolean(value);
             } else {
                 if (isReplay) {
                     // After using SET PROPERTY to modify the user property, if FE rolls back to a version without
@@ -381,6 +399,7 @@ public void update(List<Pair<String, String>> properties, boolean isReplay) thro
         this.commonProperties.setQueryTimeout(queryTimeout);
         this.commonProperties.setInsertTimeout(insertTimeout);
         this.commonProperties.setWorkloadGroup(workloadGroup);
+        this.commonProperties.setAllowResourceTagDowngrade(allowResourceTagDowngrade);
         if (newDppConfigs.containsKey(newDefaultLoadCluster)) {
             defaultLoadCluster = newDefaultLoadCluster;
         } else {
@@ -546,6 +565,9 @@ public List<List<String>> fetchProperty() {
 
         result.add(Lists.newArrayList(PROP_WORKLOAD_GROUP, String.valueOf(commonProperties.getWorkloadGroup())));
 
+        result.add(Lists.newArrayList(PROP_ALLOW_RESOURCE_TAG_DOWNGRADE,
+                String.valueOf(commonProperties.isAllowResourceTagDowngrade())));
+
         // load cluster
         if (defaultLoadCluster != null) {
             result.add(Lists.newArrayList(PROP_DEFAULT_LOAD_CLUSTER, defaultLoadCluster));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java
index 29ae1f438a10f3..b40bb92fbfae7f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java
@@ -179,6 +179,14 @@ public Set<Tag> getResourceTags(String qualifiedUser) {
         return tags;
     }
 
+    public boolean isAllowResourceTagDowngrade(String qualifiedUser) {
+        UserProperty existProperty = propertyMap.get(qualifiedUser);
+        if (existProperty == null) {
+            return false;
+        }
+        return existProperty.isAllowResourceTagDowngrade();
+    }
+
     public Pair<String, DppConfig> getLoadClusterInfo(String qualifiedUser, String cluster) throws DdlException {
         Pair<String, DppConfig> loadClusterInfo = null;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 41c055062e3e9b..92175523f227a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -888,6 +888,21 @@ private void addScanRangeLocations(Partition partition,
 
             int replicaInTablet = 0;
             long oneReplicaBytes = 0;
+
+
+            // when resource tag has no alive replica and allowResourceTagDowngrade = true,
+            // resource tag should be disabled, we should find at least one alive replica
+            boolean shouldSkipResourceTag = false;
+            boolean isAllowRgDowngrade = context.isAllowResourceTagDowngrade();
+            if (needCheckTags && isAllowRgDowngrade && !checkTagHasAvailReplica(allowedTags, replicas)) {
+                shouldSkipResourceTag = true;
+                if (ConnectContext.get() != null && LOG.isDebugEnabled()) {
+                    LOG.debug("query {} skip resource tag for table {}.",
+                            DebugUtil.printId(ConnectContext.get().queryId()),
+                            olapTable != null ? olapTable.getId() : -1);
+                }
+            }
+
             for (Replica replica : replicas) {
                 Backend backend = null;
                 long backendId = -1;
@@ -906,14 +921,15 @@ private void addScanRangeLocations(Partition partition,
                                 replica.getId());
                     }
                     String err = "replica " + replica.getId() + "'s backend " + backendId
-                            + " does not exist or not alive";
+                            + " with tag " + backend.getLocationTag() + " does not exist or not alive";
                     errs.add(err);
                     continue;
                 }
                 if (!backend.isMixNode()) {
                     continue;
                 }
-                if (needCheckTags && !allowedTags.isEmpty() && !allowedTags.contains(backend.getLocationTag())) {
+                if (!shouldSkipResourceTag && needCheckTags && !allowedTags.isEmpty() && !allowedTags.contains(
+                        backend.getLocationTag())) {
                     String err = String.format(
                             "Replica on backend %d with tag %s," + " which is not in user's resource tags: %s",
                             backend.getId(), backend.getLocationTag(), allowedTags);
@@ -954,6 +970,10 @@ private void addScanRangeLocations(Partition partition,
                 throw new UserException("tablet " + tabletId + " err: " + Joiner.on(", ").join(errs));
             }
             if (tabletIsNull) {
+                if (needCheckTags && !isAllowRgDowngrade) {
+                    errs.add("If user specified tag has no queryable replica, "
+                            + "you can set property 'allow_resource_tag_downgrade'='true' to skip resource tag.");
+                }
                 throw new UserException("tablet " + tabletId + " has no queryable replicas. err: "
                         + Joiner.on(", ").join(errs));
             }
@@ -974,6 +994,29 @@ private void addScanRangeLocations(Partition partition,
         }
     }
 
+    private boolean checkTagHasAvailReplica(Set<Tag> allowedTags, List<Replica> replicas) {
+        try {
+            for (Replica replica : replicas) {
+                long backendId = replica.getBackendId();
+                Backend backend = Env.getCurrentSystemInfo().getBackend(backendId);
+
+                if (backend == null || !backend.isAlive()) {
+                    continue;
+                }
+                if (!backend.isMixNode()) {
+                    continue;
+                }
+                if (!allowedTags.isEmpty() && allowedTags.contains(backend.getLocationTag())) {
+                    return true;
+                }
+            }
+            return false;
+        } catch (Throwable t) {
+            LOG.warn("error happens when check resource tag has avail replica ", t);
+            return true;
+        }
+    }
+
     private boolean isEnableCooldownReplicaAffinity() {
         ConnectContext connectContext = ConnectContext.get();
         if (connectContext != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index c81cf4920e1f7d..bfbb15a7a7a783 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -203,6 +203,7 @@ public enum ConnectType {
     // This property is obtained from UserProperty when the client connection is created.
     // Only when the connection is created again, the new resource tags will be retrieved from the UserProperty
     private Set<Tag> resourceTags = Sets.newHashSet();
+    private boolean allowResourceTagDowngrade = false;
     // If set to true, the resource tags set in resourceTags will be used to limit the query resources.
     // If set to false, the system will not restrict query resources.
     private boolean isResourceTagsSet = false;
@@ -999,9 +1000,14 @@ public Set<Tag> getResourceTags() {
         return resourceTags;
     }
 
-    public void setResourceTags(Set<Tag> resourceTags) {
+    public boolean isAllowResourceTagDowngrade() {
+        return allowResourceTagDowngrade;
+    }
+
+    public void setResourceTags(Set<Tag> resourceTags, boolean allowResourceTagDowngrade) {
         this.resourceTags = resourceTags;
         this.isResourceTagsSet = !this.resourceTags.isEmpty();
+        this.allowResourceTagDowngrade = allowResourceTagDowngrade;
     }
 
     public void setCurrentConnectedFEIp(String ip) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index fcc6c2362cf276..0c633186abf8ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -663,7 +663,8 @@ public TMasterOpResult proxyExecute(TMasterOpRequest request) throws TException
         }
 
         // set resource tag
-        ctx.setResourceTags(Env.getCurrentEnv().getAuth().getResourceTags(ctx.qualifiedUser));
+        ctx.setResourceTags(Env.getCurrentEnv().getAuth().getResourceTags(ctx.qualifiedUser),
+                Env.getCurrentEnv().getAuth().isAllowResourceTagDowngrade(ctx.qualifiedUser));
 
         ctx.setThreadLocalInfo();
         StmtExecutor executor = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/BeSelectionPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/system/BeSelectionPolicy.java
index 59cb4f19139b46..dbe16853bfde4b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/BeSelectionPolicy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/BeSelectionPolicy.java
@@ -44,6 +44,7 @@ public class BeSelectionPolicy {
     public boolean needLoadAvailable = false;
     // Resource tag. Empty means no need to consider resource tag.
     public Set<Tag> resourceTags = Sets.newHashSet();
+    public boolean allowResourceTagDowngrade = false;
     // storage medium. null means no need to consider storage medium.
     public TStorageMedium storageMedium = null;
     // Check if disk usage reaches limit. false means no need to check.
@@ -92,6 +93,11 @@ public Builder addTags(Set<Tag> tags) {
             return this;
         }
 
+        public Builder setAllowResourceTagDowngrade(boolean allowResourceTagDowngrade) {
+            policy.allowResourceTagDowngrade = allowResourceTagDowngrade;
+            return this;
+        }
+
         public Builder setStorageMedium(TStorageMedium medium) {
             policy.storageMedium = medium;
             return this;
@@ -137,7 +143,7 @@ public BeSelectionPolicy build() {
         }
     }
 
-    private boolean isMatch(Backend backend) {
+    private boolean isMatch(Backend backend, boolean needResourceTagAvail) {
         // Compute node is only used when preferComputeNode is set.
         if (!preferComputeNode && backend.isComputeNode()) {
             if (LOG.isDebugEnabled()) {
@@ -146,10 +152,11 @@ private boolean isMatch(Backend backend) {
             return false;
         }
 
-        if (needScheduleAvailable && !backend.isScheduleAvailable() || needQueryAvailable && !backend.isQueryAvailable()
-                || needLoadAvailable && !backend.isLoadAvailable() || !resourceTags.isEmpty() && !resourceTags.contains(
-                backend.getLocationTag()) || storageMedium != null && !backend.hasSpecifiedStorageMedium(
-                storageMedium)) {
+        if (needScheduleAvailable && !backend.isScheduleAvailable()
+                || needQueryAvailable && !backend.isQueryAvailable()
+                || needLoadAvailable && !backend.isLoadAvailable()
+                || (needResourceTagAvail && !resourceTags.isEmpty() && !resourceTags.contains(backend.getLocationTag()))
+                || storageMedium != null && !backend.hasSpecifiedStorageMedium(storageMedium)) {
             if (LOG.isDebugEnabled()) {
                 LOG.debug("Backend [{}] is not match by Other rules, policy: [{}]", backend.getHost(), this);
             }
@@ -176,7 +183,16 @@ private boolean isMatch(Backend backend) {
     }
 
     public List<Backend> getCandidateBackends(Collection<Backend> backends) {
-        List<Backend> filterBackends = backends.stream().filter(this::isMatch).collect(Collectors.toList());
+        boolean needResourceTagAvail = !this.allowResourceTagDowngrade || backends.stream()
+                .filter(backend -> resourceTags.contains(backend.getLocationTag()) && backend.isAlive())
+                .count() != 0;
+        List<Backend> filterBackends = new ArrayList<>();
+        for (Backend be : backends) {
+            if (this.isMatch(be, needResourceTagAvail)) {
+                filterBackends.add(be);
+            }
+        }
+
         List<Backend> preLocationFilterBackends = filterBackends.stream()
                 .filter(iterm -> preferredLocations.contains(iterm.getHost())).collect(Collectors.toList());
         // If preLocations were chosen, use the preLocation backends. Otherwise we just ignore this filter.
@@ -221,8 +237,9 @@ public List<Backend> getCandidateBackends(Collection<Backend> backends) {
 
     @Override
     public String toString() {
-        return String.format("computeNode=%s | query=%s | load=%s | schedule=%s | tags=%s | medium=%s",
+        return String.format("computeNode=%s | query=%s | load=%s | schedule=%s | tags=%s(%s) | medium=%s",
                 preferComputeNode, needQueryAvailable, needLoadAvailable, needScheduleAvailable,
-                resourceTags.stream().map(tag -> tag.toString()).collect(Collectors.joining(",")), storageMedium);
+                resourceTags.stream().map(tag -> tag.toString()).collect(Collectors.joining(",")),
+                this.allowResourceTagDowngrade, storageMedium);
     }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java
index 850d8b27b062af..9568841134ad45 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java
@@ -205,7 +205,7 @@ public void test() throws Exception {
         Assert.assertEquals(1, userTags.size());
 
         // update connection context and query
-        connectContext.setResourceTags(userTags);
+        connectContext.setResourceTags(userTags, false);
         String queryStr = "explain select * from test.tbl1";
         String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
         System.out.println(explainString);
@@ -221,7 +221,7 @@ public void test() throws Exception {
         }
 
         // update connection context and query, it will failed because no zone1 backend
-        connectContext.setResourceTags(userTags);
+        connectContext.setResourceTags(userTags, false);
         Assert.assertTrue(connectContext.isResourceTagsSet());
         queryStr = "explain select * from test.tbl1";
         String error = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
@@ -280,7 +280,13 @@ public void test() throws Exception {
         Assert.assertEquals(1000000, execMemLimit);
 
         List<List<String>> userProps = Env.getCurrentEnv().getAuth().getUserProperties(Auth.ROOT_USER);
-        Assert.assertEquals(13, userProps.size());
+        Assert.assertEquals(14, userProps.size());
+
+        // set resource tag downgrade
+        String setResourceTagDownStr = "set property for 'root' 'allow_resource_tag_downgrade' = 'false';";
+        ExceptionChecker.expectThrowsNoException(() -> setProperty(setResourceTagDownStr));
+        boolean tagDowngrade = Env.getCurrentEnv().getAuth().isAllowResourceTagDowngrade(Auth.ROOT_USER);
+        Assert.assertTrue(!tagDowngrade);
 
         // now :
         // be1 be2 be3 ==>tag1;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java
index 62ade50c9198f6..f582c37706c77b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java
@@ -333,6 +333,29 @@ public void testComputeNodeBackendSelect() throws Exception {
         Assert.assertEquals(3, infoService.selectBackendIdsByPolicy(policy07, 3).size());
     }
 
+    @Test
+    public void testResourceTagDowngrade() throws Exception {
+        Tag taga = Tag.create(Tag.TYPE_LOCATION, "taga");
+        addBackend(10001, "192.168.1.1", 9050);
+        Backend be1 = infoService.getBackend(10001);
+        be1.setTagMap(taga.toMap());
+        be1.setAlive(true);
+
+        addBackend(10002, "192.168.1.2", 9050);
+        Backend be2 = infoService.getBackend(10002);
+        be2.setAlive(true);
+
+        BeSelectionPolicy policy1 = new BeSelectionPolicy.Builder().addTags(Sets.newHashSet(taga)).build();
+        Assert.assertEquals(1, infoService.selectBackendIdsByPolicy(policy1, 1).size());
+
+        be1.setAlive(false);
+        Assert.assertEquals(0, infoService.selectBackendIdsByPolicy(policy1, 1).size());
+
+        BeSelectionPolicy policy2 = new BeSelectionPolicy.Builder().setAllowResourceTagDowngrade(true)
+                .addTags(Sets.newHashSet(taga)).build();
+        Assert.assertEquals(1, infoService.selectBackendIdsByPolicy(policy2, 1).size());
+    }
+
     @Test
     public void testPreferLocationsSelect() throws Exception {
         Tag taga = Tag.create(Tag.TYPE_LOCATION, "taga");
diff --git a/regression-test/data/workload_manager_p0/skip_rg_test_table.csv b/regression-test/data/workload_manager_p0/skip_rg_test_table.csv
new file mode 100644
index 00000000000000..edcc00b603a95e
--- /dev/null
+++ b/regression-test/data/workload_manager_p0/skip_rg_test_table.csv
@@ -0,0 +1,2 @@
+1|2
+3|4
\ No newline at end of file
diff --git a/regression-test/suites/workload_manager_p0/test_resource_tag.groovy b/regression-test/suites/workload_manager_p0/test_resource_tag.groovy
new file mode 100644
index 00000000000000..d6557645eb8759
--- /dev/null
+++ b/regression-test/suites/workload_manager_p0/test_resource_tag.groovy
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_resource_tag") {
+        sql "drop user if exists test_rg;"
+        sql "create user test_rg"
+        sql "GRANT SELECT_PRIV,LOAD_PRIV,ALTER_PRIV,CREATE_PRIV,DROP_PRIV ON *.*.* TO test_rg;"
+        sql "set property for test_rg 'resource_tags.location' = 'c3p0';"
+        //cloud-mode
+        if (isCloudMode()) {
+                def clusters = sql " SHOW CLUSTERS; "
+                assertTrue(!clusters.isEmpty())
+                def validCluster = clusters[0][0]
+                sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_rg""";
+        }
+
+        // test query
+        connect(user = 'test_rg', password = '', url = context.config.jdbcUrl) {
+                sql "drop table if exists test_skip_rg_bad_replica_tab;"
+                sql """
+            CREATE TABLE test_skip_rg_bad_replica_tab
+                (
+                    k1 int,
+                    k2 int,
+                )ENGINE=OLAP
+                duplicate KEY(k1)
+                DISTRIBUTED BY HASH (k1) BUCKETS 1
+                PROPERTIES(
+                'replication_allocation' = 'tag.location.default: 1'
+                );
+            """
+                sql """
+        insert into test_skip_rg_bad_replica_tab values
+        (9,10),
+        (1,2)
+        """
+                test {
+                        sql "select count(1) as t1 from test_skip_rg_bad_replica_tab;"
+                        exception "which is not in user's resource tags: [{\"location\" : \"c3p0\"}], If user specified tag has no queryable replica, you can set property 'allow_resource_tag_downgrade'='true' to skip resource tag."
+                }
+        }
+        sql "set property for test_rg 'allow_resource_tag_downgrade' = 'true';"
+
+        connect(user = 'test_rg', password = '', url = context.config.jdbcUrl) {
+                sql "select count(1) as t2 from test_skip_rg_bad_replica_tab;"
+                sql "drop table test_skip_rg_bad_replica_tab";
+        }
+
+
+        // test stream load
+        sql "set property for test_rg 'allow_resource_tag_downgrade' = 'false';"
+        sql """ DROP TABLE IF EXISTS ${context.config.defaultDb}.skip_rg_test_table """
+        sql """
+            CREATE TABLE IF NOT EXISTS ${context.config.defaultDb}.skip_rg_test_table (
+                `k1` int NULL,
+                `k2` int NULL
+            ) ENGINE=OLAP
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+            PROPERTIES ("replication_allocation" = "tag.location.default: 1");
+        """
+
+        def test_failed_command = "curl --location-trusted -u test_rg: -H column_separator:| -H Transfer-Encoding:chunked -H columns:k1,k2  -T ${context.dataPath}/skip_rg_test_table.csv http://${context.config.feHttpAddress}/api/${context.config.defaultDb}/skip_rg_test_table/_stream_load"
+        log.info("stream load skip_rg_test_table failed test cmd: ${test_failed_command}")
+        def process = test_failed_command.execute()
+        code1 = process.waitFor()
+        out1 = process.text
+        log.info("stream load skip_rg_test_table failed test result, ${out1}".toString())
+        assertTrue("${out1}".toString().contains("No backend load available") || "${out1}".toString().contains("No available backends"))
+
+        sql "set property for test_rg 'allow_resource_tag_downgrade' = 'true';"
+
+        def test_succ_command = "curl --location-trusted -u test_rg: -H column_separator:| -H Transfer-Encoding:chunked -H columns:k1,k2  -T ${context.dataPath}/skip_rg_test_table.csv http://${context.config.feHttpAddress}/api/${context.config.defaultDb}/skip_rg_test_table/_stream_load"
+        def process2 = test_succ_command.execute()
+        code2 = process2.waitFor()
+        out2 = process2.text
+        jsonRet = parseJson(out2)
+        log.info("stream load skip_rg_test_table succ test result, ${out2}".toString())
+        assertFalse("${out2}".toString().contains("No backend load available"))
+        assertTrue(jsonRet['Status'] == 'Success')
+
+
+        // clear
+        sql "drop user test_rg"
+        sql "drop table ${context.config.defaultDb}.skip_rg_test_table"
+}
\ No newline at end of file

From ed1760816b0c60b10d9f0d9055bfb0db64ca54a9 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Wed, 11 Dec 2024 00:50:52 +0800
Subject: [PATCH 260/399] [test](load) injection cases should check Exception
 is thrown (#44713)

---
 be/src/olap/rowset_builder.cpp                |  9 ++-
 .../pipeline/p0/conf/regression-conf.groovy   |  1 -
 ...tream_back_pressure_fault_injection.groovy | 71 +++++--------------
 .../test_load_stream_fault_injection.groovy   | 10 ++-
 ...eam_stub_close_wait_fault_injection.groovy |  3 +-
 ..._load_stream_stub_failure_injection.groovy |  5 +-
 .../test_memtable_flush_fault.groovy          |  1 +
 .../test_rowset_writer_fault.groovy           |  5 +-
 8 files changed, 35 insertions(+), 70 deletions(-)

diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index 9bb0df318ee11c..ec7463d5b9d75d 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -148,8 +148,13 @@ Status BaseRowsetBuilder::init_mow_context(std::shared_ptr<MowContext>& mow_cont
 }
 
 Status RowsetBuilder::check_tablet_version_count() {
-    if (!_tablet->exceed_version_limit(config::max_tablet_version_num - 100) ||
-        GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
+    bool injection = false;
+    DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
+                    { injection = true; });
+    if (injection) {
+        // do not return if injection
+    } else if (!_tablet->exceed_version_limit(config::max_tablet_version_num - 100) ||
+               GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
         return Status::OK();
     }
     //trigger compaction
diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy
index f89365ef4f4fac..4de192754251b6 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -69,7 +69,6 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as th
     "test_refresh_mtmv," +
     "test_spark_load," +
     "test_broker_load_func," +
-    "test_stream_stub_fault_injection," +
     "test_index_compaction_failure_injection," +
     "test_full_compaction_run_status," +
     "test_topn_fault_injection," + 
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_back_pressure_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_back_pressure_fault_injection.groovy
index fc2abe7fb0ad6f..bccaa8aa62f84d 100644
--- a/regression-test/suites/fault_injection_p0/test_load_stream_back_pressure_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_back_pressure_fault_injection.groovy
@@ -71,67 +71,32 @@ suite("test_load_stream_back_pressure_fault_injection", "nonConcurrent") {
 
     try {
         GetDebugPoint().enableDebugPointForAllBEs("TabletStream.append_data.long_wait")
-        def thread1 = new Thread({
-            try {
-                def res = sql "insert into test select * from baseall where k1 <= 3"
-                logger.info(res.toString())
-            } catch(Exception e) {
-                logger.info(e.getMessage())
-                assertTrue(e.getMessage().contains("Communications link failure"))
+        // the kill thread only means to end the test faster when the code does not behave as expected
+        def kill_thread = new Thread({
+            sleep(5000)
+            def processList = sql "show processlist"
+            logger.info(processList.toString())
+            processList.each { item ->
+                logger.info(item[1].toString())
+                logger.info(item[11].toString())
+                if (item[11].toString() == "insert into test select * from baseall where k1 <= 3".toString()){
+                    def res = sql "kill ${item[1]}"
+                    logger.info(res.toString())
+                }
             }
         })
-        thread1.start()
-
-        sleep(1000)
-
-        def processList = sql "show processlist"
-        logger.info(processList.toString())
-        processList.each { item ->
-            logger.info(item[1].toString())
-            logger.info(item[11].toString())
-            if (item[11].toString() == "insert into test select * from baseall where k1 <= 3".toString()){
-                def res = sql "kill ${item[1]}"
-                logger.info(res.toString())
-            }
-        }
+        kill_thread.start()
+        def res = sql "insert into test select * from baseall where k1 <= 3"
+        logger.info(res.toString())
+        assertTrue(false, "Expected exception to be thrown")
     } catch(Exception e) {
         logger.info(e.getMessage())
+        assertTrue(e.getMessage().contains("wait flush token back pressure time is more than load_stream_max_wait_flush_token_time"))
     } finally {
         GetDebugPoint().disableDebugPointForAllBEs("TabletStream.append_data.long_wait")
     }
 
-    try {
-        GetDebugPoint().enableDebugPointForAllBEs("TabletStream.add_segment.long_wait")
-        def thread1 = new Thread({
-            try {
-                def res = sql "insert into test select * from baseall where k1 <= 3"
-                logger.info(res.toString())
-            } catch(Exception e) {
-                logger.info(e.getMessage())
-                assertTrue(e.getMessage().contains("Communications link failure"))
-            }
-        })
-        thread1.start()
-
-        sleep(1000)
-
-        def processList = sql "show processlist"
-        logger.info(processList.toString())
-        processList.each { item ->
-            logger.info(item[1].toString())
-            logger.info(item[11].toString())
-            if (item[11].toString() == "insert into test select * from baseall where k1 <= 3".toString()){
-                def res = sql "kill ${item[1]}"
-                logger.info(res.toString())
-            }
-        }
-    } catch(Exception e) {
-        logger.info(e.getMessage())
-    } finally {
-        GetDebugPoint().disableDebugPointForAllBEs("TabletStream.add_segment.long_wait")
-    }
-
     sql """ DROP TABLE IF EXISTS `baseall` """
     sql """ DROP TABLE IF EXISTS `test` """
     sql """ set enable_memtable_on_sink_node=false """
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy
index d38141ed55aafe..0ba05394adf657 100644
--- a/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy
@@ -113,10 +113,11 @@ suite("load_stream_fault_injection", "nonConcurrent") {
         }
     }
 
-    def load_with_injection = { injection, expect_errmsg ->
+    def load_with_injection = { injection, expect_errmsg, success=false->
         try {
             GetDebugPoint().enableDebugPointForAllBEs(injection)
             sql "insert into test select * from baseall where k1 <= 3"
+            assertTrue(success, String.format("Expected Exception '%s', actual success", expect_errmsg))
         } catch(Exception e) {
             // assertTrue(e.getMessage().contains("Process has no memory available"))  // the msg should contain the root cause
             logger.info(e.getMessage())
@@ -125,11 +126,12 @@ suite("load_stream_fault_injection", "nonConcurrent") {
         }
     }
 
-    def load_with_injection2 = { injection1, injection2, error_msg->
+    def load_with_injection2 = { injection1, injection2, error_msg, success=false->
         try {
             GetDebugPoint().enableDebugPointForAllBEs(injection1)
             GetDebugPoint().enableDebugPointForAllBEs(injection2)
             sql "insert into test select * from baseall where k1 <= 3"
+            assertTrue(success, String.format("expected Exception '%s', actual success", expect_errmsg))
         } catch(Exception e) {
             logger.info(e.getMessage())
             assertTrue(e.getMessage().contains(error_msg))
@@ -155,8 +157,6 @@ suite("load_stream_fault_injection", "nonConcurrent") {
     load_with_injection("LoadStreamWriter.close.inverted_writers_size_not_match", "")
     load_with_injection("LoadStreamWriter.close.file_not_closed", "")
     load_with_injection("LoadStreamWriter.close.inverted_file_not_closed", "")
-    // LoadStreamWriter close_writer meet bytes_appended and real file size not match error
-    load_with_injection("FileWriter.close_writer.zero_bytes_appended", "")
     // LoadStreamWriter close_writer/add_segment meet not inited error
     load_with_injection("TabletStream.init.uninited_writer", "")
     // LoadStreamWriter init failure
@@ -168,8 +168,6 @@ suite("load_stream_fault_injection", "nonConcurrent") {
     // LoadStreamWriter add_segment meet null file writer error
     load_with_injection("LoadStreamWriter._calc_file_size.null_file_writer", "")
     load_with_injection("LoadStreamWriter._calc_file_size.file_not_closed", "")
-    // LoadStreamWriter add_segment meet bytes_appended and real file size not match error
-    load_with_injection("FileWriter.add_segment.zero_bytes_appended", "")
     // LoadStream init failed coz LoadStreamWriter init failed
     load_with_injection("RowsetBuilder.check_tablet_version_count.too_many_version", "")
     // LoadStream add_segment meet unknown segid in request header
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy
index 4a87f1daf6b2d9..58b6ba4a075e33 100644
--- a/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy
@@ -74,6 +74,7 @@ suite("test_load_stream_stub_close_wait_fault_injection", "nonConcurrent") {
         GetDebugPoint().enableDebugPointForAllBEs("LoadStreamStub::close_wait.long_wait")
         def res = sql "insert into test select * from baseall where k1 <= 3"
         logger.info(res.toString())
+        assertTrue(false, "Expected Exception to be thrown")
     } catch(Exception e) {
         logger.info(e.getMessage())
         assertTrue(e.getMessage().contains("cancel"))
@@ -85,4 +86,4 @@ suite("test_load_stream_stub_close_wait_fault_injection", "nonConcurrent") {
     sql """ DROP TABLE IF EXISTS `baseall` """
     sql """ DROP TABLE IF EXISTS `test` """
     sql """ set enable_memtable_on_sink_node=false """
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
index 5b1f9fba05b4bc..48c32883302e40 100644
--- a/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
@@ -69,10 +69,11 @@ suite("test_stream_stub_fault_injection", "nonConcurrent") {
         file "baseall.txt"
     }
 
-    def load_with_injection = { injection, error_msg->
+    def load_with_injection = { injection, error_msg, success=false->
         try {
             GetDebugPoint().enableDebugPointForAllBEs(injection)
             sql "insert into test select * from baseall where k1 <= 3"
+            assertTrue(success, String.format("Expected Exception '%s', actual success", expect_errmsg))
         } catch(Exception e) {
             logger.info(e.getMessage())
             assertTrue(e.getMessage().contains(error_msg))
@@ -87,8 +88,6 @@ suite("test_stream_stub_fault_injection", "nonConcurrent") {
     load_with_injection("StreamSinkFileWriter.finalize.finalize_failed", "failed to send segment eos to any replicas")
     // LoadStreams stream wait failed
     load_with_injection("LoadStreamStub._send_with_retry.stream_write_failed", "StreamWrite failed, err=32")
-    // LoadStreams keeping stream when release
-    load_with_injection("LoadStreams.release.keeping_streams", "")
 
     sql """ DROP TABLE IF EXISTS `baseall` """
     sql """ DROP TABLE IF EXISTS `test` """
diff --git a/regression-test/suites/fault_injection_p0/test_memtable_flush_fault.groovy b/regression-test/suites/fault_injection_p0/test_memtable_flush_fault.groovy
index 5b954f1171ce7c..0ba2f0863e2e6b 100644
--- a/regression-test/suites/fault_injection_p0/test_memtable_flush_fault.groovy
+++ b/regression-test/suites/fault_injection_p0/test_memtable_flush_fault.groovy
@@ -55,6 +55,7 @@ suite("test_memtable_flush_fault", "nonConcurrent") {
         GetDebugPoint().enableDebugPointForAllBEs("FlushToken.submit_flush_error")
         sql insert_sql
         sql "sync"
+        assertTrue(false, "Expected Exception dbug_be_memtable_submit_flush_error")
     } catch (Exception e){
         logger.info(e.getMessage())
         assertTrue(e.getMessage().contains("dbug_be_memtable_submit_flush_error"))
diff --git a/regression-test/suites/fault_injection_p0/test_rowset_writer_fault.groovy b/regression-test/suites/fault_injection_p0/test_rowset_writer_fault.groovy
index 005e4b6bc974d8..0e5113f2a6ed14 100644
--- a/regression-test/suites/fault_injection_p0/test_rowset_writer_fault.groovy
+++ b/regression-test/suites/fault_injection_p0/test_rowset_writer_fault.groovy
@@ -56,11 +56,8 @@ suite("test_rowset_writer_fault", "nonConcurrent") {
                 assertEquals("fail", json.Status.toLowerCase())
             }
         }
-    } catch(Exception e) {
-        logger.info(e.getMessage())
-        assertTrue(e.getMessage().contains(error_msg))
     } finally {
         GetDebugPoint().disableDebugPointForAllBEs(injection)
     }
     sql """ DROP TABLE IF EXISTS `baseall` """
-}
\ No newline at end of file
+}

From 94b44be3064dcb17d47190e6bb68ebadbc351ad5 Mon Sep 17 00:00:00 2001
From: wudongliang <46414265+DongLiang-0@users.noreply.github.com>
Date: Wed, 11 Dec 2024 10:25:19 +0800
Subject: [PATCH 261/399] [Enhancement] (nereids)implement
 adminCleanTrashCommand in nereids (#45051)

Issue Number: close #42846
---
 .../org/apache/doris/nereids/DorisParser.g4   |   9 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  11 ++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../commands/AdminCleanTrashCommand.java      | 120 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../test_database_management_auth.groovy      |   2 +-
 .../ddl/show_trash/test_nereids_trash.groovy  |   4 +
 7 files changed, 147 insertions(+), 5 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCleanTrashCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index fdad6baccdc056..c833e7d4b629b6 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -183,9 +183,9 @@ supportedCreateStatement
     | CREATE (EXTERNAL)? TABLE (IF NOT EXISTS)? name=multipartIdentifier
         LIKE existedTable=multipartIdentifier
         (WITH ROLLUP (rollupNames=identifierList)?)?                      #createTableLike
-    | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?    #createRole  
+    | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?    #createRole
     | CREATE WORKLOAD GROUP (IF NOT EXISTS)?
-        name=identifierOrText properties=propertyClause?                        #createWorkloadGroup          
+        name=identifierOrText properties=propertyClause?                        #createWorkloadGroup
     | CREATE ROW POLICY (IF NOT EXISTS)? name=identifier
         ON table=multipartIdentifier
         AS type=(RESTRICTIVE | PERMISSIVE)
@@ -496,6 +496,9 @@ supportedAdminStatement
     | ADMIN COMPACT TABLE baseTableRef (WHERE TYPE EQ STRING_LITERAL)?              #adminCompactTable
     | ADMIN CHECK tabletList properties=propertyClause?                             #adminCheckTablets
     | ADMIN SHOW TABLET STORAGE FORMAT VERBOSE?                                     #adminShowTabletStorageFormat
+    | ADMIN CLEAN TRASH
+        (ON LEFT_PAREN backends+=STRING_LITERAL
+              (COMMA backends+=STRING_LITERAL)* RIGHT_PAREN)?                       #adminCleanTrash
     ;
 
 supportedRecoverStatement
@@ -517,8 +520,6 @@ unsupportedAdminStatement
         (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?                              #adminRebalanceDisk
     | ADMIN CANCEL REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL
         (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?                              #adminCancelRebalanceDisk
-    | ADMIN CLEAN TRASH (ON LEFT_PAREN backends+=STRING_LITERAL
-        (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?                              #adminCleanTrash
     | ADMIN SET TABLE name=multipartIdentifier
         PARTITION VERSION properties=propertyClause?                                #adminSetPartitionVersion
     | ADMIN COPY TABLET tabletId=INTEGER_VALUE properties=propertyClause?           #adminCopyTablet
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 95dbbb504712f8..a825f80f4e08e0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -477,6 +477,7 @@
 import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
 import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
@@ -4579,6 +4580,16 @@ public LogicalPlan visitShowTrash(ShowTrashContext ctx) {
         return new ShowTrashCommand();
     }
 
+    @Override
+    public LogicalPlan visitAdminCleanTrash(DorisParser.AdminCleanTrashContext ctx) {
+        if (ctx.ON() != null) {
+            List<String> backendsQuery = Lists.newArrayList();
+            ctx.backends.forEach(backend -> backendsQuery.add(stripQuotes(backend.getText())));
+            return new AdminCleanTrashCommand(backendsQuery);
+        }
+        return new AdminCleanTrashCommand();
+    }
+
     @Override
     public LogicalPlan visitAdminShowReplicaStatus(AdminShowReplicaStatusContext ctx) {
         Expression where = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 91ec3591003500..b0e475a8161535 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -174,6 +174,7 @@ public enum PlanType {
     SHOW_CREATE_PROCEDURE_COMMAND,
     CREATE_VIEW_COMMAND,
     CLEAN_ALL_PROFILE_COMMAND,
+    CLEAN_TRASH_COMMAND,
     CREATE_ROLE_COMMAND,
     ALTER_ROLE_COMMAND,
     ALTER_VIEW_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCleanTrashCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCleanTrashCommand.java
new file mode 100644
index 00000000000000..60a293c5123ca9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCleanTrashCommand.java
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.NetUtils;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.system.Backend;
+import org.apache.doris.task.AgentBatchTask;
+import org.apache.doris.task.AgentTaskExecutor;
+import org.apache.doris.task.CleanTrashTask;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * admin clean trash
+ */
+public class AdminCleanTrashCommand extends Command implements NoForward {
+    private static final Logger LOG = LogManager.getLogger(AdminCleanTrashCommand.class);
+    private List<String> backendsQuery;
+
+    public AdminCleanTrashCommand() {
+        super(PlanType.CLEAN_TRASH_COMMAND);
+    }
+
+    public AdminCleanTrashCommand(List<String> backendsQuery) {
+        super(PlanType.CLEAN_TRASH_COMMAND);
+        this.backendsQuery = backendsQuery;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+        handleCleanTrash(backendsQuery);
+    }
+
+    private void handleCleanTrash(List<String> backendsQuery) throws AnalysisException {
+        List<Backend> backends = getNeedCleanedBackends(backendsQuery);
+        cleanTrash(backends);
+    }
+
+    private List<Backend> getNeedCleanedBackends(List<String> backendsQuery) throws AnalysisException {
+        List<Backend> needCleanedBackends = Lists.newArrayList();
+        ImmutableMap<Long, Backend> backendsInfo = Env.getCurrentSystemInfo().getAllBackendsByAllCluster();
+        if (backendsQuery == null) {
+            needCleanedBackends.addAll(backendsInfo.values());
+        } else {
+            Map<String, Long> backendsID = new HashMap<>();
+            for (Backend backend : backendsInfo.values()) {
+                backendsID.put(
+                        NetUtils.getHostPortInAccessibleFormat(backend.getHost(), backend.getHeartbeatPort()),
+                        backend.getId());
+            }
+            for (String backendQuery : backendsQuery) {
+                if (backendsID.containsKey(backendQuery)) {
+                    needCleanedBackends.add(backendsInfo.get(backendsID.get(backendQuery)));
+                    backendsID.remove(backendQuery);
+                }
+            }
+        }
+        return needCleanedBackends;
+    }
+
+    private void cleanTrash(List<Backend> backends) {
+        if (backends.isEmpty()) {
+            LOG.info("The matching be is empty, no be to clean trash.");
+            return;
+        }
+        AgentBatchTask batchTask = new AgentBatchTask();
+        for (Backend backend : backends) {
+            CleanTrashTask cleanTrashTask = new CleanTrashTask(backend.getId());
+            batchTask.addTask(cleanTrashTask);
+            LOG.info("clean trash in be {}, beId {}", backend.getHost(), backend.getId());
+        }
+        AgentTaskExecutor.submit(batchTask);
+    }
+
+    @Override
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
+        LOG.info("AdminCleanTrashCommand not supported in cloud mode");
+        throw new DdlException("Unsupported operation");
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAdminCleanTrashCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 5da741f303fed4..3634732077b7bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -19,6 +19,7 @@
 
 import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterJobStatusCommand;
@@ -221,6 +222,10 @@ default R visitAdminCompactTableCommand(AdminCompactTableCommand adminCompactTab
         return visitCommand(adminCompactTableCommand, context);
     }
 
+    default R visitAdminCleanTrashCommand(AdminCleanTrashCommand adminCleanTrashCommand, C context) {
+        return visitCommand(adminCleanTrashCommand, context);
+    }
+
     default R visitDropConstraintCommand(DropConstraintCommand dropConstraintCommand, C context) {
         return visitCommand(dropConstraintCommand, context);
     }
diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy
index b481cf8de75c00..ea10b0345244d8 100644
--- a/regression-test/suites/auth_call/test_database_management_auth.groovy
+++ b/regression-test/suites/auth_call/test_database_management_auth.groovy
@@ -111,7 +111,7 @@ suite("test_database_management_auth","p0,auth_call") {
         }
         test {
             sql """ADMIN CLEAN TRASH;"""
-            exception "denied"
+            exception "${error_in_cloud}"
         }
         test {
             sql """RECOVER DATABASE db_name;"""
diff --git a/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy b/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
index 5d795e6fd119a1..26a57c7e82d06f 100644
--- a/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
+++ b/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
@@ -19,4 +19,8 @@ suite("show_trash_nereids") {
     // can not use qt command since the output change based on cluster and backend ip
     checkNereidsExecute("""show trash;""")
     checkNereidsExecute("""show trash on "127.0.0.1:9050";""")
+
+    checkNereidsExecute("""ADMIN CLEAN TRASH;""")
+    checkNereidsExecute("""ADMIN CLEAN TRASH ON ("127.0.0.1:9050");""")
+    checkNereidsExecute("""ADMIN CLEAN TRASH ON ("192.168.0.1:9050", "192.168.0.2:9050", "192.168.0.3:9050");""")
 }

From 700719a93c570b89761a7f0ecf78c04fd1c0a4ec Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Wed, 11 Dec 2024 10:53:08 +0800
Subject: [PATCH 262/399] [feat](mtmv)Paimon queries the data in the cache
 instead of querying the latest data (#44911)

### What problem does this PR solve?
Problem Summary:
- add `PaimonMetadataCacheMgr` in `ExternalMetaCacheMgr` to manage
snapshotCache of paimon table
- move paimonSchemaCache to PaimonMetadataCacheMgr, and add schemaId as
part of key
- PaimonExternalTable overrides the methods in ExternalTable and
supports partition pruning
- PaimonExternalTable implements the MvcTable interface, supporting the
retrieval of snapshot data from the cache during queries to avoid cache
refreshes that may result in different versions of metadata being used
in a single query
- MTMVTask retrieves snapshot data of mvccTable before the task starts
to avoid cache refresh that may result in different versions of metadata
being used in a single refresh task

### Release note
Paimon queries the data in the cache instead of querying the latest data

### behavior change
behavior changes of query  paimon table:
- FE has just started and is query the latest data
- Paimon data has changed, Doris is still query the previous data
- After the snapshot cache expires, Doris will query the latest data
- desc paimon; The schema corresponding to the snapshotId in the
snapshot cache is displayed
---
 .../doris/datasource/ExternalCatalog.java     |   9 +-
 .../datasource/ExternalMetaCacheMgr.java      |  12 +
 .../doris/datasource/ExternalSchemaCache.java |   6 +-
 .../doris/datasource/ExternalTable.java       |   7 +-
 .../datasource/hive/HMSExternalTable.java     |   5 +
 .../doris/datasource/mvcc/MvccUtil.java       |  44 +++
 .../paimon/PaimonExternalTable.java           | 304 +++++++-----------
 .../paimon/PaimonMetadataCache.java           | 144 +++++++++
 .../paimon/PaimonMetadataCacheMgr.java        |  49 +++
 .../datasource/paimon/PaimonMvccSnapshot.java |  32 ++
 .../paimon/PaimonPartitionInfo.java           |   4 +-
 .../doris/datasource/paimon/PaimonSchema.java |  46 +++
 .../paimon/PaimonSchemaCacheKey.java          |  55 ++++
 .../paimon/PaimonSchemaCacheValue.java        |  25 +-
 .../datasource/paimon/PaimonSnapshot.java     |  36 +++
 .../paimon/PaimonSnapshotCacheKey.java        |  75 +++++
 .../paimon/PaimonSnapshotCacheValue.java      |  37 +++
 .../doris/datasource/paimon/PaimonUtil.java   | 122 ++++++-
 .../paimon/source/PaimonSource.java           |   3 +-
 .../doris/job/extensions/mtmv/MTMVTask.java   |  14 +
 .../doris/nereids/StatementContext.java       |  26 +-
 .../rules/rewrite/PruneFileScanPartition.java |   9 +-
 .../commands/UpdateMvByPartitionCommand.java  |   7 +-
 .../trees/plans/logical/LogicalFileScan.java  |   7 +-
 .../data/mtmv_p0/test_paimon_rewrite_mtmv.out |  16 +
 .../mtmv_p0/test_paimon_rewrite_mtmv.groovy   |  95 ++++++
 26 files changed, 950 insertions(+), 239 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccUtil.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCacheMgr.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMvccSnapshot.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchema.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshot.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheKey.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java
 create mode 100644 regression-test/data/mtmv_p0/test_paimon_rewrite_mtmv.out
 create mode 100644 regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index cde08113373aee..d7cbee18c74c7a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -41,6 +41,7 @@
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.common.security.authentication.PreExecutionAuthenticator;
 import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.ExternalSchemaCache.SchemaCacheKey;
 import org.apache.doris.datasource.es.EsExternalDatabase;
 import org.apache.doris.datasource.hive.HMSExternalCatalog;
 import org.apache.doris.datasource.hive.HMSExternalDatabase;
@@ -432,13 +433,13 @@ private void refreshOnlyCatalogCache(boolean invalidCache) {
         }
     }
 
-    public final Optional<SchemaCacheValue> getSchema(String dbName, String tblName) {
+    public final Optional<SchemaCacheValue> getSchema(SchemaCacheKey key) {
         makeSureInitialized();
-        Optional<ExternalDatabase<? extends ExternalTable>> db = getDb(dbName);
+        Optional<ExternalDatabase<? extends ExternalTable>> db = getDb(key.getDbName());
         if (db.isPresent()) {
-            Optional<? extends ExternalTable> table = db.get().getTable(tblName);
+            Optional<? extends ExternalTable> table = db.get().getTable(key.getTblName());
             if (table.isPresent()) {
-                return table.get().initSchemaAndUpdateTime();
+                return table.get().initSchemaAndUpdateTime(key);
             }
         }
         return Optional.empty();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index cc40ad292ce182..24f55e74266863 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -31,6 +31,8 @@
 import org.apache.doris.datasource.maxcompute.MaxComputeMetadataCache;
 import org.apache.doris.datasource.maxcompute.MaxComputeMetadataCacheMgr;
 import org.apache.doris.datasource.metacache.MetaCache;
+import org.apache.doris.datasource.paimon.PaimonMetadataCache;
+import org.apache.doris.datasource.paimon.PaimonMetadataCacheMgr;
 import org.apache.doris.fs.FileSystemCache;
 import org.apache.doris.nereids.exceptions.NotSupportedException;
 
@@ -92,6 +94,7 @@ public class ExternalMetaCacheMgr {
     private ExternalRowCountCache rowCountCache;
     private final IcebergMetadataCacheMgr icebergMetadataCacheMgr;
     private final MaxComputeMetadataCacheMgr maxComputeMetadataCacheMgr;
+    private final PaimonMetadataCacheMgr paimonMetadataCacheMgr;
 
     public ExternalMetaCacheMgr() {
         rowCountRefreshExecutor = ThreadPoolManager.newDaemonFixedThreadPool(
@@ -122,6 +125,7 @@ public ExternalMetaCacheMgr() {
         hudiPartitionMgr = new HudiPartitionMgr(commonRefreshExecutor);
         icebergMetadataCacheMgr = new IcebergMetadataCacheMgr(commonRefreshExecutor);
         maxComputeMetadataCacheMgr = new MaxComputeMetadataCacheMgr();
+        paimonMetadataCacheMgr = new PaimonMetadataCacheMgr(commonRefreshExecutor);
     }
 
     public ExecutorService getFileListingExecutor() {
@@ -167,6 +171,10 @@ public IcebergMetadataCache getIcebergMetadataCache() {
         return icebergMetadataCacheMgr.getIcebergMetadataCache();
     }
 
+    public PaimonMetadataCache getPaimonMetadataCache() {
+        return paimonMetadataCacheMgr.getPaimonMetadataCache();
+    }
+
     public MaxComputeMetadataCache getMaxComputeMetadataCache(long catalogId) {
         return maxComputeMetadataCacheMgr.getMaxComputeMetadataCache(catalogId);
     }
@@ -189,6 +197,7 @@ public void removeCache(long catalogId) {
         hudiPartitionMgr.removePartitionProcessor(catalogId);
         icebergMetadataCacheMgr.removeCache(catalogId);
         maxComputeMetadataCacheMgr.removeCache(catalogId);
+        paimonMetadataCacheMgr.removeCache(catalogId);
     }
 
     public void invalidateTableCache(long catalogId, String dbName, String tblName) {
@@ -204,6 +213,7 @@ public void invalidateTableCache(long catalogId, String dbName, String tblName)
         hudiPartitionMgr.cleanTablePartitions(catalogId, dbName, tblName);
         icebergMetadataCacheMgr.invalidateTableCache(catalogId, dbName, tblName);
         maxComputeMetadataCacheMgr.invalidateTableCache(catalogId, dbName, tblName);
+        paimonMetadataCacheMgr.invalidateTableCache(catalogId, dbName, tblName);
         if (LOG.isDebugEnabled()) {
             LOG.debug("invalid table cache for {}.{} in catalog {}", dbName, tblName, catalogId);
         }
@@ -222,6 +232,7 @@ public void invalidateDbCache(long catalogId, String dbName) {
         hudiPartitionMgr.cleanDatabasePartitions(catalogId, dbName);
         icebergMetadataCacheMgr.invalidateDbCache(catalogId, dbName);
         maxComputeMetadataCacheMgr.invalidateDbCache(catalogId, dbName);
+        paimonMetadataCacheMgr.invalidateDbCache(catalogId, dbName);
         if (LOG.isDebugEnabled()) {
             LOG.debug("invalid db cache for {} in catalog {}", dbName, catalogId);
         }
@@ -239,6 +250,7 @@ public void invalidateCatalogCache(long catalogId) {
         hudiPartitionMgr.cleanPartitionProcess(catalogId);
         icebergMetadataCacheMgr.invalidateCatalogCache(catalogId);
         maxComputeMetadataCacheMgr.invalidateCatalogCache(catalogId);
+        paimonMetadataCacheMgr.invalidateCatalogCache(catalogId);
         if (LOG.isDebugEnabled()) {
             LOG.debug("invalid catalog cache for {}", catalogId);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
index a0558766e81400..de3eeff75d97fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
@@ -74,7 +74,7 @@ public Long getValue() {
     }
 
     private Optional<SchemaCacheValue> loadSchema(SchemaCacheKey key) {
-        Optional<SchemaCacheValue> schema = catalog.getSchema(key.dbName, key.tblName);
+        Optional<SchemaCacheValue> schema = catalog.getSchema(key);
         if (LOG.isDebugEnabled()) {
             LOG.debug("load schema for {} in catalog {}", key, catalog.getName());
         }
@@ -83,6 +83,10 @@ private Optional<SchemaCacheValue> loadSchema(SchemaCacheKey key) {
 
     public Optional<SchemaCacheValue> getSchemaValue(String dbName, String tblName) {
         SchemaCacheKey key = new SchemaCacheKey(dbName, tblName);
+        return getSchemaValue(key);
+    }
+
+    public Optional<SchemaCacheValue> getSchemaValue(SchemaCacheKey key) {
         return schemaCache.get(key);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 5451a219edfd3c..91df061678f154 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -31,6 +31,7 @@
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.common.util.PropertyAnalyzer;
 import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.ExternalSchemaCache.SchemaCacheKey;
 import org.apache.doris.datasource.mvcc.MvccSnapshot;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
 import org.apache.doris.persist.gson.GsonPostProcessable;
@@ -317,8 +318,12 @@ public Optional<ColumnStatistic> getColumnStatistic(String colName) {
      *
      * @return
      */
-    public Optional<SchemaCacheValue> initSchemaAndUpdateTime() {
+    public Optional<SchemaCacheValue> initSchemaAndUpdateTime(SchemaCacheKey key) {
         schemaUpdateTime = System.currentTimeMillis();
+        return initSchema(key);
+    }
+
+    public Optional<SchemaCacheValue> initSchema(SchemaCacheKey key) {
         return initSchema();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 7584b5b392feb5..da4670d6d0589d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -30,6 +30,7 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.ExternalSchemaCache.SchemaCacheKey;
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.SchemaCacheValue;
 import org.apache.doris.datasource.TablePartitionValues;
@@ -501,6 +502,10 @@ public Set<String> getPartitionNames() {
     }
 
     @Override
+    public Optional<SchemaCacheValue> initSchemaAndUpdateTime(SchemaCacheKey key) {
+        return initSchemaAndUpdateTime();
+    }
+
     public Optional<SchemaCacheValue> initSchemaAndUpdateTime() {
         org.apache.hadoop.hive.metastore.api.Table table = ((HMSExternalCatalog) catalog).getClient()
                 .getTable(dbName, name);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccUtil.java
new file mode 100644
index 00000000000000..ffdaff770e21a3
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccUtil.java
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.mvcc;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.qe.ConnectContext;
+
+import java.util.Optional;
+
+public class MvccUtil {
+    /**
+     * get Snapshot From StatementContext
+     *
+     * @param tableIf
+     * @return MvccSnapshot
+     */
+    public static Optional<MvccSnapshot> getSnapshotFromContext(TableIf tableIf) {
+        ConnectContext connectContext = ConnectContext.get();
+        if (connectContext == null) {
+            return Optional.empty();
+        }
+        StatementContext statementContext = connectContext.getStatementContext();
+        if (statementContext == null) {
+            return Optional.empty();
+        }
+        return statementContext.getSnapshot(tableIf);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
index 7fe3c858448e3f..8ddfca886d9fe9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
@@ -22,13 +22,16 @@
 import org.apache.doris.catalog.MTMV;
 import org.apache.doris.catalog.PartitionItem;
 import org.apache.doris.catalog.PartitionType;
-import org.apache.doris.catalog.ScalarType;
-import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.CacheException;
+import org.apache.doris.datasource.ExternalSchemaCache;
+import org.apache.doris.datasource.ExternalSchemaCache.SchemaCacheKey;
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.SchemaCacheValue;
 import org.apache.doris.datasource.mvcc.MvccSnapshot;
+import org.apache.doris.datasource.mvcc.MvccTable;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 import org.apache.doris.mtmv.MTMVBaseTableIf;
 import org.apache.doris.mtmv.MTMVRefreshContext;
 import org.apache.doris.mtmv.MTMVRelatedTableIf;
@@ -45,25 +48,20 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.paimon.CoreOptions;
 import org.apache.paimon.catalog.Catalog;
 import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.schema.TableSchema;
-import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
 import org.apache.paimon.table.Table;
 import org.apache.paimon.table.source.Split;
-import org.apache.paimon.table.system.PartitionsTable;
-import org.apache.paimon.table.system.SnapshotsTable;
-import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.table.system.SchemasTable;
 import org.apache.paimon.types.DataField;
-import org.apache.paimon.types.DecimalType;
-import org.apache.paimon.types.MapType;
-import org.apache.paimon.types.RowType;
 
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -71,12 +69,15 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-public class PaimonExternalTable extends ExternalTable implements MTMVRelatedTableIf, MTMVBaseTableIf {
+public class PaimonExternalTable extends ExternalTable implements MTMVRelatedTableIf, MTMVBaseTableIf, MvccTable {
 
     private static final Logger LOG = LogManager.getLogger(PaimonExternalTable.class);
 
+    private final Table paimonTable;
+
     public PaimonExternalTable(long id, String name, String dbName, PaimonExternalCatalog catalog) {
         super(id, name, catalog, dbName, TableType.PAIMON_EXTERNAL_TABLE);
+        this.paimonTable = catalog.getPaimonTable(dbName, name);
     }
 
     public String getPaimonCatalogType() {
@@ -90,176 +91,27 @@ protected synchronized void makeSureInitialized() {
         }
     }
 
-    public Table getPaimonTable() {
-        makeSureInitialized();
-        Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
-        return schemaCacheValue.map(value -> ((PaimonSchemaCacheValue) value).getPaimonTable()).orElse(null);
+    public Table getPaimonTable(Optional<MvccSnapshot> snapshot) {
+        return paimonTable.copy(
+                Collections.singletonMap(CoreOptions.SCAN_VERSION.key(),
+                        String.valueOf(getOrFetchSnapshotCacheValue(snapshot).getSnapshot().getSnapshotId())));
     }
 
-    private PaimonPartitionInfo getPartitionInfoFromCache() {
-        makeSureInitialized();
-        Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
+    public PaimonSchemaCacheValue getPaimonSchemaCacheValue(long schemaId) {
+        ExternalSchemaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr().getSchemaCache(catalog);
+        Optional<SchemaCacheValue> schemaCacheValue = cache.getSchemaValue(
+                new PaimonSchemaCacheKey(dbName, name, schemaId));
         if (!schemaCacheValue.isPresent()) {
-            return new PaimonPartitionInfo();
+            throw new CacheException("failed to getSchema for: %s.%s.%s.%s",
+                    null, catalog.getName(), dbName, name, schemaId);
         }
-        return ((PaimonSchemaCacheValue) schemaCacheValue.get()).getPartitionInfo();
+        return (PaimonSchemaCacheValue) schemaCacheValue.get();
     }
 
-    private List<Column> getPartitionColumnsFromCache() {
+    private PaimonSnapshotCacheValue getPaimonSnapshotCacheValue() {
         makeSureInitialized();
-        Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
-        if (!schemaCacheValue.isPresent()) {
-            return Lists.newArrayList();
-        }
-        return ((PaimonSchemaCacheValue) schemaCacheValue.get()).getPartitionColumns();
-    }
-
-    public long getLatestSnapshotIdFromCache() throws AnalysisException {
-        makeSureInitialized();
-        Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
-        if (!schemaCacheValue.isPresent()) {
-            throw new AnalysisException("not present");
-        }
-        return ((PaimonSchemaCacheValue) schemaCacheValue.get()).getSnapshootId();
-    }
-
-    @Override
-    public Optional<SchemaCacheValue> initSchema() {
-        Table paimonTable = ((PaimonExternalCatalog) catalog).getPaimonTable(dbName, name);
-        TableSchema schema = ((FileStoreTable) paimonTable).schema();
-        List<DataField> columns = schema.fields();
-        List<Column> tmpSchema = Lists.newArrayListWithCapacity(columns.size());
-        Set<String> partitionColumnNames = Sets.newHashSet(paimonTable.partitionKeys());
-        List<Column> partitionColumns = Lists.newArrayList();
-        for (DataField field : columns) {
-            Column column = new Column(field.name().toLowerCase(),
-                    paimonTypeToDorisType(field.type()), true, null, true, field.description(), true,
-                    field.id());
-            tmpSchema.add(column);
-            if (partitionColumnNames.contains(field.name())) {
-                partitionColumns.add(column);
-            }
-        }
-        try {
-            // after 0.9.0 paimon will support table.getLatestSnapshotId()
-            long latestSnapshotId = loadLatestSnapshotId();
-            PaimonPartitionInfo partitionInfo = loadPartitionInfo(partitionColumns);
-            return Optional.of(new PaimonSchemaCacheValue(tmpSchema, partitionColumns, paimonTable, latestSnapshotId,
-                    partitionInfo));
-        } catch (IOException | AnalysisException e) {
-            LOG.warn(e);
-            return Optional.empty();
-        }
-    }
-
-    private long loadLatestSnapshotId() throws IOException {
-        Table table = ((PaimonExternalCatalog) catalog).getPaimonTable(dbName,
-                name + Catalog.SYSTEM_TABLE_SPLITTER + SnapshotsTable.SNAPSHOTS);
-        // snapshotId
-        List<InternalRow> rows = PaimonUtil.read(table, new int[][] {{0}});
-        long latestSnapshotId = 0L;
-        for (InternalRow row : rows) {
-            long snapshotId = row.getLong(0);
-            if (snapshotId > latestSnapshotId) {
-                latestSnapshotId = snapshotId;
-            }
-        }
-        return latestSnapshotId;
-    }
-
-    private PaimonPartitionInfo loadPartitionInfo(List<Column> partitionColumns) throws IOException, AnalysisException {
-        if (CollectionUtils.isEmpty(partitionColumns)) {
-            return new PaimonPartitionInfo();
-        }
-        List<PaimonPartition> paimonPartitions = loadPartitions();
-        return PaimonUtil.generatePartitionInfo(partitionColumns, paimonPartitions);
-    }
-
-    private List<PaimonPartition> loadPartitions()
-            throws IOException {
-        Table table = ((PaimonExternalCatalog) catalog).getPaimonTable(dbName,
-                name + Catalog.SYSTEM_TABLE_SPLITTER + PartitionsTable.PARTITIONS);
-        List<InternalRow> rows = PaimonUtil.read(table, null);
-        List<PaimonPartition> res = Lists.newArrayListWithCapacity(rows.size());
-        for (InternalRow row : rows) {
-            res.add(PaimonUtil.rowToPartition(row));
-        }
-        return res;
-    }
-
-    private Type paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataType dataType) {
-        int tsScale = 3; // default
-        switch (dataType.getTypeRoot()) {
-            case BOOLEAN:
-                return Type.BOOLEAN;
-            case INTEGER:
-                return Type.INT;
-            case BIGINT:
-                return Type.BIGINT;
-            case FLOAT:
-                return Type.FLOAT;
-            case DOUBLE:
-                return Type.DOUBLE;
-            case SMALLINT:
-                return Type.SMALLINT;
-            case TINYINT:
-                return Type.TINYINT;
-            case VARCHAR:
-            case BINARY:
-            case CHAR:
-            case VARBINARY:
-                return Type.STRING;
-            case DECIMAL:
-                DecimalType decimal = (DecimalType) dataType;
-                return ScalarType.createDecimalV3Type(decimal.getPrecision(), decimal.getScale());
-            case DATE:
-                return ScalarType.createDateV2Type();
-            case TIMESTAMP_WITHOUT_TIME_ZONE:
-                if (dataType instanceof org.apache.paimon.types.TimestampType) {
-                    tsScale = ((org.apache.paimon.types.TimestampType) dataType).getPrecision();
-                    if (tsScale > 6) {
-                        tsScale = 6;
-                    }
-                } else if (dataType instanceof org.apache.paimon.types.LocalZonedTimestampType) {
-                    tsScale = ((org.apache.paimon.types.LocalZonedTimestampType) dataType).getPrecision();
-                    if (tsScale > 6) {
-                        tsScale = 6;
-                    }
-                }
-                return ScalarType.createDatetimeV2Type(tsScale);
-            case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
-                if (dataType instanceof org.apache.paimon.types.LocalZonedTimestampType) {
-                    tsScale = ((org.apache.paimon.types.LocalZonedTimestampType) dataType).getPrecision();
-                    if (tsScale > 6) {
-                        tsScale = 6;
-                    }
-                }
-                return ScalarType.createDatetimeV2Type(tsScale);
-            case ARRAY:
-                ArrayType arrayType = (ArrayType) dataType;
-                Type innerType = paimonPrimitiveTypeToDorisType(arrayType.getElementType());
-                return org.apache.doris.catalog.ArrayType.create(innerType, true);
-            case MAP:
-                MapType mapType = (MapType) dataType;
-                return new org.apache.doris.catalog.MapType(
-                        paimonTypeToDorisType(mapType.getKeyType()), paimonTypeToDorisType(mapType.getValueType()));
-            case ROW:
-                RowType rowType = (RowType) dataType;
-                List<DataField> fields = rowType.getFields();
-                return new org.apache.doris.catalog.StructType(fields.stream()
-                        .map(field -> new org.apache.doris.catalog.StructField(field.name(),
-                                paimonTypeToDorisType(field.type())))
-                        .collect(Collectors.toCollection(ArrayList::new)));
-            case TIME_WITHOUT_TIME_ZONE:
-                return Type.UNSUPPORTED;
-            default:
-                LOG.warn("Cannot transform unknown type: " + dataType.getTypeRoot());
-                return Type.UNSUPPORTED;
-        }
-    }
-
-    protected Type paimonTypeToDorisType(org.apache.paimon.types.DataType type) {
-        return paimonPrimitiveTypeToDorisType(type);
+        return Env.getCurrentEnv().getExtMetaCacheMgr().getPaimonMetadataCache()
+                .getPaimonSnapshot(catalog, dbName, name);
     }
 
     @Override
@@ -289,13 +141,6 @@ public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) {
     public long fetchRowCount() {
         makeSureInitialized();
         long rowCount = 0;
-        Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
-        Table paimonTable = schemaCacheValue.map(value -> ((PaimonSchemaCacheValue) value).getPaimonTable())
-                .orElse(null);
-        if (paimonTable == null) {
-            LOG.info("Paimon table {} is null.", name);
-            return UNKNOWN_ROW_COUNT;
-        }
         List<Split> splits = paimonTable.newReadBuilder().newScan().plan().splits();
         for (Split split : splits) {
             rowCount += split.rowCount();
@@ -314,30 +159,31 @@ public void beforeMTMVRefresh(MTMV mtmv) throws DdlException {
 
     @Override
     public Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot) {
-        return Maps.newHashMap(getPartitionInfoFromCache().getNameToPartitionItem());
+        return Maps.newHashMap(getNameToPartitionItems(snapshot));
     }
 
     @Override
     public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
-        return getPartitionColumnsFromCache().size() > 0 ? PartitionType.LIST : PartitionType.UNPARTITIONED;
+        return getPartitionColumns(snapshot).size() > 0 ? PartitionType.LIST : PartitionType.UNPARTITIONED;
     }
 
     @Override
     public Set<String> getPartitionColumnNames(Optional<MvccSnapshot> snapshot) {
-        return getPartitionColumnsFromCache().stream()
+        return getPartitionColumns(snapshot).stream()
                 .map(c -> c.getName().toLowerCase()).collect(Collectors.toSet());
     }
 
     @Override
     public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
-        return getPartitionColumnsFromCache();
+        return getPaimonSchemaCacheValue(snapshot).getPartitionColumns();
     }
 
     @Override
     public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
             Optional<MvccSnapshot> snapshot)
             throws AnalysisException {
-        PaimonPartition paimonPartition = getPartitionInfoFromCache().getNameToPartition().get(partitionName);
+        PaimonPartition paimonPartition = getOrFetchSnapshotCacheValue(snapshot).getPartitionInfo().getNameToPartition()
+                .get(partitionName);
         if (paimonPartition == null) {
             throw new AnalysisException("can not find partition: " + partitionName);
         }
@@ -347,7 +193,8 @@ public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshCont
     @Override
     public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
             throws AnalysisException {
-        return new MTMVVersionSnapshot(getLatestSnapshotIdFromCache());
+        PaimonSnapshotCacheValue paimonSnapshot = getOrFetchSnapshotCacheValue(snapshot);
+        return new MTMVVersionSnapshot(paimonSnapshot.getSnapshot().getSnapshotId());
     }
 
     @Override
@@ -359,4 +206,83 @@ public boolean isPartitionColumnAllowNull() {
         // The cost is that Paimon partition writes a null value, and the materialized view cannot detect this data.
         return true;
     }
+
+    @Override
+    public MvccSnapshot loadSnapshot() {
+        return new PaimonMvccSnapshot(getPaimonSnapshotCacheValue());
+    }
+
+    @Override
+    public Map<String, PartitionItem> getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
+        return getOrFetchSnapshotCacheValue(snapshot).getPartitionInfo().getNameToPartitionItem();
+    }
+
+    @Override
+    public boolean supportInternalPartitionPruned() {
+        return true;
+    }
+
+    @Override
+    public List<Column> getFullSchema() {
+        return getPaimonSchemaCacheValue(MvccUtil.getSnapshotFromContext(this)).getSchema();
+    }
+
+    @Override
+    public Optional<SchemaCacheValue> initSchema(SchemaCacheKey key) {
+        makeSureInitialized();
+        PaimonSchemaCacheKey paimonSchemaCacheKey = (PaimonSchemaCacheKey) key;
+        try {
+            PaimonSchema schema = loadPaimonSchemaBySchemaId(paimonSchemaCacheKey);
+            List<DataField> columns = schema.getFields();
+            List<Column> dorisColumns = Lists.newArrayListWithCapacity(columns.size());
+            Set<String> partitionColumnNames = Sets.newHashSet(schema.getPartitionKeys());
+            List<Column> partitionColumns = Lists.newArrayList();
+            for (DataField field : columns) {
+                Column column = new Column(field.name().toLowerCase(),
+                        PaimonUtil.paimonTypeToDorisType(field.type()), true, null, true, field.description(), true,
+                        field.id());
+                dorisColumns.add(column);
+                if (partitionColumnNames.contains(field.name())) {
+                    partitionColumns.add(column);
+                }
+            }
+            return Optional.of(new PaimonSchemaCacheValue(dorisColumns, partitionColumns));
+        } catch (Exception e) {
+            throw new CacheException("failed to initSchema for: %s.%s.%s.%s",
+                    null, getCatalog().getName(), key.getDbName(), key.getTblName(),
+                    paimonSchemaCacheKey.getSchemaId());
+        }
+
+    }
+
+    private PaimonSchema loadPaimonSchemaBySchemaId(PaimonSchemaCacheKey key) throws IOException {
+        Table table = ((PaimonExternalCatalog) getCatalog()).getPaimonTable(key.getDbName(),
+                name + Catalog.SYSTEM_TABLE_SPLITTER + SchemasTable.SCHEMAS);
+        PredicateBuilder builder = new PredicateBuilder(table.rowType());
+        Predicate predicate = builder.equal(0, key.getSchemaId());
+        // Adding predicates will also return excess data
+        List<InternalRow> rows = PaimonUtil.read(table, new int[][] {{0}, {1}, {2}}, predicate);
+        for (InternalRow row : rows) {
+            PaimonSchema schema = PaimonUtil.rowToSchema(row);
+            if (schema.getSchemaId() == key.getSchemaId()) {
+                return schema;
+            }
+        }
+        throw new CacheException("failed to initSchema for: %s.%s.%s.%s",
+                null, getCatalog().getName(), key.getDbName(), key.getTblName(), key.getSchemaId());
+    }
+
+    private PaimonSchemaCacheValue getPaimonSchemaCacheValue(Optional<MvccSnapshot> snapshot) {
+        PaimonSnapshotCacheValue snapshotCacheValue = getOrFetchSnapshotCacheValue(snapshot);
+        return getPaimonSchemaCacheValue(snapshotCacheValue.getSnapshot().getSchemaId());
+    }
+
+    private PaimonSnapshotCacheValue getOrFetchSnapshotCacheValue(Optional<MvccSnapshot> snapshot) {
+        if (snapshot.isPresent()) {
+            return ((PaimonMvccSnapshot) snapshot.get()).getSnapshotCacheValue();
+        } else {
+            return getPaimonSnapshotCacheValue();
+        }
+    }
+
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java
new file mode 100644
index 00000000000000..5b711e070667b7
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCache.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.CacheFactory;
+import org.apache.doris.common.Config;
+import org.apache.doris.datasource.CacheException;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.ExternalMetaCacheMgr;
+
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.table.system.PartitionsTable;
+import org.apache.paimon.table.system.SnapshotsTable;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.OptionalLong;
+import java.util.concurrent.ExecutorService;
+
+public class PaimonMetadataCache {
+
+    private final LoadingCache<PaimonSnapshotCacheKey, PaimonSnapshotCacheValue> snapshotCache;
+
+    public PaimonMetadataCache(ExecutorService executor) {
+        CacheFactory snapshotCacheFactory = new CacheFactory(
+                OptionalLong.of(28800L),
+                OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
+                Config.max_external_table_cache_num,
+                true,
+                null);
+        this.snapshotCache = snapshotCacheFactory.buildCache(key -> loadSnapshot(key), null, executor);
+    }
+
+    @NotNull
+    private PaimonSnapshotCacheValue loadSnapshot(PaimonSnapshotCacheKey key) {
+        try {
+            PaimonSnapshot latestSnapshot = loadLatestSnapshot(key);
+            PaimonExternalTable table = (PaimonExternalTable) key.getCatalog().getDbOrAnalysisException(key.getDbName())
+                    .getTableOrAnalysisException(key.getTableName());
+            List<Column> partitionColumns = table.getPaimonSchemaCacheValue(latestSnapshot.getSchemaId())
+                    .getPartitionColumns();
+            PaimonPartitionInfo partitionInfo = loadPartitionInfo(key, partitionColumns);
+            return new PaimonSnapshotCacheValue(partitionInfo, latestSnapshot);
+        } catch (IOException | AnalysisException e) {
+            throw new CacheException("failed to loadSnapshot for: %s.%s.%s",
+                    e, key.getCatalog().getName(), key.getDbName(), key.getTableName());
+        }
+    }
+
+    private PaimonPartitionInfo loadPartitionInfo(PaimonSnapshotCacheKey key, List<Column> partitionColumns)
+            throws IOException, AnalysisException {
+        if (CollectionUtils.isEmpty(partitionColumns)) {
+            return new PaimonPartitionInfo();
+        }
+        List<PaimonPartition> paimonPartitions = loadPartitions(key);
+        return PaimonUtil.generatePartitionInfo(partitionColumns, paimonPartitions);
+    }
+
+    private List<PaimonPartition> loadPartitions(PaimonSnapshotCacheKey key)
+            throws IOException {
+        Table table = ((PaimonExternalCatalog) key.getCatalog()).getPaimonTable(key.getDbName(),
+                key.getTableName() + Catalog.SYSTEM_TABLE_SPLITTER + PartitionsTable.PARTITIONS);
+        List<InternalRow> rows = PaimonUtil.read(table, null, null);
+        List<PaimonPartition> res = Lists.newArrayListWithCapacity(rows.size());
+        for (InternalRow row : rows) {
+            res.add(PaimonUtil.rowToPartition(row));
+        }
+        return res;
+    }
+
+    private PaimonSnapshot loadLatestSnapshot(PaimonSnapshotCacheKey key) throws IOException {
+        Table table = ((PaimonExternalCatalog) key.getCatalog()).getPaimonTable(key.getDbName(),
+                key.getTableName() + Catalog.SYSTEM_TABLE_SPLITTER + SnapshotsTable.SNAPSHOTS);
+        // snapshotId and schemaId
+        List<InternalRow> rows = PaimonUtil.read(table, new int[][] {{0}, {1}}, null);
+        long latestSnapshotId = 0L;
+        long latestSchemaId = 0L;
+        for (InternalRow row : rows) {
+            long snapshotId = row.getLong(0);
+            if (snapshotId > latestSnapshotId) {
+                latestSnapshotId = snapshotId;
+                latestSchemaId = row.getLong(1);
+            }
+        }
+        return new PaimonSnapshot(latestSnapshotId, latestSchemaId);
+    }
+
+    public void invalidateCatalogCache(long catalogId) {
+        snapshotCache.asMap().keySet().stream()
+                .filter(key -> key.getCatalog().getId() == catalogId)
+                .forEach(snapshotCache::invalidate);
+    }
+
+    public void invalidateTableCache(long catalogId, String dbName, String tblName) {
+        snapshotCache.asMap().keySet().stream()
+                .filter(key -> key.getCatalog().getId() == catalogId && key.getDbName().equals(dbName)
+                        && key.getTableName().equals(
+                        tblName))
+                .forEach(snapshotCache::invalidate);
+    }
+
+    public void invalidateDbCache(long catalogId, String dbName) {
+        snapshotCache.asMap().keySet().stream()
+                .filter(key -> key.getCatalog().getId() == catalogId && key.getDbName().equals(dbName))
+                .forEach(snapshotCache::invalidate);
+    }
+
+    public PaimonSnapshotCacheValue getPaimonSnapshot(CatalogIf catalog, String dbName, String tbName) {
+        PaimonSnapshotCacheKey key = new PaimonSnapshotCacheKey(catalog, dbName, tbName);
+        return snapshotCache.get(key);
+    }
+
+    public Map<String, Map<String, String>> getCacheStats() {
+        Map<String, Map<String, String>> res = Maps.newHashMap();
+        res.put("paimon_snapshot_cache", ExternalMetaCacheMgr.getCacheStats(snapshotCache.stats(),
+                snapshotCache.estimatedSize()));
+        return res;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCacheMgr.java
new file mode 100644
index 00000000000000..a282fde665b197
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCacheMgr.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import java.util.concurrent.ExecutorService;
+
+public class PaimonMetadataCacheMgr {
+
+    private PaimonMetadataCache paimonMetadataCache;
+
+    public PaimonMetadataCacheMgr(ExecutorService executor) {
+        this.paimonMetadataCache = new PaimonMetadataCache(executor);
+    }
+
+    public PaimonMetadataCache getPaimonMetadataCache() {
+        return paimonMetadataCache;
+    }
+
+    public void removeCache(long catalogId) {
+        paimonMetadataCache.invalidateCatalogCache(catalogId);
+    }
+
+    public void invalidateCatalogCache(long catalogId) {
+        paimonMetadataCache.invalidateCatalogCache(catalogId);
+    }
+
+    public void invalidateTableCache(long catalogId, String dbName, String tblName) {
+        paimonMetadataCache.invalidateTableCache(catalogId, dbName, tblName);
+    }
+
+    public void invalidateDbCache(long catalogId, String dbName) {
+        paimonMetadataCache.invalidateDbCache(catalogId, dbName);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMvccSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMvccSnapshot.java
new file mode 100644
index 00000000000000..2307e91adb3911
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMvccSnapshot.java
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
+
+public class PaimonMvccSnapshot implements MvccSnapshot {
+    private final PaimonSnapshotCacheValue snapshotCacheValue;
+
+    public PaimonMvccSnapshot(PaimonSnapshotCacheValue snapshotCacheValue) {
+        this.snapshotCacheValue = snapshotCacheValue;
+    }
+
+    public PaimonSnapshotCacheValue getSnapshotCacheValue() {
+        return snapshotCacheValue;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
index 8f54f0834e481b..4d3326f8e48376 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
@@ -24,8 +24,8 @@
 import java.util.Map;
 
 public class PaimonPartitionInfo {
-    private Map<String, PartitionItem> nameToPartitionItem;
-    private Map<String, PaimonPartition> nameToPartition;
+    private final Map<String, PartitionItem> nameToPartitionItem;
+    private final Map<String, PaimonPartition> nameToPartition;
 
     public PaimonPartitionInfo() {
         this.nameToPartitionItem = Maps.newHashMap();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchema.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchema.java
new file mode 100644
index 00000000000000..ef26e1ed20879d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchema.java
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import org.apache.paimon.types.DataField;
+
+import java.util.List;
+
+public class PaimonSchema {
+    private final long schemaId;
+    private final List<DataField> fields;
+    private final List<String> partitionKeys;
+
+    public PaimonSchema(long schemaId, List<DataField> fields, List<String> partitionKeys) {
+        this.schemaId = schemaId;
+        this.fields = fields;
+        this.partitionKeys = partitionKeys;
+    }
+
+    public long getSchemaId() {
+        return schemaId;
+    }
+
+    public List<DataField> getFields() {
+        return fields;
+    }
+
+    public List<String> getPartitionKeys() {
+        return partitionKeys;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java
new file mode 100644
index 00000000000000..f74555b369b380
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import org.apache.doris.datasource.ExternalSchemaCache.SchemaCacheKey;
+
+import com.google.common.base.Objects;
+
+public class PaimonSchemaCacheKey extends SchemaCacheKey {
+    private final long schemaId;
+
+    public PaimonSchemaCacheKey(String dbName, String tableName, long schemaId) {
+        super(dbName, tableName);
+        this.schemaId = schemaId;
+    }
+
+    public long getSchemaId() {
+        return schemaId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof PaimonSchemaCacheKey)) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        PaimonSchemaCacheKey that = (PaimonSchemaCacheKey) o;
+        return schemaId == that.schemaId;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(super.hashCode(), schemaId);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheValue.java
index 20d27b2425df24..ccb530a3cbccc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheValue.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheValue.java
@@ -20,41 +20,18 @@
 import org.apache.doris.catalog.Column;
 import org.apache.doris.datasource.SchemaCacheValue;
 
-import org.apache.paimon.table.Table;
-
 import java.util.List;
 
 public class PaimonSchemaCacheValue extends SchemaCacheValue {
 
-    private Table paimonTable;
     private List<Column> partitionColumns;
-    private PaimonPartitionInfo partitionInfo;
-
-    private long snapshootId;
 
-    public PaimonSchemaCacheValue(List<Column> schema, List<Column> partitionColumns, Table paimonTable,
-            long snapshootId,
-            PaimonPartitionInfo partitionInfo) {
+    public PaimonSchemaCacheValue(List<Column> schema, List<Column> partitionColumns) {
         super(schema);
         this.partitionColumns = partitionColumns;
-        this.paimonTable = paimonTable;
-        this.snapshootId = snapshootId;
-        this.partitionInfo = partitionInfo;
-    }
-
-    public Table getPaimonTable() {
-        return paimonTable;
     }
 
     public List<Column> getPartitionColumns() {
         return partitionColumns;
     }
-
-    public PaimonPartitionInfo getPartitionInfo() {
-        return partitionInfo;
-    }
-
-    public long getSnapshootId() {
-        return snapshootId;
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshot.java
new file mode 100644
index 00000000000000..4a536dd72cc901
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshot.java
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+public class PaimonSnapshot {
+    private final long snapshotId;
+    private final long schemaId;
+
+    public PaimonSnapshot(long snapshotId, long schemaId) {
+        this.snapshotId = snapshotId;
+        this.schemaId = schemaId;
+    }
+
+    public long getSnapshotId() {
+        return snapshotId;
+    }
+
+    public long getSchemaId() {
+        return schemaId;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheKey.java
new file mode 100644
index 00000000000000..970f111a72133f
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheKey.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import org.apache.doris.datasource.CatalogIf;
+
+import java.util.Objects;
+import java.util.StringJoiner;
+
+public class PaimonSnapshotCacheKey {
+    private final CatalogIf catalog;
+    private final String dbName;
+    private final String tableName;
+
+    public PaimonSnapshotCacheKey(CatalogIf catalog, String dbName, String tableName) {
+        this.catalog = catalog;
+        this.dbName = dbName;
+        this.tableName = tableName;
+    }
+
+    public CatalogIf getCatalog() {
+        return catalog;
+    }
+
+    public String getDbName() {
+        return dbName;
+    }
+
+    public String getTableName() {
+        return tableName;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        PaimonSnapshotCacheKey that = (PaimonSnapshotCacheKey) o;
+        return catalog.getId() == that.catalog.getId()
+                && Objects.equals(dbName, that.dbName)
+                && Objects.equals(tableName, that.tableName);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(catalog.getId(), dbName, tableName);
+    }
+
+    @Override
+    public String toString() {
+        return new StringJoiner(", ", PaimonSnapshotCacheKey.class.getSimpleName() + "[", "]")
+                .add("catalog=" + catalog)
+                .add("dbName='" + dbName + "'")
+                .add("tableName='" + tableName + "'")
+                .toString();
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java
new file mode 100644
index 00000000000000..c50ecdabfde3df
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+public class PaimonSnapshotCacheValue {
+
+    private final PaimonPartitionInfo partitionInfo;
+    private final PaimonSnapshot snapshot;
+
+    public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot) {
+        this.partitionInfo = partitionInfo;
+        this.snapshot = snapshot;
+    }
+
+    public PaimonPartitionInfo getPartitionInfo() {
+        return partitionInfo;
+    }
+
+    public PaimonSnapshot getSnapshot() {
+        return snapshot;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index 8b7017cac29486..1f7576dca51d93 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -22,6 +22,7 @@
 import org.apache.doris.catalog.ListPartitionItem;
 import org.apache.doris.catalog.PartitionItem;
 import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.datasource.hive.HiveUtil;
@@ -30,12 +31,22 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.data.serializer.InternalRowSerializer;
 import org.apache.paimon.options.ConfigOption;
+import org.apache.paimon.predicate.Predicate;
 import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
 import org.apache.paimon.table.Table;
 import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DecimalType;
+import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.JsonSerdeUtil;
 import org.apache.paimon.utils.Pair;
 import org.apache.paimon.utils.Projection;
 
@@ -48,8 +59,11 @@
 import javax.annotation.Nullable;
 
 public class PaimonUtil {
+    private static final Logger LOG = LogManager.getLogger(PaimonUtil.class);
+
     public static List<InternalRow> read(
-            Table table, @Nullable int[][] projection, Pair<ConfigOption<?>, String>... dynamicOptions)
+            Table table, @Nullable int[][] projection, @Nullable Predicate predicate,
+            Pair<ConfigOption<?>, String>... dynamicOptions)
             throws IOException {
         Map<String, String> options = new HashMap<>();
         for (Pair<ConfigOption<?>, String> pair : dynamicOptions) {
@@ -60,6 +74,9 @@ public static List<InternalRow> read(
         if (projection != null) {
             readBuilder.withProjection(projection);
         }
+        if (predicate != null) {
+            readBuilder.withFilter(predicate);
+        }
         RecordReader<InternalRow> reader =
                 readBuilder.newRead().createReader(readBuilder.newScan().plan());
         InternalRowSerializer serializer =
@@ -152,4 +169,107 @@ public static ListPartitionItem toListPartitionItem(String partitionName, List<C
         ListPartitionItem listPartitionItem = new ListPartitionItem(Lists.newArrayList(key));
         return listPartitionItem;
     }
+
+    private static Type paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataType dataType) {
+        int tsScale = 3; // default
+        switch (dataType.getTypeRoot()) {
+            case BOOLEAN:
+                return Type.BOOLEAN;
+            case INTEGER:
+                return Type.INT;
+            case BIGINT:
+                return Type.BIGINT;
+            case FLOAT:
+                return Type.FLOAT;
+            case DOUBLE:
+                return Type.DOUBLE;
+            case SMALLINT:
+                return Type.SMALLINT;
+            case TINYINT:
+                return Type.TINYINT;
+            case VARCHAR:
+            case BINARY:
+            case CHAR:
+            case VARBINARY:
+                return Type.STRING;
+            case DECIMAL:
+                DecimalType decimal = (DecimalType) dataType;
+                return ScalarType.createDecimalV3Type(decimal.getPrecision(), decimal.getScale());
+            case DATE:
+                return ScalarType.createDateV2Type();
+            case TIMESTAMP_WITHOUT_TIME_ZONE:
+                if (dataType instanceof org.apache.paimon.types.TimestampType) {
+                    tsScale = ((org.apache.paimon.types.TimestampType) dataType).getPrecision();
+                    if (tsScale > 6) {
+                        tsScale = 6;
+                    }
+                } else if (dataType instanceof org.apache.paimon.types.LocalZonedTimestampType) {
+                    tsScale = ((org.apache.paimon.types.LocalZonedTimestampType) dataType).getPrecision();
+                    if (tsScale > 6) {
+                        tsScale = 6;
+                    }
+                }
+                return ScalarType.createDatetimeV2Type(tsScale);
+            case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+                if (dataType instanceof org.apache.paimon.types.LocalZonedTimestampType) {
+                    tsScale = ((org.apache.paimon.types.LocalZonedTimestampType) dataType).getPrecision();
+                    if (tsScale > 6) {
+                        tsScale = 6;
+                    }
+                }
+                return ScalarType.createDatetimeV2Type(tsScale);
+            case ARRAY:
+                ArrayType arrayType = (ArrayType) dataType;
+                Type innerType = paimonPrimitiveTypeToDorisType(arrayType.getElementType());
+                return org.apache.doris.catalog.ArrayType.create(innerType, true);
+            case MAP:
+                MapType mapType = (MapType) dataType;
+                return new org.apache.doris.catalog.MapType(
+                        paimonTypeToDorisType(mapType.getKeyType()), paimonTypeToDorisType(mapType.getValueType()));
+            case ROW:
+                RowType rowType = (RowType) dataType;
+                List<DataField> fields = rowType.getFields();
+                return new org.apache.doris.catalog.StructType(fields.stream()
+                        .map(field -> new org.apache.doris.catalog.StructField(field.name(),
+                                paimonTypeToDorisType(field.type())))
+                        .collect(Collectors.toCollection(ArrayList::new)));
+            case TIME_WITHOUT_TIME_ZONE:
+                return Type.UNSUPPORTED;
+            default:
+                LOG.warn("Cannot transform unknown type: " + dataType.getTypeRoot());
+                return Type.UNSUPPORTED;
+        }
+    }
+
+    public static Type paimonTypeToDorisType(org.apache.paimon.types.DataType type) {
+        return paimonPrimitiveTypeToDorisType(type);
+    }
+
+    /**
+     * https://paimon.apache.org/docs/0.9/maintenance/system-tables/#schemas-table
+     * demo:
+     * 0
+     * [{"id":0,"name":"user_id","type":"BIGINT NOT NULL"},
+     * {"id":1,"name":"item_id","type":"BIGINT"},
+     * {"id":2,"name":"behavior","type":"STRING"},
+     * {"id":3,"name":"dt","type":"STRING NOT NULL"},
+     * {"id":4,"name":"hh","type":"STRING NOT NULL"}]
+     * ["dt"]
+     * ["dt","hh","user_id"]
+     * {"owner":"hadoop","provider":"paimon"}
+     * 2024-12-03 15:38:14.734
+     *
+     * @param row
+     * @return
+     */
+    public static PaimonSchema rowToSchema(InternalRow row) {
+        long schemaId = row.getLong(0);
+        String fieldsStr = row.getString(1).toString();
+        String partitionKeysStr = row.getString(2).toString();
+        List<DataField> fields = JsonSerdeUtil.fromJson(fieldsStr, new TypeReference<List<DataField>>() {
+        });
+        List<String> partitionKeys = JsonSerdeUtil.fromJson(partitionKeysStr, new TypeReference<List<String>>() {
+        });
+        return new PaimonSchema(schemaId, fields, partitionKeys);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java
index 885eba06ed956d..a8bb814f1d353b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java
@@ -21,6 +21,7 @@
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.UserException;
 import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 import org.apache.doris.datasource.paimon.PaimonExternalTable;
 import org.apache.doris.datasource.property.constants.PaimonProperties;
 import org.apache.doris.thrift.TFileAttributes;
@@ -36,7 +37,7 @@ public class PaimonSource {
     public PaimonSource(TupleDescriptor desc) {
         this.desc = desc;
         this.paimonExtTable = (PaimonExternalTable) desc.getTable();
-        this.originTable = paimonExtTable.getPaimonTable();
+        this.originTable = paimonExtTable.getPaimonTable(MvccUtil.getSnapshotFromContext(paimonExtTable));
     }
 
     public TupleDescriptor getDesc() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index 68ed0cc9b23592..353e024ed97348 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -29,6 +29,9 @@
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.DebugUtil;
 import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
+import org.apache.doris.datasource.mvcc.MvccTable;
+import org.apache.doris.datasource.mvcc.MvccTableInfo;
 import org.apache.doris.job.common.TaskStatus;
 import org.apache.doris.job.exception.JobException;
 import org.apache.doris.job.task.AbstractTask;
@@ -70,6 +73,7 @@
 import java.math.RoundingMode;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
@@ -141,6 +145,8 @@ public enum MTMVTaskRefreshMode {
     private StmtExecutor executor;
     private Map<String, MTMVRefreshPartitionSnapshot> partitionSnapshots;
 
+    private final Map<MvccTableInfo, MvccSnapshot> snapshots = Maps.newHashMap();
+
     public MTMVTask() {
     }
 
@@ -218,6 +224,9 @@ private void exec(Set<String> refreshPartitionNames,
             throws Exception {
         ConnectContext ctx = MTMVPlanUtil.createMTMVContext(mtmv);
         StatementContext statementContext = new StatementContext();
+        for (Entry<MvccTableInfo, MvccSnapshot> entry : snapshots.entrySet()) {
+            statementContext.setSnapshot(entry.getKey(), entry.getValue());
+        }
         ctx.setStatementContext(statementContext);
         TUniqueId queryId = generateQueryId();
         lastQueryId = DebugUtil.printId(queryId);
@@ -305,6 +314,11 @@ private void beforeMTMVRefresh() throws AnalysisException, DdlException {
                 MTMVBaseTableIf baseTableIf = (MTMVBaseTableIf) tableIf;
                 baseTableIf.beforeMTMVRefresh(mtmv);
             }
+            if (tableIf instanceof MvccTable) {
+                MvccTable mvccTable = (MvccTable) tableIf;
+                MvccSnapshot mvccSnapshot = mvccTable.loadSnapshot();
+                snapshots.put(new MvccTableInfo(mvccTable), mvccSnapshot);
+            }
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 008a2c8ac70da2..f21600ac585763 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -555,7 +555,11 @@ public void loadSnapshots(Map<List<String>, TableIf> tables) {
         }
         for (TableIf tableIf : tables.values()) {
             if (tableIf instanceof MvccTable) {
-                snapshots.put(new MvccTableInfo(tableIf), ((MvccTable) tableIf).loadSnapshot());
+                MvccTableInfo mvccTableInfo = new MvccTableInfo(tableIf);
+                // may be set by MTMV, we can not load again
+                if (!snapshots.containsKey(mvccTableInfo)) {
+                    snapshots.put(mvccTableInfo, ((MvccTable) tableIf).loadSnapshot());
+                }
             }
         }
     }
@@ -563,11 +567,25 @@ public void loadSnapshots(Map<List<String>, TableIf> tables) {
     /**
      * Obtain snapshot information of mvcc
      *
-     * @param mvccTable mvccTable
+     * @param tableIf tableIf
      * @return MvccSnapshot
      */
-    public MvccSnapshot getSnapshot(MvccTable mvccTable) {
-        return snapshots.get(new MvccTableInfo(mvccTable));
+    public Optional<MvccSnapshot> getSnapshot(TableIf tableIf) {
+        if (!(tableIf instanceof MvccTable)) {
+            return Optional.empty();
+        }
+        MvccTableInfo mvccTableInfo = new MvccTableInfo(tableIf);
+        return Optional.ofNullable(snapshots.get(mvccTableInfo));
+    }
+
+    /**
+     * Obtain snapshot information of mvcc
+     *
+     * @param mvccTableInfo mvccTableInfo
+     * @param snapshot snapshot
+     */
+    public void setSnapshot(MvccTableInfo mvccTableInfo, MvccSnapshot snapshot) {
+        snapshots.put(mvccTableInfo, snapshot);
     }
 
     private static class CloseableResource implements Closeable {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
index ba8b270d1f397d..e99906f5e13dc4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
@@ -36,7 +36,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -74,8 +73,8 @@ public Rule build() {
     private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable,
             LogicalFilter<LogicalFileScan> filter, LogicalFileScan scan, CascadesContext ctx) {
         Map<String, PartitionItem> selectedPartitionItems = Maps.newHashMap();
-        // todo: real snapshotId
-        if (CollectionUtils.isEmpty(externalTable.getPartitionColumns(Optional.empty()))) {
+        if (CollectionUtils.isEmpty(externalTable.getPartitionColumns(
+                ctx.getStatementContext().getSnapshot(externalTable)))) {
             // non partitioned table, return NOT_PRUNED.
             // non partition table will be handled in HiveScanNode.
             return SelectedPartitions.NOT_PRUNED;
@@ -83,8 +82,8 @@ private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable,
         Map<String, Slot> scanOutput = scan.getOutput()
                 .stream()
                 .collect(Collectors.toMap(slot -> slot.getName().toLowerCase(), Function.identity()));
-        // todo: real snapshotId
-        List<Slot> partitionSlots = externalTable.getPartitionColumns(Optional.empty())
+        List<Slot> partitionSlots = externalTable.getPartitionColumns(
+                        ctx.getStatementContext().getSnapshot(externalTable))
                 .stream()
                 .map(column -> scanOutput.get(column.getName().toLowerCase()))
                 .collect(Collectors.toList());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
index 869c2d0b38bd4c..b5be72950cd47a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
@@ -29,6 +29,7 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.UserException;
 import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 import org.apache.doris.mtmv.BaseTableInfo;
 import org.apache.doris.mtmv.MTMVRelatedTableIf;
 import org.apache.doris.nereids.analyzer.UnboundRelation;
@@ -315,10 +316,8 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation,
                         }
                         if (targetTable instanceof ExternalTable) {
                             // Add filter only when partition has data when external table
-                            // TODO: 2024/12/4 real snapshot
-                            partitionHasDataItems.add(
-                                    ((ExternalTable) targetTable).getNameToPartitionItems(Optional.empty())
-                                            .get(partitionName));
+                            partitionHasDataItems.add(((ExternalTable) targetTable).getNameToPartitionItems(
+                                    MvccUtil.getSnapshotFromContext(targetTable)).get(partitionName));
                         }
                     }
                     if (partitionHasDataItems.isEmpty()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index 96b8e032d11274..1f5f71f7bafe59 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -20,6 +20,7 @@
 import org.apache.doris.analysis.TableSnapshot;
 import org.apache.doris.catalog.PartitionItem;
 import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 import org.apache.doris.nereids.memo.GroupExpression;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.trees.TableSample;
@@ -60,10 +61,10 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List<String> quali
     }
 
     public LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
-                           Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
-        // todo: real snapshotId
+            Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
         this(id, table, qualifier, Optional.empty(), Optional.empty(),
-                table.initSelectedPartitions(Optional.empty()), tableSample, tableSnapshot);
+                table.initSelectedPartitions(MvccUtil.getSnapshotFromContext(table)),
+                tableSample, tableSnapshot);
     }
 
     public SelectedPartitions getSelectedPartitions() {
diff --git a/regression-test/data/mtmv_p0/test_paimon_rewrite_mtmv.out b/regression-test/data/mtmv_p0/test_paimon_rewrite_mtmv.out
new file mode 100644
index 00000000000000..63bda82c1db5bd
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_paimon_rewrite_mtmv.out
@@ -0,0 +1,16 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !refresh_one_partition --
+a	10
+
+-- !refresh_one_partition_rewrite --
+a	10
+b	10
+
+-- !refresh_auto --
+a	10
+b	10
+
+-- !refresh_all_partition_rewrite --
+a	10
+b	10
+
diff --git a/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy b/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
new file mode 100644
index 00000000000000..985443875c7b26
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_rewrite_mtmv", "p0,external,mtmv,external_docker,external_docker_doris") {
+    String enabled = context.config.otherConfigs.get("enablePaimonTest")
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+        logger.info("disabled paimon test")
+        return
+    }
+    String suiteName = "test_paimon_rewrite_mtmv"
+    String catalogName = "${suiteName}_catalog"
+    String mvName = "${suiteName}_mv"
+    String dbName = context.config.getDbNameByFile(context.file)
+
+    String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+    String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+    sql """set materialized_view_rewrite_enable_contain_external_table=true;"""
+    String mvSql = "SELECT par,count(*) as num FROM ${catalogName}.`test_paimon_spark`.test_tb_mix_format group by par;";
+
+    sql """drop catalog if exists ${catalogName}"""
+    sql """CREATE CATALOG ${catalogName} PROPERTIES (
+            'type'='paimon',
+            'warehouse' = 's3://warehouse/wh/',
+            "s3.access_key" = "admin",
+            "s3.secret_key" = "password",
+            "s3.endpoint" = "http://${externalEnvIp}:${minio_port}",
+            "s3.region" = "us-east-1"
+        );"""
+
+    sql """analyze table ${catalogName}.`test_paimon_spark`.test_tb_mix_format with sync"""
+    sql """alter table ${catalogName}.`test_paimon_spark`.test_tb_mix_format modify column par set stats ('row_count'='20');"""
+
+    sql """drop materialized view if exists ${mvName};"""
+
+    sql """
+        CREATE MATERIALIZED VIEW ${mvName}
+            BUILD DEFERRED REFRESH AUTO ON MANUAL
+            partition by(`par`)
+            DISTRIBUTED BY RANDOM BUCKETS 2
+            PROPERTIES ('replication_num' = '1')
+            AS
+             ${mvSql}
+        """
+    def showPartitionsResult = sql """show partitions from ${mvName}"""
+    logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+    assertTrue(showPartitionsResult.toString().contains("p_a"))
+    assertTrue(showPartitionsResult.toString().contains("p_b"))
+
+    // refresh one partitions
+    sql """
+            REFRESH MATERIALIZED VIEW ${mvName} partitions(p_a);
+        """
+    waitingMTMVTaskFinishedByMvName(mvName)
+    order_qt_refresh_one_partition "SELECT * FROM ${mvName} "
+
+    def explainOnePartition = sql """ explain  ${mvSql} """
+    logger.info("explainOnePartition: " + explainOnePartition.toString())
+    assertTrue(explainOnePartition.toString().contains("VUNION"))
+    order_qt_refresh_one_partition_rewrite "${mvSql}"
+
+    mv_rewrite_success("${mvSql}", "${mvName}")
+
+    //refresh auto
+    sql """
+            REFRESH MATERIALIZED VIEW ${mvName} auto
+        """
+    waitingMTMVTaskFinishedByMvName(mvName)
+    order_qt_refresh_auto "SELECT * FROM ${mvName} "
+
+    def explainAllPartition = sql """ explain  ${mvSql}; """
+    logger.info("explainAllPartition: " + explainAllPartition.toString())
+    assertTrue(explainAllPartition.toString().contains("VOlapScanNode"))
+    order_qt_refresh_all_partition_rewrite "${mvSql}"
+
+    mv_rewrite_success("${mvSql}", "${mvName}")
+
+    sql """drop materialized view if exists ${mvName};"""
+    sql """drop catalog if exists ${catalogName}"""
+}
+

From 55db27733e16a6036aa5dac051c06071c391ffd6 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Wed, 11 Dec 2024 11:12:26 +0800
Subject: [PATCH 263/399] [regression-test](fix) fix case with invalid grammer
 (#45211)

Problem Summary:
Defining global variables in a test suite can easily introduce errors,
such as variable conflicts in concurrent scenarios.
In #44576 , a method for detecting global variables was introduced.

That approach conflicts with the function calling like
connect(user=user, password="pwd"). Furthermore, the connect(user=user,
password="pwd") method cannot specify assignments to parameters with
default values; values still need to be assigned according to the order
of parameters. Therefore, modifying these calling method.
---
 .../suites/account_p0/test_alter_user.groovy  | 46 ++++++-------
 .../account_p0/test_auth_compatibility.groovy |  2 +-
 .../suites/account_p0/test_auth_show.groovy   |  4 +-
 .../suites/account_p0/test_grant_priv.groovy  |  4 +-
 .../test_grant_priv_resource.groovy           |  4 +-
 .../test_grant_priv_workload.groovy           |  4 +-
 .../test_nereids_authentication.groovy        | 14 ++--
 .../account_p0/test_nereids_row_policy.groovy |  4 +-
 .../suites/account_p0/test_role.groovy        |  4 +-
 .../test_account_management_grant_auth.groovy |  6 +-
 .../test_account_management_role_auth.groovy  |  4 +-
 .../test_account_management_user_auth.groovy  |  6 +-
 .../test_assistant_command_auth.groovy        |  6 +-
 .../test_cluster_management_auth.groovy       |  6 +-
 .../test_database_management_auth.groovy      |  2 +-
 .../auth_call/test_ddl_backup_auth.groovy     |  4 +-
 .../auth_call/test_ddl_catalog_auth.groovy    | 14 ++--
 .../test_ddl_colocate_group_auth.groovy       |  4 +-
 .../auth_call/test_ddl_database_auth.groovy   | 14 ++--
 .../auth_call/test_ddl_encryptkey_auth.groovy |  4 +-
 .../auth_call/test_ddl_file_auth.groovy       |  6 +-
 .../auth_call/test_ddl_function_auth.groovy   |  6 +-
 .../auth_call/test_ddl_index_auth.groovy      |  4 +-
 .../suites/auth_call/test_ddl_job_auth.groovy |  4 +-
 .../auth_call/test_ddl_mask_view_auth.groovy  |  4 +-
 .../auth_call/test_ddl_mtmv_auth.groovy       | 22 +++----
 .../suites/auth_call/test_ddl_mv_auth.groovy  | 10 +--
 .../auth_call/test_ddl_part_table_auth.groovy |  6 +-
 .../test_ddl_policy_storage_auth.groovy       |  4 +-
 .../auth_call/test_ddl_repository_auth.groovy |  4 +-
 .../auth_call/test_ddl_resource_auth.groovy   |  4 +-
 .../auth_call/test_ddl_restore_auth.groovy    |  6 +-
 .../auth_call/test_ddl_row_policy_auth.groovy |  6 +-
 .../test_ddl_sql_block_rule_auth.groovy       |  4 +-
 .../auth_call/test_ddl_table_auth.groovy      | 38 +++++------
 .../auth_call/test_ddl_view_auth.groovy       | 20 +++---
 .../test_ddl_workload_group_auth.groovy       |  4 +-
 .../auth_call/test_dml_analyze_auth.groovy    |  6 +-
 .../test_dml_broker_load_auth.groovy          |  6 +-
 .../test_dml_cancel_profile_auth.groovy       |  4 +-
 .../test_dml_delete_table_auth.groovy         | 12 ++--
 .../test_dml_export_table_auth.groovy         |  6 +-
 .../auth_call/test_dml_insert_auth.groovy     |  8 +--
 .../test_dml_multi_routine_load_auth.groovy   |  8 +--
 .../auth_call/test_dml_mysql_load_auth.groovy |  4 +-
 .../auth_call/test_dml_outfile_auth.groovy    |  4 +-
 .../test_dml_routine_load_auth.groovy         |  8 +--
 .../auth_call/test_dml_select_udf_auth.groovy |  4 +-
 .../test_dml_stream_load_auth.groovy          |  4 +-
 .../test_dml_update_table_auth.groovy         |  6 +-
 .../test_grant_show_view_priv_auth.groovy     |  6 +-
 .../auth_call/test_hive_base_case_auth.groovy | 24 +++----
 .../auth_call/test_show_backend_auth.groovy   |  4 +-
 .../auth_call/test_show_broker_auth.groovy    |  4 +-
 .../auth_call/test_show_charset_auth.groovy   |  6 +-
 .../test_show_convert_light_sc_auth.groovy    |  4 +-
 .../test_show_create_table_auth.groovy        | 10 +--
 .../auth_call/test_show_data_auth.groovy      |  6 +-
 .../test_show_database_id_auth.groovy         |  4 +-
 .../test_show_dynamic_table_auth.groovy       |  4 +-
 .../auth_call/test_show_frontend_auth.groovy  |  4 +-
 .../auth_call/test_show_grant_auth.groovy     |  4 +-
 .../auth_call/test_show_proc_auth.groovy      |  4 +-
 .../test_show_query_stats_auth.groovy         |  2 +-
 .../test_show_repository_auth.groovy          |  4 +-
 .../auth_call/test_show_tablet_auth.groovy    |  4 +-
 .../auth_call/test_show_typecast_auth.groovy  |  2 +-
 .../test_admin_copy_tablet_auth.groovy        |  2 +-
 .../auth_p0/test_alter_policy_auth.groovy     |  2 +-
 .../auth_p0/test_alter_view_auth.groovy       |  4 +-
 .../test_cancel_alter_system_auth.groovy      |  2 +-
 .../auth_p0/test_create_policy_auth.groovy    |  2 +-
 .../auth_p0/test_create_view_auth.groovy      |  4 +-
 .../test_drop_materialized_view_auth.groovy   |  2 +-
 .../auth_p0/test_drop_policy_auth.groovy      |  2 +-
 .../test_master_slave_consistency_auth.groovy | 64 +++++++++----------
 .../auth_p0/test_select_column_auth.groovy    | 16 ++---
 .../auth_p0/test_select_view_auth.groovy      |  4 +-
 .../test_set_ldap_admin_password_auth.groovy  |  2 +-
 .../test_show_catalog_recycle_bin_auth.groovy |  2 +-
 .../test_show_create_database_auth.groovy     |  2 +-
 .../test_show_create_repository_auth.groovy   |  2 +-
 .../suites/auth_p0/test_show_data_auth.groovy |  2 +-
 .../auth_p0/test_show_encryptkeys_auth.groovy |  2 +-
 .../auth_p0/test_show_plugins_auth.groovy     |  2 +-
 .../test_show_repositories_auth.groovy        |  2 +-
 .../auth_p0/test_show_snapshot_auth.groovy    |  2 +-
 .../auth_p0/test_show_stream_load_auth.groovy |  4 +-
 .../test_show_tablets_belong_auth.groovy      |  2 +-
 .../auth_p0/test_show_transaction_auth.groovy |  2 +-
 .../auth_p0/test_use_encryptkey_auth.groovy   |  4 +-
 .../suites/auth_up_down_p0/load.groovy        | 22 +++----
 .../test_grant_revoke_auth.groovy             | 12 ++--
 .../test_disable_revoke_admin_auth.groovy     |  2 +-
 .../test_grant_revoke_cluster_role.groovy     |  6 +-
 .../test_grant_revoke_cluster_to_user.groovy  | 30 ++++-----
 ..._grant_revoke_compute_group_to_user.groovy | 30 ++++-----
 .../multi_cluster/test_auto_start.groovy      |  2 +-
 ...test_compaction_uniq_keys_row_store.groovy |  4 +-
 ...t_compaction_uniq_keys_row_store_ck.groovy |  4 +-
 .../delete_p0/test_delete_on_mor.groovy       |  2 +-
 .../suites/demo_p0/connect_action.groovy      |  2 +-
 .../hive/test_external_catalog_hive.groovy    |  2 +-
 .../hive/test_external_sql_block_rule.groovy  |  6 +-
 .../hive/test_hive_other.groovy               |  2 +-
 .../iceberg/iceberg_drop_rest_table.groovy    |  2 +-
 .../jdbc/test_jdbc_call.groovy                |  4 +-
 .../jdbc/test_mysql_jdbc_catalog.groovy       |  4 +-
 .../hive/test_trino_hive_other.groovy         |  2 +-
 .../tvf/test_catalogs_tvf.groovy              |  6 +-
 ...st_insert_from_tvf_with_common_user.groovy |  2 +-
 .../tvf/test_s3_tvf_with_resource.groovy      |  4 +-
 .../test_cloud_mow_insert_with_retry.groovy   |  2 +-
 .../insert_p0/insert_group_commit_into.groovy | 10 +--
 ..._group_commit_into_max_filter_ratio.groovy |  2 +-
 .../insert_group_commit_into_unique.groovy    |  6 +-
 ..._group_commit_into_unique_sync_mode.groovy |  6 +-
 ...insert_group_commit_with_large_data.groovy |  2 +-
 .../suites/insert_p0/prepare_insert.groovy    | 10 +--
 ...st_group_commit_data_bytes_property.groovy |  2 +-
 ...t_group_commit_interval_ms_property.groovy |  2 +-
 .../suites/insert_p0/test_jdbc.groovy         |  4 +-
 .../insert_p0/transaction/txn_insert.groovy   |  2 +-
 .../transaction/txn_insert_restart_fe.groovy  |  4 +-
 .../operate_internal_schema.groovy            |  2 +-
 .../javaudf_p0/test_javaudf_auth.groovy       |  4 +-
 .../test_routine_load_with_user.groovy        |  2 +-
 .../manager/test_manager_interface_3.groovy   | 28 ++++----
 .../authorization/column_authorization.groovy |  2 +-
 .../authorization/view_authorization.groovy   |  2 +-
 .../cache/parse_sql_from_sql_cache.groovy     | 18 +++---
 .../ddl/account/test_nereids_role.groovy      |  4 +-
 .../nereids_p0/delete/delete_cte.groovy       |  2 +-
 .../nereids_p0/delete/delete_cte_ck.groovy    |  2 +-
 .../delete/delete_mow_partial_update.groovy   |  2 +-
 .../nereids_p0/delete/delete_using.groovy     |  2 +-
 .../insert_into_table/insert_auth.groovy      |  6 +-
 .../insert_into_table/partial_update.groovy   |  2 +-
 .../partial_update_complex.groovy             |  2 +-
 .../partial_update_seq_col.groovy             |  2 +-
 .../with_auth/with_select_table_auth.groovy   |  4 +-
 .../point_query_p0/test_point_query.groovy    |  6 +-
 .../point_query_p0/test_point_query_ck.groovy |  6 +-
 .../test_point_query_partition.groovy         |  6 +-
 .../test_point_query_partition_ck.groovy      |  6 +-
 .../point_query_p0/test_rowstore.groovy       |  2 +-
 .../point_query_p0/test_rowstore_ck.groovy    |  2 +-
 .../prepared_stmt_p0/prepared_stmt.groovy     |  2 +-
 .../prepared_stmt_in_list.groovy              |  2 +-
 .../system/test_partitions_schema.groovy      |  6 +-
 .../query_p0/system/test_table_options.groovy |  6 +-
 .../system/test_table_properties.groovy       |  6 +-
 .../suites/query_p0/test_row_policy.groovy    |  2 +-
 .../test_alter_table_column_rename.groovy     |  2 +-
 .../show_p0/test_show_backend_config.groovy   |  2 +-
 .../suites/show_p0/test_show_backends.groovy  |  4 +-
 .../show_p0/test_show_statistic_proc.groovy   |  4 +-
 .../statistics/test_analyze_mtmv.groovy       |  2 +-
 .../suites/statistics/test_analyze_mv.groovy  |  2 +-
 .../test_auto_analyze_black_white_list.groovy |  2 +-
 .../statistics/test_partition_stats.groovy    |  2 +-
 .../test_mix_partial_update.groovy            |  2 +-
 .../test_new_partial_update_delete.groovy     |  4 +-
 .../partial_update/test_partial_update.groovy |  2 +-
 ...st_partial_update_2pc_schema_change.groovy |  2 +-
 .../test_partial_update_after_delete.groovy   |  2 +-
 .../test_partial_update_auto_inc.groovy       |  2 +-
 ...t_partial_update_case_insensitivity.groovy |  2 +-
 .../test_partial_update_complex_type.groovy   |  2 +-
 ...l_update_complex_type_schema_change.groovy |  2 +-
 .../test_partial_update_default_value.groovy  |  2 +-
 .../test_partial_update_delete.groovy         |  2 +-
 .../test_partial_update_delete_sign.groovy    |  2 +-
 ...l_update_insert_light_schema_change.groovy |  2 +-
 .../test_partial_update_insert_seq_col.groovy |  2 +-
 ...t_partial_update_native_insert_stmt.groovy |  2 +-
 ...l_update_native_insert_stmt_complex.groovy |  2 +-
 .../test_partial_update_seq_col.groovy        |  2 +-
 .../test_partial_update_seq_col_delete.groovy |  2 +-
 .../test_partial_update_seq_type.groovy       |  2 +-
 ...test_partial_update_seq_type_delete.groovy |  2 +-
 .../test_partial_update_strict_mode.groovy    |  2 +-
 .../test_partial_update_upsert.groovy         |  2 +-
 ...est_partial_update_with_delete_stmt.groovy |  2 +-
 ..._partial_update_with_inverted_index.groovy |  2 +-
 ...est_partial_update_with_update_stmt.groovy |  2 +-
 .../test_delete_sign.groovy                   |  2 +-
 .../legacy/test_f_2pc_schema_change.groovy    |  2 +-
 .../test_f_complex_type_schema_change.groovy  |  2 +-
 .../test_mix_partial_update.groovy            |  2 +-
 .../test_new_partial_update_delete.groovy     |  4 +-
 .../partial_update/test_partial_update.groovy |  2 +-
 ...st_partial_update_2pc_schema_change.groovy |  2 +-
 .../test_partial_update_after_delete.groovy   |  2 +-
 .../test_partial_update_auto_inc.groovy       |  2 +-
 ...t_partial_update_case_insensitivity.groovy |  2 +-
 .../test_partial_update_complex_type.groovy   |  2 +-
 ...l_update_complex_type_schema_change.groovy |  2 +-
 .../test_partial_update_default_value.groovy  |  2 +-
 .../test_partial_update_delete.groovy         |  2 +-
 .../test_partial_update_delete_sign.groovy    |  2 +-
 ...l_update_insert_light_schema_change.groovy |  2 +-
 .../test_partial_update_insert_seq_col.groovy |  2 +-
 .../test_partial_update_merge_type.groovy     |  2 +-
 ...t_partial_update_native_insert_stmt.groovy |  2 +-
 ...l_update_native_insert_stmt_complex.groovy |  2 +-
 .../test_partial_update_only_keys.groovy      |  2 +-
 .../test_partial_update_seq_col.groovy        |  2 +-
 .../test_partial_update_seq_col_delete.groovy |  2 +-
 .../test_partial_update_seq_type.groovy       |  2 +-
 ...test_partial_update_seq_type_delete.groovy |  2 +-
 .../test_partial_update_strict_mode.groovy    |  2 +-
 .../test_partial_update_upsert.groovy         |  2 +-
 ...est_partial_update_with_delete_stmt.groovy |  2 +-
 ..._partial_update_with_inverted_index.groovy |  2 +-
 ...est_partial_update_with_update_stmt.groovy |  2 +-
 .../test_delete_sign.groovy                   |  2 +-
 .../variant_p0/variant_with_rowstore.groovy   |  4 +-
 .../privilege/test_vault_privilege.groovy     | 18 +++---
 .../test_vault_privilege_restart.groovy       | 12 ++--
 .../test_vault_privilege_with_role.groovy     |  6 +-
 .../workload_manager_p0/test_curd_wlg.groovy  |  8 +--
 .../test_workload_sched_policy.groovy         | 12 ++--
 223 files changed, 577 insertions(+), 577 deletions(-)

diff --git a/regression-test/suites/account_p0/test_alter_user.groovy b/regression-test/suites/account_p0/test_alter_user.groovy
index 2cfa5b082b3994..7a2c4649a0667e 100644
--- a/regression-test/suites/account_p0/test_alter_user.groovy
+++ b/regression-test/suites/account_p0/test_alter_user.groovy
@@ -37,7 +37,7 @@ suite("test_alter_user", "account,nonConcurrent") {
     sql """alter user test_auth_user2 password_history 0"""
     sql """set password for 'test_auth_user2' = password('12345')"""
     
-    def result1 = connect(user = 'test_auth_user2', password = '12345', url = context.config.jdbcUrl) {
+    def result1 = connect('test_auth_user2', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
 
@@ -48,7 +48,7 @@ suite("test_alter_user", "account,nonConcurrent") {
         sql """alter user test_auth_user2 identified by 'abc12345'"""
         exception "Cannot use these credentials for 'test_auth_user2'@'%' because they contradict the password history policy"
     }
-    result1 = connect(user = 'test_auth_user2', password = 'abc123456', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user2', 'abc123456', context.config.jdbcUrl) {
         sql 'select 1'
     }
     sql """set global password_history=0""" // set to disabled
@@ -58,31 +58,31 @@ suite("test_alter_user", "account,nonConcurrent") {
     sql """grant all on *.* to test_auth_user3"""
          
     // login success in multi times
-    result1 = connect(user = 'test_auth_user3', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user3', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
-    result1 = connect(user = 'test_auth_user3', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user3', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
-    result1 = connect(user = 'test_auth_user3', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user3', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
     // login failed in 2 times
     try {
-        connect(user = 'test_auth_user3', password = 'wrong', url = context.config.jdbcUrl) {}
+        connect('test_auth_user3', 'wrong',context.config.jdbcUrl) {}
         assertTrue(false. "should not be able to login")
     } catch (Exception e) {
         assertTrue(e.getMessage().contains("Access denied for user 'test_auth_user3"), e.getMessage())
     } 
     try {
-        connect(user = 'test_auth_user3', password = 'wrong', url = context.config.jdbcUrl) {}
+        connect('test_auth_user3', 'wrong', context.config.jdbcUrl) {}
         assertTrue(false. "should not be able to login")
     } catch (Exception e) {
         assertTrue(e.getMessage().contains("Access denied for user 'test_auth_user3"), e.getMessage())
     } 
     // login with correct password but also failed
     try {
-        connect(user = 'test_auth_user3', password = '12345', url = context.config.jdbcUrl) {}
+        connect('test_auth_user3', '12345', context.config.jdbcUrl) {}
         assertTrue(false. "should not be able to login")
     } catch (Exception e) {
         assertTrue(e.getMessage().contains("Access denied for user 'test_auth_user3'@'%'. Account is blocked for 86400 second(s) (86400 second(s) remaining) due to 2 consecutive failed logins."), e.getMessage())
@@ -90,7 +90,7 @@ suite("test_alter_user", "account,nonConcurrent") {
 
     // unlock user and login again
     sql """alter user test_auth_user3 account_unlock"""
-    result1 = connect(user = 'test_auth_user3', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user3', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
 
@@ -98,27 +98,27 @@ suite("test_alter_user", "account,nonConcurrent") {
     sql """alter user test_auth_user3 PASSWORD_LOCK_TIME 5 SECOND"""
     // login failed in 2 times to lock the accout again
     try {
-        connect(user = 'test_auth_user3', password = 'wrong', url = context.config.jdbcUrl) {}
+        connect('test_auth_user3', 'wrong', context.config.jdbcUrl) {}
         assertTrue(false. "should not be able to login")
     } catch (Exception e) {
         assertTrue(e.getMessage().contains("Access denied for user 'test_auth_user3"), e.getMessage())
     } 
     try {
-        connect(user = 'test_auth_user3', password = 'wrong', url = context.config.jdbcUrl) {}
+        connect('test_auth_user3', 'wrong', context.config.jdbcUrl) {}
         assertTrue(false. "should not be able to login")
     } catch (Exception e) {
         assertTrue(e.getMessage().contains("Access denied for user 'test_auth_user3"), e.getMessage())
     } 
     // login with correct password but also failed
     try {
-        connect(user = 'test_auth_user3', password = '12345', url = context.config.jdbcUrl) {}
+        connect('test_auth_user3', '12345', context.config.jdbcUrl) {}
         assertTrue(false. "should not be able to login")
     } catch (Exception e) {
         assertTrue(e.getMessage().contains("Access denied for user 'test_auth_user3'@'%'. Account is blocked for 5 second(s) (5 second(s) remaining) due to 2 consecutive failed logins."), e.getMessage())
     } 
     // sleep 5 second to unlock account
     sleep(5000)
-    result1 = connect(user = 'test_auth_user3', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user3', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
 
@@ -134,7 +134,7 @@ suite("test_alter_user", "account,nonConcurrent") {
     }
 
     sql """set password for 'test_auth_user3' = password('Ab1234567^')"""
-    result1 = connect(user = 'test_auth_user3', password = 'Ab1234567^', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user3', 'Ab1234567^', context.config.jdbcUrl) {
         sql 'select 1'
     }
     sql """set global validate_password_policy=NONE"""
@@ -142,12 +142,12 @@ suite("test_alter_user", "account,nonConcurrent") {
     // 5. test expire
     sql """create user test_auth_user4 identified by '12345' PASSWORD_EXPIRE INTERVAL 5 SECOND"""
     sql """grant all on *.* to test_auth_user4"""
-    result1 = connect(user = 'test_auth_user4', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user4', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
     sleep(6000)
     try {
-        connect(user = 'test_auth_user4', password = '12345', url = context.config.jdbcUrl) {}
+        connect('test_auth_user4', '12345', context.config.jdbcUrl) {}
         assertTrue(false. "should not be able to login")
     } catch (Exception e) {
         assertTrue(e.getMessage().contains("Your password has expired. To log in you must change it using a client that supports expired passwords."), e.getMessage())
@@ -157,7 +157,7 @@ suite("test_alter_user", "account,nonConcurrent") {
     sql """drop user test_auth_user4"""
     sql """create user test_auth_user4 identified by '12345'"""
     sql """grant all on *.* to test_auth_user4"""
-    result1 = connect(user = 'test_auth_user4', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user4', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
 
@@ -165,17 +165,17 @@ suite("test_alter_user", "account,nonConcurrent") {
     sql """drop user test_auth_user4"""
     sql """create user test_auth_user4 identified by '12345' PASSWORD_EXPIRE INTERVAL 5 SECOND"""
     sql """grant all on *.* to test_auth_user4"""
-    result1 = connect(user = 'test_auth_user4', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user4', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
     sleep(6000)
     sql """set password for 'test_auth_user4' = password('123')"""
-    result2 = connect(user = 'test_auth_user4', password = '123', url = context.config.jdbcUrl) {
+    result2 = connect('test_auth_user4', '123', context.config.jdbcUrl) {
         sql 'select 1'
     }
     sleep(6000)
     try {
-        connect(user = 'test_auth_user4', password = '123', url = context.config.jdbcUrl) {}
+        connect('test_auth_user4', '123', context.config.jdbcUrl) {}
         assertTrue(false. "should not be able to login")
     } catch (Exception e) {
         assertTrue(e.getMessage().contains("Your password has expired. To log in you must change it using a client that supports expired passwords."), e.getMessage())
@@ -185,11 +185,11 @@ suite("test_alter_user", "account,nonConcurrent") {
     sql """drop user test_auth_user4"""
     sql """create user test_auth_user4 identified by '12345'"""
     sql """grant all on *.* to test_auth_user4"""
-    result1 = connect(user = 'test_auth_user4', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user4', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
     sleep(1000)
-    result2 = connect(user = 'test_auth_user4', password = '12345', url = context.config.jdbcUrl) {
+    result2 = connect('test_auth_user4', '12345', context.config.jdbcUrl) {
         sql 'select 1'
     }
 
@@ -197,7 +197,7 @@ suite("test_alter_user", "account,nonConcurrent") {
     sql """drop user if exists test_auth_user4"""
     sql """create user test_auth_user4 identified by '12345'"""
     sql """grant SELECT_PRIV on regression_test.* to test_auth_user4"""
-    result1 = connect(user = 'test_auth_user4', password = '12345', url = context.config.jdbcUrl) {
+    result1 = connect('test_auth_user4', '12345', context.config.jdbcUrl) {
         sql 'select 1'
         sql 'use information_schema'
         sql 'use mysql'
diff --git a/regression-test/suites/account_p0/test_auth_compatibility.groovy b/regression-test/suites/account_p0/test_auth_compatibility.groovy
index c4f780b01751c1..f3118680e50433 100644
--- a/regression-test/suites/account_p0/test_auth_compatibility.groovy
+++ b/regression-test/suites/account_p0/test_auth_compatibility.groovy
@@ -66,7 +66,7 @@ suite("test_auth_compatibility", "account") {
         GRANT SHOW_VIEW_PRIV on ${dbName}.${viewName} TO ${user}
     """
 
-    connect(user = user, password = pwd, url = context.config.jdbcUrl) {
+    connect(user, pwd, context.config.jdbcUrl) {
         sql """USE ${dbName}"""
         // auth ok, no exception
         sql """SHOW CREATE TABLE ${viewName}"""
diff --git a/regression-test/suites/account_p0/test_auth_show.groovy b/regression-test/suites/account_p0/test_auth_show.groovy
index c847fd1715aaeb..d9d4f3fa7f16ec 100644
--- a/regression-test/suites/account_p0/test_auth_show.groovy
+++ b/regression-test/suites/account_p0/test_auth_show.groovy
@@ -45,7 +45,7 @@ suite("test_auth_show", "account") {
 
     // With select priv for table, should be able to see db
     sql """GRANT SELECT_PRIV ON ${dbName}.${tableName} TO ${user}"""
-    def result1 = connect(user=user, password="${pwd}", url=url) {
+    def result1 = connect(user, "${pwd}", url) {
         sql """show databases like '${dbName}'"""
     }
     assertEquals(result1.size(), 1)
@@ -53,7 +53,7 @@ suite("test_auth_show", "account") {
 
     // With show_view priv for table, should be able to see db
     sql """GRANT SHOW_VIEW_PRIV ON ${dbName}.${tableName} TO ${user}"""
-    def result2 = connect(user=user, password="${pwd}", url=url) {
+    def result2 = connect(user, "${pwd}", url) {
         sql """show databases like '${dbName}'"""
     }
     assertEquals(result2.size(), 1)
diff --git a/regression-test/suites/account_p0/test_grant_priv.groovy b/regression-test/suites/account_p0/test_grant_priv.groovy
index abb95e9d1228aa..d2e04945905bee 100644
--- a/regression-test/suites/account_p0/test_grant_priv.groovy
+++ b/regression-test/suites/account_p0/test_grant_priv.groovy
@@ -38,7 +38,7 @@ suite("test_grant_priv") {
 
     // test only have select_priv, can not grant to other user
     sql """grant select_priv on ${dbName}.* to ${user1}"""
-    connect(user=user1, password="${pwd}", url=url) {
+    connect(user1, "${pwd}", url) {
         try {
             sql """grant select_priv on ${dbName}.* to ${user2}"""
             Assert.fail("can not grant to other user");
@@ -49,7 +49,7 @@ suite("test_grant_priv") {
 
     // test both have select_priv and grant_priv , can grant to other user
     sql """grant grant_priv on ${dbName}.* to ${user1}"""
-    connect(user=user1, password="${pwd}", url=url) {
+    connect(user1, "${pwd}", url) {
         try {
             sql """grant select_priv on ${dbName}.* to ${user2}"""
         } catch (Exception e) {
diff --git a/regression-test/suites/account_p0/test_grant_priv_resource.groovy b/regression-test/suites/account_p0/test_grant_priv_resource.groovy
index f679835e1ca96e..c8eea500a07dc5 100644
--- a/regression-test/suites/account_p0/test_grant_priv_resource.groovy
+++ b/regression-test/suites/account_p0/test_grant_priv_resource.groovy
@@ -33,7 +33,7 @@ suite("test_grant_priv_resource") {
 
     // test only have USAGE_PRIV, can not grant to other user
     sql """grant USAGE_PRIV on RESOURCE ${resource1} to ${user1}"""
-    connect(user=user1, password="${pwd}", url=url) {
+    connect(user1, "${pwd}", url) {
         try {
             sql """grant USAGE_PRIV on RESOURCE ${resource1} to ${user2}"""
             Assert.fail("can not grant to other user");
@@ -44,7 +44,7 @@ suite("test_grant_priv_resource") {
 
     // test both have USAGE_PRIV and grant_priv , can grant to other user
     sql """grant grant_priv on RESOURCE * to ${user1}"""
-    connect(user=user1, password="${pwd}", url=url) {
+    connect(user1, "${pwd}", url) {
         try {
             sql """grant USAGE_PRIV on RESOURCE ${resource1} to ${user2}"""
         } catch (Exception e) {
diff --git a/regression-test/suites/account_p0/test_grant_priv_workload.groovy b/regression-test/suites/account_p0/test_grant_priv_workload.groovy
index 291cfe1fa4cd02..12149f0fa7a8b7 100644
--- a/regression-test/suites/account_p0/test_grant_priv_workload.groovy
+++ b/regression-test/suites/account_p0/test_grant_priv_workload.groovy
@@ -33,7 +33,7 @@ suite("test_grant_priv_workload") {
 
     // test only have USAGE_PRIV, can not grant to other user
     sql """grant USAGE_PRIV on WORKLOAD GROUP ${workload1} to ${user1}"""
-    connect(user=user1, password="${pwd}", url=url) {
+    connect(user1, "${pwd}", url) {
         try {
             sql """grant USAGE_PRIV on WORKLOAD GROUP ${workload1} to ${user2}"""
             Assert.fail("can not grant to other user");
@@ -44,7 +44,7 @@ suite("test_grant_priv_workload") {
 
     // test both have USAGE_PRIV and grant_priv , can grant to other user
     sql """grant grant_priv on WORKLOAD GROUP ${workload1} to ${user1}"""
-    connect(user=user1, password="${pwd}", url=url) {
+    connect(user1, "${pwd}", url) {
         try {
            sql """grant USAGE_PRIV on WORKLOAD GROUP ${workload1} to ${user2}"""
         } catch (Exception e) {
diff --git a/regression-test/suites/account_p0/test_nereids_authentication.groovy b/regression-test/suites/account_p0/test_nereids_authentication.groovy
index a0d9c4b4514c08..c2ee5f12e8cb1d 100644
--- a/regression-test/suites/account_p0/test_nereids_authentication.groovy
+++ b/regression-test/suites/account_p0/test_nereids_authentication.groovy
@@ -54,26 +54,26 @@ suite("test_nereids_authentication", "query") {
     
     def tokens = context.config.jdbcUrl.split('/')
     def url=tokens[0] + "//" + tokens[2] + "/" + dbName + "?"
-    def result = connect(user=user, password='Doris_123456', url=url) {
+    def result = connect(user, 'Doris_123456', url) {
         sql "SELECT * FROM ${tableName1}"
     }
     assertEquals(result.size(), 0)
 
-    connect(user=user, password='Doris_123456', url=url) {
+    connect(user, 'Doris_123456', url) {
         test {
             sql "SELECT * FROM ${tableName2}"
             exception "denied"
         }
     }
 
-    connect(user=user, password='Doris_123456', url=url) {
+    connect(user, 'Doris_123456', url) {
         test {
             sql "SELECT count(*) FROM ${tableName2}"
             exception "denied"
         }
     }
 
-    connect(user=user, password='Doris_123456', url=url) {
+    connect(user, 'Doris_123456', url) {
         test {
             sql "SELECT * FROM ${tableName1}, ${tableName2} WHERE ${tableName1}.`key` = ${tableName2}.`key`"
             exception "denied"
@@ -81,14 +81,14 @@ suite("test_nereids_authentication", "query") {
     }
 
     sql "GRANT SELECT_PRIV ON internal.${dbName}.${tableName2} TO ${user}"
-    connect(user=user, password='Doris_123456', url=url) {
+    connect(user, 'Doris_123456', url) {
         sql "SELECT * FROM ${tableName2}"
     }
     assertEquals(result.size(), 0)
-    connect(user=user, password='Doris_123456', url=url) {
+    connect(user, 'Doris_123456', url) {
         sql "SELECT count(*) FROM ${tableName2}"
     }
-    connect(user=user, password='Doris_123456', url=url) {
+    connect(user, 'Doris_123456', url) {
         sql "SELECT * FROM ${tableName1}, ${tableName2} WHERE ${tableName1}.`key` = ${tableName2}.`key`"
     }
     assertEquals(result.size(), 0)
diff --git a/regression-test/suites/account_p0/test_nereids_row_policy.groovy b/regression-test/suites/account_p0/test_nereids_row_policy.groovy
index ea7cf99fb28840..38426dc09e6ef4 100644
--- a/regression-test/suites/account_p0/test_nereids_row_policy.groovy
+++ b/regression-test/suites/account_p0/test_nereids_row_policy.groovy
@@ -32,10 +32,10 @@ suite("test_nereids_row_policy") {
     }
 
     def assertQueryResult = { size ->
-        def result = connect(user=user, password='123abc!@#', url=url) {
+        def result = connect(user, '123abc!@#', url) {
             sql "SELECT * FROM ${tableName}"
         }
-        connect(user=user, password='123abc!@#', url=url) {
+        connect(user, '123abc!@#', url) {
             test {
                 sql "SELECT * FROM ${viewName}"
                 exception "does not have privilege for"
diff --git a/regression-test/suites/account_p0/test_role.groovy b/regression-test/suites/account_p0/test_role.groovy
index 6d88238ebf5f03..24b4e6859795b2 100644
--- a/regression-test/suites/account_p0/test_role.groovy
+++ b/regression-test/suites/account_p0/test_role.groovy
@@ -32,13 +32,13 @@ suite("test_role", "account") {
     sql """GRANT SELECT_PRIV ON ${context.config.defaultDb} TO ROLE '${role}'"""
     sql """GRANT SELECT_PRIV ON ${dbName} TO ROLE '${role}'"""
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}' DEFAULT ROLE '${role}'"""
-    def result1 = connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    def result1 = connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "show databases like '${dbName}'"
     }
     assertEquals(result1.size(), 1)
 
     sql """REVOKE SELECT_PRIV ON ${dbName} FROM ROLE '${role}'"""
-    def result2 = connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    def result2 = connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "show databases like '${dbName}'"
     }
     assertEquals(result2.size(), 0)
diff --git a/regression-test/suites/auth_call/test_account_management_grant_auth.groovy b/regression-test/suites/auth_call/test_account_management_grant_auth.groovy
index 28cb71cdeacadb..eff62e64f88e8a 100644
--- a/regression-test/suites/auth_call/test_account_management_grant_auth.groovy
+++ b/regression-test/suites/auth_call/test_account_management_grant_auth.groovy
@@ -42,7 +42,7 @@ suite("test_account_management_grant_auth","p0,auth_call") {
     sql """grant select_priv on regression_test to ${user}"""
     sql """create database ${dbName}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE USER ${user_derive} IDENTIFIED BY '${pwd}';"""
             exception "denied"
@@ -69,7 +69,7 @@ suite("test_account_management_grant_auth","p0,auth_call") {
         }
     }
     sql """grant grant_priv on *.*.* to '${user}'"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE USER ${user_derive} IDENTIFIED BY '${pwd}';"""
         sql """ALTER USER ${user_derive} IDENTIFIED BY "${pwd}";"""
         sql """SET PASSWORD FOR '${user_derive}' = PASSWORD('${pwd}')"""
@@ -85,7 +85,7 @@ suite("test_account_management_grant_auth","p0,auth_call") {
     }
     sql """revoke grant_priv on *.*.* from '${user}'"""
     sql """grant admin_priv on *.*.* to '${user}'"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE USER ${user_derive} IDENTIFIED BY '${pwd}';"""
         sql """ALTER USER ${user_derive} IDENTIFIED BY "${pwd}";"""
         sql """SET PASSWORD FOR '${user_derive}' = PASSWORD('${pwd}')"""
diff --git a/regression-test/suites/auth_call/test_account_management_role_auth.groovy b/regression-test/suites/auth_call/test_account_management_role_auth.groovy
index 2a90eeedf67bc6..c3b278100815f8 100644
--- a/regression-test/suites/auth_call/test_account_management_role_auth.groovy
+++ b/regression-test/suites/auth_call/test_account_management_role_auth.groovy
@@ -45,7 +45,7 @@ suite("test_account_management_role_auth","p0,auth_call") {
     sql """create database ${dbName}"""
     sql """GRANT '${role}' TO ${user};"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE ROLE ${role_derive}"""
             exception "denied"
@@ -60,7 +60,7 @@ suite("test_account_management_role_auth","p0,auth_call") {
         }
     }
     sql """grant grant_priv on *.*.* to ROLE '${role}'"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE ROLE ${role_derive}"""
         sql """ALTER ROLE ${role_derive} COMMENT "this is my first role";"""
         sql """DROP ROLE ${role_derive}"""
diff --git a/regression-test/suites/auth_call/test_account_management_user_auth.groovy b/regression-test/suites/auth_call/test_account_management_user_auth.groovy
index 936071b1e9d9ef..fe6d6805f47f9b 100644
--- a/regression-test/suites/auth_call/test_account_management_user_auth.groovy
+++ b/regression-test/suites/auth_call/test_account_management_user_auth.groovy
@@ -41,7 +41,7 @@ suite("test_account_management_user_auth","p0,auth_call") {
     sql """grant select_priv on regression_test to ${user}"""
     sql """create database ${dbName}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE USER ${user_derive} IDENTIFIED BY '${pwd}';"""
             exception "denied"
@@ -69,7 +69,7 @@ suite("test_account_management_user_auth","p0,auth_call") {
     }
     sql """grant grant_priv on *.*.* to '${user}'"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE USER ${user_derive} IDENTIFIED BY '${pwd}';"""
         sql """ALTER USER ${user_derive} IDENTIFIED BY "${pwd}";"""
         sql """SET PASSWORD FOR '${user_derive}' = PASSWORD('${pwd}')"""
@@ -85,7 +85,7 @@ suite("test_account_management_user_auth","p0,auth_call") {
     }
     sql """revoke grant_priv on *.*.* from '${user}'"""
     sql """grant admin_priv on *.*.* to '${user}'"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE USER ${user_derive} IDENTIFIED BY '${pwd}';"""
         sql """ALTER USER ${user_derive} IDENTIFIED BY "${pwd}";"""
         sql """SET PASSWORD FOR '${user_derive}' = PASSWORD('${pwd}')"""
diff --git a/regression-test/suites/auth_call/test_assistant_command_auth.groovy b/regression-test/suites/auth_call/test_assistant_command_auth.groovy
index 339bd86a6c1645..49a149a28ce51f 100644
--- a/regression-test/suites/auth_call/test_assistant_command_auth.groovy
+++ b/regression-test/suites/auth_call/test_assistant_command_auth.groovy
@@ -60,7 +60,7 @@ suite("test_assistant_command_auth","p0,auth_call") {
         );"""
 
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """use ${dbName}"""
             exception "denied"
@@ -81,13 +81,13 @@ suite("test_assistant_command_auth","p0,auth_call") {
     }
 
     sql """grant select_PRIV on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """DESC ${dbName}.${tableName} ALL;"""
     }
 
     sql """grant select_PRIV on ${catalogName}.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, password="${pwd}", context.config.jdbcUrl) {
         sql """REFRESH CATALOG ${catalogName};"""
     }
 
diff --git a/regression-test/suites/auth_call/test_cluster_management_auth.groovy b/regression-test/suites/auth_call/test_cluster_management_auth.groovy
index 396bcdc0f9a544..2061b9dbca773a 100644
--- a/regression-test/suites/auth_call/test_cluster_management_auth.groovy
+++ b/regression-test/suites/auth_call/test_cluster_management_auth.groovy
@@ -75,7 +75,7 @@ suite ("test_cluster_management_auth","nonConcurrent,p0,auth_call") {
 
     // pipeline can't support delete node, it can affect other case
     if (is_exists_follower()) {
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """show frontends"""
                 exception "denied"
@@ -92,7 +92,7 @@ suite ("test_cluster_management_auth","nonConcurrent,p0,auth_call") {
     }
 
     if (is_exists_observer()) {
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """show frontends"""
                 exception "denied"
@@ -109,7 +109,7 @@ suite ("test_cluster_management_auth","nonConcurrent,p0,auth_call") {
     }
 
     if (is_exists_backends()) {
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """show backends"""
                 exception "denied"
diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy
index ea10b0345244d8..de4a55ad5973cf 100644
--- a/regression-test/suites/auth_call/test_database_management_auth.groovy
+++ b/regression-test/suites/auth_call/test_database_management_auth.groovy
@@ -40,7 +40,7 @@ suite("test_database_management_auth","p0,auth_call") {
     sql """grant select_priv on regression_test to ${user}"""
     sql """create database ${dbName}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW FRONTEND CONFIG"""
             exception "denied"
diff --git a/regression-test/suites/auth_call/test_ddl_backup_auth.groovy b/regression-test/suites/auth_call/test_ddl_backup_auth.groovy
index 3e2d8d005d32ce..fdb2d6b2221973 100644
--- a/regression-test/suites/auth_call/test_ddl_backup_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_backup_auth.groovy
@@ -76,7 +76,7 @@ suite("test_ddl_backup_auth","p0,auth_call") {
             )"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """BACKUP SNAPSHOT ${dbName}.${backupLabelName}
                     TO ${repositoryName}
@@ -100,7 +100,7 @@ suite("test_ddl_backup_auth","p0,auth_call") {
         }
     }
     sql """grant LOAD_PRIV on ${dbName}.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """BACKUP SNAPSHOT ${dbName}.${backupLabelName}
                 TO ${repositoryName}
                 ON (${tableName})
diff --git a/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy b/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy
index 87c256c770e3b3..80f71a2e16f94b 100644
--- a/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy
@@ -43,7 +43,7 @@ suite("test_ddl_catalog_auth","p0,auth_call") {
     sql """grant select_priv on regression_test to ${user}"""
 
     // ddl create
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """create catalog if not exists ${catalogName} properties (
                     'type'='hms'
@@ -58,7 +58,7 @@ suite("test_ddl_catalog_auth","p0,auth_call") {
         );"""
     sql """grant Create_priv on ${catalogName}.*.* to ${user}"""
     sql """drop catalog ${catalogName}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """create catalog if not exists ${catalogName} properties (
             'type'='hms'
         );"""
@@ -69,14 +69,14 @@ suite("test_ddl_catalog_auth","p0,auth_call") {
 
     // ddl alter
     // user alter
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """ALTER CATALOG ${catalogName} RENAME ${catalogNameNew};"""
             exception "denied"
         }
     }
     sql """grant ALTER_PRIV on ${catalogName}.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """ALTER CATALOG ${catalogName} RENAME ${catalogNameNew};"""
         test {
             sql """show create catalog ${catalogNameNew}"""
@@ -87,21 +87,21 @@ suite("test_ddl_catalog_auth","p0,auth_call") {
     }
     // root alter
     sql """ALTER CATALOG ${catalogNameNew} RENAME ${catalogName};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """show create catalog ${catalogName}"""
         def ctl_res = sql """show catalogs;"""
         assertTrue(ctl_res.size() == 2)
     }
 
     // ddl drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """drop CATALOG ${catalogName};"""
             exception "denied"
         }
     }
     sql """grant DROP_PRIV on ${catalogName}.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """drop CATALOG ${catalogName};"""
         def ctl_res = sql """show catalogs;"""
         assertTrue(ctl_res.size() == 1)
diff --git a/regression-test/suites/auth_call/test_ddl_colocate_group_auth.groovy b/regression-test/suites/auth_call/test_ddl_colocate_group_auth.groovy
index 3c243facdb4fbb..3d393a31db97dd 100644
--- a/regression-test/suites/auth_call/test_ddl_colocate_group_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_colocate_group_auth.groovy
@@ -45,7 +45,7 @@ suite("test_ddl_colocate_group_auth","p0,auth_call") {
             );"""
 
     // ddl create
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """ALTER COLOCATE GROUP ${dbName}.${colocateGroupName}
                     SET (
@@ -55,7 +55,7 @@ suite("test_ddl_colocate_group_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """ALTER COLOCATE GROUP ${dbName}.${colocateGroupName}
                 SET (
                    "replication_num"="1"
diff --git a/regression-test/suites/auth_call/test_ddl_database_auth.groovy b/regression-test/suites/auth_call/test_ddl_database_auth.groovy
index 80e6b1b6e76695..2821a375ddb559 100644
--- a/regression-test/suites/auth_call/test_ddl_database_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_database_auth.groovy
@@ -40,7 +40,7 @@ suite("test_ddl_database_auth","p0,auth_call") {
     sql """grant select_priv on regression_test to ${user}"""
 
     // ddl create
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """create database ${dbName};"""
             exception "denied"
@@ -51,7 +51,7 @@ suite("test_ddl_database_auth","p0,auth_call") {
     sql """create database ${dbName};"""
     sql """grant Create_priv on ${dbName}.* to ${user}"""
     sql """drop database ${dbName};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """create database ${dbName};"""
         sql """show create database ${dbName}"""
         def db_res = sql """show databases;"""
@@ -60,14 +60,14 @@ suite("test_ddl_database_auth","p0,auth_call") {
 
     // ddl alter
     // user alter
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """ALTER database ${dbName} RENAME ${dbNameNew};"""
             exception "denied"
         }
     }
     sql """grant ALTER_PRIV on ${dbName}.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """ALTER database ${dbName} RENAME ${dbNameNew};"""
         test {
             sql """show create database ${dbNameNew}"""
@@ -78,21 +78,21 @@ suite("test_ddl_database_auth","p0,auth_call") {
     }
     // root alter
     sql """ALTER database ${dbNameNew} RENAME ${dbName};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """show create database ${dbName}"""
         def db_res = sql """show databases;"""
         assertTrue(db_res.size() == 4 || db_res.size() == 2)
     }
 
     // ddl drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """drop database ${dbName};"""
             exception "denied"
         }
     }
     sql """grant DROP_PRIV on ${dbName}.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """drop database ${dbName};"""
         def ctl_res = sql """show databases;"""
         assertTrue(ctl_res.size() == 3 || ctl_res.size() == 1)
diff --git a/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy b/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy
index e24c88d55864b5..0749bdab71780a 100644
--- a/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy
@@ -38,7 +38,7 @@ suite("test_ddl_encryptkey_auth","p0,auth_call") {
     sql """create database ${dbName}"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE ENCRYPTKEY ${encryptkeyName} AS "ABCD123456789";"""
             exception "denied"
@@ -53,7 +53,7 @@ suite("test_ddl_encryptkey_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """CREATE ENCRYPTKEY ${encryptkeyName} AS "ABCD123456789";"""
         def res = sql """SHOW ENCRYPTKEYS FROM ${dbName}"""
diff --git a/regression-test/suites/auth_call/test_ddl_file_auth.groovy b/regression-test/suites/auth_call/test_ddl_file_auth.groovy
index 73aa9204b54c02..e32c26ecc220d0 100644
--- a/regression-test/suites/auth_call/test_ddl_file_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_file_auth.groovy
@@ -42,7 +42,7 @@ suite("test_ddl_file_auth","p0,auth_call") {
     def dataFilePath = "https://"+"${bucket}"+"."+"${s3_endpoint}"+"/regression/auth_test.key"
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE FILE "${fileName}" IN ${dbName}
                 PROPERTIES
@@ -62,13 +62,13 @@ suite("test_ddl_file_auth","p0,auth_call") {
         }
     }
     sql """grant select_priv on ${dbName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """SHOW FILE FROM ${dbName};"""
     }
     sql """revoke select_priv on ${dbName} from ${user}"""
 
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         checkNereidsExecute("""CREATE FILE "${fileName}" IN ${dbName}
                 PROPERTIES
                 (
diff --git a/regression-test/suites/auth_call/test_ddl_function_auth.groovy b/regression-test/suites/auth_call/test_ddl_function_auth.groovy
index f8fd51cf1cf641..2fa524bf424bd0 100644
--- a/regression-test/suites/auth_call/test_ddl_function_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_function_auth.groovy
@@ -40,7 +40,7 @@ suite("test_ddl_function_auth","p0,auth_call") {
     sql """grant select_priv on ${dbName}.* to ${user}"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE ALIAS FUNCTION ${dbName}.${functionName}(INT) WITH PARAMETER(id)  AS CONCAT(LEFT(id, 3), '****', RIGHT(id, 4));"""
             exception "denied"
@@ -56,7 +56,7 @@ suite("test_ddl_function_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE ALIAS FUNCTION ${dbName}.${functionName}(INT) WITH PARAMETER(id)  AS CONCAT(LEFT(id, 3), '****', RIGHT(id, 4));"""
         sql """use ${dbName}"""
         def res = sql """show functions"""
@@ -74,7 +74,7 @@ suite("test_ddl_function_auth","p0,auth_call") {
     sql """use ${dbName}"""
     def func_res = sql """show functions"""
     assertTrue(func_res.size() == 1)
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         def res = sql """SHOW CREATE FUNCTION ${dbName}.${functionName}(INT)"""
         logger.info("res: " + res)
diff --git a/regression-test/suites/auth_call/test_ddl_index_auth.groovy b/regression-test/suites/auth_call/test_ddl_index_auth.groovy
index baa8d79f8c79c8..53ba3a0d4b3123 100644
--- a/regression-test/suites/auth_call/test_ddl_index_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_index_auth.groovy
@@ -80,7 +80,7 @@ suite("test_ddl_index_auth","p0,auth_call") {
     assertTrue(index_res_tmp.size() == 0)
 
     // ddl create
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE INDEX IF NOT EXISTS ${indexName} ON ${dbName}.${tableName} (username3) USING INVERTED COMMENT 'balabala';"""
             exception "denied"
@@ -97,7 +97,7 @@ suite("test_ddl_index_auth","p0,auth_call") {
         }
     }
     sql """grant ALTER_PRIV on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def ss = sql """show grants"""
         logger.info("ss:" + ss)
         sql """use ${dbName}"""
diff --git a/regression-test/suites/auth_call/test_ddl_job_auth.groovy b/regression-test/suites/auth_call/test_ddl_job_auth.groovy
index 376cf8efbe90a6..45798191e48568 100644
--- a/regression-test/suites/auth_call/test_ddl_job_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_job_auth.groovy
@@ -50,7 +50,7 @@ suite("test_ddl_job_auth","p0,auth_call") {
     sql """create table ${dbName}.${tableNameDst} like ${dbName}.${tableName}"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE JOB ${jobName} ON SCHEDULE AT '2020-01-01 00:00:00' DO INSERT INTO ${dbName}.${tableNameDst} SELECT * FROM ${dbName}.${tableName};"""
             exception "denied"
@@ -75,7 +75,7 @@ suite("test_ddl_job_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE JOB ${jobName} ON SCHEDULE AT '2100-01-01 00:00:00' DO INSERT INTO ${dbName}.${tableNameDst} SELECT * FROM ${dbName}.${tableName};"""
         def res = sql """select * from jobs("type"="insert") where Name="${jobName}";"""
         assertTrue(res.size() == 1)
diff --git a/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy b/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy
index 319eec2963ee4d..590e75781f19f2 100644
--- a/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy
@@ -55,14 +55,14 @@ suite("test_ddl_mask_view_auth","p0,auth_call") {
                 AS
                 SELECT mask(id) as k1, mask(username) as v1 FROM ${dbName}.${tableName} GROUP BY k1, v1;"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """select * from ${dbName}.${viewName};"""
             exception "denied"
         }
     }
     sql """grant select_PRIV on ${dbName}.${viewName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """select * from ${dbName}.${viewName};"""
         assertTrue(res[0][0] == "n")
         assertTrue(res[0][1] == "nnnxxxXXX")
diff --git a/regression-test/suites/auth_call/test_ddl_mtmv_auth.groovy b/regression-test/suites/auth_call/test_ddl_mtmv_auth.groovy
index 3aa146f58890e5..4db2177ee6ce18 100644
--- a/regression-test/suites/auth_call/test_ddl_mtmv_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_mtmv_auth.groovy
@@ -48,7 +48,7 @@ suite("test_ddl_mtmv_auth","p0,auth_call") {
             );"""
 
     // ddl create
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE MATERIALIZED VIEW ${dbName}.${mtmvName} 
                 BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
@@ -64,7 +64,7 @@ suite("test_ddl_mtmv_auth","p0,auth_call") {
                 PROPERTIES ('replication_num' = '1') 
                 AS select username, sum(id) as sum_id from ${dbName}.${tableName} group by username"""
     sql """grant Create_priv on ${dbName}.${mtmvName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE MATERIALIZED VIEW ${dbName}.${mtmvName} 
                 BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
@@ -76,7 +76,7 @@ suite("test_ddl_mtmv_auth","p0,auth_call") {
     }
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
     sql """drop MATERIALIZED VIEW ${dbName}.${mtmvName};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE MATERIALIZED VIEW ${dbName}.${mtmvName} 
             BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
             DISTRIBUTED BY RANDOM BUCKETS 1 
@@ -88,7 +88,7 @@ suite("test_ddl_mtmv_auth","p0,auth_call") {
         assertTrue(tb_res.size() == 2)
     }
     sql """revoke select_priv on ${dbName}.${tableName} from ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """refresh MATERIALIZED VIEW ${mtmvName} auto;"""
 
@@ -122,7 +122,7 @@ suite("test_ddl_mtmv_auth","p0,auth_call") {
     // ddl alter
     // user alter
     sql """revoke Create_priv on ${dbName}.${mtmvName} from ${user};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """ALTER MATERIALIZED VIEW ${mtmvName} rename ${mtmvNameNew};"""
             exception "denied"
@@ -133,7 +133,7 @@ suite("test_ddl_mtmv_auth","p0,auth_call") {
         }
     }
     sql """grant ALTER_PRIV on ${dbName}.${mtmvName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """show create materialized view ${mtmvName}"""
         sql """ALTER MATERIALIZED VIEW ${mtmvName} rename ${mtmvNameNew};"""
@@ -148,7 +148,7 @@ suite("test_ddl_mtmv_auth","p0,auth_call") {
     // root alter
     sql """use ${dbName}"""
     sql """ALTER MATERIALIZED VIEW ${mtmvNameNew} RENAME ${mtmvName};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """show create materialized view ${mtmvName}"""
         def db_res = sql """show tables;"""
@@ -156,20 +156,20 @@ suite("test_ddl_mtmv_auth","p0,auth_call") {
     }
 
     // dml select
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """select username from ${dbName}.${mtmvName}"""
             exception "denied"
         }
     }
     sql """grant select_priv(username) on ${dbName}.${mtmvName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """select username from ${dbName}.${mtmvName}"""
     }
     sql """revoke select_priv(username) on ${dbName}.${mtmvName} from ${user}"""
 
     // ddl drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """use ${dbName}"""
             sql """drop materialized view ${mtmvName};"""
@@ -177,7 +177,7 @@ suite("test_ddl_mtmv_auth","p0,auth_call") {
         }
     }
     sql """grant DROP_PRIV on ${dbName}.${mtmvName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """drop materialized view ${mtmvName};"""
         def ctl_res = sql """show tables;"""
diff --git a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
index 485343df67392c..4dbf54fdf0df72 100644
--- a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
@@ -54,7 +54,7 @@ suite("test_ddl_mv_auth","p0,auth_call") {
         """
 
     // ddl create
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """create materialized view ${mvName} as select username from ${dbName}.${tableName};"""
             exception "denied"
@@ -65,7 +65,7 @@ suite("test_ddl_mv_auth","p0,auth_call") {
         }
     }
     sql """grant select_priv(username) on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         test {
             sql """create materialized view ${mvName} as select username from ${dbName}.${tableName};"""
@@ -77,7 +77,7 @@ suite("test_ddl_mv_auth","p0,auth_call") {
         }
     }
     sql """grant alter_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """create materialized view ${mvName} as select username from ${dbName}.${tableName};"""
         waitingMVTaskFinishedByMvName(dbName, tableName)
@@ -92,7 +92,7 @@ suite("test_ddl_mv_auth","p0,auth_call") {
     sql """revoke select_priv(username) on ${dbName}.${tableName} from ${user}"""
 
     // ddl drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName} ON ${dbName}.${tableName};"""
             exception "denied"
@@ -103,7 +103,7 @@ suite("test_ddl_mv_auth","p0,auth_call") {
         }
     }
     sql """grant alter_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName} ON ${tableName};"""
         sql """ALTER TABLE ${dbName}.${tableName} DROP ROLLUP ${rollupName};"""
diff --git a/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy b/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy
index 0d1bfb8551fb11..34e4766e19e5e2 100644
--- a/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy
@@ -52,7 +52,7 @@ suite("test_ddl_part_table_auth","p0,auth_call") {
     logger.info("insert_res: " + insert_res)
 
     def partition_info = sql """show partitions from ${dbName}.${tableName}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """show partitions from ${dbName}.${tableName}"""
             exception "denied"
@@ -63,14 +63,14 @@ suite("test_ddl_part_table_auth","p0,auth_call") {
         }
     }
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """show partitions from ${dbName}.${tableName}"""
         sql """show query stats"""
     }
     sql """revoke select_priv on ${dbName}.${tableName} from ${user}"""
 
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """show partition ${partition_info[0][0]}"""
         assertTrue(res.size() == 1)
     }
diff --git a/regression-test/suites/auth_call/test_ddl_policy_storage_auth.groovy b/regression-test/suites/auth_call/test_ddl_policy_storage_auth.groovy
index 0b79c6d7f08d3a..59b7bf2bc6e2c7 100644
--- a/regression-test/suites/auth_call/test_ddl_policy_storage_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_policy_storage_auth.groovy
@@ -53,7 +53,7 @@ suite("test_ddl_policy_storage_auth","p0,auth_call") {
             );"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use regression_test;"""
         test {
             sql """CREATE STORAGE POLICY ${storagePolicyName}
@@ -77,7 +77,7 @@ suite("test_ddl_policy_storage_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE STORAGE POLICY ${storagePolicyName}
                 PROPERTIES(
                   "storage_resource" = "${resourceName}",
diff --git a/regression-test/suites/auth_call/test_ddl_repository_auth.groovy b/regression-test/suites/auth_call/test_ddl_repository_auth.groovy
index 8e127778f5680e..bfd1d9cf3008f4 100644
--- a/regression-test/suites/auth_call/test_ddl_repository_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_repository_auth.groovy
@@ -42,7 +42,7 @@ suite("test_ddl_repository_auth","p0,auth_call") {
     sql """create database ${dbName}"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE REPOSITORY `${repositoryName}`
                     WITH S3
@@ -67,7 +67,7 @@ suite("test_ddl_repository_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE REPOSITORY `${repositoryName}`
                 WITH S3
                 ON LOCATION "s3://${bucket}/${repositoryName}"
diff --git a/regression-test/suites/auth_call/test_ddl_resource_auth.groovy b/regression-test/suites/auth_call/test_ddl_resource_auth.groovy
index a9d64ed901735f..25cd53d8e19fe7 100644
--- a/regression-test/suites/auth_call/test_ddl_resource_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_resource_auth.groovy
@@ -36,7 +36,7 @@ suite("test_ddl_resource_auth","p0,auth_call") {
     sql """create database ${dbName}"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE RESOURCE IF NOT EXISTS "${resourceName}"
                     PROPERTIES(
@@ -68,7 +68,7 @@ suite("test_ddl_resource_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE RESOURCE IF NOT EXISTS "${resourceName}"
                 PROPERTIES(
                     "type" = "s3",
diff --git a/regression-test/suites/auth_call/test_ddl_restore_auth.groovy b/regression-test/suites/auth_call/test_ddl_restore_auth.groovy
index 91b7c1378fd9a0..635c57a585acf8 100644
--- a/regression-test/suites/auth_call/test_ddl_restore_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_restore_auth.groovy
@@ -117,14 +117,14 @@ suite("test_ddl_restore_auth","p0,auth_call") {
     sql """truncate table ${dbName}.`${tableName}`"""
 
     sql """grant admin_PRIV on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def show_snapshot_res = sql """SHOW SNAPSHOT ON ${repositoryName};"""
         logger.info("show_snapshot_res: " + show_snapshot_res)
     }
     sql """revoke admin_PRIV on *.*.* from ${user}"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW SNAPSHOT ON ${repositoryName};"""
             exception "denied"
@@ -154,7 +154,7 @@ suite("test_ddl_restore_auth","p0,auth_call") {
         }
     }
     sql """grant LOAD_PRIV on ${dbName}.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """RESTORE SNAPSHOT ${dbName}.`${restoreLabelName}`
                 FROM `${repositoryName}`
                 ON ( `${tableName}` )
diff --git a/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy b/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy
index 106e22d20e9024..af1e074f8d7d8c 100644
--- a/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy
@@ -48,7 +48,7 @@ suite("test_ddl_row_policy_auth","p0,auth_call") {
             );"""
 
     // ddl create
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE ROW POLICY ${rowPolicyName} ON ${dbName}.${tableName} AS RESTRICTIVE TO ${user} USING (id = 1);"""
             exception "denied"
@@ -64,7 +64,7 @@ suite("test_ddl_row_policy_auth","p0,auth_call") {
 
     }
     sql """grant grant_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE ROW POLICY ${rowPolicyName} ON ${dbName}.${tableName} AS RESTRICTIVE TO ${user} USING (id = 1);"""
 
         test {
@@ -75,7 +75,7 @@ suite("test_ddl_row_policy_auth","p0,auth_call") {
         sql """DROP ROW POLICY ${rowPolicyName} on ${dbName}.${tableName} for ${user}"""
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE ROW POLICY ${rowPolicyName} ON ${dbName}.${tableName} AS RESTRICTIVE TO ${user} USING (id = 1);"""
         def res = sql """SHOW ROW POLICY FOR ${user}"""
         assertTrue(res.size() == 1)
diff --git a/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy b/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy
index f5693686fada17..3941897a5e9dd5 100644
--- a/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy
@@ -48,7 +48,7 @@ suite("test_ddl_sql_block_rule_auth","p0,auth_call") {
             );"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE SQL_BLOCK_RULE ${sqlBlockRuleName} 
                     PROPERTIES(
@@ -75,7 +75,7 @@ suite("test_ddl_sql_block_rule_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE SQL_BLOCK_RULE ${sqlBlockRuleName} 
                     PROPERTIES(
                       "sql"="select \\\\* from ${dbName}\\\\.${tableName}",
diff --git a/regression-test/suites/auth_call/test_ddl_table_auth.groovy b/regression-test/suites/auth_call/test_ddl_table_auth.groovy
index 4c74699f55fcbf..c96aeb0d5e2cad 100644
--- a/regression-test/suites/auth_call/test_ddl_table_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_table_auth.groovy
@@ -68,7 +68,7 @@ suite("test_ddl_table_auth","p0,auth_call") {
     }
 
     // ddl create
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """create table ${dbName}.${tableName} (
                     id BIGINT,
@@ -98,7 +98,7 @@ suite("test_ddl_table_auth","p0,auth_call") {
         );"""
     sql """grant Create_priv on ${dbName}.${tableName} to ${user}"""
     sql """drop table ${dbName}.${tableName};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """create table ${dbName}.${tableName} (
                     id BIGINT,
                     username VARCHAR(20)
@@ -119,7 +119,7 @@ suite("test_ddl_table_auth","p0,auth_call") {
 
     // ddl alter
     // user alter
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """ALTER table ${tableName} RENAME ${tableNameNew};"""
             exception "denied"
@@ -132,7 +132,7 @@ suite("test_ddl_table_auth","p0,auth_call") {
         assertTrue(res.size() == 0)
     }
     sql """grant ALTER_PRIV on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """ALTER table ${tableName} RENAME ${tableNameNew};"""
 
@@ -146,7 +146,7 @@ suite("test_ddl_table_auth","p0,auth_call") {
     // root alter
     sql """use ${dbName}"""
     sql """ALTER table ${tableNameNew} RENAME ${tableName};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """show create table ${tableName}"""
         def db_res = sql """show tables;"""
@@ -154,7 +154,7 @@ suite("test_ddl_table_auth","p0,auth_call") {
     }
 
     // show
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """ALTER TABLE ${tableName} ADD COLUMN new_col INT KEY DEFAULT "0";"""
         def res = sql """SHOW ALTER TABLE COLUMN;"""
@@ -162,20 +162,20 @@ suite("test_ddl_table_auth","p0,auth_call") {
     }
 
     // dml select
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """select id from ${dbName}.${tableName}"""
             exception "denied"
         }
     }
     sql """grant select_priv(id) on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """select id from ${dbName}.${tableName}"""
     }
     sql """revoke select_priv(id) on ${dbName}.${tableName} from ${user}"""
 
     // ddl create table like
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """create table ${cteLikeDstDb}.${cteLikeDstTb} like ${dbName}.${tableName};"""
             exception "denied"
@@ -185,21 +185,21 @@ suite("test_ddl_table_auth","p0,auth_call") {
     sql """create table ${cteLikeDstDb}.${cteLikeDstTb} like ${dbName}.${tableName};"""
     sql """grant Create_priv on ${cteLikeDstDb}.${cteLikeDstTb} to ${user}"""
     sql """drop table ${cteLikeDstDb}.${cteLikeDstTb};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """create table ${cteLikeDstDb}.${cteLikeDstTb} like ${dbName}.${tableName};"""
             exception "denied"
         }
     }
     sql """grant SELECT_PRIV on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """create table ${cteLikeDstDb}.${cteLikeDstTb} like ${dbName}.${tableName};"""
     }
     sql """revoke SELECT_PRIV on ${dbName}.${tableName} from ${user}"""
 
     // ddl create table select
     sql """create database ${cteSelectDstDb}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """create table ${cteSelectDstDb}.${cteSelectDstTb}(username) PROPERTIES("replication_num" = "1") as select username from ${dbName}.${tableName};"""
             exception "denied"
@@ -208,7 +208,7 @@ suite("test_ddl_table_auth","p0,auth_call") {
     sql """create table ${cteSelectDstDb}.${cteSelectDstTb}(username) PROPERTIES("replication_num" = "1") as select username from ${dbName}.${tableName};"""
     sql """grant Create_priv on ${cteSelectDstDb}.${cteSelectDstTb} to ${user}"""
     sql """drop table ${cteSelectDstDb}.${cteSelectDstTb}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """create table ${cteSelectDstDb}.${cteSelectDstTb}(username) PROPERTIES("replication_num" = "1") as select username from ${dbName}.${tableName};"""
             exception "denied"
@@ -217,20 +217,20 @@ suite("test_ddl_table_auth","p0,auth_call") {
     sql """create table ${cteSelectDstDb}.${cteSelectDstTb}(username) PROPERTIES("replication_num" = "1") as select username from ${dbName}.${tableName};"""
     sql """grant LOAD_PRIV on ${cteSelectDstDb}.${cteSelectDstTb} to ${user}"""
     sql """drop table ${cteSelectDstDb}.${cteSelectDstTb}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """create table ${cteSelectDstDb}.${cteSelectDstTb}(username) PROPERTIES("replication_num" = "1") as select username from ${dbName}.${tableName};"""
             exception "denied"
         }
     }
     sql """grant select_priv(username) on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """create table ${cteSelectDstDb}.${cteSelectDstTb}(username) PROPERTIES("replication_num" = "1") as select username from ${dbName}.${tableName};"""
     }
 
     waitingChangeTaskFinished(dbName)
     // ddl truncate
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """use ${dbName}"""
             sql """truncate table ${tableName};"""
@@ -238,13 +238,13 @@ suite("test_ddl_table_auth","p0,auth_call") {
         }
     }
     sql """grant LOAD_PRIV on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """truncate table ${tableName};"""
     }
 
     // ddl drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """use ${dbName}"""
             sql """drop table ${tableName};"""
@@ -252,7 +252,7 @@ suite("test_ddl_table_auth","p0,auth_call") {
         }
     }
     sql """grant DROP_PRIV on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """drop table ${tableName};"""
         def ctl_res = sql """show tables;"""
diff --git a/regression-test/suites/auth_call/test_ddl_view_auth.groovy b/regression-test/suites/auth_call/test_ddl_view_auth.groovy
index bf967e54fcb02c..05f263ada20066 100644
--- a/regression-test/suites/auth_call/test_ddl_view_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_view_auth.groovy
@@ -53,7 +53,7 @@ suite("test_ddl_view_auth","p0,auth_call") {
         """
 
     // ddl create
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE VIEW ${dbName}.${viewName} (k1, v1)
                 AS
@@ -67,7 +67,7 @@ suite("test_ddl_view_auth","p0,auth_call") {
         }
     }
     sql """grant select_priv(id) on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE VIEW ${dbName}.${viewName} (k1, v1)
                 AS
@@ -84,7 +84,7 @@ suite("test_ddl_view_auth","p0,auth_call") {
             WHERE id = 1 GROUP BY k1;"""
     sql """grant Create_priv on ${dbName}.${viewName} to ${user}"""
     sql """drop view ${dbName}.${viewName}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE VIEW ${dbName}.${viewName} (k1, v1)
             AS
             SELECT id as k1, SUM(id) FROM ${dbName}.${tableName}
@@ -93,7 +93,7 @@ suite("test_ddl_view_auth","p0,auth_call") {
         def res = sql """SHOW VIEW from ${tableName} from ${dbName}"""
         assertTrue(res.size() == 1)
     }
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE VIEW ${dbName}.${viewName} (k1, v1)
                 AS
@@ -105,7 +105,7 @@ suite("test_ddl_view_auth","p0,auth_call") {
 
     // ddl alter
     // user alter
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """alter VIEW ${dbName}.${viewName} (k1, v1)
                 AS
@@ -115,7 +115,7 @@ suite("test_ddl_view_auth","p0,auth_call") {
         }
     }
     sql """grant ALTER_PRIV on ${dbName}.${viewName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """alter VIEW ${dbName}.${viewName} (k1, v1)
                 AS
                 SELECT id as k1, SUM(id) FROM ${dbName}.${tableName}
@@ -123,26 +123,26 @@ suite("test_ddl_view_auth","p0,auth_call") {
     }
 
     // dml show
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """select * from ${dbName}.${viewName};"""
             exception "denied"
         }
     }
     sql """grant select_PRIV on ${dbName}.${viewName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """select * from ${dbName}.${viewName};"""
     }
 
     // ddl drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """drop table ${dbName}.${viewName};"""
             exception 'denied'
         }
     }
     sql """grant DROP_PRIV on ${dbName}.${viewName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """drop view ${viewName};"""
         def ctl_res = sql """show tables;"""
diff --git a/regression-test/suites/auth_call/test_ddl_workload_group_auth.groovy b/regression-test/suites/auth_call/test_ddl_workload_group_auth.groovy
index 7657055d9d4fcb..32589515c6baa2 100644
--- a/regression-test/suites/auth_call/test_ddl_workload_group_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_workload_group_auth.groovy
@@ -39,7 +39,7 @@ suite("test_ddl_workload_group_auth","p0,auth_call") {
     sql """create database ${dbName}"""
 
     // ddl create,show,drop
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """CREATE WORKLOAD GROUP "${workloadGroupName}"
                     PROPERTIES (
@@ -64,7 +64,7 @@ suite("test_ddl_workload_group_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE WORKLOAD GROUP "${workloadGroupName}"
                 PROPERTIES (
                     "cpu_share"="10"
diff --git a/regression-test/suites/auth_call/test_dml_analyze_auth.groovy b/regression-test/suites/auth_call/test_dml_analyze_auth.groovy
index 5db5da61f945f0..59706f140e69b1 100644
--- a/regression-test/suites/auth_call/test_dml_analyze_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_analyze_auth.groovy
@@ -48,7 +48,7 @@ suite("test_dml_analyze_auth","p0,auth_call") {
                 "replication_num" = "1"
             );"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """
                 analyze table ${dbName}.${tableName} with sync;
@@ -69,7 +69,7 @@ suite("test_dml_analyze_auth","p0,auth_call") {
         }
     }
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """
             analyze table ${dbName}.${tableName} with sync;
             """
@@ -80,7 +80,7 @@ suite("test_dml_analyze_auth","p0,auth_call") {
         sql """show table stats ${dbName}.${tableName};"""
     }
     sql """grant select_priv on ${dbName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """show table status from ${dbName};"""
     }
 
diff --git a/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy b/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy
index 524513a570c0dd..3ec26146699bce 100644
--- a/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy
@@ -68,7 +68,7 @@ suite("test_dml_broker_load_auth","p0,auth_call") {
             );"""
 
     sql """use ${dbName}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """
             LOAD LABEL ${loadLabelName} (
@@ -104,7 +104,7 @@ suite("test_dml_broker_load_auth","p0,auth_call") {
         }
     }
     sql """grant load_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName};"""
         sql """
         LOAD LABEL ${loadLabelName} (
@@ -149,7 +149,7 @@ suite("test_dml_broker_load_auth","p0,auth_call") {
     }
 
     sql """grant load_priv on ${dbName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """SHOW LOAD FROM ${dbName} WHERE LABEL LIKE '${loadLabelName}'"""
         logger.info("res: " + res)
         assertTrue(res.size() == 1)
diff --git a/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy b/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy
index 84c5a5ef5ba20e..f1aa5d0fa95105 100644
--- a/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy
@@ -35,7 +35,7 @@ suite("test_dml_cancel_profile_auth","p0,auth_call,nonConcurrent") {
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """
                 CLEAN ALL PROFILE;
@@ -44,7 +44,7 @@ suite("test_dml_cancel_profile_auth","p0,auth_call,nonConcurrent") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         checkNereidsExecute("CLEAN ALL PROFILE")
         sql """
             CLEAN ALL PROFILE;
diff --git a/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy b/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy
index 541902ebd68b92..d325250f2ce696 100644
--- a/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy
@@ -54,7 +54,7 @@ suite("test_dml_delete_table_auth","p0,auth_call") {
         (3, "333");
         """
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """DELETE FROM ${dbName}.${tableName} WHERE id = 3;"""
             exception "denied"
@@ -64,7 +64,7 @@ suite("test_dml_delete_table_auth","p0,auth_call") {
         assertTrue(del_res.size() == 0)
     }
     sql """grant load_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """DELETE FROM ${dbName}.${tableName} WHERE id = 3;"""
             exception "denied"
@@ -73,7 +73,7 @@ suite("test_dml_delete_table_auth","p0,auth_call") {
         assertTrue(del_res.size() == 0)
     }
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """DELETE FROM ${dbName}.${tableName} WHERE id = 3;"""
         def del_res = sql """show DELETE from ${dbName}"""
         logger.info("del_res: " + del_res)
@@ -114,7 +114,7 @@ suite("test_dml_delete_table_auth","p0,auth_call") {
         (4),
         (5);"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """DELETE FROM ${dbName}.${tableName1} 
                 USING ${dbName}.${tableName2} INNER JOIN ${dbName}.${tableName3} 
@@ -124,7 +124,7 @@ suite("test_dml_delete_table_auth","p0,auth_call") {
         }
     }
     sql """grant load_priv on ${dbName}.${tableName1} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """DELETE FROM ${dbName}.${tableName1} 
                 USING ${dbName}.${tableName2} INNER JOIN ${dbName}.${tableName3} 
@@ -136,7 +136,7 @@ suite("test_dml_delete_table_auth","p0,auth_call") {
     sql """grant select_priv on ${dbName}.${tableName1} to ${user}"""
     sql """grant select_priv on ${dbName}.${tableName2} to ${user}"""
     sql """grant select_priv on ${dbName}.${tableName3} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """DELETE FROM ${dbName}.${tableName1} 
             USING ${dbName}.${tableName2} INNER JOIN ${dbName}.${tableName3} 
             ON ${dbName}.${tableName2}.id = ${dbName}.${tableName3}.id
diff --git a/regression-test/suites/auth_call/test_dml_export_table_auth.groovy b/regression-test/suites/auth_call/test_dml_export_table_auth.groovy
index 8d13bc4bddb71b..12812fdf0dde09 100644
--- a/regression-test/suites/auth_call/test_dml_export_table_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_export_table_auth.groovy
@@ -66,7 +66,7 @@ suite("test_dml_export_table_auth","p0,auth_call") {
     String region = getS3Region()
     String bucket = context.config.otherConfigs.get("s3BucketName");
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """EXPORT TABLE ${dbName}.${tableName} TO "s3://${bucket}/test_outfile/exp_${exportLabel}"
                 PROPERTIES(
@@ -91,7 +91,7 @@ suite("test_dml_export_table_auth","p0,auth_call") {
         }
     }
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """EXPORT TABLE ${dbName}.${tableName} TO "s3://${bucket}/test_outfile/exp_${exportLabel}"
                 PROPERTIES(
                     "format" = "csv",
@@ -109,7 +109,7 @@ suite("test_dml_export_table_auth","p0,auth_call") {
         assertTrue(res.size() == 1)
     }
     sql """grant select_priv on ${dbName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         def res = sql """show export;"""
         logger.info("res: " + res)
diff --git a/regression-test/suites/auth_call/test_dml_insert_auth.groovy b/regression-test/suites/auth_call/test_dml_insert_auth.groovy
index 6a04281b1c0aeb..6cfe66cb10c6d5 100644
--- a/regression-test/suites/auth_call/test_dml_insert_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_insert_auth.groovy
@@ -63,7 +63,7 @@ suite("test_dml_insert_auth","p0,auth_call") {
                    (3, "333")
             """
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def insert_res = sql """SHOW LAST INSERT"""
         logger.info("insert_res: " + insert_res)
         test {
@@ -83,7 +83,7 @@ suite("test_dml_insert_auth","p0,auth_call") {
         }
     }
     sql """grant load_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """
                 INSERT INTO ${dbName}.${tableName} (id, username)
                 VALUES (1, "111"),
@@ -101,7 +101,7 @@ suite("test_dml_insert_auth","p0,auth_call") {
     assertTrue(rows[0][0] == 3)
 
     // insert overwrite
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """
             INSERT OVERWRITE table ${dbName}.${tableName} VALUES (4, "444");
             """
@@ -110,7 +110,7 @@ suite("test_dml_insert_auth","p0,auth_call") {
     assertTrue(rows[0][0] == 1)
 
     sql """grant select_priv on ${dbName}.${srcTableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """INSERT OVERWRITE table ${dbName}.${tableName} SELECT * FROM ${dbName}.${srcTableName};"""
     }
     rows = sql """select count() from ${dbName}.${tableName}"""
diff --git a/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy b/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy
index 1d9c6b8395603d..11fd6c43db4602 100644
--- a/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy
@@ -69,7 +69,7 @@ suite("test_dml_multi_routine_load_auth","p0,auth_call") {
         sql new File("""${context.file.parent}/ddl/${tableName1}.sql""").text
         sql new File("""${context.file.parent}/ddl/${tableName2}.sql""").text
 
-        connect(user = user, password = "${pwd}", url = context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """
                 CREATE ROUTINE LOAD ${dbName}.${labelName} 
@@ -92,7 +92,7 @@ suite("test_dml_multi_routine_load_auth","p0,auth_call") {
         }
 
         sql """grant load_priv on ${dbName}.${tableName1} to ${user}"""
-        connect(user = user, password = "${pwd}", url = context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """
                 CREATE ROUTINE LOAD ${dbName}.${labelName} 
@@ -114,7 +114,7 @@ suite("test_dml_multi_routine_load_auth","p0,auth_call") {
             }
         }
         sql """grant load_priv on ${dbName}.${tableName2} to ${user}"""
-        connect(user = user, password = "${pwd}", url = context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """
                 CREATE ROUTINE LOAD ${dbName}.${labelName} 
@@ -136,7 +136,7 @@ suite("test_dml_multi_routine_load_auth","p0,auth_call") {
             }
         }
         sql """grant load_priv on ${dbName}.* to ${user}"""
-        connect(user = user, password = "${pwd}", url = context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             sql """
                 CREATE ROUTINE LOAD ${dbName}.${labelName} 
                 COLUMNS TERMINATED BY "|"
diff --git a/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy b/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy
index 6010b4b8b744d9..3963fe9433ef45 100644
--- a/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy
@@ -50,7 +50,7 @@ suite("test_dml_mysql_load_auth","p0,auth_call") {
 
     sql """use ${dbName}"""
     def path_file = "${context.file.parent}/../../data/auth_call/stream_load_data.csv"
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """
                 LOAD DATA LOCAL
@@ -64,7 +64,7 @@ suite("test_dml_mysql_load_auth","p0,auth_call") {
         }
     }
     sql """grant load_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName};"""
         sql """
                 LOAD DATA LOCAL
diff --git a/regression-test/suites/auth_call/test_dml_outfile_auth.groovy b/regression-test/suites/auth_call/test_dml_outfile_auth.groovy
index 5318b1dbe31ccf..7edf476883a93b 100644
--- a/regression-test/suites/auth_call/test_dml_outfile_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_outfile_auth.groovy
@@ -58,7 +58,7 @@ suite("test_dml_outfile_auth","p0,auth_call") {
     String region = getS3Region()
     String bucket = context.config.otherConfigs.get("s3BucketName")
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """
                 SELECT * FROM ${dbName}.${tableName} t ORDER BY id
@@ -74,7 +74,7 @@ suite("test_dml_outfile_auth","p0,auth_call") {
         }
     }
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """
             SELECT * FROM ${dbName}.${tableName} t ORDER BY id
             INTO OUTFILE "s3://${bucket}/outfile/auth/exp_"
diff --git a/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy b/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy
index 8ee5236960baa7..db6698b01af194 100644
--- a/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy
@@ -68,7 +68,7 @@ suite("test_dml_routine_load_auth","p0,auth_call") {
             PROPERTIES("replication_num" = "1");
             ;"""
 
-        connect(user = user, password = "${pwd}", url = context.config.jdbcUrl) {
+        connect(user,  "${pwd}",  context.config.jdbcUrl) {
             test {
                 sql """
                 CREATE ROUTINE LOAD ${dbName}.${labelName} ON ${tableName}
@@ -86,7 +86,7 @@ suite("test_dml_routine_load_auth","p0,auth_call") {
         }
 
         sql """grant load_priv on ${dbName}.${tableName} to ${user}"""
-        connect(user = user, password = "${pwd}", url = context.config.jdbcUrl) {
+        connect(user,  "${pwd}",  context.config.jdbcUrl) {
             sql """
                 CREATE ROUTINE LOAD ${dbName}.${labelName} ON ${tableName}
                 COLUMNS(a,b),
@@ -101,7 +101,7 @@ suite("test_dml_routine_load_auth","p0,auth_call") {
         }
         sql """revoke load_priv on ${dbName}.${tableName} from ${user}"""
 
-        connect(user = user, password = "${pwd}", url = context.config.jdbcUrl) {
+        connect(user,  "${pwd}",  context.config.jdbcUrl) {
             test {
                 sql """
                 ALTER ROUTINE LOAD FOR ${dbName}.${labelName}
@@ -130,7 +130,7 @@ suite("test_dml_routine_load_auth","p0,auth_call") {
             }
         }
         sql """grant load_priv on ${dbName}.${tableName} to ${user}"""
-        connect(user = user, password = "${pwd}", url = context.config.jdbcUrl) {
+        connect(user,  "${pwd}",  context.config.jdbcUrl) {
             sql """PAUSE ROUTINE LOAD FOR ${dbName}.${labelName};"""
             sql """
             ALTER ROUTINE LOAD FOR ${dbName}.${labelName}
diff --git a/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy b/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy
index 72413b3e292ceb..b2f565fc02ef8a 100644
--- a/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy
@@ -61,14 +61,14 @@ suite("test_dml_select_udf_auth","p0,auth_call") {
         """
     sql """ INSERT INTO ${dbName}.${tableName} VALUES ("abc"), ("123"), ("123"); """
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """ SELECT ${dbName}.${udfName}(col_1) as a FROM ${dbName}.${tableName} ORDER BY a; """
             exception "Can not found function"
         }
     }
     sql """grant select_priv on ${dbName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """ SELECT ${dbName}.${udfName}(col_1) as a FROM ${dbName}.${tableName} ORDER BY a; """
     }
 
diff --git a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy
index 30f01c0e72d9c5..26ee5526f1ea69 100644
--- a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy
@@ -90,7 +90,7 @@ suite("test_dml_stream_load_auth","p0,auth_call") {
     int pos3 = sout.indexOf(":", pos1)
     def tsc_id = sout.substring(pos3+2, pos2)
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW TRANSACTION FROM ${dbName} WHERE ID=${tsc_id};"""
             exception "denied"
@@ -105,7 +105,7 @@ suite("test_dml_stream_load_auth","p0,auth_call") {
 
     sql """grant admin_priv on *.*.* to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def transaction_res = sql """SHOW TRANSACTION FROM ${dbName} WHERE ID=${tsc_id};"""
         assertTrue(transaction_res.size() == 1)
     }
diff --git a/regression-test/suites/auth_call/test_dml_update_table_auth.groovy b/regression-test/suites/auth_call/test_dml_update_table_auth.groovy
index 10c9a3fcb80f2a..153dad5026a930 100644
--- a/regression-test/suites/auth_call/test_dml_update_table_auth.groovy
+++ b/regression-test/suites/auth_call/test_dml_update_table_auth.groovy
@@ -55,21 +55,21 @@ suite("test_dml_update_table_auth","p0,auth_call") {
         (3, "333");
         """
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """UPDATE ${dbName}.${tableName} SET username = "444" WHERE id=1;"""
             exception "denied"
         }
     }
     sql """grant load_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """UPDATE ${dbName}.${tableName} SET username = "444" WHERE id=1;"""
             exception "denied"
         }
     }
     sql """grant select_priv(id) on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """UPDATE ${dbName}.${tableName} SET username = "444" WHERE id=1;"""
     }
 
diff --git a/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy b/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy
index 59a7dc1efa8b5e..3a736fe4703816 100644
--- a/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy
+++ b/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy
@@ -64,11 +64,11 @@ suite("test_grant_show_view_priv_auth","p0,auth_call") {
 
     sql """grant grant_priv on *.*.* to '${user}'"""
     sql """grant SHOW_VIEW_PRIV on ${dbName}.${viewName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """CREATE USER ${user_derive} IDENTIFIED BY '${pwd}';"""
         sql """grant select_priv on regression_test to ${user_derive}"""
 
-        connect(user=user_derive, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user_derive, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """show create table ${dbName}.${viewName};"""
                 exception "denied"
@@ -79,7 +79,7 @@ suite("test_grant_show_view_priv_auth","p0,auth_call") {
             }
         }
         sql """grant SHOW_VIEW_PRIV on ${dbName}.${viewName} to ${user_derive}"""
-        connect(user=user_derive, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user_derive, "${pwd}", context.config.jdbcUrl) {
             sql """show create table ${dbName}.${viewName};"""
             test {
                 sql """select * from ${dbName}.${viewName};"""
diff --git a/regression-test/suites/auth_call/test_hive_base_case_auth.groovy b/regression-test/suites/auth_call/test_hive_base_case_auth.groovy
index b1f432ddfb293d..271015bec7c154 100644
--- a/regression-test/suites/auth_call/test_hive_base_case_auth.groovy
+++ b/regression-test/suites/auth_call/test_hive_base_case_auth.groovy
@@ -52,7 +52,7 @@ suite("test_hive_base_case_auth", "p0,auth_call") {
         sql """grant select_priv on regression_test to ${user}"""
 
         // create catalog
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """create catalog if not exists ${catalogName} properties (
                     'type'='hms'
@@ -67,7 +67,7 @@ suite("test_hive_base_case_auth", "p0,auth_call") {
         );"""
         sql """grant Create_priv on ${catalogName}.*.* to ${user}"""
         try_sql """drop catalog if exists ${catalogName}"""
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             sql """create catalog if not exists ${catalogName} properties (
                 'type'='hms',
                 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
@@ -81,7 +81,7 @@ suite("test_hive_base_case_auth", "p0,auth_call") {
         sql """revoke Create_priv on ${catalogName}.*.* from ${user}"""
 
         // create database
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """create database ${catalogName}.${dbName};"""
                 exception "denied"
@@ -91,13 +91,13 @@ suite("test_hive_base_case_auth", "p0,auth_call") {
         sql """grant Create_priv on ${catalogName}.${dbName}.* to ${user}"""
         sql """drop table if exists ${catalogName}.${dbName}.${tableName};"""
         sql """drop database ${catalogName}.${dbName};"""
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             sql """create database ${catalogName}.${dbName};"""
         }
         sql """revoke Create_priv on ${catalogName}.${dbName}.* from ${user}"""
 
         // create table
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """create table ${catalogName}.${dbName}.${tableName} (
                     id BIGINT,
@@ -118,7 +118,7 @@ suite("test_hive_base_case_auth", "p0,auth_call") {
         );"""
         sql """grant Create_priv on ${catalogName}.${dbName}.${tableName} to ${user}"""
         sql """drop table ${catalogName}.${dbName}.${tableName}"""
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             sql """create table ${catalogName}.${dbName}.${tableName} (
                 id BIGINT,
                 username VARCHAR(20)
@@ -135,7 +135,7 @@ suite("test_hive_base_case_auth", "p0,auth_call") {
         sql """revoke Create_priv on ${catalogName}.${dbName}.${tableName} from ${user}"""
 
         // load
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """
                 insert into ${catalogName}.${dbName}.${tableName} values 
@@ -146,7 +146,7 @@ suite("test_hive_base_case_auth", "p0,auth_call") {
             }
         }
         sql """grant LOAD_PRIV on ${catalogName}.${dbName}.${tableName} to ${user}"""
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             sql """
             insert into ${catalogName}.${dbName}.${tableName} values 
             (1, "111"),
@@ -156,21 +156,21 @@ suite("test_hive_base_case_auth", "p0,auth_call") {
         sql """revoke LOAD_PRIV on ${catalogName}.${dbName}.${tableName} from ${user}"""
 
         // alter
-//        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+//        connect(user, "${pwd}", context.config.jdbcUrl) {
 //            test {
 //                sql """ALTER table ${catalogName}.${dbName}.${tableName} RENAME ${tableNameNew};"""
 //                exception "denied"
 //            }
 //        }
 //        sql """grant ALTER_PRIV on ${catalogName}.${dbName}.${tableName} to ${user}"""
-//        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+//        connect(user, "${pwd}", context.config.jdbcUrl) {
 //            sql """ALTER table ${catalogName}.${dbName}.${tableName} RENAME ${tableNameNew};"""
 //        }
 //        sql """revoke ALTER_PRIV on ${catalogName}.${dbName}.${tableName} from ${user}"""
 //        sql """ALTER table ${catalogName}.${dbName}.${tableNameNew} RENAME ${tableName};"""
 
         // drop
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             test {
                 sql """drop catalog ${catalogName}"""
                 exception "denied"
@@ -185,7 +185,7 @@ suite("test_hive_base_case_auth", "p0,auth_call") {
             }
         }
         sql """grant DROP_PRIV on ${catalogName}.*.* to ${user}"""
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             sql """drop table ${catalogName}.${dbName}.${tableName}"""
             sql """drop database ${catalogName}.${dbName}"""
             sql """drop catalog ${catalogName}"""
diff --git a/regression-test/suites/auth_call/test_show_backend_auth.groovy b/regression-test/suites/auth_call/test_show_backend_auth.groovy
index 16df2a6c10b398..adccf3637e6e2a 100644
--- a/regression-test/suites/auth_call/test_show_backend_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_backend_auth.groovy
@@ -33,14 +33,14 @@ suite("test_show_backend_auth","p0,auth_call") {
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW BACKENDS"""
             exception "denied"
         }
     }
     sql """grant node_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """SHOW BACKENDS"""
         assertTrue(res.size() > 0)
     }
diff --git a/regression-test/suites/auth_call/test_show_broker_auth.groovy b/regression-test/suites/auth_call/test_show_broker_auth.groovy
index 6ce1c4b361ac66..d8fa9bd10ebe01 100644
--- a/regression-test/suites/auth_call/test_show_broker_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_broker_auth.groovy
@@ -33,14 +33,14 @@ suite("test_show_broker_auth","p0,auth_call") {
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW BROKER;"""
             exception "denied"
         }
     }
     sql """grant node_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """SHOW BROKER;"""
     }
 
diff --git a/regression-test/suites/auth_call/test_show_charset_auth.groovy b/regression-test/suites/auth_call/test_show_charset_auth.groovy
index 803810dc204ab2..3ca23f7f6becf7 100644
--- a/regression-test/suites/auth_call/test_show_charset_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_charset_auth.groovy
@@ -36,7 +36,7 @@ suite("test_show_no_auth","p0,auth_call") {
     sql """CREATE USER '${user1}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """SHOW CHARSET"""
         sql """SHOW DATA TYPES"""
         sql """SHOW ENGINES"""
@@ -57,7 +57,7 @@ suite("test_show_no_auth","p0,auth_call") {
         }
     }
     sql """grant grant_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """show PROPERTY for ${user1}"""
         logger.info("res: " + res)
         assertTrue(res.size() > 0)
@@ -68,7 +68,7 @@ suite("test_show_no_auth","p0,auth_call") {
     }
     sql """revoke grant_priv on *.*.* from ${user}"""
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """SHOW TRASH;"""
         logger.info("res: " + res)
         assertTrue(res.size() >= 1)
diff --git a/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy b/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy
index c7a765fe9a3599..d54862f1710845 100644
--- a/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy
@@ -33,14 +33,14 @@ suite("test_show_convert_light_sc_auth","p0,auth_call") {
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW CONVERT_LIGHT_SCHEMA_CHANGE_PROCESS;"""
             exception "denied"
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """SHOW CONVERT_LIGHT_SCHEMA_CHANGE_PROCESS;"""
     }
 
diff --git a/regression-test/suites/auth_call/test_show_create_table_auth.groovy b/regression-test/suites/auth_call/test_show_create_table_auth.groovy
index 6152806ce6fbb6..2a85ea42e6c8da 100644
--- a/regression-test/suites/auth_call/test_show_create_table_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_create_table_auth.groovy
@@ -45,7 +45,7 @@ suite("test_show_create_table_auth","p0,auth_call") {
                 "replication_num" = "1"
             );"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """show create table ${dbName}.${tableName}"""
             exception "denied"
@@ -56,7 +56,7 @@ suite("test_show_create_table_auth","p0,auth_call") {
         }
     }
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """show create table ${tableName}"""
         sql """SHOW DATA SKEW FROM ${tableName};"""
@@ -64,7 +64,7 @@ suite("test_show_create_table_auth","p0,auth_call") {
     sql """revoke select_priv on ${dbName}.${tableName} from ${user}"""
 
     sql """grant create_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """show create table ${tableName}"""
         sql """SHOW DATA SKEW FROM ${tableName};"""
@@ -72,7 +72,7 @@ suite("test_show_create_table_auth","p0,auth_call") {
     sql """revoke create_priv on ${dbName}.${tableName} from ${user}"""
 
     sql """grant drop_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """show create table ${tableName}"""
         sql """SHOW DATA SKEW FROM ${tableName};"""
@@ -80,7 +80,7 @@ suite("test_show_create_table_auth","p0,auth_call") {
     sql """revoke drop_priv on ${dbName}.${tableName} from ${user}"""
 
     sql """grant alter_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """show create table ${tableName}"""
         sql """SHOW DATA SKEW FROM ${tableName};"""
diff --git a/regression-test/suites/auth_call/test_show_data_auth.groovy b/regression-test/suites/auth_call/test_show_data_auth.groovy
index 84bf9497f546e8..c6f3b6dd1536ba 100644
--- a/regression-test/suites/auth_call/test_show_data_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_data_auth.groovy
@@ -55,7 +55,7 @@ suite("test_show_data_auth","p0,auth_call") {
                 "replication_num" = "1"
             );"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """show data from ${dbName}.${tableName}"""
             exception "denied"
@@ -68,7 +68,7 @@ suite("test_show_data_auth","p0,auth_call") {
     }
 
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use ${dbName}"""
         sql """show data from ${dbName}.${tableName}"""
         test {
@@ -80,7 +80,7 @@ suite("test_show_data_auth","p0,auth_call") {
     sql """revoke select_priv on ${dbName}.${tableName} from ${user}"""
 
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """SHOW DATA;"""
     }
 
diff --git a/regression-test/suites/auth_call/test_show_database_id_auth.groovy b/regression-test/suites/auth_call/test_show_database_id_auth.groovy
index 0a0d44be07476d..e30dc8d0db74a4 100644
--- a/regression-test/suites/auth_call/test_show_database_id_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_database_id_auth.groovy
@@ -37,7 +37,7 @@ suite("test_show_database_id_auth","p0,auth_call") {
     sql """grant select_priv on regression_test to ${user}"""
     sql """create database ${dbName}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """show database 1001"""
             exception "denied"
@@ -52,7 +52,7 @@ suite("test_show_database_id_auth","p0,auth_call") {
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """show database 1001"""
         sql """show table 1001"""
         sql """SHOW CATALOG RECYCLE BIN;"""
diff --git a/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy b/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy
index 248e1d1c3559d7..727d705664311b 100644
--- a/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy
@@ -52,12 +52,12 @@ suite("test_show_dynamic_table_auth","p0,auth_call") {
                 "dynamic_partition.start_day_of_month" = "3"
             );"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """SHOW DYNAMIC PARTITION TABLES from ${dbName};"""
         assertTrue(res.size() == 0)
     }
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         
         checkNereidsExecute("SHOW DYNAMIC PARTITION TABLES from ${dbName};");
 
diff --git a/regression-test/suites/auth_call/test_show_frontend_auth.groovy b/regression-test/suites/auth_call/test_show_frontend_auth.groovy
index 2844ee992c2a98..f4a9bc15b074ec 100644
--- a/regression-test/suites/auth_call/test_show_frontend_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_frontend_auth.groovy
@@ -33,7 +33,7 @@ suite("test_show_frontend_auth","p0,auth_call") {
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW frontends"""
             exception "denied"
@@ -44,7 +44,7 @@ suite("test_show_frontend_auth","p0,auth_call") {
         }
     }
     sql """grant node_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """SHOW frontends"""
         assertTrue(res.size() > 0)
 
diff --git a/regression-test/suites/auth_call/test_show_grant_auth.groovy b/regression-test/suites/auth_call/test_show_grant_auth.groovy
index 5c6c3f069e32bf..1223faf6739d4a 100644
--- a/regression-test/suites/auth_call/test_show_grant_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_grant_auth.groovy
@@ -33,7 +33,7 @@ suite("test_show_grant_auth","p0,auth_call") {
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """show grants;"""
         test {
             sql """show all grants;"""
@@ -46,7 +46,7 @@ suite("test_show_grant_auth","p0,auth_call") {
         }
     }
     sql """grant grant_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """show grants;"""
         sql """show all grants;"""
         sql """show roles;"""
diff --git a/regression-test/suites/auth_call/test_show_proc_auth.groovy b/regression-test/suites/auth_call/test_show_proc_auth.groovy
index 725a9785a7af5c..1daf8d97f703f2 100644
--- a/regression-test/suites/auth_call/test_show_proc_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_proc_auth.groovy
@@ -33,14 +33,14 @@ suite("test_show_proc_auth","p0,auth_call") {
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW PROC "/";"""
             exception "denied"
         }
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """SHOW PROC "/";"""
     }
 
diff --git a/regression-test/suites/auth_call/test_show_query_stats_auth.groovy b/regression-test/suites/auth_call/test_show_query_stats_auth.groovy
index 1a9e8898fff432..ba951a47465ee3 100644
--- a/regression-test/suites/auth_call/test_show_query_stats_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_query_stats_auth.groovy
@@ -34,7 +34,7 @@ suite("test_show_query_stats_auth","p0,auth_call") {
     sql """grant select_priv on regression_test to ${user}"""
 
     logger.info("context.config.jdbcUrl: " + context.config.jdbcUrl)
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """show query stats;"""
         logger.info("res:" + res)
     }
diff --git a/regression-test/suites/auth_call/test_show_repository_auth.groovy b/regression-test/suites/auth_call/test_show_repository_auth.groovy
index 5d7c51ed59c3b1..43f6785e0ea7f5 100644
--- a/regression-test/suites/auth_call/test_show_repository_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_repository_auth.groovy
@@ -52,7 +52,7 @@ suite("test_show_repository_auth","p0,auth_call") {
                 "s3.secret_key" = "${sk}"
             )"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW CREATE REPOSITORY for ${repositoryName};"""
             exception "denied"
@@ -64,7 +64,7 @@ suite("test_show_repository_auth","p0,auth_call") {
 
     }
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """SHOW CREATE REPOSITORY for ${repositoryName};"""
 
         def res = sql """SHOW REPOSITORIES;"""
diff --git a/regression-test/suites/auth_call/test_show_tablet_auth.groovy b/regression-test/suites/auth_call/test_show_tablet_auth.groovy
index d071dcae074dec..3ac938d8462228 100644
--- a/regression-test/suites/auth_call/test_show_tablet_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_tablet_auth.groovy
@@ -52,7 +52,7 @@ suite("test_show_tablet_auth","p0,auth_call") {
         """
 
     sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
             sql """SHOW TABLETS FROM ${dbName}.${tableName}"""
             exception "denied"
@@ -73,7 +73,7 @@ suite("test_show_tablet_auth","p0,auth_call") {
     sql """revoke select_priv on ${dbName}.${tableName} from ${user}"""
 
     sql """grant admin_priv on *.*.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         def res = sql """SHOW TABLETS FROM ${dbName}.${tableName}"""
 
         def tablet_res = sql """SHOW TABLET ${res[0][0]}"""
diff --git a/regression-test/suites/auth_call/test_show_typecast_auth.groovy b/regression-test/suites/auth_call/test_show_typecast_auth.groovy
index 104e8107e1fb76..8df4a2e2dc18c2 100644
--- a/regression-test/suites/auth_call/test_show_typecast_auth.groovy
+++ b/regression-test/suites/auth_call/test_show_typecast_auth.groovy
@@ -33,7 +33,7 @@ suite("test_show_typecast_auth","p0,auth_call") {
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """use regression_test"""
         sql """show type_cast;"""
     }
diff --git a/regression-test/suites/auth_p0/test_admin_copy_tablet_auth.groovy b/regression-test/suites/auth_p0/test_admin_copy_tablet_auth.groovy
index c040dfb8e956c3..9bfffadb730e66 100644
--- a/regression-test/suites/auth_p0/test_admin_copy_tablet_auth.groovy
+++ b/regression-test/suites/auth_p0/test_admin_copy_tablet_auth.groovy
@@ -23,7 +23,7 @@ suite("test_admin_copy_tablet_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "ADMIN COPY TABLET 10010 PROPERTIES('backend_id' = '10001');"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_alter_policy_auth.groovy b/regression-test/suites/auth_p0/test_alter_policy_auth.groovy
index 3b1e9af6e2c609..c5b720dbf92d23 100644
--- a/regression-test/suites/auth_p0/test_alter_policy_auth.groovy
+++ b/regression-test/suites/auth_p0/test_alter_policy_auth.groovy
@@ -23,7 +23,7 @@ suite("test_alter_policy_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql """
                 ALTER STORAGE POLICY has_test_policy_to_alter PROPERTIES("cooldown_datetime" = "2023-06-08 00:00:00");
diff --git a/regression-test/suites/auth_p0/test_alter_view_auth.groovy b/regression-test/suites/auth_p0/test_alter_view_auth.groovy
index 52bf8ac75c7217..137af46f32fb16 100644
--- a/regression-test/suites/auth_p0/test_alter_view_auth.groovy
+++ b/regression-test/suites/auth_p0/test_alter_view_auth.groovy
@@ -42,7 +42,7 @@ suite("test_alter_view_auth","p0,auth") {
         """
     sql """grant select_priv on regression_test to ${user}"""
     sql """create view ${dbName}.${viewName} as select * from ${dbName}.${tableName};"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "alter view ${dbName}.${viewName} as select * from ${dbName}.${tableName};"
         } catch (Exception e) {
@@ -51,7 +51,7 @@ suite("test_alter_view_auth","p0,auth") {
         }
     }
     sql """grant Alter_priv on ${dbName}.${viewName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "alter view ${dbName}.${viewName} as select * from ${dbName}.${tableName};"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_cancel_alter_system_auth.groovy b/regression-test/suites/auth_p0/test_cancel_alter_system_auth.groovy
index 0e630fdae8e9cc..6dc93a817ad942 100644
--- a/regression-test/suites/auth_p0/test_cancel_alter_system_auth.groovy
+++ b/regression-test/suites/auth_p0/test_cancel_alter_system_auth.groovy
@@ -23,7 +23,7 @@ suite("test_cancel_alter_system_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "CANCEL DECOMMISSION BACKEND 'id1';"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_create_policy_auth.groovy b/regression-test/suites/auth_p0/test_create_policy_auth.groovy
index 8debfaedb273a7..db7968dc64ddf8 100644
--- a/regression-test/suites/auth_p0/test_create_policy_auth.groovy
+++ b/regression-test/suites/auth_p0/test_create_policy_auth.groovy
@@ -23,7 +23,7 @@ suite("test_create_policy_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "CREATE ROW POLICY test_create_policy_auth ON test.table1 AS RESTRICTIVE TO test USING (c1 = 'a');"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_create_view_auth.groovy b/regression-test/suites/auth_p0/test_create_view_auth.groovy
index 4e708c8e694579..17e61e0a97d78d 100644
--- a/regression-test/suites/auth_p0/test_create_view_auth.groovy
+++ b/regression-test/suites/auth_p0/test_create_view_auth.groovy
@@ -39,7 +39,7 @@ suite("test_create_view_auth","p0,auth") {
         );
         """
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "create view ${dbName}.v1 as select * from ${dbName}.t1;"
         } catch (Exception e) {
@@ -48,7 +48,7 @@ suite("test_create_view_auth","p0,auth") {
         }
     }
     sql """grant create_priv on ${dbName}.* to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "create view ${dbName}.v1 as select * from ${dbName}.${tableName};"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_drop_materialized_view_auth.groovy b/regression-test/suites/auth_p0/test_drop_materialized_view_auth.groovy
index 4d55b1482cff12..47ca778de217d9 100644
--- a/regression-test/suites/auth_p0/test_drop_materialized_view_auth.groovy
+++ b/regression-test/suites/auth_p0/test_drop_materialized_view_auth.groovy
@@ -23,7 +23,7 @@ suite("test_drop_materialized_view_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "DROP MATERIALIZED VIEW  mv_name ON table_name;"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_drop_policy_auth.groovy b/regression-test/suites/auth_p0/test_drop_policy_auth.groovy
index 0d6a9e103d173f..748a56bf7b45bf 100644
--- a/regression-test/suites/auth_p0/test_drop_policy_auth.groovy
+++ b/regression-test/suites/auth_p0/test_drop_policy_auth.groovy
@@ -23,7 +23,7 @@ suite("test_drop_policy_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "DROP ROW POLICY test_row_policy_1 on table1;"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy b/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy
index f6e36bc0e463a2..90228ebf3a5130 100644
--- a/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy
+++ b/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy
@@ -115,7 +115,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
 
         logger.info("url_tmp1:" + url_tmp1)
         logger.info("new_jdbc_url:" + new_jdbc_url)
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             try {
                 sql "SHOW CATALOG RECYCLE BIN WHERE NAME = '${catalog_name}'"
             } catch (Exception e) {
@@ -123,7 +123,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
                 assertTrue(e.getMessage().contains("denied"))
             }
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             try {
                 sql "SHOW CATALOG RECYCLE BIN WHERE NAME = '${catalog_name}'"
             } catch (Exception e) {
@@ -131,7 +131,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
                 assertTrue(e.getMessage().contains("denied"))
             }
         }
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             try {
                 sql "SHOW DATA"
             } catch (Exception e) {
@@ -139,7 +139,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
                 assertTrue(e.getMessage().contains("denied"))
             }
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             try {
                 sql "SHOW DATA"
             } catch (Exception e) {
@@ -148,7 +148,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
             }
         }
 
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             try {
                 sql "select username from ${dbName}.${tableName}"
             } catch (Exception e) {
@@ -156,7 +156,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
                 assertTrue(e.getMessage().contains("denied"))
             }
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             try {
                 sql "select username from ${dbName}.${tableName}"
             } catch (Exception e) {
@@ -165,15 +165,15 @@ suite ("test_follower_consistent_auth","p0,auth") {
             }
         }
         sql """grant select_priv(username) on ${dbName}.${tableName} to ${user}"""
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             sql "select username from ${dbName}.${tableName}"
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             sql "select username from ${dbName}.${tableName}"
         }
 
 
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             try {
                 sql "select username from ${dbName}.${view_name}"
             } catch (Exception e) {
@@ -181,7 +181,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
                 assertTrue(e.getMessage().contains("denied"))
             }
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             try {
                 sql "select username from ${dbName}.${view_name}"
             } catch (Exception e) {
@@ -190,15 +190,15 @@ suite ("test_follower_consistent_auth","p0,auth") {
             }
         }
         sql """grant select_priv(username) on ${dbName}.${view_name} to ${user}"""
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             sql "select username from ${dbName}.${view_name}"
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             sql "select username from ${dbName}.${view_name}"
         }
 
 
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             try {
                 sql "select username from ${dbName}.${mtmv_name}"
             } catch (Exception e) {
@@ -206,7 +206,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
                 assertTrue(e.getMessage().contains("denied"))
             }
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             try {
                 sql "select username from ${dbName}.${mtmv_name}"
             } catch (Exception e) {
@@ -215,10 +215,10 @@ suite ("test_follower_consistent_auth","p0,auth") {
             }
         }
         sql """grant select_priv(username) on ${dbName}.${mtmv_name} to ${user}"""
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             sql "select username from ${dbName}.${mtmv_name}"
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             sql "select username from ${dbName}.${mtmv_name}"
         }
 
@@ -228,15 +228,15 @@ suite ("test_follower_consistent_auth","p0,auth") {
 
         // user
         sql """grant select_priv on ${dbName}.${tableName} to ${user}"""
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             sql "select username from ${dbName}.${tableName}"
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             sql "select username from ${dbName}.${tableName}"
         }
 
         sql """revoke select_priv on ${dbName}.${tableName} from ${user}"""
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             try {
                 sql "select username from ${dbName}.${tableName}"
             } catch (Exception e) {
@@ -244,7 +244,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
                 assertTrue(e.getMessage().contains("denied"))
             }
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             try {
                 sql "select username from ${dbName}.${tableName}"
             } catch (Exception e) {
@@ -257,17 +257,17 @@ suite ("test_follower_consistent_auth","p0,auth") {
         sql """grant select_priv on ${dbName}.${tableName} to ROLE '${role}'"""
         sql """grant Load_priv on ${dbName}.${tableName} to ROLE '${role}'"""
         sql """grant '${role}' to '${user}'"""
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             sql "select username from ${dbName}.${tableName}"
             sql """insert into ${dbName}.`${tableName}` values (4, "444")"""
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             sql "select username from ${dbName}.${tableName}"
             sql """insert into ${dbName}.`${tableName}` values (4, "444")"""
         }
 
         sql """revoke '${role}' from '${user}'"""
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             try {
                 sql "select username from ${dbName}.${tableName}"
             } catch (Exception e) {
@@ -275,7 +275,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
                 assertTrue(e.getMessage().contains("denied"))
             }
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             try {
                 sql "select username from ${dbName}.${tableName}"
             } catch (Exception e) {
@@ -286,7 +286,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
 
 
         // workload group
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             sql """set workload_group = '${wg}';"""
             try {
                 sql "select username from ${dbName}.${tableName}"
@@ -295,7 +295,7 @@ suite ("test_follower_consistent_auth","p0,auth") {
                 assertTrue(e.getMessage().contains("denied"))
             }
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             sql """set workload_group = '${wg}';"""
             try {
                 sql "select username from ${dbName}.${tableName}"
@@ -305,31 +305,31 @@ suite ("test_follower_consistent_auth","p0,auth") {
             }
         }
         sql """GRANT USAGE_PRIV ON WORKLOAD GROUP '${wg}' TO '${user}';"""
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             sql """set workload_group = '${wg}';"""
             sql """select username from ${dbName}.${tableName}"""
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             sql """set workload_group = '${wg}';"""
             sql """select username from ${dbName}.${tableName}"""
         }
 
         // resource group
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             def res = sql """SHOW RESOURCES;"""
             assertTrue(res == [])
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             def res = sql """SHOW RESOURCES;"""
             assertTrue(res == [])
         }
         sql """GRANT USAGE_PRIV ON RESOURCE ${rg} TO ${user};"""
-        connect(user=user, password="${pwd}", url=url_tmp1) {
+        connect(user, "${pwd}", url_tmp1) {
             ArrayList res = sql """SHOW RESOURCES;"""
             logger.info("res:" + res)
             assertTrue(res.size() == 10)
         }
-        connect(user=user, password="${pwd}", url=new_jdbc_url) {
+        connect(user, "${pwd}", new_jdbc_url) {
             ArrayList res = sql """SHOW RESOURCES;"""
             logger.info("res:" + res)
             assertTrue(res.size() == 10)
diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy b/regression-test/suites/auth_p0/test_select_column_auth.groovy
index 3c3216d0500bb0..52f1dc02697dd4 100644
--- a/regression-test/suites/auth_p0/test_select_column_auth.groovy
+++ b/regression-test/suites/auth_p0/test_select_column_auth.groovy
@@ -74,7 +74,7 @@ suite("test_select_column_auth","p0,auth") {
     sql """grant select_priv on regression_test to ${user}"""
 
     // table column
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "select username from ${dbName}.${tableName}"
         } catch (Exception e) {
@@ -83,12 +83,12 @@ suite("test_select_column_auth","p0,auth") {
         }
     }
     sql """grant select_priv(username) on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "select username from ${dbName}.${tableName}"
     }
 
     // view column
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "select username from ${dbName}.${mv_name}"
         } catch (Exception e) {
@@ -97,12 +97,12 @@ suite("test_select_column_auth","p0,auth") {
         }
     }
     sql """grant select_priv(username) on ${dbName}.${mv_name} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "select username from ${dbName}.${mv_name}"
     }
 
     // mtmv column
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "select username from ${dbName}.${mtmv_name}"
         } catch (Exception e) {
@@ -111,13 +111,13 @@ suite("test_select_column_auth","p0,auth") {
         }
     }
     sql """grant select_priv(username) on ${dbName}.${mtmv_name} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "select username from ${dbName}.${mtmv_name}"
     }
 
 
     // mtmv hit
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "SET enable_materialized_view_rewrite=true"
         try {
             sql "select username, sum(id) from ${dbName}.${tableName} group by username"
@@ -129,7 +129,7 @@ suite("test_select_column_auth","p0,auth") {
     sql """grant select_priv(username) on ${dbName}.${mtmv_name} to ${user}"""
     sql """grant select_priv(sum_id) on ${dbName}.${mtmv_name} to ${user}"""
     sql """grant select_priv(id) on ${dbName}.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "SET enable_materialized_view_rewrite=true"
         explain {
             sql("""select username, sum(id) from ${dbName}.${tableName} group by username""")
diff --git a/regression-test/suites/auth_p0/test_select_view_auth.groovy b/regression-test/suites/auth_p0/test_select_view_auth.groovy
index 87ec8cf0aeb758..9a5020ea163ce6 100644
--- a/regression-test/suites/auth_p0/test_select_view_auth.groovy
+++ b/regression-test/suites/auth_p0/test_select_view_auth.groovy
@@ -68,7 +68,7 @@ suite("test_select_view_auth","p0,auth") {
     sql """grant select_priv on regression_test to ${user}"""
 
     // table column
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "select * from ${dbName}.${viewName}"
         } catch (Exception e) {
@@ -77,7 +77,7 @@ suite("test_select_view_auth","p0,auth") {
         }
     }
     sql """grant select_priv on ${dbName}.${viewName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "select * from ${dbName}.${viewName}"
     }
 
diff --git a/regression-test/suites/auth_p0/test_set_ldap_admin_password_auth.groovy b/regression-test/suites/auth_p0/test_set_ldap_admin_password_auth.groovy
index 6c5032f7ef9867..e72e1e1774c8a5 100644
--- a/regression-test/suites/auth_p0/test_set_ldap_admin_password_auth.groovy
+++ b/regression-test/suites/auth_p0/test_set_ldap_admin_password_auth.groovy
@@ -23,7 +23,7 @@ suite("test_set_ldap_admin_password_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SET LDAP_ADMIN_PASSWORD = PASSWORD('plain password')"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_show_catalog_recycle_bin_auth.groovy b/regression-test/suites/auth_p0/test_show_catalog_recycle_bin_auth.groovy
index aa3ba7f8e2bfa2..9c9d7a07cdd8f3 100644
--- a/regression-test/suites/auth_p0/test_show_catalog_recycle_bin_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_catalog_recycle_bin_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_catalog_recycle_bin_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SHOW CATALOG RECYCLE BIN WHERE NAME = 'test'"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_show_create_database_auth.groovy b/regression-test/suites/auth_p0/test_show_create_database_auth.groovy
index c4efff7d662478..4562a301fc3986 100644
--- a/regression-test/suites/auth_p0/test_show_create_database_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_create_database_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_create_database_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SHOW CREATE DATABASE db_name"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_show_create_repository_auth.groovy b/regression-test/suites/auth_p0/test_show_create_repository_auth.groovy
index 5e787aead258bf..717b70a723e435 100644
--- a/regression-test/suites/auth_p0/test_show_create_repository_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_create_repository_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_create_repository_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SHOW CREATE REPOSITORY for repository_name"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_show_data_auth.groovy b/regression-test/suites/auth_p0/test_show_data_auth.groovy
index fdd42d2759664f..75176d14a442c4 100644
--- a/regression-test/suites/auth_p0/test_show_data_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_data_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_data_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SHOW DATA"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_show_encryptkeys_auth.groovy b/regression-test/suites/auth_p0/test_show_encryptkeys_auth.groovy
index 1e2cf4c27fd212..ccc6de1b7213b9 100644
--- a/regression-test/suites/auth_p0/test_show_encryptkeys_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_encryptkeys_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_encryptkeys_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "use regression_test"
             sql "SHOW ENCRYPTKEYS"
diff --git a/regression-test/suites/auth_p0/test_show_plugins_auth.groovy b/regression-test/suites/auth_p0/test_show_plugins_auth.groovy
index b5b4de84265162..bb91edc2ea5d58 100644
--- a/regression-test/suites/auth_p0/test_show_plugins_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_plugins_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_plugins_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SHOW PLUGINS"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_show_repositories_auth.groovy b/regression-test/suites/auth_p0/test_show_repositories_auth.groovy
index 507d8768669973..4b4e68ed6cae08 100644
--- a/regression-test/suites/auth_p0/test_show_repositories_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_repositories_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_repositories_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SHOW REPOSITORIES"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_show_snapshot_auth.groovy b/regression-test/suites/auth_p0/test_show_snapshot_auth.groovy
index f90e1fdc57aaab..9ef808e7fee2f7 100644
--- a/regression-test/suites/auth_p0/test_show_snapshot_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_snapshot_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_snapshot_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SHOW SNAPSHOT ON example_repo"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_show_stream_load_auth.groovy b/regression-test/suites/auth_p0/test_show_stream_load_auth.groovy
index cb2446f04405f3..f64f0fa73dd220 100644
--- a/regression-test/suites/auth_p0/test_show_stream_load_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_stream_load_auth.groovy
@@ -59,13 +59,13 @@ suite("test_show_stream_load_auth","p0,auth") {
 
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
        res = sql "SHOW STREAM LOAD from regression_test_auth_p0 where label = '${label}'"
        log.info(res.toString())
        assertFalse(res.toString().contains("${label}"))
     }
     sql """grant load_priv on regression_test_auth_p0.${tableName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
        res = sql "SHOW STREAM LOAD from regression_test_auth_p0 where label = '${label}'"
        log.info(res.toString())
        assertTrue(res.toString().contains("${label}"))
diff --git a/regression-test/suites/auth_p0/test_show_tablets_belong_auth.groovy b/regression-test/suites/auth_p0/test_show_tablets_belong_auth.groovy
index b7d66af9348f0d..b86a840bae1be9 100644
--- a/regression-test/suites/auth_p0/test_show_tablets_belong_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_tablets_belong_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_tablets_belong_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SHOW TABLETS BELONG 27028"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_show_transaction_auth.groovy b/regression-test/suites/auth_p0/test_show_transaction_auth.groovy
index f4e564c52889a0..9e6303a2a925b8 100644
--- a/regression-test/suites/auth_p0/test_show_transaction_auth.groovy
+++ b/regression-test/suites/auth_p0/test_show_transaction_auth.groovy
@@ -23,7 +23,7 @@ suite("test_show_transaction_auth","p0,auth") {
     try_sql("DROP USER ${user}")
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
     sql """grant select_priv on regression_test to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "SHOW TRANSACTION WHERE ID=4005;"
         } catch (Exception e) {
diff --git a/regression-test/suites/auth_p0/test_use_encryptkey_auth.groovy b/regression-test/suites/auth_p0/test_use_encryptkey_auth.groovy
index 78fc2e4c8a8caa..965bd4b3b162bb 100644
--- a/regression-test/suites/auth_p0/test_use_encryptkey_auth.groovy
+++ b/regression-test/suites/auth_p0/test_use_encryptkey_auth.groovy
@@ -36,7 +36,7 @@ suite("test_use_encryptkey_auth","p0,auth") {
     }
     sql """CREATE ENCRYPTKEY ${key} AS 'ABCD123456789'"""
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         test {
               sql """
                   SELECT HEX(AES_ENCRYPT("Doris is Great", KEY ${dbName}.${key}));
@@ -45,7 +45,7 @@ suite("test_use_encryptkey_auth","p0,auth") {
           }
     }
     sql """grant select_priv on ${dbName} to ${user}"""
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
           sql """
               SELECT HEX(AES_ENCRYPT("Doris is Great", KEY ${dbName}.${key}));
           """
diff --git a/regression-test/suites/auth_up_down_p0/load.groovy b/regression-test/suites/auth_up_down_p0/load.groovy
index df164a7c2a1331..7ac11b627abe9e 100644
--- a/regression-test/suites/auth_up_down_p0/load.groovy
+++ b/regression-test/suites/auth_up_down_p0/load.groovy
@@ -128,15 +128,15 @@ suite("test_upgrade_downgrade_prepare_auth","p0,auth,restart_fe") {
     // user
     sql """grant select_priv on ${dbName}.${tableName1} to ${user1}"""
     sql """grant select_priv on ${dbName}.${tableName2} to ${user1}"""
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         sql "select username from ${dbName}.${tableName1}"
     }
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         sql "select username from ${dbName}.${tableName2}"
     }
 
     sql """revoke select_priv on ${dbName}.${tableName1} from ${user1}"""
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "select username from ${dbName}.${tableName1}"
         } catch (Exception e) {
@@ -144,7 +144,7 @@ suite("test_upgrade_downgrade_prepare_auth","p0,auth,restart_fe") {
             assertTrue(e.getMessage().contains("denied"))
         }
     }
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         sql "select username from ${dbName}.${tableName2}"
     }
 
@@ -152,13 +152,13 @@ suite("test_upgrade_downgrade_prepare_auth","p0,auth,restart_fe") {
     sql """grant select_priv on ${dbName}.${tableName1} to ROLE '${role1}'"""
     sql """grant Load_priv on ${dbName}.${tableName1} to ROLE '${role2}'"""
     sql """grant '${role1}', '${role2}' to '${user2}'"""
-    connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user2, "${pwd}", context.config.jdbcUrl) {
         sql "select username from ${dbName}.${tableName1}"
         sql """insert into ${dbName}.`${tableName1}` values (4, "444")"""
     }
 
     sql """revoke '${role1}' from '${user2}'"""
-    connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user2, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "select username from ${dbName}.${tableName1}"
         } catch (Exception e) {
@@ -166,12 +166,12 @@ suite("test_upgrade_downgrade_prepare_auth","p0,auth,restart_fe") {
             assertTrue(e.getMessage().contains("denied"))
         }
     }
-    connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user2, "${pwd}", context.config.jdbcUrl) {
         sql """insert into ${dbName}.`${tableName1}` values (5, "555")"""
     }
 
     // workload group
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         sql """set workload_group = '${wg1}';"""
         try {
             sql "select username from ${dbName}.${tableName2}"
@@ -181,18 +181,18 @@ suite("test_upgrade_downgrade_prepare_auth","p0,auth,restart_fe") {
         }
     }
     sql """GRANT USAGE_PRIV ON WORKLOAD GROUP '${wg1}' TO '${user1}';"""
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         sql """set workload_group = '${wg1}';"""
         sql """select username from ${dbName}.${tableName2}"""
     }
 
     // resource group
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         def res = sql """SHOW RESOURCES;"""
         assertTrue(res == [])
     }
     sql """GRANT USAGE_PRIV ON RESOURCE ${rg1} TO ${user1};"""
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         def res = sql """SHOW RESOURCES;"""
         assertTrue(res.size() == 10)
     }
diff --git a/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy b/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy
index 9a57cf146d31e5..61b953e03706fe 100644
--- a/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy
+++ b/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy
@@ -45,7 +45,7 @@ suite("test_upgrade_downgrade_compatibility_auth","p0,auth,restart_fe") {
     }
 
     // user
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "select username from ${dbName}.${tableName1}"
         } catch (Exception e) {
@@ -53,12 +53,12 @@ suite("test_upgrade_downgrade_compatibility_auth","p0,auth,restart_fe") {
             assertTrue(e.getMessage().contains("denied"))
         }
     }
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         sql "select username from ${dbName}.${tableName2}"
     }
 
     // role
-    connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user2, "${pwd}", context.config.jdbcUrl) {
         try {
             sql "select username from ${dbName}.${tableName1}"
         } catch (Exception e) {
@@ -66,18 +66,18 @@ suite("test_upgrade_downgrade_compatibility_auth","p0,auth,restart_fe") {
             assertTrue(e.getMessage().contains("denied"))
         }
     }
-    connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user2, "${pwd}", context.config.jdbcUrl) {
         sql """insert into ${dbName}.`${tableName1}` values (5, "555")"""
     }
 
     // workload group
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         sql """set workload_group = '${wg1}';"""
         sql """select username from ${dbName}.${tableName2}"""
     }
 
     // resource group
-    connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user1, "${pwd}", context.config.jdbcUrl) {
         def res = sql """SHOW RESOURCES;"""
         assertTrue(res.size() == 10)
     }
diff --git a/regression-test/suites/cloud_p0/auth/test_disable_revoke_admin_auth.groovy b/regression-test/suites/cloud_p0/auth/test_disable_revoke_admin_auth.groovy
index 2a36ecf8879d9d..19add1d2c87503 100644
--- a/regression-test/suites/cloud_p0/auth/test_disable_revoke_admin_auth.groovy
+++ b/regression-test/suites/cloud_p0/auth/test_disable_revoke_admin_auth.groovy
@@ -30,7 +30,7 @@ suite("test_disable_revoke_admin_auth", "cloud_auth") {
     }
 
     try {
-        result = connect(user = "${user}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+        result = connect("${user}", 'Cloud12345', context.config.jdbcUrl) {
              sql """
                 revoke 'admin' from 'admin'
              """
diff --git a/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_role.groovy b/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_role.groovy
index bd08353381e64a..f5594c226e8713 100644
--- a/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_role.groovy
+++ b/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_role.groovy
@@ -138,7 +138,7 @@ suite("test_grant_revoke_cluster_stage_to_role", "cloud_auth") {
         insert into ${tbl} (k1, k2) values (1, "10");
     """
 
-    connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
         sql """use @${validCluster}"""
         def sqlRet = sql """SELECT * FROM ${db}.${tbl}"""
         assertEquals(sqlRet[0][0] as int, 1)
@@ -276,7 +276,7 @@ suite("test_grant_revoke_cluster_stage_to_role", "cloud_auth") {
     assertEquals(m.values().toUnique().asList().get(0) as String, "Cluster_usage_priv")
 
     // still can select, because have global * cluster
-    connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
         sql """use @${validCluster}"""
         def sqlRet = sql """SELECT * FROM ${db}.${tbl}"""
         assertEquals(sqlRet[0][0] as int, 1)
@@ -312,7 +312,7 @@ suite("test_grant_revoke_cluster_stage_to_role", "cloud_auth") {
     commonAuth result, "'${user1}'@'%'", "Yes", "testRole", "Select_priv"
 
     // can not use @cluster, because no cluster auth
-    connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """use @${validCluster}"""
             exception "USAGE denied to user"
diff --git a/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_to_user.groovy b/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_to_user.groovy
index 916da2490f1562..a61b4f96954e9c 100644
--- a/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_to_user.groovy
+++ b/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_to_user.groovy
@@ -97,14 +97,14 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
 
     sql """create user ${user3} identified by 'Cloud12345'"""
     sql """GRANT SELECT_PRIV ON *.*.* TO '${user3}'@'%'"""
-    result = connect(user = "${user3}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    result = connect("${user3}", 'Cloud12345', context.config.jdbcUrl) {
             sql """SHOW CLUSTERS"""
     }
     // not grant any cluster to user3
     assertTrue(result.isEmpty())
     def db = context.dbName
 
-    connect(user = "${user3}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user3}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """select * from ${db}.${tbl}"""
             exception "the user is not granted permission to the compute group"
@@ -118,7 +118,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
     sql "sync"
 
     // admin role user can grant cluster to use
-    result = connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    result = connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
             sql """GRANT USAGE_PRIV ON CLUSTER '${cluster1}' TO '${user1}'"""
     }
 
@@ -132,7 +132,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
     commonAuth result, "'${user1}'@'%'" as String, "Yes", "admin", "Admin_priv"
     assertTrue((result.CloudClusterPrivs as String).contains("${cluster1}: Cluster_usage_priv"))
 
-    connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """use @${cluster1}"""
             exception "${cluster1} not exist"
@@ -145,7 +145,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
 
     sql """GRANT USAGE_PRIV ON CLUSTER '${cluster1}' TO '${user2}'"""
     try {
-        result = connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+        result = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
              sql """GRANT USAGE_PRIV ON CLUSTER '${cluster1}' TO '${user1}'"""
         }
     } catch (Exception e) {
@@ -163,7 +163,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
     result = getProperty("default_cloud_cluster", "${user1}")
     assertEquals(result.Value as String, "${validCluster}" as String)
 
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         result = sql """use @${validCluster}"""
         assertEquals(result[0][0], 0)
         result = getProperty("default_cloud_cluster", "")
@@ -171,7 +171,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
     }
         // set default_cloud_cluster to ''
     sql """SET PROPERTY FOR '${user2}' 'default_cloud_cluster' = ''"""
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         result = getProperty("default_cloud_cluster", "")
         assertEquals(result.Value as String, "" as String) 
     } 
@@ -179,14 +179,14 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
     sql """SET PROPERTY FOR '${user2}' 'default_cloud_cluster' = '${validCluster}'"""
     result = sql """REVOKE USAGE_PRIV ON CLUSTER '${validCluster}' FROM '${user2}'"""
     assertEquals(result[0][0], 0)
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """use @${cluster1}"""
             exception "USAGE denied to user"
         }
     }
 
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """use @${validCluster}"""
             exception "USAGE denied to user"
@@ -195,20 +195,20 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
 
     sql """GRANT USAGE_PRIV ON CLUSTER '${cluster1}' TO '${user2}'"""
     sql """GRANT USAGE_PRIV ON CLUSTER '${validCluster}' TO '${user2}'"""
-    show_cluster_2 = connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    show_cluster_2 = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
             getCluster(validCluster)
     }
 
     assertTrue(show_cluster_2[2].equals(user2), "Expect just only have user ${user2}")
 
-    result = connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    result = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
             sql """USE @${validCluster}"""
     }
     assertEquals(result[0][0], 0)
 
     sql """REVOKE USAGE_PRIV ON CLUSTER '${validCluster}' FROM '${user2}'"""
 
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """use @${validCluster}"""
             exception "USAGE denied to user"
@@ -231,7 +231,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
     sql "sync"
     // 3. revoke cluster
     // admin role user can revoke cluster
-    result = connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    result = connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
             sql """REVOKE USAGE_PRIV ON CLUSTER '${cluster1}' FROM '${user1}'"""
     }
 
@@ -242,7 +242,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
     
     // general user can't revoke cluster
     try {
-        result = connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+        result = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
              sql """REVOKE USAGE_PRIV ON CLUSTER '${cluster1}' FROM '${user2}'"""
         }
     } catch (Exception e) {
@@ -267,7 +267,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
 
     // user1 no admin auth, so failed to set other default cloud cluster
     try {
-        result = connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+        result = connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
             sql """SET PROPERTY FOR '${user2}' 'default_cloud_cluster' = '${validCluster}'""" 
         }
     } catch (Exception e) {
diff --git a/regression-test/suites/cloud_p0/auth/test_grant_revoke_compute_group_to_user.groovy b/regression-test/suites/cloud_p0/auth/test_grant_revoke_compute_group_to_user.groovy
index 1256028ed317d3..09205433a3988e 100644
--- a/regression-test/suites/cloud_p0/auth/test_grant_revoke_compute_group_to_user.groovy
+++ b/regression-test/suites/cloud_p0/auth/test_grant_revoke_compute_group_to_user.groovy
@@ -104,14 +104,14 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
 
     logAndExecuteSql """create user ${user3} identified by 'Cloud12345'"""
     logAndExecuteSql """GRANT SELECT_PRIV ON *.*.* TO '${user3}'@'%'"""
-    result = connect(user = "${user3}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    result = connect("${user3}", 'Cloud12345', context.config.jdbcUrl) {
             sql """SHOW COMPUTE GROUPS"""
     }
     // not grant any group to user3
     assertTrue(result.isEmpty())
     def db = context.dbName
 
-    connect(user = "${user3}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user3}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """select * from ${db}.${tbl}"""
             exception "the user is not granted permission to the compute group"
@@ -125,7 +125,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
     logAndExecuteSql "sync"
 
     // admin role user can grant group to use
-    result = connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    result = connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
             sql """GRANT USAGE_PRIV ON COMPUTE GROUP '${group1}' TO '${user1}'"""
     }
 
@@ -139,7 +139,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
     commonAuth result, "'${user1}'@'%'" as String, "Yes", "admin", "Admin_priv"
     assertTrue((result.ComputeGroupPrivs as String).contains("${group1}: Cluster_usage_priv"))
 
-    connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """use @${group1}"""
             exception "${group1} not exist"
@@ -152,7 +152,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
 
     logAndExecuteSql """GRANT USAGE_PRIV ON COMPUTE GROUP '${group1}' TO '${user2}'"""
     try {
-        result = connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+        result = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
              sql """GRANT USAGE_PRIV ON COMPUTE GROUP '${group1}' TO '${user1}'"""
         }
     } catch (Exception e) {
@@ -170,7 +170,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
     result = getProperty("default_compute_group", "${user1}")
     assertEquals(result.Value as String, "${validCluster}" as String)
 
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         result = sql """use @${validCluster}"""
         assertEquals(result[0][0], 0)
         result = getProperty("default_compute_group", "")
@@ -178,7 +178,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
     }
         // set default_compute_group to ''
     logAndExecuteSql """SET PROPERTY FOR '${user2}' 'default_compute_group' = ''"""
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         result = getProperty("default_compute_group", "")
         assertEquals(result.Value as String, "" as String) 
     } 
@@ -186,14 +186,14 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
     logAndExecuteSql """SET PROPERTY FOR '${user2}' 'default_compute_group' = '${validCluster}'"""
     result = logAndExecuteSql """REVOKE USAGE_PRIV ON COMPUTE GROUP '${validCluster}' FROM '${user2}'"""
     assertEquals(result[0][0], 0)
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """use @${group1}"""
             exception "USAGE denied to user"
         }
     }
 
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """use @${validCluster}"""
             exception "USAGE denied to user"
@@ -202,20 +202,20 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
 
     logAndExecuteSql """GRANT USAGE_PRIV ON COMPUTE GROUP '${group1}' TO '${user2}'"""
     logAndExecuteSql """GRANT USAGE_PRIV ON COMPUTE GROUP '${validCluster}' TO '${user2}'"""
-    show_group_2 = connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    show_group_2 = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
             getCluster(validCluster)
     }
 
     assertTrue(show_group_2[2].equals(user2), "Expect just only have user ${user2}")
 
-    result = connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    result = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
             sql """USE @${validCluster}"""
     }
     assertEquals(result[0][0], 0)
 
     logAndExecuteSql """REVOKE USAGE_PRIV ON COMPUTE GROUP '${validCluster}' FROM '${user2}'"""
 
-    connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
         test {
             sql """use @${validCluster}"""
             exception "USAGE denied to user"
@@ -238,7 +238,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
     logAndExecuteSql "sync"
     // 3. revoke group
     // admin role user can revoke group
-    result = connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+    result = connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
             sql """REVOKE USAGE_PRIV ON COMPUTE GROUP '${group1}' FROM '${user1}'"""
     }
 
@@ -249,7 +249,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
     
     // general user can't revoke group
     try {
-        result = connect(user = "${user2}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+        result = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
              sql """REVOKE USAGE_PRIV ON COMPUTE GROUP '${group1}' FROM '${user2}'"""
         }
     } catch (Exception e) {
@@ -274,7 +274,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
 
     // user1 no admin auth, so failed to set other default compute group
     try {
-        result = connect(user = "${user1}", password = 'Cloud12345', url = context.config.jdbcUrl) {
+        result = connect("${user1}", 'Cloud12345', context.config.jdbcUrl) {
             sql """SET PROPERTY FOR '${user2}' 'default_compute_group' = '${validCluster}'""" 
         }
     } catch (Exception e) {
diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy
index d9b105ec92d552..ce45bcfe093972 100644
--- a/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy
+++ b/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy
@@ -141,7 +141,7 @@ suite('test_auto_start_in_cloud', 'multi_cluster, docker') {
             def begin = System.currentTimeMillis();
             // root cant resume, due to deamon thread use root
             def connInfo = context.threadLocalConn.get()
-            result = connect(user = 'admin', password = '', url = connInfo.conn.getMetaData().getURL()) {
+            result = connect('admin', '', connInfo.conn.getMetaData().getURL()) {
                 sql 'SELECT * FROM table1'
             }
             def cost = System.currentTimeMillis() - begin;
diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy
index e00e8c3b764289..82b8f2b8d5a06b 100644
--- a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy
+++ b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy
@@ -75,7 +75,7 @@ suite("test_compaction_uniq_keys_row_store", "p0") {
             }
             // set server side prepared statment url
             def url="jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb + "?&useServerPrepStmts=true"
-            def result1 = connect(user=user, password=password, url=url) {
+            def result1 = connect(user, password, url) {
                 def stmt = prepareStatement """ SELECT  /*+ SET_VAR(enable_nereids_planner=true,enable_fallback_to_original_planner=false) */ * FROM ${tableName} t where user_id = ? and date = ? and datev2 = ? and datetimev2_1 = ? and datetimev2_2 = ? and city = ? and age = ? and sex = ?; """
                 setPrepareStmtArgs stmt, 1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.21', '2017-10-01 11:11:11.11', 'Beijing', 10, 1
                 qe_point_select stmt
@@ -95,7 +95,7 @@ suite("test_compaction_uniq_keys_row_store", "p0") {
                 qe_point_select stmt
             }
 
-            def result2 = connect(user=user, password=password, url=url) {
+            def result2 = connect(user, password, url) {
                 def stmt = prepareStatement """ SELECT datetimev2_1,datetime_val1,datetime_val2,max_dwell_time FROM ${tableName} t where user_id = ? and date = ? and datev2 = ? and datetimev2_1 = ? and datetimev2_2 = ? and city = ? and age = ? and sex = ?; """
                 setPrepareStmtArgs stmt, 1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.21', '2017-10-01 11:11:11.11', 'Beijing', 10, 1
                 qe_point_select stmt
diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store_ck.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store_ck.groovy
index b37c58e2be0a1c..5145e810c1b020 100644
--- a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store_ck.groovy
+++ b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store_ck.groovy
@@ -75,7 +75,7 @@ suite("test_compaction_uniq_keys_row_store_ck", "p0") {
             }
             // set server side prepared statment url
             def url="jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb + "?&useServerPrepStmts=true"
-            def result1 = connect(user=user, password=password, url=url) {
+            def result1 = connect(user, password, url) {
                 def stmt = prepareStatement """ SELECT  /*+ SET_VAR(enable_nereids_planner=true,enable_fallback_to_original_planner=false) */ * FROM ${tableName} t where user_id = ? and date = ? and datev2 = ? and datetimev2_1 = ? and datetimev2_2 = ? and city = ? and age = ? and sex = ?; """
                 setPrepareStmtArgs stmt, 1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.21', '2017-10-01 11:11:11.11', 'Beijing', 10, 1
                 qe_point_select stmt
@@ -95,7 +95,7 @@ suite("test_compaction_uniq_keys_row_store_ck", "p0") {
                 qe_point_select stmt
             }
 
-            def result2 = connect(user=user, password=password, url=url) {
+            def result2 = connect(user, password, url) {
                 def stmt = prepareStatement """ SELECT datetimev2_1,datetime_val1,datetime_val2,max_dwell_time FROM ${tableName} t where user_id = ? and date = ? and datev2 = ? and datetimev2_1 = ? and datetimev2_2 = ? and city = ? and age = ? and sex = ?; """
                 setPrepareStmtArgs stmt, 1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.21', '2017-10-01 11:11:11.11', 'Beijing', 10, 1
                 qe_point_select stmt
diff --git a/regression-test/suites/delete_p0/test_delete_on_mor.groovy b/regression-test/suites/delete_p0/test_delete_on_mor.groovy
index b39642058d8cf8..79658bf9524724 100644
--- a/regression-test/suites/delete_p0/test_delete_on_mor.groovy
+++ b/regression-test/suites/delete_p0/test_delete_on_mor.groovy
@@ -19,7 +19,7 @@ suite("test_delete_on_mor") {
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
 
-    connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+    connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
         sql "use ${db};"
 
         def tableA = "test_delete_on_mor1"
diff --git a/regression-test/suites/demo_p0/connect_action.groovy b/regression-test/suites/demo_p0/connect_action.groovy
index 437b8fe6651bb8..f269ae35225129 100644
--- a/regression-test/suites/demo_p0/connect_action.groovy
+++ b/regression-test/suites/demo_p0/connect_action.groovy
@@ -17,7 +17,7 @@
 
 suite("connect_action") {
     logger.info("ok")
-    def result1 = connect(user = 'admin', password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+    def result1 = connect('admin', context.config.jdbcPassword, context.config.jdbcUrl) {
         // execute sql with admin user
         sql 'select 99 + 1'
     }
diff --git a/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy b/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy
index b71c72957e9c02..16c8c1f6707d04 100644
--- a/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy
@@ -154,7 +154,7 @@ suite("test_external_catalog_hive", "p0,external,hive,external_docker,external_d
         try_sql("DROP USER ${user}")
         sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
         sql """GRANT SELECT_PRIV on *.*.* to '${user}'"""
-        connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+        connect(user, "${pwd}", context.config.jdbcUrl) {
             sql """switch hms;"""
             test {
                 sql "show tables"
diff --git a/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy b/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy
index 37e7993b1cf2ea..f9a94f92b8d4a1 100644
--- a/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy
@@ -61,21 +61,21 @@ suite("test_external_sql_block_rule", "external_docker,hive,external_docker_hive
     sql """grant all on *.*.* to external_block_user3;"""
 
     // login as external_block_user1 
-    def result1 = connect(user = 'external_block_user1', password = '', url = context.config.jdbcUrl) {
+    def result1 = connect('external_block_user1', '', context.config.jdbcUrl) {
         test {
             sql """select * from ${catalog_name}.`default`.parquet_partition_table order by l_linenumber limit 10;"""
             exception """sql hits sql block rule: external_hive_partition, reach partition_num : 3"""
         }
     }
     // login as external_block_user2
-    def result2 = connect(user = 'external_block_user2', password = '', url = context.config.jdbcUrl) {
+    def result2 = connect('external_block_user2', '', context.config.jdbcUrl) {
         test {
             sql """select * from ${catalog_name}.`default`.parquet_partition_table order by l_linenumber limit 10;"""
             exception """sql hits sql block rule: external_hive_partition2, reach tablet_num : 3"""
         }
     }
     // login as external_block_user3
-    def result3 = connect(user = 'external_block_user3', password = '', url = context.config.jdbcUrl) {
+    def result3 = connect('external_block_user3', '', context.config.jdbcUrl) {
         test {
             sql """select * from ${catalog_name}.`default`.parquet_partition_table order by l_linenumber limit 10;"""
             exception """sql hits sql block rule: external_hive_partition3, reach cardinality : 3"""
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_other.groovy b/regression-test/suites/external_table_p0/hive/test_hive_other.groovy
index 936790c6dc23d9..198c746d42e3f0 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_other.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_other.groovy
@@ -74,7 +74,7 @@ suite("test_hive_other", "p0,external,hive,external_docker,external_docker_hive"
         sql """create user ext_catalog_user identified by '12345'"""
         sql """grant all on internal.${context.config.defaultDb}.* to ext_catalog_user"""
         sql """grant all on ${catalog_name}.*.* to ext_catalog_user"""
-        connect(user = 'ext_catalog_user', password = '12345', url = context.config.jdbcUrl) {
+        connect('ext_catalog_user', '12345', context.config.jdbcUrl) {
             def database_lists = sql """show databases from ${catalog_name}"""
             boolean ok = false;
             for (int i = 0; i < database_lists.size(); ++i) {
diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_drop_rest_table.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_drop_rest_table.groovy
index 35293a22177e2c..8bb779d759c317 100644
--- a/regression-test/suites/external_table_p0/iceberg/iceberg_drop_rest_table.groovy
+++ b/regression-test/suites/external_table_p0/iceberg/iceberg_drop_rest_table.groovy
@@ -65,7 +65,7 @@ suite("iceberg_drop_rest_table", "p0,external,doris,external_docker,external_doc
     sql """ grant all on internal.*.* to user1 """
     sql """ grant SELECT_PRIV on ${catalog_name}.${db}.${tb} to user1 """
 
-    def result1 = connect(user = 'user1', password = '12345', url = context.config.jdbcUrl) {
+    def result1 = connect('user1', '12345', context.config.jdbcUrl) {
         sql """ use ${catalog_name}.${db} """
         qt_q1 """ select * from ${tb} """
         test {
diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_call.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_call.groovy
index bb9e39562cf657..e9e00b7084fba0 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_call.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_call.groovy
@@ -122,7 +122,7 @@ suite("test_jdbc_call", "p0,external,doris,external_docker,external_docker_doris
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user2}""";
     }    
 
-    def result1 = connect(user="${user1}", password="", url=context.config.jdbcUrl) {
+    def result1 = connect("${user1}", "", context.config.jdbcUrl) {
         sql """set enable_nereids_planner=true;"""
         sql """set enable_fallback_to_original_planner=false;"""
         test {
@@ -131,7 +131,7 @@ suite("test_jdbc_call", "p0,external,doris,external_docker,external_docker_doris
         }
     }
 
-    def result2 = connect(user="${user2}", password="", url=context.config.jdbcUrl) {
+    def result2 = connect("${user2}", "", context.config.jdbcUrl) {
         sql """set enable_nereids_planner=true;"""
         sql """set enable_fallback_to_original_planner=false;"""
         sql """call execute_stmt("${catalog_name}", "insert into ${internal_db_name}.${internal_tbl_name} values (9, 10)")"""
diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
index 3db6c227555f31..5845cf79479c98 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
@@ -192,7 +192,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
 
         // test insert
         String uuid1 = UUID.randomUUID().toString();
-        connect(user=user, password="${pwd}", url=url) {
+        connect(user, "${pwd}", url) {
             try {
                 sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """
                 fail()
@@ -203,7 +203,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
 
         sql """GRANT LOAD_PRIV ON ${catalog_name}.${ex_db_name}.${test_insert} TO ${user}"""
 
-        connect(user=user, password="${pwd}", url=url) {
+        connect(user, "${pwd}", url) {
             try {
                 sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """
             } catch (Exception e) {
diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_other.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_other.groovy
index 6d410b2cb9ad27..a54866931eff7e 100644
--- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_other.groovy
+++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_other.groovy
@@ -80,7 +80,7 @@ suite("test_trino_hive_other", "external,hive,external_docker,external_docker_hi
         sql """create user ext_catalog_user identified by '12345'"""
         sql """grant all on internal.${context.config.defaultDb}.* to ext_catalog_user"""
         sql """grant all on ${catalog_name}.*.* to ext_catalog_user"""
-        connect(user = 'ext_catalog_user', password = '12345', url = context.config.jdbcUrl) {
+        connect('ext_catalog_user', '12345', context.config.jdbcUrl) {
             def database_lists = sql """show databases from ${catalog_name}"""
             boolean ok = false;
             for (int i = 0; i < database_lists.size(); ++j) {
diff --git a/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy
index 748a7e49d14c77..2d68c2078172ba 100644
--- a/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy
@@ -108,7 +108,7 @@ suite("test_catalogs_tvf","p0,external,tvf,external_docker") {
 
 
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """ switch internal """
         order_qt_test_15 """ select  CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and   Property= "type" """ 
         order_qt_test_16 """ select  CatalogName,CatalogType,Property,Value from catalogs()  """ 
@@ -117,7 +117,7 @@ suite("test_catalogs_tvf","p0,external,tvf,external_docker") {
     sql """GRANT SELECT_PRIV on `catalog_tvf_test_dlf`.``.`` to '${user}'"""
 
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """ switch internal """
 
         order_qt_test_17 """ select  CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and   Property= "dlf.secret_key" """ 
@@ -129,7 +129,7 @@ suite("test_catalogs_tvf","p0,external,tvf,external_docker") {
     sql """REVOKE SELECT_PRIV on `catalog_tvf_test_dlf`.``.`` FROM '${user}'"""
 
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """ switch internal """
 
         order_qt_test_21 """ select  CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and   Property= "dlf.secret_key" """ 
diff --git a/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy b/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy
index c10fc1a12228c5..c23878b4d1b64f 100644
--- a/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy
@@ -96,7 +96,7 @@ suite("test_insert_from_tvf_with_common_user", "p0") {
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${common_user}""";
     }
 
-    connect(user = "${common_user}", password = '12345', url = context.config.jdbcUrl) {
+    connect("${common_user}", '12345', context.config.jdbcUrl) {
         sql """ use regression_test_external_table_p0_tvf """
 
         sql """ INSERT INTO ${load_table_name}
diff --git a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy
index 23116cca9330bd..bbee5fccc4ca07 100644
--- a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy
@@ -206,7 +206,7 @@ suite("test_s3_tvf_with_resource", "p0") {
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}""";
     }
     // not have usage priv, can not select tvf with resource
-    connect(user=user, password="${pwd}", url=url) {
+    connect(user, "${pwd}", url) {
         test {
                 sql """
                     SELECT * FROM S3 (
@@ -221,7 +221,7 @@ suite("test_s3_tvf_with_resource", "p0") {
     }
 
     // only have select_priv of view,can select view with resource
-    connect(user=user, password="${pwd}", url=url) {
+    connect(user, "${pwd}", url) {
             sql """SELECT * FROM ${viewName};"""
     }
 
diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_insert_with_retry.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_insert_with_retry.groovy
index 99eb18a4917066..0247e2ce4d614a 100644
--- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_insert_with_retry.groovy
+++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_insert_with_retry.groovy
@@ -42,7 +42,7 @@ suite("test_cloud_mow_insert_with_retry", "nonConcurrent") {
                         "enable_unique_key_merge_on_write" = "true",
                         "disable_auto_compaction" = "true",
                         "replication_num" = "1"); """
-            connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl + "&useLocalSessionState=true") {
+            connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl + "&useLocalSessionState=true") {
                 def timeout = 2000
                 def now = System.currentTimeMillis()
                 sql "insert into ${table1} values(1,1,1);"
diff --git a/regression-test/suites/insert_p0/insert_group_commit_into.groovy b/regression-test/suites/insert_p0/insert_group_commit_into.groovy
index 67907b5a69b923..ae6262c3597294 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_into.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_into.groovy
@@ -120,7 +120,7 @@ suite("insert_group_commit_into") {
             );
             """
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl + "&useLocalSessionState=true") {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl + "&useLocalSessionState=true") {
             sql """ set group_commit = async_mode; """
 
             // 1. insert into
@@ -258,7 +258,7 @@ suite("insert_group_commit_into") {
             if (observer_fe != null) {
                 def url = "jdbc:mysql://${observer_fe.Host}:${observer_fe.QueryPort}/"
                 logger.info("observer url: " + url)
-                connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = url) {
+                connect( context.config.jdbcUser,  context.config.jdbcPassword,  url) {
                     sql """ set group_commit = async_mode; """
 
                     // 1. insert into
@@ -320,7 +320,7 @@ suite("insert_group_commit_into") {
             PROPERTIES ("replication_allocation" = "tag.location.default: 1", "group_commit_interval_ms" = "200")
             """
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = async_mode; """
 
             // 1. insert into
@@ -390,7 +390,7 @@ suite("insert_group_commit_into") {
             select ordernum,max(dnt) as dnt from ${table}
             group by ordernum
             ORDER BY ordernum;""")
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = async_mode; """
 
             // 1. insert into
@@ -487,7 +487,7 @@ suite("insert_group_commit_into") {
                 ); 
             """
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = async_mode; """
             group_commit_insert """ insert into ${table} values(1, 'test'); """, 1
             group_commit_insert """ insert into ${table}(k1,`or`) values (2,"or"); """, 1
diff --git a/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy b/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
index fb936fdc54faff..a2dad1b686c39e 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
@@ -169,7 +169,7 @@ suite("insert_group_commit_into_max_filter_ratio") {
     // 100 rows(success, fail), 10000 rows(success, fail), 15000 rows(success, fail)
     // async mode, sync mode, off mode
     sql """ truncate table ${tableName} """
-    connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+    connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
 
         sql """ set group_commit = sync_mode; """
         group_commit_insert """ insert into ${dbTableName} values (1, 'a', 10); """, 1
diff --git a/regression-test/suites/insert_p0/insert_group_commit_into_unique.groovy b/regression-test/suites/insert_p0/insert_group_commit_into_unique.groovy
index 74f2f345780200..f4ab4e41065bcf 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_into_unique.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_into_unique.groovy
@@ -91,7 +91,7 @@ suite("insert_group_commit_into_unique") {
             """
 
         // 1. insert into
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = async_mode; """
             group_commit_insert """ insert into ${dbTableName} values (1, 'a', 10),(5, 'q', 50); """, 2
             group_commit_insert """ insert into ${dbTableName}(id) values(6); """, 1
@@ -170,7 +170,7 @@ suite("insert_group_commit_into_unique") {
             """
 
         // 1. insert into
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = async_mode; """
 
             group_commit_insert """ insert into ${dbTableName} values (1, 'a', 10),(5, 'q', 50); """, 2
@@ -251,7 +251,7 @@ suite("insert_group_commit_into_unique") {
             """
 
         // 1. insert into
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = async_mode; """
 
             group_commit_insert """ insert into ${dbTableName}(id, name, score, __DORIS_SEQUENCE_COL__) values (1, 'a', 10, 100),(5, 'q', 50, 500); """, 2
diff --git a/regression-test/suites/insert_p0/insert_group_commit_into_unique_sync_mode.groovy b/regression-test/suites/insert_p0/insert_group_commit_into_unique_sync_mode.groovy
index 50586c829345ae..0af8611922a68e 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_into_unique_sync_mode.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_into_unique_sync_mode.groovy
@@ -128,7 +128,7 @@ suite("insert_group_commit_into_unique_sync_mode") {
             """
 
         // 1. insert into
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = sync_mode; """
 
             group_commit_insert """ insert into ${dbTableName} values (1, 'a', 10),(5, 'q', 50); """, 2
@@ -209,7 +209,7 @@ suite("insert_group_commit_into_unique_sync_mode") {
             """
 
         // 1. insert into
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = sync_mode; """
 
             group_commit_insert """ insert into ${dbTableName} values (1, 'a', 10),(5, 'q', 50); """, 2
@@ -293,7 +293,7 @@ suite("insert_group_commit_into_unique_sync_mode") {
             """
 
         // 1. insert into
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = sync_mode; """
 
             group_commit_insert """ insert into ${dbTableName}(id, name, score, __DORIS_SEQUENCE_COL__) values (1, 'a', 10, 100),(5, 'q', 50, 500); """, 2
diff --git a/regression-test/suites/insert_p0/insert_group_commit_with_large_data.groovy b/regression-test/suites/insert_p0/insert_group_commit_with_large_data.groovy
index 989446fd097656..8fa975f56ed0c2 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_with_large_data.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_with_large_data.groovy
@@ -66,7 +66,7 @@ suite("insert_group_commit_with_large_data") {
             );
             """
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql """ set group_commit = async_mode; """
             sql """ use ${db}; """
 
diff --git a/regression-test/suites/insert_p0/prepare_insert.groovy b/regression-test/suites/insert_p0/prepare_insert.groovy
index ee4b5c3d654bd8..5d9457b9d9e515 100644
--- a/regression-test/suites/insert_p0/prepare_insert.groovy
+++ b/regression-test/suites/insert_p0/prepare_insert.groovy
@@ -101,7 +101,7 @@ suite("prepare_insert") {
         }
     }
 
-    def result1 = connect(user = user, password = password, url = url) {
+    def result1 = connect(user, password, url) {
         def stmt = prepareStatement "insert into ${tableName} values(?, ?, ?)"
         check {assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, stmt.class)}
         stmt.setInt(1, 1)
@@ -142,7 +142,7 @@ suite("prepare_insert") {
     }
 
     // insert with null
-    result1 = connect(user = user, password = password, url = url) {
+    result1 = connect(user,  password, url) {
         def stmt = prepareStatement "insert into ${tableName} values(?, ?, ?)"
         check {assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, stmt.class)}
         stmt.setNull(1, java.sql.Types.INTEGER)
@@ -157,7 +157,7 @@ suite("prepare_insert") {
 
     // insert with label
     def label = "insert_" + System.currentTimeMillis()
-    result1 = connect(user = user, password = password, url = url) {
+    result1 = connect(user,  password, url) {
         def stmt = prepareStatement "insert into ${tableName} with label ${label} values(?, ?, ?)"
         assertEquals(com.mysql.cj.jdbc.ClientPreparedStatement, stmt.class)
         stmt.setInt(1, 5)
@@ -171,7 +171,7 @@ suite("prepare_insert") {
     }
 
     url += "&rewriteBatchedStatements=true"
-    result1 = connect(user = user, password = password, url = url) {
+    result1 = connect(user,  password, url) {
         def stmt = prepareStatement "insert into ${tableName} values(?, ?, ?)"
         check {assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, stmt.class)}
         stmt.setInt(1, 10)
@@ -211,7 +211,7 @@ suite("prepare_insert") {
     }
 
     url += "&cachePrepStmts=true"
-    result1 = connect(user = user, password = password, url = url) {
+    result1 = connect(user,  password, url) {
         def stmt = prepareStatement "insert into ${tableName} values(?, ?, ?)"
         check {assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, stmt.class)}
         stmt.setInt(1, 10)
diff --git a/regression-test/suites/insert_p0/test_group_commit_data_bytes_property.groovy b/regression-test/suites/insert_p0/test_group_commit_data_bytes_property.groovy
index 2fc08b68fb1edc..461c60cd089198 100644
--- a/regression-test/suites/insert_p0/test_group_commit_data_bytes_property.groovy
+++ b/regression-test/suites/insert_p0/test_group_commit_data_bytes_property.groovy
@@ -59,7 +59,7 @@ suite("test_group_commit_data_bytes_property") {
                 );
             """
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
 
             sql """ set group_commit = async_mode; """
 
diff --git a/regression-test/suites/insert_p0/test_group_commit_interval_ms_property.groovy b/regression-test/suites/insert_p0/test_group_commit_interval_ms_property.groovy
index 688c8575366374..e05acca703aeb7 100644
--- a/regression-test/suites/insert_p0/test_group_commit_interval_ms_property.groovy
+++ b/regression-test/suites/insert_p0/test_group_commit_interval_ms_property.groovy
@@ -56,7 +56,7 @@ suite("test_group_commit_interval_ms_property") {
                 );
             """
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
 
             sql """ set group_commit = async_mode; """
 
diff --git a/regression-test/suites/insert_p0/test_jdbc.groovy b/regression-test/suites/insert_p0/test_jdbc.groovy
index acd275983aaeae..9beadaff1c4393 100644
--- a/regression-test/suites/insert_p0/test_jdbc.groovy
+++ b/regression-test/suites/insert_p0/test_jdbc.groovy
@@ -62,7 +62,7 @@ suite("test_jdbc") {
     ]
 
     def insert = { jdbc_url ->
-        connect(user = user, password = password, url = jdbc_url) {
+        connect(user, password, jdbc_url) {
             logger.info("insert url: {}", jdbc_url)
             def ps = prepareStatement "insert into ${tableName} values(?, ?)"
             for (int i = 0; i < batchSize; i++) {
@@ -79,7 +79,7 @@ suite("test_jdbc") {
     }
 
     def update = { jdbc_url ->
-        connect(user = user, password = password, url = jdbc_url) {
+        connect(user, password, jdbc_url) {
             logger.info("update url: {}", jdbc_url)
             def ps = prepareStatement "update ${tableName} set phone = ? where id = ?";
             for (int i = 0; i < batchSize; i++) {
diff --git a/regression-test/suites/insert_p0/transaction/txn_insert.groovy b/regression-test/suites/insert_p0/transaction/txn_insert.groovy
index b54fc66e714fbd..424a03ee173f5a 100644
--- a/regression-test/suites/insert_p0/transaction/txn_insert.groovy
+++ b/regression-test/suites/insert_p0/transaction/txn_insert.groovy
@@ -291,7 +291,7 @@ suite("txn_insert") {
             def observer_fe_url = get_observer_fe_url()
             if (observer_fe_url != null) {
                 logger.info("observer url: $observer_fe_url")
-                connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = observer_fe_url) {
+                connect( context.config.jdbcUser,  context.config.jdbcPassword,  observer_fe_url) {
                     result = sql """ select count() from regression_test_insert_p0_transaction.${table}_0 """
                     logger.info("select from observer result: $result")
                     assertEquals(79, result[0][0])
diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe.groovy
index 9a743bfc9859ba..def0df33bdf455 100644
--- a/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe.groovy
+++ b/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe.groovy
@@ -85,7 +85,7 @@ suite("txn_insert_restart_fe", 'docker') {
         def observer_fe_url = get_observer_fe_url()
         if (observer_fe_url != null) {
             logger.info("observer url: $observer_fe_url")
-            connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = observer_fe_url) {
+            connect( context.config.jdbcUser,  context.config.jdbcPassword,  observer_fe_url) {
                 order_qt_select_observer """ select * from ${dbName}.tbl_2 """
             }
         }
@@ -110,7 +110,7 @@ suite("txn_insert_restart_fe", 'docker') {
 
         if (observer_fe_url != null) {
             logger.info("observer url: $observer_fe_url")
-            connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = observer_fe_url) {
+            connect( context.config.jdbcUser,  context.config.jdbcPassword,  observer_fe_url) {
                 order_qt_select_observer_2 """ select * from ${dbName}.tbl_2 """
             }
         }
diff --git a/regression-test/suites/internal_schema_p0/operate_internal_schema.groovy b/regression-test/suites/internal_schema_p0/operate_internal_schema.groovy
index 1f89fa921b87d9..2ad3be51d260bb 100644
--- a/regression-test/suites/internal_schema_p0/operate_internal_schema.groovy
+++ b/regression-test/suites/internal_schema_p0/operate_internal_schema.groovy
@@ -58,7 +58,7 @@ suite("operate_internal_schema") {
     sql """GRANT ADMIN_PRIV ON *.*.* TO ${user}"""
     def tokens = context.config.jdbcUrl.split('/')
     def url=tokens[0] + "//" + tokens[2] + "/" + "__internal_schema" + "?"
-    connect(user=user, password="${pwd}", url=url) {
+    connect(user, "${pwd}", url) {
             sql "use __internal_schema;"
             if (!isCloudMode()) {
                 try {
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_auth.groovy b/regression-test/suites/javaudf_p0/test_javaudf_auth.groovy
index f961b2ec6e570e..0729f14bb333f1 100644
--- a/regression-test/suites/javaudf_p0/test_javaudf_auth.groovy
+++ b/regression-test/suites/javaudf_p0/test_javaudf_auth.groovy
@@ -58,7 +58,7 @@ suite("test_javaudf_auth") {
         "symbol"="org.apache.doris.udf.IntTest",
         "type"="JAVA_UDF"
     ); """
-    connect(user=user, password="${pwd}", url=url) {
+    connect(user, "${pwd}", url) {
         try {
             sql "select ${dbName}.java_udf_auth_test(1)"
             fail()
@@ -68,7 +68,7 @@ suite("test_javaudf_auth") {
     }
 
     sql """GRANT SELECT_PRIV ON ${dbName}.* TO ${user}"""
-    connect(user=user, password="${pwd}", url=url) {
+    connect(user, "${pwd}", url) {
         try {
             sql "select ${dbName}.java_udf_auth_test(1)"
         } catch (Exception e) {
diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy
index 4f9522c73bccc6..7b01dbacc6fa2d 100644
--- a/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy
+++ b/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy
@@ -68,7 +68,7 @@ suite("test_routine_load_with_user","p0") {
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}""";
     }
 
-    connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """ DROP TABLE IF EXISTS ${tableName}"""
         sql """ CREATE TABLE IF NOT EXISTS ${tableName}
                 (
diff --git a/regression-test/suites/manager/test_manager_interface_3.groovy b/regression-test/suites/manager/test_manager_interface_3.groovy
index 587fa48f9ebb01..22af98d5648ca7 100644
--- a/regression-test/suites/manager/test_manager_interface_3.groovy
+++ b/regression-test/suites/manager/test_manager_interface_3.groovy
@@ -93,7 +93,7 @@ suite('test_manager_interface_3',"p0") {
             sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user2}""";
         }
 
-        connect(user=user1, password="${pwd}", url=url) {
+        connect(user1, "${pwd}", url) {
             test {
                 sql """ select 1"""
                 result(
@@ -131,7 +131,7 @@ suite('test_manager_interface_3',"p0") {
         sql """grant  DROP_PRIV on ${dbName} TO ROLE '${role1}' """
         sql """grant  CREATE_PRIV on ${dbName} TO  '${user1}' """
         
-        connect(user=user1, password="${pwd}", url=url) {
+        connect(user1, "${pwd}", url) {
     
             sql """ create table test_manager_tb_2 (
                     k1 TINYINT,
@@ -145,7 +145,7 @@ suite('test_manager_interface_3',"p0") {
 
         sql """grant  LOAD_PRIV on ${dbName} TO  '${user2}' """
         sql """ grant "${role1}" to '${user2}' """  
-        connect(user=user2, password="${pwd}", url=url) {
+        connect(user2, "${pwd}", url) {
     
             test {  
                 sql """ create table test_manager_tb_2 (
@@ -214,7 +214,7 @@ suite('test_manager_interface_3',"p0") {
         assertTrue(x == 4)
         
         sql """ revoke CREATE_PRIV on ${dbName}  from '${user1}' """ 
-        connect(user=user1, password="${pwd}", url=url) {
+        connect(user1, "${pwd}", url) {
             test {  
                 sql """ create table test_manager_tb_2 (
                     k1 TINYINT,
@@ -228,7 +228,7 @@ suite('test_manager_interface_3',"p0") {
         }
 
         sql """ revoke LOAD_PRIV on ${dbName}  from '${user2}' """ 
-        connect(user=user2, password="${pwd}", url=url) {
+        connect(user2, "${pwd}", url) {
             test{
                 sql """ insert into test_manager_tb values(1,"2"); """
                 exception """LOAD command denied to user"""
@@ -317,14 +317,14 @@ suite('test_manager_interface_3',"p0") {
                 PROPERTIES ('replication_num' = '1');"""
         
 
-        connect(user=user1, password="${pwd}", url=url) {
+        connect(user1, "${pwd}", url) {
             test {
                 sql """ Drop table ${dbName}.test_manager_tb_2"""
                 exception "Access denied; you need (at least one of) the (DROP) privilege(s) for this operation"
             }
         }
 
-        connect(user=user2, password="${pwd}", url=url) {
+        connect(user2, "${pwd}", url) {
             test{
                 sql """ Drop table ${dbName}.test_manager_tb_2"""
                 exception "Access denied; you need (at least one of) the (DROP) privilege(s) for this operation"
@@ -333,13 +333,13 @@ suite('test_manager_interface_3',"p0") {
 
         sql """set password for '${user2}' = password('${new_pwd}')"""
         try {
-            connect(user =user2, password = '${pwd}', url = url) {}
+            connect(user2, '${pwd}', url) {}
             assertTrue(false. "should not be able to login")
         } catch (Exception e) {
             assertTrue(e.getMessage().contains("Access denied for user"), e.getMessage())
         } 
 
-        connect(user=user2, password="${new_pwd}", url=url) {            
+        connect(user2, "${new_pwd}", url) {            
             result =  sql """ select k1 from ${dbName}.${tbName} order by k1 desc limit 1"""
             assertTrue(result[0][0] == 3) 
         
@@ -359,7 +359,7 @@ suite('test_manager_interface_3',"p0") {
         sql """ revoke "${role1}" from "${user2}" """ 
 
         try {
-            connect(user =user2, password = '${pwd}', url = url) {}
+            connect(user2, '${pwd}', url) {}
             assertTrue(false. "should not be able to login")
         } catch (Exception e) {
             assertTrue(e.getMessage().contains("Access denied for user"), e.getMessage())
@@ -426,7 +426,7 @@ suite('test_manager_interface_3',"p0") {
         }
         assertTrue(x == 20)
 
-        connect(user=user, password="${pwd}", url=url) { 
+        connect(user, "${pwd}", url) { 
             result = sql """ show resources """
             x = 0
             for(int i = 0;i<result.size();i++) {
@@ -456,7 +456,7 @@ suite('test_manager_interface_3',"p0") {
 
 
         sql """ revoke USAGE_PRIV on RESOURCE  ${resource_name} FROM ROLE '${role}' """
-        connect(user=user, password="${pwd}", url=url) { 
+        connect(user, "${pwd}", url) { 
             result = sql """ show resources """
             x = 0
             for(int i = 0;i<result.size();i++) {
@@ -469,7 +469,7 @@ suite('test_manager_interface_3',"p0") {
         }
 
         sql """grant  USAGE_PRIV on RESOURCE  ${resource_name} TO '${user}' """
-        connect(user=user, password="${pwd}", url=url) { 
+        connect(user, "${pwd}", url) { 
             result = sql """ show resources """
             x = 0
             for(int i = 0;i<result.size();i++) {
@@ -612,7 +612,7 @@ suite('test_manager_interface_3',"p0") {
             sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}""";
         }
         
-        connect(user=user, password="${pwd}", url=url) { 
+        connect(user, "${pwd}", url) { 
             List<List<Object>> result = sql """ show property like  "max_query_instances" """
             assertTrue(result[0][0]=="max_query_instances")
             assertTrue(result[0][1]=="-1")
diff --git a/regression-test/suites/nereids_p0/authorization/column_authorization.groovy b/regression-test/suites/nereids_p0/authorization/column_authorization.groovy
index 493aa03243681e..c2e22f10c22678 100644
--- a/regression-test/suites/nereids_p0/authorization/column_authorization.groovy
+++ b/regression-test/suites/nereids_p0/authorization/column_authorization.groovy
@@ -56,7 +56,7 @@ suite("column_authorization") {
 
     def defaultDbUrl = context.config.jdbcUrl.substring(0, context.config.jdbcUrl.lastIndexOf("/"))
     logger.info("connect to ${defaultDbUrl}".toString())
-    connect(user = user1, password = null, url = defaultDbUrl) {
+    connect(user1, null, defaultDbUrl) {
         sql "set enable_fallback_to_original_planner=false"
 
         // no privilege to name
diff --git a/regression-test/suites/nereids_p0/authorization/view_authorization.groovy b/regression-test/suites/nereids_p0/authorization/view_authorization.groovy
index cd11f8ed0e15d2..51503c3cd2d737 100644
--- a/regression-test/suites/nereids_p0/authorization/view_authorization.groovy
+++ b/regression-test/suites/nereids_p0/authorization/view_authorization.groovy
@@ -59,7 +59,7 @@ suite("view_authorization") {
 
     def defaultDbUrl = context.config.jdbcUrl.substring(0, context.config.jdbcUrl.lastIndexOf("/"))
     logger.info("connect to ${defaultDbUrl}".toString())
-    connect(user = user1, password = null, url = defaultDbUrl) {
+    connect(user1, null, defaultDbUrl) {
         sql "set enable_fallback_to_original_planner=false"
 
         // no privilege to base table
diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
index 3635936e8be083..8c9e77bba7876e 100644
--- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
+++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
@@ -309,7 +309,7 @@ suite("parse_sql_from_sql_cache") {
 
 
             extraThread("test_cache_user1_thread", {
-                connect(user = "test_cache_user1", password="DORIS@2024") {
+                connect("test_cache_user1", "DORIS@2024") {
                     sql "use ${dbName}"
                     sql "set enable_nereids_planner=true"
                     sql "set enable_fallback_to_original_planner=false"
@@ -345,7 +345,7 @@ suite("parse_sql_from_sql_cache") {
             sql "sync"
 
             extraThread("test_cache_user2_thread", {
-                connect(user = "test_cache_user2", password="DORIS@2024") {
+                connect("test_cache_user2", "DORIS@2024") {
                     sql "use ${dbName}"
                     sql "set enable_nereids_planner=true"
                     sql "set enable_fallback_to_original_planner=false"
@@ -367,7 +367,7 @@ suite("parse_sql_from_sql_cache") {
 
             // after row policy changed, the cache is invalidate
             extraThread("test_cache_user2_thread2", {
-                connect(user = "test_cache_user2", password="DORIS@2024") {
+                connect("test_cache_user2", "DORIS@2024") {
                     sql "use ${dbName}"
                     sql "set enable_nereids_planner=true"
                     sql "set enable_fallback_to_original_planner=false"
@@ -409,7 +409,7 @@ suite("parse_sql_from_sql_cache") {
             sleep(10000)
 
             extraThread("test_cache_user3_thread", {
-                connect(user = "test_cache_user3", password="DORIS@2024") {
+                connect("test_cache_user3", "DORIS@2024") {
                     sql "use ${dbName}"
                     sql "set enable_nereids_planner=true"
                     sql "set enable_fallback_to_original_planner=false"
@@ -430,7 +430,7 @@ suite("parse_sql_from_sql_cache") {
 
             // after row policy changed, the cache is invalidate
             extraThread("test_cache_user3_thread2", {
-                connect(user = "test_cache_user3", password="DORIS@2024") {
+                connect("test_cache_user3", "DORIS@2024") {
                     sql "use ${dbName}"
                     sql "set enable_nereids_planner=true"
                     sql "set enable_fallback_to_original_planner=false"
@@ -463,7 +463,7 @@ suite("parse_sql_from_sql_cache") {
             sql "sync"
 
             extraThread("test_cache_user4_thread", {
-                connect(user = "test_cache_user4", password="DORIS@2024") {
+                connect("test_cache_user4", "DORIS@2024") {
                     sql "use ${dbName}"
                     sql "set enable_nereids_planner=true"
                     sql "set enable_fallback_to_original_planner=false"
@@ -481,7 +481,7 @@ suite("parse_sql_from_sql_cache") {
 
             // after privileges changed, the cache is invalidate
             extraThread("test_cache_user4_thread2", {
-                connect(user = "test_cache_user4", password="DORIS@2024") {
+                connect("test_cache_user4", "DORIS@2024") {
                     sql "set enable_nereids_planner=true"
                     sql "set enable_fallback_to_original_planner=false"
                     sql "set enable_sql_cache=true"
@@ -638,7 +638,7 @@ suite("parse_sql_from_sql_cache") {
             def dbName = context.config.getDbNameByFile(context.file)
 
             log.info("connect to fe: ${fe1}")
-            connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = "jdbc:mysql://${fe1}") {
+            connect( context.config.jdbcUser,  context.config.jdbcPassword,  "jdbc:mysql://${fe1}") {
                 sql  "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')"
 
                 sql "use ${dbName}"
@@ -660,7 +660,7 @@ suite("parse_sql_from_sql_cache") {
             }
 
             log.info("connect to fe: ${fe2}")
-            connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = "jdbc:mysql://${fe2}") {
+            connect( context.config.jdbcUser,  context.config.jdbcPassword,  "jdbc:mysql://${fe2}") {
                 sql  "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')"
 
                 sql "use ${dbName}"
diff --git a/regression-test/suites/nereids_p0/ddl/account/test_nereids_role.groovy b/regression-test/suites/nereids_p0/ddl/account/test_nereids_role.groovy
index a3bee25cf5822a..71798f226d79e1 100644
--- a/regression-test/suites/nereids_p0/ddl/account/test_nereids_role.groovy
+++ b/regression-test/suites/nereids_p0/ddl/account/test_nereids_role.groovy
@@ -32,13 +32,13 @@ suite("test_nereids_role") {
     sql """GRANT SELECT_PRIV ON ${context.config.defaultDb} TO ROLE '${role}'"""
     sql """GRANT SELECT_PRIV ON ${dbName} TO ROLE '${role}'"""
     sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}' DEFAULT ROLE '${role}'"""
-    def result1 = connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    def result1 = connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "show databases like '${dbName}'"
     }
     assertEquals(result1.size(), 1)
 
     sql """REVOKE SELECT_PRIV ON ${dbName} FROM ROLE '${role}'"""
-    def result2 = connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+    def result2 = connect(user, "${pwd}", context.config.jdbcUrl) {
         sql "show databases like '${dbName}'"
     }
     assertEquals(result2.size(), 0)
diff --git a/regression-test/suites/nereids_p0/delete/delete_cte.groovy b/regression-test/suites/nereids_p0/delete/delete_cte.groovy
index 711ba1dac4f752..8d58866216b69d 100644
--- a/regression-test/suites/nereids_p0/delete/delete_cte.groovy
+++ b/regression-test/suites/nereids_p0/delete/delete_cte.groovy
@@ -23,7 +23,7 @@ suite('nereids_delete_cte') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def t1 = 't1_cte'
diff --git a/regression-test/suites/nereids_p0/delete/delete_cte_ck.groovy b/regression-test/suites/nereids_p0/delete/delete_cte_ck.groovy
index 2e9bd1f8fb3fa9..090b3fb15529dc 100644
--- a/regression-test/suites/nereids_p0/delete/delete_cte_ck.groovy
+++ b/regression-test/suites/nereids_p0/delete/delete_cte_ck.groovy
@@ -23,7 +23,7 @@ suite('nereids_delete_cte_ck') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def t1 = 't1_cte_ck'
diff --git a/regression-test/suites/nereids_p0/delete/delete_mow_partial_update.groovy b/regression-test/suites/nereids_p0/delete/delete_mow_partial_update.groovy
index ec78ec35f43719..a03a9a92fa746e 100644
--- a/regression-test/suites/nereids_p0/delete/delete_mow_partial_update.groovy
+++ b/regression-test/suites/nereids_p0/delete/delete_mow_partial_update.groovy
@@ -23,7 +23,7 @@ suite('nereids_delete_mow_partial_update') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             sql 'set enable_nereids_planner=true'
diff --git a/regression-test/suites/nereids_p0/delete/delete_using.groovy b/regression-test/suites/nereids_p0/delete/delete_using.groovy
index 061d748d8a3321..d38c89277214b8 100644
--- a/regression-test/suites/nereids_p0/delete/delete_using.groovy
+++ b/regression-test/suites/nereids_p0/delete/delete_using.groovy
@@ -23,7 +23,7 @@ suite('nereids_delete_using') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             sql 'drop table if exists t1'
diff --git a/regression-test/suites/nereids_p0/insert_into_table/insert_auth.groovy b/regression-test/suites/nereids_p0/insert_into_table/insert_auth.groovy
index 490ff5cbee473e..4c9968746d13c1 100644
--- a/regression-test/suites/nereids_p0/insert_into_table/insert_auth.groovy
+++ b/regression-test/suites/nereids_p0/insert_into_table/insert_auth.groovy
@@ -56,7 +56,7 @@ suite('nereids_insert_auth') {
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}""";
     }
 
-    connect(user=user, password="${pwd}", url=url) {
+    connect(user, "${pwd}", url) {
         try {
             sql """ insert into ${db}.${t1} values (1, 1) """
             fail()
@@ -67,7 +67,7 @@ suite('nereids_insert_auth') {
 
     sql """GRANT LOAD_PRIV ON ${db}.${t1} TO ${user}"""
 
-    connect(user=user, password="${pwd}", url=url) {
+    connect(user, "${pwd}", url) {
         try {
             sql """ insert into ${db}.${t1} values (1, 1) """
         } catch (Exception e) {
@@ -76,7 +76,7 @@ suite('nereids_insert_auth') {
         }
     }
 
-    connect(user=user, password="${pwd}", url=url) {
+    connect(user, "${pwd}", url) {
         try {
             sql """ insert overwrite table ${db}.${t1} values (2, 2) """
         } catch (Exception e) {
diff --git a/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy b/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy
index 153cbe83938de3..6a54d6a5e71702 100644
--- a/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy
+++ b/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy
@@ -24,7 +24,7 @@ suite("nereids_partial_update_native_insert_stmt", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             sql "set enable_nereids_dml=true;"
diff --git a/regression-test/suites/nereids_p0/insert_into_table/partial_update_complex.groovy b/regression-test/suites/nereids_p0/insert_into_table/partial_update_complex.groovy
index 7b33dca57a4d52..1d5a827160d0d1 100644
--- a/regression-test/suites/nereids_p0/insert_into_table/partial_update_complex.groovy
+++ b/regression-test/suites/nereids_p0/insert_into_table/partial_update_complex.groovy
@@ -23,7 +23,7 @@ suite("nereids_partial_update_native_insert_stmt_complex", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             sql "set enable_nereids_dml=true;"
diff --git a/regression-test/suites/nereids_p0/insert_into_table/partial_update_seq_col.groovy b/regression-test/suites/nereids_p0/insert_into_table/partial_update_seq_col.groovy
index 8e3ae2f422445c..6ae4fd78e2a87e 100644
--- a/regression-test/suites/nereids_p0/insert_into_table/partial_update_seq_col.groovy
+++ b/regression-test/suites/nereids_p0/insert_into_table/partial_update_seq_col.groovy
@@ -24,7 +24,7 @@ suite("nereids_partial_update_native_insert_seq_col", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             sql "set enable_nereids_dml=true;"
diff --git a/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy b/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy
index c51dea2f5a7403..09296bbbc1b69e 100644
--- a/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy
@@ -135,7 +135,7 @@ suite("with_select_table_auth","p0,auth") {
             l_suppkey;
     """)
 
-    connect(user=user_name, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user_name, "${pwd}", context.config.jdbcUrl) {
         sql "use ${db}"
         mv_rewrite_success(
             """
@@ -156,7 +156,7 @@ suite("with_select_table_auth","p0,auth") {
         )
     }
 
-    connect(user=user_name, password="${pwd}", url=context.config.jdbcUrl) {
+    connect(user_name, "${pwd}", context.config.jdbcUrl) {
         sql "use ${db}"
         test {
             sql """select * from mv1;"""
diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy b/regression-test/suites/point_query_p0/test_point_query.groovy
index f84012a8fd77d3..7c3a255b10eb73 100644
--- a/regression-test/suites/point_query_p0/test_point_query.groovy
+++ b/regression-test/suites/point_query_p0/test_point_query.groovy
@@ -63,7 +63,7 @@ suite("test_point_query", "nonConcurrent") {
 
         def nprep_sql = { sql_str ->
             def url_without_prep = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb
-            connect(user = user, password = password, url = url_without_prep) {
+            connect(user, password, url_without_prep) {
                 // set to false to invalid cache correcly
                 sql "set enable_memtable_on_sink_node = false"
                 sql sql_str
@@ -137,7 +137,7 @@ suite("test_point_query", "nonConcurrent") {
             sql """ INSERT INTO ${tableName} VALUES(252, 120939.11130, "${generateString(252)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 252, "7022-01-01 11:30:38", 0, 90696620686827832.374, [0], null) """
             sql """ INSERT INTO ${tableName} VALUES(298, 120939.11130, "${generateString(298)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 298, "7022-01-01 11:30:38", 1, 90696620686827832.374, [], []) """
 
-            def result1 = connect(user=user, password=password, url=prepare_url) {
+            def result1 = connect(user, password, prepare_url) {
                 def stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=true) */ * from ${tableName} where k1 = ? and k2 = ? and k3 = ?"
                 assertEquals(stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
                 stmt.setInt(1, 1231)
@@ -224,7 +224,7 @@ suite("test_point_query", "nonConcurrent") {
                 qe_point_select stmt
             }
             // disable useServerPrepStmts
-            def result2 = connect(user=user, password=password, url=context.config.jdbcUrl) {
+            def result2 = connect(user, password, context.config.jdbcUrl) {
                 qt_sql """select /*+ SET_VAR(enable_nereids_planner=true) */ * from ${tableName} where k1 = 1231 and k2 = 119291.11 and k3 = 'ddd'"""
                 qt_sql """select /*+ SET_VAR(enable_nereids_planner=true) */ * from ${tableName} where k1 = 1237 and k2 = 120939.11130 and k3 = 'a    ddd'"""
                 qt_sql """select /*+ SET_VAR(enable_nereids_planner=true) */ hex(k3), hex(k4), k7 + 10.1 from ${tableName} where k1 = 1237 and k2 = 120939.11130 and k3 = 'a    ddd'"""
diff --git a/regression-test/suites/point_query_p0/test_point_query_ck.groovy b/regression-test/suites/point_query_p0/test_point_query_ck.groovy
index 33c31dc0970929..f7c53c7207e99e 100644
--- a/regression-test/suites/point_query_p0/test_point_query_ck.groovy
+++ b/regression-test/suites/point_query_p0/test_point_query_ck.groovy
@@ -63,7 +63,7 @@ suite("test_point_query_ck") {
 
         def nprep_sql = { sql_str ->
             def url_without_prep = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb
-            connect(user = user, password = password, url = url_without_prep) {
+            connect(user, password, url_without_prep) {
                 // set to false to invalid cache correcly
                 sql "set enable_memtable_on_sink_node = false"
                 sql sql_str
@@ -138,7 +138,7 @@ suite("test_point_query_ck") {
             sql """ INSERT INTO ${tableName} VALUES(252, 120939.11130, "${generateString(252)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 252, "7022-01-01 11:30:38", 0, 90696620686827832.374, [0], null) """
             sql """ INSERT INTO ${tableName} VALUES(298, 120939.11130, "${generateString(298)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 298, "7022-01-01 11:30:38", 1, 90696620686827832.374, [], []) """
 
-            def result1 = connect(user=user, password=password, url=prepare_url) {
+            def result1 = connect(user, password, prepare_url) {
                 def stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=true) */ * from ${tableName} where k1 = ? and k2 = ? and k3 = ?"
                 assertEquals(stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
                 stmt.setInt(1, 1231)
@@ -225,7 +225,7 @@ suite("test_point_query_ck") {
                 qe_point_select stmt
             }
             // disable useServerPrepStmts
-            def result2 = connect(user=user, password=password, url=context.config.jdbcUrl) {
+            def result2 = connect(user, password, context.config.jdbcUrl) {
                 qt_sql """select /*+ SET_VAR(enable_nereids_planner=true) */ * from ${tableName} where k1 = 1231 and k2 = 119291.11 and k3 = 'ddd'"""
                 qt_sql """select /*+ SET_VAR(enable_nereids_planner=true) */ * from ${tableName} where k1 = 1237 and k2 = 120939.11130 and k3 = 'a    ddd'"""
                 qt_sql """select /*+ SET_VAR(enable_nereids_planner=true) */ hex(k3), hex(k4), k7 + 10.1 from ${tableName} where k1 = 1237 and k2 = 120939.11130 and k3 = 'a    ddd'"""
diff --git a/regression-test/suites/point_query_p0/test_point_query_partition.groovy b/regression-test/suites/point_query_p0/test_point_query_partition.groovy
index 5062677c842b29..848729b44232a7 100644
--- a/regression-test/suites/point_query_p0/test_point_query_partition.groovy
+++ b/regression-test/suites/point_query_p0/test_point_query_partition.groovy
@@ -48,7 +48,7 @@ suite("test_point_query_partition") {
 
     def nprep_sql = { sql_str ->
         def url_without_prep = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb
-        connect(user = user, password = password, url = url_without_prep) {
+        connect(user, password, url_without_prep) {
             sql sql_str
         }
     }
@@ -85,7 +85,7 @@ suite("test_point_query_partition") {
     sql """INSERT INTO ${tableName} VALUES (33, 'f')"""
     sql """INSERT INTO ${tableName} VALUES (45, 'g')"""
     sql """INSERT INTO ${tableName} VALUES (999, 'h')"""
-    def result1 = connect(user=user, password=password, url=prepare_url) {
+    def result1 = connect(user, password, prepare_url) {
         def stmt = prepareStatement "select * from ${tableName} where k1 = ?"
         assertEquals(stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
         stmt.setInt(1, 1)
@@ -139,7 +139,7 @@ suite("test_point_query_partition") {
     """  
     sql """insert into regression_test_serving_p0.customer(customer_key, customer_value_0, customer_value_1) values(686612, "686612", "686612")"""
     sql """insert into regression_test_serving_p0.customer(customer_key, customer_value_0, customer_value_1) values(686613, "686613", "686613")"""
-    def result3 = connect(user=user, password=password, url=prepare_url) {
+    def result3 = connect(user, password, prepare_url) {
         def stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=true) */ * from regression_test_serving_p0.customer where customer_key = ?"
         stmt.setInt(1, 686612)
         qe_point_selectxxx stmt 
diff --git a/regression-test/suites/point_query_p0/test_point_query_partition_ck.groovy b/regression-test/suites/point_query_p0/test_point_query_partition_ck.groovy
index 5428b1a6f16aaa..658a0ecfb71be6 100644
--- a/regression-test/suites/point_query_p0/test_point_query_partition_ck.groovy
+++ b/regression-test/suites/point_query_p0/test_point_query_partition_ck.groovy
@@ -48,7 +48,7 @@ suite("test_point_query_partition_ck") {
 
     def nprep_sql = { sql_str ->
         def url_without_prep = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb
-        connect(user = user, password = password, url = url_without_prep) {
+        connect(user, password, url_without_prep) {
             sql sql_str
         }
     }
@@ -86,7 +86,7 @@ suite("test_point_query_partition_ck") {
     sql """INSERT INTO ${tableName} VALUES (33, 'f')"""
     sql """INSERT INTO ${tableName} VALUES (45, 'g')"""
     sql """INSERT INTO ${tableName} VALUES (999, 'h')"""
-    def result1 = connect(user=user, password=password, url=prepare_url) {
+    def result1 = connect(user, password, prepare_url) {
         def stmt = prepareStatement "select * from ${tableName} where k1 = ?"
         assertEquals(stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
         stmt.setInt(1, 1)
@@ -142,7 +142,7 @@ suite("test_point_query_partition_ck") {
     """  
     sql """insert into ${tableName}(customer_key, customer_value_0, customer_value_1) values(686612, "686612", "686612")"""
     sql """insert into ${tableName}(customer_key, customer_value_0, customer_value_1) values(686613, "686613", "686613")"""
-    def result3 = connect(user=user, password=password, url=prepare_url) {
+    def result3 = connect(user, password, prepare_url) {
         def stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=true) */ * from ${tableName} where customer_key = ?"
         stmt.setInt(1, 686612)
         qe_point_selectxxx stmt 
diff --git a/regression-test/suites/point_query_p0/test_rowstore.groovy b/regression-test/suites/point_query_p0/test_rowstore.groovy
index d24d6767d48b4f..13279e3ce877fc 100644
--- a/regression-test/suites/point_query_p0/test_rowstore.groovy
+++ b/regression-test/suites/point_query_p0/test_rowstore.groovy
@@ -197,7 +197,7 @@ suite("test_rowstore", "p0,nonConcurrent") {
         """
 
     // set server side prepared statement url
-    connect(user = user, password = password, url = prepare_url) {
+    connect(user, password, prepare_url) {
         def prep_sql = { sql_str, k ->
             def stmt = prepareStatement sql_str
             stmt.setInt(1, k)
diff --git a/regression-test/suites/point_query_p0/test_rowstore_ck.groovy b/regression-test/suites/point_query_p0/test_rowstore_ck.groovy
index 82968e2aae5352..957aa976ee62c0 100644
--- a/regression-test/suites/point_query_p0/test_rowstore_ck.groovy
+++ b/regression-test/suites/point_query_p0/test_rowstore_ck.groovy
@@ -203,7 +203,7 @@ suite("test_rowstore_ck", "p0,nonConcurrent") {
         """
 
     // set server side prepared statement url
-    connect(user = user, password = password, url = prepare_url) {
+    connect(user, password, prepare_url) {
         def prep_sql = { sql_str, k ->
             def stmt = prepareStatement sql_str
             stmt.setInt(1, k)
diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy
index e01e574976f1bc..d32fbd1532ee4d 100644
--- a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy
+++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy
@@ -24,7 +24,7 @@ suite("test_prepared_stmt", "nonConcurrent") {
     def password = context.config.jdbcPassword
     // def url = context.config.jdbcUrl + "&useServerPrepStmts=true&useCursorFetch=true"
     String url = getServerPrepareJdbcUrl(context.config.jdbcUrl, "regression_test_prepared_stmt_p0")
-    def result1 = connect(user=user, password=password, url=url) {
+    def result1 = connect(user, password, url) {
         sql """DROP TABLE IF EXISTS ${tableName} """
         sql """
              CREATE TABLE IF NOT EXISTS ${tableName} (
diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt_in_list.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt_in_list.groovy
index 3c972e83fd9fa0..54ec1efa4b373d 100644
--- a/regression-test/suites/prepared_stmt_p0/prepared_stmt_in_list.groovy
+++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt_in_list.groovy
@@ -24,7 +24,7 @@ suite("test_prepared_stmt_in_list", "nonConcurrent") {
     // def url = context.config.jdbcUrl + "&useServerPrepStmts=true"
     String url = getServerPrepareJdbcUrl(context.config.jdbcUrl, "regression_test_prepared_stmt_p0")
     sql """set global max_prepared_stmt_count = 1024"""
-    def result1 = connect(user=user, password=password, url=url) {
+    def result1 = connect(user, password, url) {
         sql """DROP TABLE IF EXISTS ${tableName} """
         sql """
              CREATE TABLE IF NOT EXISTS ${tableName} (
diff --git a/regression-test/suites/query_p0/system/test_partitions_schema.groovy b/regression-test/suites/query_p0/system/test_partitions_schema.groovy
index 0cf83d67e66995..98b12c3705c777 100644
--- a/regression-test/suites/query_p0/system/test_partitions_schema.groovy
+++ b/regression-test/suites/query_p0/system/test_partitions_schema.groovy
@@ -184,17 +184,17 @@ suite("test_partitions_schema") {
     def tokens = context.config.jdbcUrl.split('/')
     def url=tokens[0] + "//" + tokens[2] + "/" + "information_schema" + "?"
 
-    connect(user=user, password='123abc!@#', url=url) {
+    connect(user, '123abc!@#', url) {
         order_qt_select_check_3 """select $listOfColum from information_schema.partitions where table_schema=\"${dbName}\""""
     }
 
     sql "GRANT SELECT_PRIV ON ${dbName}.duplicate_table  TO ${user}"
-    connect(user=user, password='123abc!@#', url=url) {
+    connect(user, '123abc!@#', url) {
         order_qt_select_check_4 """select TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PARTITION_NAME from information_schema.partitions where table_schema=\"${dbName}\""""
     }
     
     sql "REVOKE SELECT_PRIV ON ${dbName}.duplicate_table  FROM ${user}"
-    connect(user=user, password='123abc!@#', url=url) {
+    connect(user, '123abc!@#', url) {
         order_qt_select_check_5 """select TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PARTITION_NAME from information_schema.partitions where table_schema=\"${dbName}\""""
     }
 
diff --git a/regression-test/suites/query_p0/system/test_table_options.groovy b/regression-test/suites/query_p0/system/test_table_options.groovy
index 9d2e99ab974413..fef118a82b280b 100644
--- a/regression-test/suites/query_p0/system/test_table_options.groovy
+++ b/regression-test/suites/query_p0/system/test_table_options.groovy
@@ -199,17 +199,17 @@ suite("test_table_options") {
     def tokens = context.config.jdbcUrl.split('/')
     def url=tokens[0] + "//" + tokens[2] + "/" + "information_schema" + "?"
 
-    connect(user=user, password='123abc!@#', url=url) {
+    connect(user, '123abc!@#', url) {
        qt_select_check_3 """select * from information_schema.table_options ORDER BY TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,TABLE_MODEL,TABLE_MODEL_KEY,DISTRIBUTE_KEY,DISTRIBUTE_TYPE,BUCKETS_NUM,PARTITION_NUM; """
     }
 
     sql "GRANT SELECT_PRIV ON ${dbName}.duplicate_table  TO ${user}"
-    connect(user=user, password='123abc!@#', url=url) {
+    connect(user, '123abc!@#', url) {
        qt_select_check_4 """select * from information_schema.table_options ORDER BY TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,TABLE_MODEL,TABLE_MODEL_KEY,DISTRIBUTE_KEY,DISTRIBUTE_TYPE,BUCKETS_NUM,PARTITION_NUM; """
     }
     
     sql "REVOKE SELECT_PRIV ON ${dbName}.duplicate_table  FROM ${user}"
-    connect(user=user, password='123abc!@#', url=url) {
+    connect(user, '123abc!@#', url) {
        qt_select_check_5 """select * from information_schema.table_options ORDER BY TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,TABLE_MODEL,TABLE_MODEL_KEY,DISTRIBUTE_KEY,DISTRIBUTE_TYPE,BUCKETS_NUM,PARTITION_NUM; """
     }
 
diff --git a/regression-test/suites/query_p0/system/test_table_properties.groovy b/regression-test/suites/query_p0/system/test_table_properties.groovy
index 3314975d689c0f..1861ae4d6280d6 100644
--- a/regression-test/suites/query_p0/system/test_table_properties.groovy
+++ b/regression-test/suites/query_p0/system/test_table_properties.groovy
@@ -105,17 +105,17 @@ suite("test_table_properties") {
     def tokens = context.config.jdbcUrl.split('/')
     def url=tokens[0] + "//" + tokens[2] + "/" + "information_schema" + "?"
 
-    connect(user=user, password='123abc!@#', url=url) {
+    connect(user, '123abc!@#', url) {
        qt_select_check_4 """select * from information_schema.table_properties where PROPERTY_NAME != "default.replication_allocation" ORDER BY TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PROPERTY_NAME,PROPERTY_VALUE"""
     }
 
     sql "GRANT SELECT_PRIV ON ${dbName}.duplicate_table  TO ${user}"
-    connect(user=user, password='123abc!@#', url=url) {
+    connect(user, '123abc!@#', url) {
        qt_select_check_5 """select * from information_schema.table_properties where PROPERTY_NAME != "default.replication_allocation" ORDER BY TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PROPERTY_NAME,PROPERTY_VALUE"""
     }
  
     sql "REVOKE SELECT_PRIV ON ${dbName}.duplicate_table  FROM ${user}"
-    connect(user=user, password='123abc!@#', url=url) {
+    connect(user, '123abc!@#', url) {
        qt_select_check_6 """select * from information_schema.table_properties where PROPERTY_NAME != "default.replication_allocation" ORDER BY TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PROPERTY_NAME,PROPERTY_VALUE"""
     }
 
diff --git a/regression-test/suites/query_p0/test_row_policy.groovy b/regression-test/suites/query_p0/test_row_policy.groovy
index 01766e9a084057..4af498e55e3b46 100644
--- a/regression-test/suites/query_p0/test_row_policy.groovy
+++ b/regression-test/suites/query_p0/test_row_policy.groovy
@@ -37,7 +37,7 @@ suite("test_row_policy") {
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}""";
     }
 
-    connect(user=user, password='123456', url=url) {
+    connect(user, '123456', url) {
         sql "set enable_nereids_planner = true"
         sql "set enable_fallback_to_original_planner = false"
         sql "SELECT * FROM ${tableName} a JOIN ${tableName} b ON a.id = b.id"
diff --git a/regression-test/suites/schema_change_p0/test_alter_table_column_rename.groovy b/regression-test/suites/schema_change_p0/test_alter_table_column_rename.groovy
index 3650b93ccbb282..f2c2ae657913b3 100644
--- a/regression-test/suites/schema_change_p0/test_alter_table_column_rename.groovy
+++ b/regression-test/suites/schema_change_p0/test_alter_table_column_rename.groovy
@@ -39,7 +39,7 @@ suite("test_alter_table_column_rename") {
     for (row : result) {
         //println row
         String jdbcUrl = "jdbc:mysql://" + row[1] + ":" + row[4]
-        def result1 = connect(user = 'root', password = '', jdbcUrl) {
+        def result1 = connect('root', '', jdbcUrl) {
             sql """ SYNC """
             sql """ use regression_test_schema_change_p0 """
             sql """ select * from ${tbName} where new_col = 2 """
diff --git a/regression-test/suites/show_p0/test_show_backend_config.groovy b/regression-test/suites/show_p0/test_show_backend_config.groovy
index b2e0bea26c1fe6..326b6104257eea 100644
--- a/regression-test/suites/show_p0/test_show_backend_config.groovy
+++ b/regression-test/suites/show_p0/test_show_backend_config.groovy
@@ -33,7 +33,7 @@ suite("test_show_backend_config") {
         }
     }
 
-    connect(user = userName, password = passwd, url = context.config.jdbcUrl) {
+    connect(userName, passwd, context.config.jdbcUrl) {
         def backends = sql_return_maparray """ show backends """
         def beId = backends[0].BackendId
         def bePort = backends[0].BePort
diff --git a/regression-test/suites/show_p0/test_show_backends.groovy b/regression-test/suites/show_p0/test_show_backends.groovy
index 0c41cd68ec7ac2..7a503f09b3282b 100644
--- a/regression-test/suites/show_p0/test_show_backends.groovy
+++ b/regression-test/suites/show_p0/test_show_backends.groovy
@@ -23,11 +23,11 @@ suite("test_show_backends", "show") {
 
     sql """grant ADMIN_PRIV on *.*.* to test_show_backends_user1"""
 
-    def result1 = connect(user = 'test_show_backends_user1', password = '12345', url = context.config.jdbcUrl) {
+    def result1 = connect('test_show_backends_user1', '12345', context.config.jdbcUrl) {
         sql """ show backends """
     }
     log.info(result1.toString())
-    def result2 = connect(user = 'test_show_backends_user1', password = '12345', url = context.config.jdbcUrl) {
+    def result2 = connect('test_show_backends_user1', '12345', context.config.jdbcUrl) {
             sql """ show proc '/backends' """
         }
     log.info(result2.toString())
diff --git a/regression-test/suites/show_p0/test_show_statistic_proc.groovy b/regression-test/suites/show_p0/test_show_statistic_proc.groovy
index 50825868789bb4..4a37b92fc0f172 100644
--- a/regression-test/suites/show_p0/test_show_statistic_proc.groovy
+++ b/regression-test/suites/show_p0/test_show_statistic_proc.groovy
@@ -26,10 +26,10 @@ suite("test_show_statistic_proc", "nonConcurrent") {
     sql """drop database if exists test_statistic_proc_db"""
     sql """create database test_statistic_proc_db"""
 
-    def result1 = connect(user = 'test_show_statistic_proc_user1', password = '12345', url = context.config.jdbcUrl) {
+    def result1 = connect('test_show_statistic_proc_user1', '12345', context.config.jdbcUrl) {
         sql """ show proc '/statistic' """
     }
-    def result2 = connect(user = 'test_show_statistic_proc_user1', password = '12345', url = context.config.jdbcUrl) {
+    def result2 = connect('test_show_statistic_proc_user1', '12345', context.config.jdbcUrl) {
         sql """ show databases """
     }
     assertEquals(result1.size(), result2.size())
diff --git a/regression-test/suites/statistics/test_analyze_mtmv.groovy b/regression-test/suites/statistics/test_analyze_mtmv.groovy
index 49e1b75a476dbb..614a04ba7e5770 100644
--- a/regression-test/suites/statistics/test_analyze_mtmv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mtmv.groovy
@@ -31,7 +31,7 @@ suite("test_analyze_mtmv") {
         def tokens = context.config.jdbcUrl.split('/')
         def url=tokens[0] + "//" + host + ":" + port
         logger.info("Master url is " + url)
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, url) {
             sql """use ${db}"""
             result = sql """show frontends;"""
             logger.info("show frontends result master: " + result)
diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy b/regression-test/suites/statistics/test_analyze_mv.groovy
index 64e86439be1ce5..657eb235453808 100644
--- a/regression-test/suites/statistics/test_analyze_mv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mv.groovy
@@ -49,7 +49,7 @@ suite("test_analyze_mv") {
         def tokens = context.config.jdbcUrl.split('/')
         def url=tokens[0] + "//" + host + ":" + port
         logger.info("Master url is " + url)
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, url) {
             sql """use ${db}"""
             result = sql """show frontends;"""
             logger.info("show frontends result master: " + result)
diff --git a/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy b/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy
index f27ecdc5e1616b..223a69cfa380f9 100644
--- a/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy
+++ b/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy
@@ -31,7 +31,7 @@ suite("test_auto_analyze_black_white_list") {
         def tokens = context.config.jdbcUrl.split('/')
         def url=tokens[0] + "//" + host + ":" + port
         logger.info("Master url is " + url)
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, url) {
             sql """use ${db}"""
             result = sql """show frontends;"""
             logger.info("show frontends result master: " + result)
diff --git a/regression-test/suites/statistics/test_partition_stats.groovy b/regression-test/suites/statistics/test_partition_stats.groovy
index 96be61296eadf0..fac20fa6e20293 100644
--- a/regression-test/suites/statistics/test_partition_stats.groovy
+++ b/regression-test/suites/statistics/test_partition_stats.groovy
@@ -34,7 +34,7 @@ suite("test_partition_stats") {
         def tokens = context.config.jdbcUrl.split('/')
         def url=tokens[0] + "//" + host + ":" + port
         logger.info("Master url is " + url)
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url) {
+        connect(context.config.jdbcUser, context.config.jdbcPassword, url) {
             sql """use ${db}"""
             result = sql """show frontends;"""
             logger.info("show frontends result master: " + result)
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_mix_partial_update.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_mix_partial_update.groovy
index 7bcda3debb6694..4a6d587d3f72ae 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_mix_partial_update.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_mix_partial_update.groovy
@@ -23,7 +23,7 @@ suite('test_mix_partial_update') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             sql "set enable_nereids_planner=true"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_new_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_new_partial_update_delete.groovy
index 22d5fe9cade6bd..76772ad19b3966 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_new_partial_update_delete.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_new_partial_update_delete.groovy
@@ -23,7 +23,7 @@ suite('test_new_partial_update_delete') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             sql "set enable_nereids_planner=true"
@@ -146,7 +146,7 @@ suite('test_new_partial_update_delete') {
         }
     }
 
-    connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+    connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
         sql "use ${db};"
         try {
             def tableAggName = "test_new_partial_update_agg_delete"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update.groovy
index 34a62e6da0c990..9a04f731ac5c9c 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_primary_key_partial_update"
             // create table
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_2pc_schema_change.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_2pc_schema_change.groovy
index e576cb86cb48b3..e7205620ef319f 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_2pc_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_2pc_schema_change.groovy
@@ -42,7 +42,7 @@ suite("test_partial_update_2pc_schema_change", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_partial_update_2pc_schema_change"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_after_delete.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_after_delete.groovy
index 066f76fb372589..c00bba7ae341c5 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_after_delete.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_after_delete.groovy
@@ -23,7 +23,7 @@ suite("test_partial_update_after_delete", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             sql "SET enable_nereids_planner=true;"
             sql "SET enable_fallback_to_original_planner=false;"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_auto_inc.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_auto_inc.groovy
index 14e26ccab8818d..63a14ec70eba1d 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_auto_inc.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_auto_inc.groovy
@@ -23,7 +23,7 @@ suite("test_partial_update_auto_inc") {
     for (def use_mow : [/*false,*/ true]) {
         for (def use_nereids_planner : [false, true]) {
             logger.info("current params: use_mow: ${use_mow}, use_nereids_planner: ${use_nereids_planner}")
-            connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+            connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
                 sql "use ${db};"
 
                 if (use_nereids_planner) {
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_case_insensitivity.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_case_insensitivity.groovy
index a5766f478d366b..5431ec266662c3 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_case_insensitivity.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_case_insensitivity.groovy
@@ -23,7 +23,7 @@ suite("test_partial_update_case_insensitivity", "p0") {
     for (def use_row_store : [false, true]) {
         for (def use_nereids_planner : [false, true]) {
             logger.info("current params: use_row_store: ${use_row_store}, use_nereids_planner: ${use_nereids_planner}")
-            connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+            connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
                 sql "use ${db};"
                 if (use_nereids_planner) {
                     sql """ set enable_nereids_dml = true; """
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_complex_type.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_complex_type.groovy
index 1b22d404e4c5f2..910bdeacd68232 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_complex_type.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_complex_type.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_complex_type", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_primary_key_partial_update_complex_type"
             // NOTE:
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_complex_type_schema_change.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_complex_type_schema_change.groovy
index 8a2ed39cf7055e..de74fe6ad13224 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_complex_type_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_complex_type_schema_change.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_complex_type_schema_change", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_primary_key_partial_update_complex_type_schema_change"
             // create table
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_default_value.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_default_value.groovy
index a688ccab2fc167..1a66075d2ab5de 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_default_value.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_default_value.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_default_value", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_default_value"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete.groovy
index 3d6d4582629aac..99fb042bb685d2 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete.groovy
@@ -23,7 +23,7 @@ suite('test_partial_update_delete') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName1 = "test_partial_update_delete1"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete_sign.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete_sign.groovy
index ba38e92eb15909..2805b0b669c3e1 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete_sign.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete_sign.groovy
@@ -23,7 +23,7 @@ suite('test_partial_update_delete_sign') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName1 = "test_partial_update_delete_sign1"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_light_schema_change.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
index cc6faa03dc0d79..135c18f4fc7141 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
@@ -29,7 +29,7 @@ suite("test_partial_update_insert_light_schema_change", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             // ===== light schema change =====
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_seq_col.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_seq_col.groovy
index 44396dc5254725..6dc7169ed2bd25 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_seq_col.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_insert_seq_col.groovy
@@ -24,7 +24,7 @@ suite("test_partial_update_native_insert_seq_col", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             sql "sync;"
 
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt.groovy
index 604978c31472ca..76a65d21728781 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt.groovy
@@ -24,7 +24,7 @@ suite("test_partial_update_native_insert_stmt", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             sql "sync;"
 
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt_complex.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt_complex.groovy
index 1593030b7af472..c03eff973db1ae 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt_complex.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt_complex.groovy
@@ -23,7 +23,7 @@ suite("test_partial_update_native_insert_stmt_complex", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             sql "sync;"
 
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col.groovy
index 884882b7c93934..ef8829bcb8ab35 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col.groovy
@@ -27,7 +27,7 @@ suite("test_primary_key_partial_update_seq_col", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_seq_col"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.groovy
index b37cdc68525d77..5193d876cf89ed 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.groovy
@@ -27,7 +27,7 @@ suite("test_primary_key_partial_update_seq_col_delete", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_seq_col_delete"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type.groovy
index 3075faa022240f..49d40ed8426a97 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type.groovy
@@ -27,7 +27,7 @@ suite("test_primary_key_partial_update_seq_type", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_seq_type"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.groovy
index 142feebef559c1..2d256c6369fae1 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.groovy
@@ -27,7 +27,7 @@ suite("test_primary_key_partial_update_seq_type_delete", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_seq_type_delete"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy
index 313fc97d06ba55..3cf0ef814e9998 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy
@@ -27,7 +27,7 @@ suite("test_partial_update_strict_mode", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_partial_update_strict_mode"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy
index c8203cb0f9da53..b6b72287c3f8ab 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy
@@ -27,7 +27,7 @@ suite("test_partial_update_upsert", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_partial_update_upsert1"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_delete_stmt.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_delete_stmt.groovy
index b853ce2d328295..6dfd3bad4a5026 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_delete_stmt.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_delete_stmt.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_with_delete_stmt", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_with_delete_stmt"
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_inverted_index.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_inverted_index.groovy
index 3395757d876516..0b01d421a29278 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_inverted_index.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_inverted_index.groovy
@@ -27,7 +27,7 @@ suite("test_partial_update_with_inverted_index", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_partial_update_with_inverted_index"
             // create table
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_update_stmt.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_update_stmt.groovy
index 0a29b382b306ae..60fe8167dc9ce5 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_update_stmt.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_with_update_stmt.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_with_update_stmt", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_with_update_stmt"
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_delete_sign.groovy b/regression-test/suites/unique_with_mow_c_p0/test_delete_sign.groovy
index 8192aa2de1d7b6..0437bf709d5b9f 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_delete_sign.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_delete_sign.groovy
@@ -24,7 +24,7 @@ suite("test_delete_sign", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_delete_sign"
             // test delete sigin X sequence column
diff --git a/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_2pc_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_2pc_schema_change.groovy
index 5d099fc0d102f8..23a9dce1001eb0 100644
--- a/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_2pc_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_2pc_schema_change.groovy
@@ -44,7 +44,7 @@ suite("test_f_2pc_schema_change", "p0") {
 
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_f_2pc_schema_change"
diff --git a/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_complex_type_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_complex_type_schema_change.groovy
index 71f2f22cf78185..6f8002d7f15e99 100644
--- a/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_complex_type_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_complex_type_schema_change.groovy
@@ -22,7 +22,7 @@ suite("test_f_partial_update_complex_type_schema_change", "p0") {
 
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_f_partial_update_complex_type_schema_change"
             sql """ DROP TABLE IF EXISTS ${tableName} """
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_mix_partial_update.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_mix_partial_update.groovy
index 35b5e0cce81177..8e82e051562333 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_mix_partial_update.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_mix_partial_update.groovy
@@ -23,7 +23,7 @@ suite('test_mix_partial_update') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             sql "set enable_nereids_planner=true"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy
index 32cb8eedc1279c..b856f6a452f01b 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy
@@ -23,7 +23,7 @@ suite('test_new_partial_update_delete') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             sql "set enable_nereids_planner=true"
@@ -143,7 +143,7 @@ suite('test_new_partial_update_delete') {
         }
     }
 
-    connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+    connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
         sql "use ${db};"
         try {
             def tableAggName = "test_new_partial_update_agg_delete"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy
index 9643f9501ed48a..5e91790baf932a 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_primary_key_partial_update"
             // create table
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy
index 6f000786fa9cc1..9b323dec19c7a8 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy
@@ -42,7 +42,7 @@ suite("test_partial_update_2pc_schema_change", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_partial_update_2pc_schema_change"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy
index 9757b9e7685846..e12674407d6b33 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy
@@ -23,7 +23,7 @@ suite("test_partial_update_after_delete", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             sql "set enable_unique_key_partial_update=false;"
             sql "set enable_insert_strict=true;"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
index d689deeb8afe02..6f79ddf7f9e532 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
@@ -19,7 +19,7 @@ suite("test_partial_update_auto_inc") {
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
 
-    connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+    connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
         sql "use ${db};"
 
         sql """ DROP TABLE IF EXISTS test_primary_key_partial_update_auto_inc """
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_case_insensitivity.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_case_insensitivity.groovy
index 4e114349ce025c..4d6db47fac869e 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_case_insensitivity.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_case_insensitivity.groovy
@@ -21,7 +21,7 @@ suite("test_partial_update_case_insensitivity", "p0") {
     sql "select 1;" // to create database
 
     for (def use_row_store : [false, true]) {
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_partial_update_case_insensitivity"
             sql """ DROP TABLE IF EXISTS ${tableName} """
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy
index ba13bfbeef9b13..ebd32310fd2b45 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_complex_type", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_primary_key_partial_update_complex_type"
             // NOTE:
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.groovy
index 6604ced55bcad7..43e6a2a3f9de13 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type_schema_change.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_complex_type_schema_change", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_primary_key_partial_update_complex_type_schema_change"
             // create table
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_default_value.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_default_value.groovy
index cacdc5113fd4fa..28d1d0ed42d556 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_default_value.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_default_value.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_default_value", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_default_value"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy
index 7b25982f896849..bf49c95d38f0bb 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy
@@ -22,7 +22,7 @@ suite('test_partial_update_delete') {
 
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             sql "sync;"
 
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
index f2d93d9d715413..c6ac7f501f1bb1 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
@@ -23,7 +23,7 @@ suite('test_partial_update_delete_sign') {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName1 = "test_partial_update_delete_sign1"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
index f085a1bb7d3a20..d92dc41a68bc34 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
@@ -26,7 +26,7 @@ suite("test_partial_update_insert_light_schema_change", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             // ===== light schema change =====
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_seq_col.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_seq_col.groovy
index 4c6cbaaa9ccbf8..05725604f330a9 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_seq_col.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_seq_col.groovy
@@ -24,7 +24,7 @@ suite("test_partial_update_native_insert_seq_col", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             sql "sync;"
 
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy
index f1919e8b066a80..dfaa9aca3068f0 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy
@@ -34,7 +34,7 @@ suite("test_partial_update_merge_type", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_partial_update_merge_type"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy
index 457961e2fec60c..c3e25a3a690c44 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy
@@ -24,7 +24,7 @@ suite("test_partial_update_native_insert_stmt", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             sql "sync;"
 
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt_complex.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt_complex.groovy
index f014beb309efcd..350d0f9b00827a 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt_complex.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt_complex.groovy
@@ -23,7 +23,7 @@ suite("test_partial_update_native_insert_stmt_complex", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             sql "sync;"
 
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
index 29f1257f9cb4ad..722f1fccb7643c 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
@@ -24,7 +24,7 @@ suite("test_partial_update_only_keys", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_partial_update_only_keys"
             sql """ DROP TABLE IF EXISTS ${tableName} force"""
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy
index 4479a66fdc8e9a..900c01d80e3f8e 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_seq_col", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_seq_col"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.groovy
index a4163f44cc0feb..f59605211ae66b 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_seq_col_delete", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_seq_col_delete"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type.groovy
index f38d39c91e8209..09a93c2ca5b071 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_seq_type", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_seq_type"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.groovy
index ab38bb9929b09e..76d9cdecc900ca 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_seq_type_delete", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_seq_type_delete"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
index fe47a5ebc89450..c4d26baee8f481 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
@@ -24,7 +24,7 @@ suite("test_partial_update_strict_mode", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_partial_update_strict_mode"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_upsert.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_upsert.groovy
index 5e4a02f5407904..ad2e4b2f4ff229 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_upsert.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_upsert.groovy
@@ -24,7 +24,7 @@ suite("test_partial_update_upsert", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_partial_update_upsert1"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_delete_stmt.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_delete_stmt.groovy
index b853ce2d328295..6dfd3bad4a5026 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_delete_stmt.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_delete_stmt.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_with_delete_stmt", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_with_delete_stmt"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_inverted_index.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_inverted_index.groovy
index f72160bb594ee9..5247a7db707cae 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_inverted_index.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_inverted_index.groovy
@@ -24,7 +24,7 @@ suite("test_partial_update_with_inverted_index", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_partial_update_with_inverted_index"
             // create table
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_update_stmt.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_update_stmt.groovy
index c04c535c080d88..d597ff968ea6c0 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_update_stmt.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_with_update_stmt.groovy
@@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_with_update_stmt", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_primary_key_partial_update_with_update_stmt"
diff --git a/regression-test/suites/unique_with_mow_p0/test_delete_sign.groovy b/regression-test/suites/unique_with_mow_p0/test_delete_sign.groovy
index c067b757c091a2..94564b3409179e 100644
--- a/regression-test/suites/unique_with_mow_p0/test_delete_sign.groovy
+++ b/regression-test/suites/unique_with_mow_p0/test_delete_sign.groovy
@@ -24,7 +24,7 @@ suite("test_delete_sign", "p0") {
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
             sql "use ${db};"
             def tableName = "test_delete_sign"
             // test delete sigin X sequence column
diff --git a/regression-test/suites/variant_p0/variant_with_rowstore.groovy b/regression-test/suites/variant_p0/variant_with_rowstore.groovy
index 69957c25859da3..e3df1866309672 100644
--- a/regression-test/suites/variant_p0/variant_with_rowstore.groovy
+++ b/regression-test/suites/variant_p0/variant_with_rowstore.groovy
@@ -92,7 +92,7 @@ suite("regression_test_variant_rowstore", "variant_type"){
             properties("replication_num" = "1", "disable_auto_compaction" = "false", "store_row_column" = "true", "enable_unique_key_merge_on_write" = "true");
     """
     sql """insert into ${table_name} select k, cast(v as string), cast(v as string) from var_rowstore"""
-    def result1 = connect(user=user, password=password, url=prepare_url) {
+    def result1 = connect(user, password, prepare_url) {
         def stmt = prepareStatement "select * from var_rs_pq where k = ?"
         assertEquals(stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
         stmt.setInt(1, -3)
@@ -124,7 +124,7 @@ suite("regression_test_variant_rowstore", "variant_type"){
         );
     """
     sql """insert into table_rs_invalid_json values (1, '1|[""]')"""
-    def result2 = connect(user=user, password=password, url=prepare_url) {
+    def result2 = connect(user, password, prepare_url) {
         def stmt = prepareStatement "select * from table_rs_invalid_json where col0 = ?"
         stmt.setInt(1, 1)
         qe_point_select stmt
diff --git a/regression-test/suites/vault_p0/privilege/test_vault_privilege.groovy b/regression-test/suites/vault_p0/privilege/test_vault_privilege.groovy
index feedbadb3b714e..3238a4d4857469 100644
--- a/regression-test/suites/vault_p0/privilege/test_vault_privilege.groovy
+++ b/regression-test/suites/vault_p0/privilege/test_vault_privilege.groovy
@@ -77,7 +77,7 @@ suite("test_vault_privilege", "nonConcurrent") {
 
         def vault2 = "test_privilege_vault2"
         // Only users with admin role can create storage vault
-        connect(user = user1, password = 'Cloud12345', url = context.config.jdbcUrl) {
+        connect(user1, 'Cloud12345', context.config.jdbcUrl) {
             expectExceptionLike({
                 sql """
                     CREATE STORAGE VAULT IF NOT EXISTS ${vault2}
@@ -91,14 +91,14 @@ suite("test_vault_privilege", "nonConcurrent") {
         }
 
         // Only users with admin role can set/unset default storage vault
-        connect(user = user1, password = 'Cloud12345', url = context.config.jdbcUrl) {
+        connect(user1, 'Cloud12345', context.config.jdbcUrl) {
             expectExceptionLike({
                 sql """
                     SET ${vault1} AS DEFAULT STORAGE VAULT
                 """
             }, "denied")
         }
-        connect(user = user1, password = 'Cloud12345', url = context.config.jdbcUrl) {
+        connect(user1, 'Cloud12345', context.config.jdbcUrl) {
             expectExceptionLike({
                 sql """
                     UNSET DEFAULT STORAGE VAULT
@@ -106,7 +106,7 @@ suite("test_vault_privilege", "nonConcurrent") {
             }, "denied")
         }
 
-        def result = connect(user = user1, password = 'Cloud12345', url = context.config.jdbcUrl) {
+        def result = connect(user1, 'Cloud12345', context.config.jdbcUrl) {
                 sql " SHOW STORAGE VAULT; "
         }
         assertTrue(result.isEmpty())
@@ -114,7 +114,7 @@ suite("test_vault_privilege", "nonConcurrent") {
         sql """
             DROP TABLE IF EXISTS ${table2};
         """
-        connect(user = user1, password = 'Cloud12345', url = context.config.jdbcUrl) {
+        connect(user1, 'Cloud12345', context.config.jdbcUrl) {
             expectExceptionLike({
                 sql """
                     CREATE TABLE IF NOT EXISTS ${table2} (
@@ -135,13 +135,13 @@ suite("test_vault_privilege", "nonConcurrent") {
             GRANT usage_priv ON STORAGE VAULT '${vault1}' TO '${user1}';
         """
 
-        result = connect(user = user1, password = 'Cloud12345', url = context.config.jdbcUrl) {
+        result = connect(user1, 'Cloud12345', context.config.jdbcUrl) {
                 sql " SHOW STORAGE VAULT; "
         }
         storageVaults = result.stream().map(row -> row[0]).collect(Collectors.toSet())
         assertTrue(storageVaults.contains(vault1))
 
-        connect(user = user1, password = 'Cloud12345', url = context.config.jdbcUrl) {
+        connect(user1, 'Cloud12345', context.config.jdbcUrl) {
             sql """
                 CREATE TABLE IF NOT EXISTS ${table2} (
                         C_CUSTKEY     INTEGER NOT NULL,
@@ -160,7 +160,7 @@ suite("test_vault_privilege", "nonConcurrent") {
             REVOKE usage_priv ON STORAGE VAULT '${vault1}' FROM '${user1}';
         """
 
-        result = connect(user = user1, password = 'Cloud12345', url = context.config.jdbcUrl) {
+        result = connect(user1, 'Cloud12345', context.config.jdbcUrl) {
                 sql " SHOW STORAGE VAULT; "
         }
         assertTrue(result.isEmpty())
@@ -168,7 +168,7 @@ suite("test_vault_privilege", "nonConcurrent") {
         sql """
             DROP TABLE IF EXISTS ${table3};
         """
-        connect(user = user1, password = 'Cloud12345', url = context.config.jdbcUrl) {
+        connect(user1, 'Cloud12345', context.config.jdbcUrl) {
             expectExceptionLike({
                 sql """
                     CREATE TABLE IF NOT EXISTS ${table3} (
diff --git a/regression-test/suites/vault_p0/privilege/test_vault_privilege_restart.groovy b/regression-test/suites/vault_p0/privilege/test_vault_privilege_restart.groovy
index 7ff5ec0792b90d..fa5475c1762e96 100644
--- a/regression-test/suites/vault_p0/privilege/test_vault_privilege_restart.groovy
+++ b/regression-test/suites/vault_p0/privilege/test_vault_privilege_restart.groovy
@@ -112,7 +112,7 @@ suite("test_vault_privilege_restart", "nonConcurrent") {
 
         // Test user privilege, the newly created user cannot create or set default vault
         // Only users with admin role can create storage vault
-        connect(user = user1, password = passwd, url = context.config.jdbcUrl) {
+        connect(user1, passwd, context.config.jdbcUrl) {
             sql """use ${db}"""
             expectExceptionLike({
                 sql """
@@ -126,7 +126,7 @@ suite("test_vault_privilege_restart", "nonConcurrent") {
             }, "denied")
         }
         // Only users with admin role can set/unset default storage vault
-        connect(user = user1, password = passwd, url = context.config.jdbcUrl) {
+        connect(user1, passwd, context.config.jdbcUrl) {
             sql """use ${db}"""
             expectExceptionLike({
                 sql """
@@ -134,7 +134,7 @@ suite("test_vault_privilege_restart", "nonConcurrent") {
                 """
             }, "denied")
         }
-        connect(user = user1, password = passwd, url = context.config.jdbcUrl) {
+        connect(user1, passwd, context.config.jdbcUrl) {
             sql """use ${db}"""
             expectExceptionLike({
                 sql """
@@ -144,7 +144,7 @@ suite("test_vault_privilege_restart", "nonConcurrent") {
         }
 
         // user1 should see vault1
-        def result = connect(user = user1, password = passwd, url = context.config.jdbcUrl) {
+        def result = connect(user1, passwd, context.config.jdbcUrl) {
             sql """use ${db}"""
             sql " SHOW STORAGE VAULT; "
         }
@@ -159,7 +159,7 @@ suite("test_vault_privilege_restart", "nonConcurrent") {
         sql """
             DROP TABLE IF EXISTS ${table2} force;
         """
-        connect(user = user1, password = passwd, url = context.config.jdbcUrl) {
+        connect(user1, passwd, context.config.jdbcUrl) {
             sql """use ${db}"""
             sql """
                 CREATE TABLE ${table2} (
@@ -175,7 +175,7 @@ suite("test_vault_privilege_restart", "nonConcurrent") {
             """
         }
 
-        result = connect(user = user1, password = passwd, url = context.config.jdbcUrl) {
+        result = connect(user1, passwd, context.config.jdbcUrl) {
             sql """use ${db}"""
             sql " SHOW create table ${table2}; "
         }
diff --git a/regression-test/suites/vault_p0/privilege/test_vault_privilege_with_role.groovy b/regression-test/suites/vault_p0/privilege/test_vault_privilege_with_role.groovy
index 4d541989c154f3..7d406cf471a714 100644
--- a/regression-test/suites/vault_p0/privilege/test_vault_privilege_with_role.groovy
+++ b/regression-test/suites/vault_p0/privilege/test_vault_privilege_with_role.groovy
@@ -55,7 +55,7 @@ suite("test_vault_privilege_with_role", "nonConcurrent") {
     sql """CREATE USER ${userName} identified by '${userPassword}' DEFAULT ROLE '${roleName}'"""
     sql """GRANT create_priv ON *.*.* TO '${userName}'; """
 
-    connect(user = userName, password = userPassword, url = context.config.jdbcUrl) {
+    connect(userName, userPassword, context.config.jdbcUrl) {
         expectExceptionLike({
             sql """
                 CREATE TABLE IF NOT EXISTS ${dbName}.${tableName} (
@@ -74,7 +74,7 @@ suite("test_vault_privilege_with_role", "nonConcurrent") {
 
     sql """ GRANT usage_priv ON STORAGE VAULT '${vaultName}' TO ROLE '${roleName}';"""
 
-    connect(user = userName, password = userPassword, url = context.config.jdbcUrl) {
+    connect(userName, userPassword, context.config.jdbcUrl) {
         sql """
             CREATE TABLE IF NOT EXISTS ${dbName}.${tableName} (
                     C_CUSTKEY     INTEGER NOT NULL,
@@ -93,7 +93,7 @@ suite("test_vault_privilege_with_role", "nonConcurrent") {
         REVOKE usage_priv ON STORAGE VAULT '${vaultName}' FROM ROLE '${roleName}';
     """
 
-    connect(user = userName, password = userPassword, url = context.config.jdbcUrl) {
+    connect(userName, userPassword, context.config.jdbcUrl) {
         expectExceptionLike({
             sql """
                 CREATE TABLE IF NOT EXISTS ${dbName}.${tableName}_2 (
diff --git a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy
index 5c3e0aafe76857..be54bec5e2ecda 100644
--- a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy
+++ b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy
@@ -341,11 +341,11 @@ suite("test_crud_wlg") {
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_wlg_user""";
     }
 
-    connect(user = 'test_wlg_user', password = '12345', url = context.config.jdbcUrl) {
+    connect('test_wlg_user', '12345', context.config.jdbcUrl) {
             sql """ select count(1) from information_schema.backend_active_tasks; """
     }
 
-    connect(user = 'test_wlg_user', password = '12345', url = context.config.jdbcUrl) {
+    connect('test_wlg_user', '12345', context.config.jdbcUrl) {
         sql """ set workload_group = test_group; """
         test {
             sql """ select count(1) from information_schema.backend_active_tasks; """
@@ -355,7 +355,7 @@ suite("test_crud_wlg") {
 
     sql "GRANT USAGE_PRIV ON WORKLOAD GROUP 'test_group' TO 'test_wlg_user'@'%';"
 
-    connect(user = 'test_wlg_user', password = '12345', url = context.config.jdbcUrl) {
+    connect('test_wlg_user', '12345', context.config.jdbcUrl) {
         sql """ set workload_group = test_group; """
         sql """ select count(1) from information_schema.backend_active_tasks; """
     }
@@ -731,7 +731,7 @@ suite("test_crud_wlg") {
         def validCluster = clusters[0][0]
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_wg_priv_user2""";
     }
-    connect(user = 'test_wg_priv_user2', password = '', url = context.config.jdbcUrl) {
+    connect('test_wg_priv_user2', '', context.config.jdbcUrl) {
         qt_select_wgp_11 "select GRANTEE,WORKLOAD_GROUP_NAME,PRIVILEGE_TYPE,IS_GRANTABLE from information_schema.workload_group_privileges where grantee like '%test_wg_priv%' order by GRANTEE,WORKLOAD_GROUP_NAME,PRIVILEGE_TYPE,IS_GRANTABLE; "
     }
 
diff --git a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
index 6f80ca3490a608..06e10688397689 100644
--- a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
+++ b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
@@ -157,7 +157,7 @@ suite("test_workload_sched_policy") {
     // 1 create test_set_var_policy
     sql "create workload policy test_set_var_policy conditions(username='test_workload_sched_user')" +
             "actions(set_session_variable 'workload_group=test_set_session_wg');"
-    def result1 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) {
+    def result1 = connect('test_workload_sched_user', '12345', context.config.jdbcUrl) {
         logger.info("begin sleep 15s to wait")
         Thread.sleep(15000)
         sql "show variables like 'workload_group';"
@@ -168,7 +168,7 @@ suite("test_workload_sched_policy") {
     // 2 create test_set_var_policy2 with higher priority
     sql "create workload policy test_set_var_policy2 conditions(username='test_workload_sched_user') " +
             "actions(set_session_variable 'workload_group=test_set_session_wg2') properties('priority'='10');"
-    def result2 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) {
+    def result2 = connect('test_workload_sched_user', '12345', context.config.jdbcUrl) {
         Thread.sleep(3000)
         sql "show variables like 'workload_group';"
     }
@@ -177,7 +177,7 @@ suite("test_workload_sched_policy") {
 
     // 3 disable test_set_var_policy2
     sql "alter workload policy test_set_var_policy2 properties('enabled'='false');"
-    def result3 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) {
+    def result3 = connect('test_workload_sched_user', '12345', context.config.jdbcUrl) {
         Thread.sleep(3000)
         sql "show variables like 'workload_group';"
     }
@@ -242,7 +242,7 @@ suite("test_workload_sched_policy") {
         def curTime = System.currentTimeMillis()
         def totalTime = 30 * 60 * 1000 // 30min
 
-        connect(user = 'test_policy_user', password = '12345', url = context.config.jdbcUrl) {
+        connect('test_policy_user', '12345', context.config.jdbcUrl) {
             sql "set workload_group=policy_group"
             boolean flag = false
             long lastTime = System.currentTimeMillis()
@@ -250,13 +250,13 @@ suite("test_workload_sched_policy") {
             while (curTime - startTime <= totalTime) {
                 if (curTime - lastTime > 20000) {
                     if (flag) {
-                        connect(user = 'root', password = '', url = context.config.jdbcUrl) {
+                        connect('root', '', context.config.jdbcUrl) {
                             sql "alter workload policy test_cancel_query_policy properties('workload_group'='policy_group2');"
                             sql "alter workload policy test_cancel_query_policy2 properties('workload_group'='policy_group');"
                         }
                         flag = false
                     } else {
-                        connect(user = 'root', password = '', url = context.config.jdbcUrl) {
+                        connect('root', '', context.config.jdbcUrl) {
                             sql "alter workload policy test_cancel_query_policy properties('workload_group'='policy_group');"
                             sql "alter workload policy test_cancel_query_policy2 properties('workload_group'='policy_group2');"
                         }

From e8aab052bd6ba299d8f585e8b5d23096a924d7b8 Mon Sep 17 00:00:00 2001
From: Tiewei Fang <fangtiewei@selectdb.com>
Date: Wed, 11 Dec 2024 11:19:47 +0800
Subject: [PATCH 264/399] [fix](Outfile) add more regression-test cases for
 `Select into outfile` (#45230)

Problem Summary:

1. add more regression-test cases for `Select into outfile`
2. Support to export the NULL type data in Doris to Orc file format.
---
 .../apache/doris/analysis/OutFileClause.java  |   6 +
 .../outfile/test_outfile_constant.out         |  76 +++++++++++
 .../outfile/test_outfile_null_type.out        |  12 ++
 .../outfile/test_outfile_constant.groovy      | 121 ++++++++++++++++++
 .../outfile/test_outfile_null_type.groovy     |  21 ++-
 5 files changed, 225 insertions(+), 11 deletions(-)
 create mode 100644 regression-test/data/export_p0/outfile/test_outfile_constant.out
 create mode 100644 regression-test/suites/export_p0/outfile/test_outfile_constant.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index 026e4da29b59da..d05a52ec9054da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -289,6 +289,9 @@ private void genOrcSchema(List<Expr> resultExprs, List<String> colLabels) throws
     private String dorisTypeToOrcTypeMap(Type dorisType) throws AnalysisException {
         String orcType = "";
         switch (dorisType.getPrimitiveType()) {
+            case NULL_TYPE:
+                orcType = "tinyint";
+                break;
             case BOOLEAN:
             case TINYINT:
             case SMALLINT:
@@ -413,6 +416,9 @@ private void analyzeForOrcFormat(List<Expr> resultExprs, List<String> colLabels)
             Pair<String, String> schema = this.orcSchemas.get(i);
             Type resultType = resultExprs.get(i).getType();
             switch (resultType.getPrimitiveType()) {
+                case NULL_TYPE:
+                    checkOrcType(schema.second, "tinyint", true, resultType.getPrimitiveType().toString());
+                    break;
                 case BOOLEAN:
                 case TINYINT:
                 case SMALLINT:
diff --git a/regression-test/data/export_p0/outfile/test_outfile_constant.out b/regression-test/data/export_p0/outfile/test_outfile_constant.out
new file mode 100644
index 00000000000000..855a66a129f4db
--- /dev/null
+++ b/regression-test/data/export_p0/outfile/test_outfile_constant.out
@@ -0,0 +1,76 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_export --
+1	ftw-1	19
+2	ftw-2	20
+3	ftw-3	21
+4	ftw-4	22
+5	ftw-5	23
+6	ftw-6	24
+7	ftw-7	25
+8	ftw-8	26
+9	ftw-9	27
+10	\N	\N
+
+-- !s3_select --
+1
+
+-- !s3_select --
+3.1415926
+
+-- !s3_select --
+2.718281828459045
+
+-- !s3_select --
+3.141592653589793
+
+-- !s3_select --
+2003
+
+-- !s3_select --
+2024-12-09
+
+-- !s3_select --
+\N
+
+-- !s3_select --
+1
+
+-- !s3_select --
+3.1415926
+
+-- !s3_select --
+2.718281828459045
+
+-- !s3_select --
+3.141592653589793
+
+-- !s3_select --
+2003
+
+-- !s3_select --
+2024-12-09
+
+-- !s3_select --
+\N
+
+-- !s3_select --
+1
+
+-- !s3_select --
+3.1415926
+
+-- !s3_select --
+2.718281828459045
+
+-- !s3_select --
+3.141592653589793
+
+-- !s3_select --
+2003
+
+-- !s3_select --
+2024-12-09
+
+-- !s3_select --
+\N
+
diff --git a/regression-test/data/export_p0/outfile/test_outfile_null_type.out b/regression-test/data/export_p0/outfile/test_outfile_null_type.out
index 1777bba72c1b5c..a39131b61e2f82 100644
--- a/regression-test/data/export_p0/outfile/test_outfile_null_type.out
+++ b/regression-test/data/export_p0/outfile/test_outfile_null_type.out
@@ -23,6 +23,18 @@
 8	ftw-8	26	\N
 9	ftw-9	27	\N
 
+-- !select_load_orc --
+1	ftw-1	19	\N
+10	\N	\N	\N
+2	ftw-2	20	\N
+3	ftw-3	21	\N
+4	ftw-4	22	\N
+5	ftw-5	23	\N
+6	ftw-6	24	\N
+7	ftw-7	25	\N
+8	ftw-8	26	\N
+9	ftw-9	27	\N
+
 -- !select_load_csv --
 1	ftw-1	19	\N
 10	\N	\N	\N
diff --git a/regression-test/suites/export_p0/outfile/test_outfile_constant.groovy b/regression-test/suites/export_p0/outfile/test_outfile_constant.groovy
new file mode 100644
index 00000000000000..3df294992376bf
--- /dev/null
+++ b/regression-test/suites/export_p0/outfile/test_outfile_constant.groovy
@@ -0,0 +1,121 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_outfile_constant", "p0") {
+    String ak = getS3AK()
+    String sk = getS3SK()
+    String s3_endpoint = getS3Endpoint()
+    String region = getS3Region()
+    String bucket = context.config.otherConfigs.get("s3BucketName");
+
+    def export_table_name = "test_outfile_constant"
+    def outFilePath = "${bucket}/outfile/constant_type/exp_"
+
+    def outfile_to_S3 = { select_stmt, format ->
+        // select ... into outfile ...
+        def res = sql """
+            ${select_stmt}
+            INTO OUTFILE "s3://${outFilePath}"
+            FORMAT AS ${format}
+            PROPERTIES (
+                "s3.endpoint" = "${s3_endpoint}",
+                "s3.region" = "${region}",
+                "s3.secret_key"="${sk}",
+                "s3.access_key" = "${ak}"
+            );
+        """
+
+        return res[0][3]
+    }
+
+    def s3_tvf = { outfile_url, format ->
+       order_qt_s3_select """ SELECT * FROM S3 (
+            "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+            "ACCESS_KEY"= "${ak}",
+            "SECRET_KEY" = "${sk}",
+            "format" = "${format}",
+            "region" = "${region}"
+        );
+        """ 
+    }
+
+    sql """ DROP TABLE IF EXISTS ${export_table_name} """
+    sql """
+        CREATE TABLE `${export_table_name}` (
+            `id` int(11) NULL,
+            `Name` string NULL,
+            `age` int(11) NULL
+        ) ENGINE=OLAP
+        DISTRIBUTED BY HASH(`id`)
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+        """
+
+    StringBuilder sb = new StringBuilder()
+    int i = 1
+    for (; i < 10; i ++) {
+        sb.append("""
+            (${i}, 'ftw-${i}', ${i + 18}),
+        """)
+    }
+    sb.append("""
+            (${i}, NULL, NULL)
+        """)
+    sql """ INSERT INTO ${export_table_name} VALUES
+            ${sb.toString()}
+        """
+    def insert_res = sql "show last insert;"
+    logger.info("insert result: " + insert_res.toString())
+    qt_select_export """ SELECT * FROM ${export_table_name} t ORDER BY id; """
+
+
+    def test_outfile_constant = { format ->
+        // select 1
+        def outfile_url = outfile_to_S3("Select 1", "${format}")
+        s3_tvf("${outfile_url}", "${format}")
+
+        // select 3.1415926
+        outfile_url = outfile_to_S3("Select 3.1415926", "${format}")
+        s3_tvf("${outfile_url}", "${format}")
+
+        // select e()
+        outfile_url = outfile_to_S3("Select e()", "${format}")
+        s3_tvf("${outfile_url}", "${format}")
+
+        // select Pi()
+        outfile_url = outfile_to_S3("Select Pi()", "${format}")
+        s3_tvf("${outfile_url}", "${format}")
+
+        // select 2024-12-09;
+        outfile_url = outfile_to_S3("Select 2024-12-09", "${format}")
+        s3_tvf("${outfile_url}", "${format}")
+
+        // select "2024-12-09";
+        outfile_url = outfile_to_S3("""Select "2024-12-09" """, "${format}")
+        s3_tvf("${outfile_url}", "${format}")
+
+        // select null
+        outfile_url = outfile_to_S3("Select null", "${format}")
+        s3_tvf("${outfile_url}", "${format}")
+    }
+
+    test_outfile_constant("parquet");
+    test_outfile_constant("orc");
+    test_outfile_constant("csv");
+    
+}
\ No newline at end of file
diff --git a/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy b/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
index 492b6770195341..fcd75f63c5b969 100644
--- a/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
+++ b/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
@@ -84,18 +84,17 @@ suite("test_outfile_null_type", "p0") {
             );
             """
 
-    // TODO(ftw): orc supports to export data of NULL type.
     // orc file foramt
-    // format = "orc"
-    // outfile_url = outfile_to_S3("${format}")
-    // qt_select_load_orc """ SELECT * FROM S3 (
-    //             "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
-    //             "ACCESS_KEY"= "${ak}",
-    //             "SECRET_KEY" = "${sk}",
-    //             "format" = "${format}",
-    //             "region" = "${region}"
-    //         );
-    //         """
+    format = "orc"
+    outfile_url = outfile_to_S3("${format}")
+    order_qt_select_load_orc """ SELECT * FROM S3 (
+                "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.${format}",
+                "ACCESS_KEY"= "${ak}",
+                "SECRET_KEY" = "${sk}",
+                "format" = "${format}",
+                "region" = "${region}"
+            );
+            """
 
     // csv file foramt
     format = "csv"

From 4ec8db51b79baf26990907e03dd7d3e04a4a5faf Mon Sep 17 00:00:00 2001
From: Tiewei Fang <fangtiewei@selectdb.com>
Date: Wed, 11 Dec 2024 11:20:56 +0800
Subject: [PATCH 265/399] [fix](Outfile) fix the unclear error code (#45119)

Problem Summary:

If errors occur with the arrow SDK, they are being converted to the
`Status::InvalidArgument` error in Doris. However, this is not clear. It
would be more appropriate to use the `Status::InternalError` code
instead.
---
 be/src/util/arrow/utils.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/util/arrow/utils.cpp b/be/src/util/arrow/utils.cpp
index 742f5bd0fc33f3..0201373beb0125 100644
--- a/be/src/util/arrow/utils.cpp
+++ b/be/src/util/arrow/utils.cpp
@@ -28,7 +28,7 @@ Status to_doris_status(const arrow::Status& status) {
     if (status.ok()) {
         return Status::OK();
     } else {
-        return Status::InvalidArgument(status.ToString());
+        return Status::InternalError(status.ToString());
     }
 }
 

From cc51e99dae339381c22a494460914e8a0431e6fb Mon Sep 17 00:00:00 2001
From: yujun <yujun@selectdb.com>
Date: Wed, 11 Dec 2024 11:51:55 +0800
Subject: [PATCH 266/399] [feat](nereids) expression add min max scope for slot
 reference (#45081)

expression add min max scope for slot reference, so that olap scan can
reduce field's searching scope.

detail: in simplify range rule, record min and max range of the slot
reference, then add 'e > min and e < max' to the expression.

example:

```
TA >= 10 and TA <= 20 or TA >= 50 and TA <= 60 or TA >= 100 and TA <= 120
    => (TA <= 20 or TA >= 50 and TA <= 60 or TA >= 100) AND TA >= 10 and TA <= 120

TA in (10, 50, 100) or TA >= 70 and TA <= 90
    => (TA in (10, 50, 100) or TA >= 70 AND TA <= 90) AND TA >= 10 AND TA <= 100

TA between 10 and 20 and TB between 10 and 20 or TA between 100 and 120 and TB between 100 and 120
    => (TA <= 20 and TB <= 20 or TA >= 100 and TB >= 100) AND TA >= 10 AND TA <= 120 AND TB >= 10 AND TB <= 120

ISNULL (TA > 10) and TA > 10 and TA < 20 or TA > 50 and TA < 60 or TA > 100 and TA < 120
    => (ISNULL(TA > 10) and TA < 20 or TA > 50 and TA < 60 or TA > 100) AND TA > 10 and TA < 120
```

benchmark on tpch tpcds 1T,  performance no change.
---
 ...xpressionNormalizationAndOptimization.java |   1 +
 .../expression/ExpressionOptimization.java    |  15 +
 .../rules/expression/rules/AddMinMax.java     | 378 ++++++++++++
 .../expression/rules/RangeInference.java      | 478 ++++++++++++++++
 .../rules/expression/rules/SimplifyRange.java | 536 +++---------------
 .../doris/nereids/util/ExpressionUtils.java   |  52 +-
 .../expression/ExpressionRewriteTest.java     |  62 ++
 .../ExpressionRewriteTestHelper.java          |  12 +-
 .../rules/expression/SimplifyRangeTest.java   |   2 +-
 .../doris/nereids/sqltest/InferTest.java      |  17 +-
 .../nereids_hint_tpcds_p0/shape/query13.out   |   4 +-
 .../nereids_hint_tpcds_p0/shape/query34.out   |   2 +-
 .../nereids_hint_tpcds_p0/shape/query48.out   |   4 +-
 .../nereids_hint_tpcds_p0/shape/query85.out   |   4 +-
 .../nereids_hint_tpcds_p0/shape/query88.out   |  16 +-
 .../data/nereids_hint_tpch_p0/shape/q19.out   |   4 +-
 .../bs_downgrade_shape/query13.out            |   4 +-
 .../shape/query13.out                         |   4 +-
 .../shape/query34.out                         |  12 +-
 .../shape/query48.out                         |   4 +-
 .../shape/query85.out                         |   4 +-
 .../shape/query88.out                         |  16 +-
 .../noStatsRfPrune/query13.out                |   4 +-
 .../noStatsRfPrune/query34.out                |   2 +-
 .../noStatsRfPrune/query48.out                |   4 +-
 .../noStatsRfPrune/query85.out                |   4 +-
 .../noStatsRfPrune/query88.out                |  16 +-
 .../no_stats_shape/query13.out                |   4 +-
 .../no_stats_shape/query34.out                |   2 +-
 .../no_stats_shape/query48.out                |   4 +-
 .../no_stats_shape/query85.out                |   4 +-
 .../no_stats_shape/query88.out                |  16 +-
 .../rf_prune/query13.out                      |   4 +-
 .../rf_prune/query34.out                      |  12 +-
 .../rf_prune/query48.out                      |   4 +-
 .../rf_prune/query85.out                      |   4 +-
 .../rf_prune/query88.out                      |  16 +-
 .../shape/query13.out                         |   4 +-
 .../shape/query34.out                         |  12 +-
 .../shape/query48.out                         |   4 +-
 .../shape/query85.out                         |   4 +-
 .../shape/query88.out                         |  16 +-
 .../shape/query13.out                         |   4 +-
 .../shape/query34.out                         |   2 +-
 .../shape/query48.out                         |   4 +-
 .../shape/query85.out                         |   4 +-
 .../shape/query88.out                         |  16 +-
 .../nostats_rf_prune/q19.out                  |   4 +-
 .../rf_prune/q19.out                          |   4 +-
 .../shape/q19.out                             |   4 +-
 .../shape_no_stats/q19.out                    |   4 +-
 .../new_shapes_p0/hint_tpch/shape/q19.out     |   4 +-
 .../tpcds_sf100/noStatsRfPrune/query13.out    |   4 +-
 .../tpcds_sf100/noStatsRfPrune/query34.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query48.out    |   4 +-
 .../tpcds_sf100/noStatsRfPrune/query85.out    |   4 +-
 .../tpcds_sf100/noStatsRfPrune/query88.out    |  16 +-
 .../tpcds_sf100/no_stats_shape/query13.out    |   4 +-
 .../tpcds_sf100/no_stats_shape/query34.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query48.out    |   4 +-
 .../tpcds_sf100/no_stats_shape/query85.out    |   4 +-
 .../tpcds_sf100/no_stats_shape/query88.out    |  16 +-
 .../tpcds_sf100/rf_prune/query13.out          |   4 +-
 .../tpcds_sf100/rf_prune/query34.out          |  12 +-
 .../tpcds_sf100/rf_prune/query48.out          |   4 +-
 .../tpcds_sf100/rf_prune/query85.out          |   4 +-
 .../tpcds_sf100/rf_prune/query88.out          |  16 +-
 .../tpcds_sf100/shape/query13.out             |   4 +-
 .../tpcds_sf100/shape/query34.out             |  12 +-
 .../tpcds_sf100/shape/query48.out             |   4 +-
 .../tpcds_sf100/shape/query85.out             |   4 +-
 .../tpcds_sf100/shape/query88.out             |  16 +-
 .../bs_downgrade_shape/query13.out            |   4 +-
 .../tpcds_sf1000/shape/query13.out            |   4 +-
 .../tpcds_sf1000/shape/query34.out            |  12 +-
 .../tpcds_sf1000/shape/query48.out            |   4 +-
 .../tpcds_sf1000/shape/query85.out            |   4 +-
 .../tpcds_sf1000/shape/query88.out            |  16 +-
 .../tpch_sf1000/nostats_rf_prune/q19.out      |   4 +-
 .../tpch_sf1000/rf_prune/q19.out              |   4 +-
 .../new_shapes_p0/tpch_sf1000/shape/q19.out   |   4 +-
 .../tpch_sf1000/shape_no_stats/q19.out        |   4 +-
 82 files changed, 1318 insertions(+), 703 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/AddMinMax.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalizationAndOptimization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalizationAndOptimization.java
index d694062ef1f049..1bb698b07ee4a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalizationAndOptimization.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalizationAndOptimization.java
@@ -27,6 +27,7 @@ public ExpressionNormalizationAndOptimization() {
                 ImmutableList.<ExpressionRewriteRule>builder()
                         .addAll(ExpressionNormalization.NORMALIZE_REWRITE_RULES)
                         .addAll(ExpressionOptimization.OPTIMIZE_REWRITE_RULES)
+                        .addAll(ExpressionOptimization.ADD_RANGE)
                         .build()
         ));
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
index 5d45b2430c42b0..21c56f516ada7b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
@@ -17,6 +17,7 @@
 
 package org.apache.doris.nereids.rules.expression;
 
+import org.apache.doris.nereids.rules.expression.rules.AddMinMax;
 import org.apache.doris.nereids.rules.expression.rules.ArrayContainToArrayOverlap;
 import org.apache.doris.nereids.rules.expression.rules.BetweenToEqual;
 import org.apache.doris.nereids.rules.expression.rules.CaseWhenToIf;
@@ -58,6 +59,20 @@ public class ExpressionOptimization extends ExpressionRewrite {
                     BetweenToEqual.INSTANCE
             )
     );
+
+    /**
+     * don't use it with PushDownFilterThroughJoin, it may cause dead loop:
+     *   LogicalFilter(origin expr)
+     *      => LogicalFilter((origin expr) and (add min max range))
+     *      => LogicalFilter((origin expr)) // use PushDownFilterThroughJoin
+     *      => ...
+     */
+    public static final List<ExpressionRewriteRule> ADD_RANGE = ImmutableList.of(
+            bottomUp(
+                    AddMinMax.INSTANCE
+            )
+    );
+
     private static final ExpressionRuleExecutor EXECUTOR = new ExpressionRuleExecutor(OPTIMIZE_REWRITE_RULES);
 
     public ExpressionOptimization() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/AddMinMax.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/AddMinMax.java
new file mode 100644
index 00000000000000..639616244ec0ab
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/AddMinMax.java
@@ -0,0 +1,378 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.DiscreteValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.EmptyValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.RangeValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.UnknownValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.ValueDesc;
+import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
+import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
+import org.apache.doris.nereids.trees.expressions.LessThan;
+import org.apache.doris.nereids.trees.expressions.LessThanEqual;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.collect.BoundType;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
+import com.google.common.collect.Sets;
+import org.apache.commons.lang3.NotImplementedException;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * This class implements the function to add min max to or expression.
+ * for example:
+ *
+ * a > 10 and a < 20 or a > 30 and a < 40 or a > 50 and a < 60
+ *   => (a < 20 or a > 30 and a < 40 or a > 50) and a > 10 and a < 60
+ *
+ * a between 10 and 20 and b between 10 and 20 or a between 100 and 200 and b between 100 and 200
+ *   => (a <= 20 and b <= 20 or a >= 100 and b >= 100) and a >= 10 and a <= 200 and b >= 10 and b <= 200
+ */
+public class AddMinMax implements ExpressionPatternRuleFactory {
+    public static final AddMinMax INSTANCE = new AddMinMax();
+
+    @Override
+    public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
+        return ImmutableList.of(
+                matchesTopType(CompoundPredicate.class)
+                        .thenApply(ctx -> rewrite(ctx.expr, ctx.rewriteContext))
+        );
+    }
+
+    /** rewrite */
+    public Expression rewrite(CompoundPredicate expr, ExpressionRewriteContext context) {
+        ValueDesc valueDesc = (new RangeInference()).getValue(expr, context);
+        Map<Expression, MinMaxValue> exprMinMaxValues = getExprMinMaxValues(valueDesc);
+        removeUnnecessaryMinMaxValues(expr, exprMinMaxValues);
+        if (!exprMinMaxValues.isEmpty()) {
+            return addExprMinMaxValues(expr, context, exprMinMaxValues);
+        }
+        return expr;
+    }
+
+    private enum MatchMinMax {
+        MATCH_MIN,
+        MATCH_MAX,
+        MATCH_NONE,
+    }
+
+    private static class MinMaxValue {
+        // min max range, if range = null means empty
+        Range<Literal> range;
+
+        // expression in range is discrete value
+        boolean isDiscrete;
+
+        // expr relative order, for keep order after add min-max to the expression
+        int exprOrderIndex;
+
+        public MinMaxValue(Range<Literal> range, boolean isDiscrete, int exprOrderIndex) {
+            this.range = range;
+            this.isDiscrete = isDiscrete;
+            this.exprOrderIndex = exprOrderIndex;
+        }
+    }
+
+    private void removeUnnecessaryMinMaxValues(Expression expr, Map<Expression, MinMaxValue> exprMinMaxValues) {
+        exprMinMaxValues.entrySet().removeIf(entry -> entry.getValue().isDiscrete || entry.getValue().range == null
+                || (!entry.getValue().range.hasLowerBound() && !entry.getValue().range.hasUpperBound()));
+        if (exprMinMaxValues.isEmpty()) {
+            return;
+        }
+
+        // keep original expression order, don't rewrite a sub expression if it's in original conjunctions.
+        // example: if original expression is:  '(a >= 100) AND (...)',  and after visiting got a's range is [100, 200],
+        // because 'a >= 100' is in expression's conjunctions, don't add 'a >= 100' to expression,
+        // then the rewritten expression is '((a >= 100) AND (...)) AND (a <= 200)'
+        List<Expression> conjuncts = ExpressionUtils.extractConjunction(expr);
+        for (Expression conjunct : conjuncts) {
+            List<Expression> disjunctions = ExpressionUtils.extractDisjunction(conjunct);
+            if (disjunctions.isEmpty() || !(disjunctions.get(0) instanceof ComparisonPredicate)) {
+                continue;
+            }
+            Expression targetExpr = disjunctions.get(0).child(0);
+            boolean matchMin = false;
+            boolean matchMax = false;
+            for (Expression disjunction : disjunctions) {
+                MatchMinMax match = getExprMatchMinMax(disjunction, exprMinMaxValues);
+                if (match == MatchMinMax.MATCH_NONE || !disjunction.child(0).equals(targetExpr)) {
+                    matchMin = false;
+                    matchMax = false;
+                    break;
+                }
+                if (match == MatchMinMax.MATCH_MIN) {
+                    matchMin = true;
+                } else if (match == MatchMinMax.MATCH_MAX) {
+                    matchMax = true;
+                }
+            }
+            MinMaxValue targetValue = exprMinMaxValues.get(targetExpr);
+            if (matchMin) {
+                // remove targetValue's lower bound
+                if (targetValue.range.hasUpperBound()) {
+                    targetValue.range = Range.upTo(targetValue.range.upperEndpoint(),
+                        targetValue.range.upperBoundType());
+                } else {
+                    exprMinMaxValues.remove(targetExpr);
+                }
+            }
+            if (matchMax) {
+                // remove targetValue's upper bound
+                if (targetValue.range.hasLowerBound()) {
+                    targetValue.range = Range.downTo(targetValue.range.lowerEndpoint(),
+                        targetValue.range.lowerBoundType());
+                } else {
+                    exprMinMaxValues.remove(targetExpr);
+                }
+            }
+        }
+    }
+
+    private Expression addExprMinMaxValues(Expression expr, ExpressionRewriteContext context,
+            Map<Expression, MinMaxValue> exprMinMaxValues) {
+        List<Map.Entry<Expression, MinMaxValue>> minMaxExprs = exprMinMaxValues.entrySet().stream()
+                .sorted((a, b) -> Integer.compare(a.getValue().exprOrderIndex, b.getValue().exprOrderIndex))
+                .collect(Collectors.toList());
+        List<Expression> addExprs = Lists.newArrayListWithExpectedSize(minMaxExprs.size() * 2);
+        for (Map.Entry<Expression, MinMaxValue> entry : minMaxExprs) {
+            Expression targetExpr = entry.getKey();
+            Range<Literal> range = entry.getValue().range;
+            if (range.hasLowerBound() && range.hasUpperBound()
+                    && range.lowerEndpoint().equals(range.upperEndpoint())
+                    && range.lowerBoundType() == BoundType.CLOSED
+                    && range.upperBoundType() == BoundType.CLOSED) {
+                Expression cmp = new EqualTo(targetExpr, range.lowerEndpoint());
+                addExprs.add(cmp);
+                continue;
+            }
+            if (range.hasLowerBound()) {
+                Literal literal = range.lowerEndpoint();
+                Expression cmp = range.lowerBoundType() == BoundType.CLOSED
+                        ? new GreaterThanEqual(targetExpr, literal) : new GreaterThan(targetExpr, literal);
+                addExprs.add(cmp);
+            }
+            if (range.hasUpperBound()) {
+                Literal literal = range.upperEndpoint();
+                Expression cmp = range.upperBoundType() == BoundType.CLOSED
+                        ? new LessThanEqual(targetExpr, literal) : new LessThan(targetExpr, literal);
+                addExprs.add(cmp);
+            }
+        }
+
+        // later will add `addExprs` to original expr, before doing that, remove duplicate expr in original expr
+        Expression replaceOriginExpr = replaceCmpMinMax(expr, Sets.newHashSet(addExprs));
+
+        addExprs.add(0, replaceOriginExpr);
+        Expression result = FoldConstantRuleOnFE.evaluate(ExpressionUtils.and(addExprs), context);
+        if (result.equals(expr)) {
+            return expr;
+        }
+
+        return result;
+    }
+
+    private Expression replaceCmpMinMax(Expression expr, Set<Expression> cmpMinMaxExprs) {
+        // even if expr is nullable, replace it to true is ok because expression will 'AND' it later
+        if (cmpMinMaxExprs.contains(expr)) {
+            return BooleanLiteral.TRUE;
+        }
+
+        // only replace those expression whose all its ancestors are AND / OR
+        if (!(expr instanceof CompoundPredicate)) {
+            return expr;
+        }
+
+        ImmutableList.Builder<Expression> newChildren = ImmutableList.builderWithExpectedSize(expr.arity());
+        boolean changed = false;
+        for (Expression child : expr.children()) {
+            Expression newChild = replaceCmpMinMax(child, cmpMinMaxExprs);
+            if (child != newChild) {
+                changed = true;
+            }
+            newChildren.add(newChild);
+        }
+
+        if (changed) {
+            return expr.withChildren(newChildren.build());
+        } else {
+            return expr;
+        }
+    }
+
+    private MatchMinMax getExprMatchMinMax(Expression expr,
+            Map<Expression, MinMaxValue> exprMinMaxValues) {
+        if (!(expr instanceof ComparisonPredicate)) {
+            return MatchMinMax.MATCH_NONE;
+        }
+
+        ComparisonPredicate cp = (ComparisonPredicate) expr;
+        Expression left = cp.left();
+        Expression right = cp.right();
+        if (!(right instanceof Literal)) {
+            return MatchMinMax.MATCH_NONE;
+        }
+
+        MinMaxValue value = exprMinMaxValues.get(left);
+        if (value == null) {
+            return MatchMinMax.MATCH_NONE;
+        }
+
+        if (cp instanceof GreaterThan || cp instanceof GreaterThanEqual) {
+            if (value.range.hasLowerBound() && value.range.lowerEndpoint().equals(right)) {
+                BoundType boundType = value.range.lowerBoundType();
+                if ((boundType == BoundType.CLOSED && expr instanceof GreaterThanEqual)
+                        || (boundType == BoundType.OPEN && expr instanceof GreaterThan)) {
+                    return MatchMinMax.MATCH_MIN;
+                }
+            }
+        } else if (expr instanceof LessThan || expr instanceof LessThanEqual) {
+            if (value.range.hasUpperBound() && value.range.upperEndpoint().equals(right)) {
+                BoundType boundType = value.range.upperBoundType();
+                if ((boundType == BoundType.CLOSED && expr instanceof LessThanEqual)
+                        || (boundType == BoundType.OPEN && expr instanceof LessThan)) {
+                    return MatchMinMax.MATCH_MAX;
+                }
+            }
+        }
+
+        return MatchMinMax.MATCH_NONE;
+    }
+
+    private boolean isExprNeedAddMinMax(Expression expr) {
+        return (expr instanceof SlotReference) && ((SlotReference) expr).getColumn().isPresent();
+    }
+
+    private Map<Expression, MinMaxValue> getExprMinMaxValues(ValueDesc value) {
+        if (value instanceof EmptyValue) {
+            return getExprMinMaxValues((EmptyValue) value);
+        } else if (value instanceof DiscreteValue) {
+            return getExprMinMaxValues((DiscreteValue) value);
+        } else if (value instanceof RangeValue) {
+            return getExprMinMaxValues((RangeValue) value);
+        } else if (value instanceof UnknownValue) {
+            return getExprMinMaxValues((UnknownValue) value);
+        } else {
+            throw new NotImplementedException("not implements");
+        }
+    }
+
+    private Map<Expression, MinMaxValue> getExprMinMaxValues(EmptyValue value) {
+        Expression reference = value.getReference();
+        Map<Expression, MinMaxValue> exprMinMaxValues = Maps.newHashMap();
+        if (isExprNeedAddMinMax(reference)) {
+            exprMinMaxValues.put(reference, new MinMaxValue(null, true, 0));
+        }
+        return exprMinMaxValues;
+    }
+
+    private Map<Expression, MinMaxValue> getExprMinMaxValues(DiscreteValue value) {
+        Expression reference = value.getReference();
+        Map<Expression, MinMaxValue> exprMinMaxValues = Maps.newHashMap();
+        if (isExprNeedAddMinMax(reference)) {
+            exprMinMaxValues.put(reference, new MinMaxValue(Range.encloseAll(value.getValues()), true, 0));
+        }
+        return exprMinMaxValues;
+    }
+
+    private Map<Expression, MinMaxValue> getExprMinMaxValues(RangeValue value) {
+        Expression reference = value.getReference();
+        Map<Expression, MinMaxValue> exprMinMaxValues = Maps.newHashMap();
+        if (isExprNeedAddMinMax(reference)) {
+            exprMinMaxValues.put(reference, new MinMaxValue(value.getRange(), false, 0));
+        }
+        return exprMinMaxValues;
+    }
+
+    private Map<Expression, MinMaxValue> getExprMinMaxValues(UnknownValue valueDesc) {
+        List<ValueDesc> sourceValues = valueDesc.getSourceValues();
+        if (sourceValues.isEmpty()) {
+            return Maps.newHashMap();
+        }
+        Map<Expression, MinMaxValue> result = Maps.newHashMap(getExprMinMaxValues(sourceValues.get(0)));
+        int nextExprOrderIndex = result.values().stream().mapToInt(k -> k.exprOrderIndex).max().orElse(0);
+        for (int i = 1; i < sourceValues.size(); i++) {
+            // process in sourceValues[i]
+            Map<Expression, MinMaxValue> minMaxValues = getExprMinMaxValues(sourceValues.get(i));
+            for (Map.Entry<Expression, MinMaxValue> entry : minMaxValues.entrySet()) {
+                Expression expr = entry.getKey();
+                MinMaxValue value = result.get(expr);
+                MinMaxValue otherValue = entry.getValue();
+                if (valueDesc.isAnd()) {
+                    if (value == null) { // value = null means range for all
+                        nextExprOrderIndex++;
+                        value = otherValue;
+                        value.exprOrderIndex = nextExprOrderIndex;
+                        result.put(expr, value);
+                    } else if (otherValue.range == null) { // range = null means empty range
+                        value.range = null;
+                    } else if (value.range != null) {
+                        if (value.range.isConnected(otherValue.range)) {
+                            value.range = value.range.intersection(otherValue.range);
+                            value.isDiscrete = value.isDiscrete && otherValue.isDiscrete;
+                        } else {
+                            value.range = null;
+                        }
+                    }
+                } else {
+                    if (value == null) { // value = null means range for all
+                        nextExprOrderIndex++;
+                        value = new MinMaxValue(Range.all(), false, nextExprOrderIndex);
+                        result.put(expr, value);
+                    } else if (value.range == null) { // range = null means empty range
+                        value.range = otherValue.range;
+                        value.isDiscrete = otherValue.isDiscrete;
+                    } else if (otherValue.range != null) {
+                        value.range = value.range.span(otherValue.range);
+                        value.isDiscrete = value.isDiscrete && otherValue.isDiscrete;
+                    }
+                }
+            }
+
+            // process not in sourceValues[i]
+            if (!valueDesc.isAnd()) {
+                for (Map.Entry<Expression, MinMaxValue> entry : result.entrySet()) {
+                    Expression expr = entry.getKey();
+                    MinMaxValue value = entry.getValue();
+                    if (!minMaxValues.containsKey(expr)) {
+                        value.range = Range.all();
+                        value.isDiscrete = false;
+                    }
+                }
+            }
+        }
+        return result;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java
new file mode 100644
index 00000000000000..74099df1123f25
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java
@@ -0,0 +1,478 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.trees.expressions.And;
+import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
+import org.apache.doris.nereids.trees.expressions.InPredicate;
+import org.apache.doris.nereids.trees.expressions.LessThan;
+import org.apache.doris.nereids.trees.expressions.LessThanEqual;
+import org.apache.doris.nereids.trees.expressions.Or;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Range;
+import com.google.common.collect.Sets;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.function.BinaryOperator;
+import java.util.stream.Collectors;
+
+/**
+ * collect range of expression
+ */
+public class RangeInference extends ExpressionVisitor<RangeInference.ValueDesc, ExpressionRewriteContext> {
+
+    /*
+     * get expression's value desc.
+     */
+    public ValueDesc getValue(Expression expr, ExpressionRewriteContext context) {
+        return expr.accept(new RangeInference(), context);
+    }
+
+    @Override
+    public ValueDesc visit(Expression expr, ExpressionRewriteContext context) {
+        return new UnknownValue(context, expr);
+    }
+
+    private ValueDesc buildRange(ExpressionRewriteContext context, ComparisonPredicate predicate) {
+        Expression right = predicate.child(1);
+        if (right.isNullLiteral()) {
+            return new UnknownValue(context, predicate);
+        }
+        // only handle `NumericType` and `DateLikeType`
+        if (right.isLiteral() && (right.getDataType().isNumericType() || right.getDataType().isDateLikeType())) {
+            return ValueDesc.range(context, predicate);
+        }
+        return new UnknownValue(context, predicate);
+    }
+
+    @Override
+    public ValueDesc visitGreaterThan(GreaterThan greaterThan, ExpressionRewriteContext context) {
+        return buildRange(context, greaterThan);
+    }
+
+    @Override
+    public ValueDesc visitGreaterThanEqual(GreaterThanEqual greaterThanEqual, ExpressionRewriteContext context) {
+        return buildRange(context, greaterThanEqual);
+    }
+
+    @Override
+    public ValueDesc visitLessThan(LessThan lessThan, ExpressionRewriteContext context) {
+        return buildRange(context, lessThan);
+    }
+
+    @Override
+    public ValueDesc visitLessThanEqual(LessThanEqual lessThanEqual, ExpressionRewriteContext context) {
+        return buildRange(context, lessThanEqual);
+    }
+
+    @Override
+    public ValueDesc visitEqualTo(EqualTo equalTo, ExpressionRewriteContext context) {
+        return buildRange(context, equalTo);
+    }
+
+    @Override
+    public ValueDesc visitInPredicate(InPredicate inPredicate, ExpressionRewriteContext context) {
+        // only handle `NumericType` and `DateLikeType`
+        if (ExpressionUtils.isAllNonNullLiteral(inPredicate.getOptions())
+                && (ExpressionUtils.matchNumericType(inPredicate.getOptions())
+                || ExpressionUtils.matchDateLikeType(inPredicate.getOptions()))) {
+            return ValueDesc.discrete(context, inPredicate);
+        }
+        return new UnknownValue(context, inPredicate);
+    }
+
+    @Override
+    public ValueDesc visitAnd(And and, ExpressionRewriteContext context) {
+        return simplify(context, and, ExpressionUtils.extractConjunction(and),
+                ValueDesc::intersect, true);
+    }
+
+    @Override
+    public ValueDesc visitOr(Or or, ExpressionRewriteContext context) {
+        return simplify(context, or, ExpressionUtils.extractDisjunction(or),
+                ValueDesc::union, false);
+    }
+
+    private ValueDesc simplify(ExpressionRewriteContext context,
+            Expression originExpr, List<Expression> predicates,
+            BinaryOperator<ValueDesc> op, boolean isAnd) {
+
+        Multimap<Expression, ValueDesc> groupByReference
+                = Multimaps.newListMultimap(new LinkedHashMap<>(), ArrayList::new);
+        for (Expression predicate : predicates) {
+            ValueDesc valueDesc = predicate.accept(this, context);
+            List<ValueDesc> valueDescs = (List<ValueDesc>) groupByReference.get(valueDesc.reference);
+            valueDescs.add(valueDesc);
+        }
+
+        List<ValueDesc> valuePerRefs = Lists.newArrayList();
+        for (Entry<Expression, Collection<ValueDesc>> referenceValues : groupByReference.asMap().entrySet()) {
+            List<ValueDesc> valuePerReference = (List) referenceValues.getValue();
+
+            // merge per reference
+            ValueDesc simplifiedValue = valuePerReference.get(0);
+            for (int i = 1; i < valuePerReference.size(); i++) {
+                simplifiedValue = op.apply(simplifiedValue, valuePerReference.get(i));
+            }
+
+            valuePerRefs.add(simplifiedValue);
+        }
+
+        if (valuePerRefs.size() == 1) {
+            return valuePerRefs.get(0);
+        }
+
+        // use UnknownValue to wrap different references
+        return new UnknownValue(context, originExpr, valuePerRefs, isAnd);
+    }
+
+    /**
+     * value desc
+     */
+    public abstract static class ValueDesc {
+        ExpressionRewriteContext context;
+        Expression toExpr;
+        Expression reference;
+
+        public ValueDesc(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
+            this.context = context;
+            this.toExpr = toExpr;
+            this.reference = reference;
+        }
+
+        public Expression getReference() {
+            return reference;
+        }
+
+        public Expression getOriginExpr() {
+            return toExpr;
+        }
+
+        public ExpressionRewriteContext getExpressionRewriteContext() {
+            return context;
+        }
+
+        public abstract ValueDesc union(ValueDesc other);
+
+        /** or */
+        public static ValueDesc union(ExpressionRewriteContext context,
+                RangeValue range, DiscreteValue discrete, boolean reverseOrder) {
+            long count = discrete.values.stream().filter(x -> range.range.test(x)).count();
+            if (count == discrete.values.size()) {
+                return range;
+            }
+            Expression toExpr = FoldConstantRuleOnFE.evaluate(
+                    new Or(range.toExpr, discrete.toExpr), context);
+            List<ValueDesc> sourceValues = reverseOrder
+                    ? ImmutableList.of(discrete, range)
+                    : ImmutableList.of(range, discrete);
+            return new UnknownValue(context, toExpr, sourceValues, false);
+        }
+
+        /** intersect */
+        public abstract ValueDesc intersect(ValueDesc other);
+
+        /** intersect */
+        public static ValueDesc intersect(ExpressionRewriteContext context, RangeValue range, DiscreteValue discrete) {
+            DiscreteValue result = new DiscreteValue(context, discrete.reference, discrete.toExpr);
+            discrete.values.stream().filter(x -> range.range.contains(x)).forEach(result.values::add);
+            if (!result.values.isEmpty()) {
+                return result;
+            }
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(new And(range.toExpr, discrete.toExpr), context);
+            return new EmptyValue(context, range.reference, originExpr);
+        }
+
+        private static ValueDesc range(ExpressionRewriteContext context, ComparisonPredicate predicate) {
+            Literal value = (Literal) predicate.right();
+            if (predicate instanceof EqualTo) {
+                return new DiscreteValue(context, predicate.left(), predicate, value);
+            }
+            Range<Literal> range = null;
+            if (predicate instanceof GreaterThanEqual) {
+                range = Range.atLeast(value);
+            } else if (predicate instanceof GreaterThan) {
+                range = Range.greaterThan(value);
+            } else if (predicate instanceof LessThanEqual) {
+                range = Range.atMost(value);
+            } else if (predicate instanceof LessThan) {
+                range = Range.lessThan(value);
+            }
+
+            return new RangeValue(context, predicate.left(), predicate, range);
+        }
+
+        public static ValueDesc discrete(ExpressionRewriteContext context, InPredicate in) {
+            // Set<Literal> literals = (Set) Utils.fastToImmutableSet(in.getOptions());
+            Set<Literal> literals = in.getOptions().stream().map(Literal.class::cast).collect(Collectors.toSet());
+            return new DiscreteValue(context, in.getCompareExpr(), in, literals);
+        }
+    }
+
+    /**
+     * empty range
+     */
+    public static class EmptyValue extends ValueDesc {
+
+        public EmptyValue(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
+            super(context, reference, toExpr);
+        }
+
+        @Override
+        public ValueDesc union(ValueDesc other) {
+            return other;
+        }
+
+        @Override
+        public ValueDesc intersect(ValueDesc other) {
+            return this;
+        }
+    }
+
+    /**
+     * use @see com.google.common.collect.Range to wrap `ComparisonPredicate`
+     * for example:
+     * a > 1 => (1...+∞)
+     */
+    public static class RangeValue extends ValueDesc {
+        Range<Literal> range;
+
+        public RangeValue(ExpressionRewriteContext context, Expression reference,
+                Expression toExpr, Range<Literal> range) {
+            super(context, reference, toExpr);
+            this.range = range;
+        }
+
+        public Range<Literal> getRange() {
+            return range;
+        }
+
+        @Override
+        public ValueDesc union(ValueDesc other) {
+            if (other instanceof EmptyValue) {
+                return other.union(this);
+            }
+            if (other instanceof RangeValue) {
+                Expression originExpr = FoldConstantRuleOnFE.evaluate(new Or(toExpr, other.toExpr), context);
+                RangeValue o = (RangeValue) other;
+                if (range.isConnected(o.range)) {
+                    return new RangeValue(context, reference, originExpr, range.span(o.range));
+                }
+                return new UnknownValue(context, originExpr,
+                        ImmutableList.of(this, other), false);
+            }
+            if (other instanceof DiscreteValue) {
+                return union(context, this, (DiscreteValue) other, false);
+            }
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(new Or(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), false);
+        }
+
+        @Override
+        public ValueDesc intersect(ValueDesc other) {
+            if (other instanceof EmptyValue) {
+                return other.intersect(this);
+            }
+            if (other instanceof RangeValue) {
+                Expression originExpr = FoldConstantRuleOnFE.evaluate(new And(toExpr, other.toExpr), context);
+                RangeValue o = (RangeValue) other;
+                if (range.isConnected(o.range)) {
+                    return new RangeValue(context, reference, originExpr, range.intersection(o.range));
+                }
+                return new EmptyValue(context, reference, originExpr);
+            }
+            if (other instanceof DiscreteValue) {
+                return intersect(context, this, (DiscreteValue) other);
+            }
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(new And(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), true);
+        }
+
+        @Override
+        public String toString() {
+            return range == null ? "UnknownRange" : range.toString();
+        }
+    }
+
+    /**
+     * use `Set` to wrap `InPredicate`
+     * for example:
+     * a in (1,2,3) => [1,2,3]
+     */
+    public static class DiscreteValue extends ValueDesc {
+        Set<Literal> values;
+
+        public DiscreteValue(ExpressionRewriteContext context,
+                Expression reference, Expression toExpr, Literal... values) {
+            this(context, reference, toExpr, Arrays.asList(values));
+        }
+
+        public DiscreteValue(ExpressionRewriteContext context,
+                Expression reference, Expression toExpr, Collection<Literal> values) {
+            super(context, reference, toExpr);
+            this.values = Sets.newTreeSet(values);
+        }
+
+        public Set<Literal> getValues() {
+            return values;
+        }
+
+        @Override
+        public ValueDesc union(ValueDesc other) {
+            if (other instanceof EmptyValue) {
+                return other.union(this);
+            }
+            if (other instanceof DiscreteValue) {
+                Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                        ExpressionUtils.or(toExpr, other.toExpr), context);
+                DiscreteValue discreteValue = new DiscreteValue(context, reference, originExpr);
+                discreteValue.values.addAll(((DiscreteValue) other).values);
+                discreteValue.values.addAll(this.values);
+                return discreteValue;
+            }
+            if (other instanceof RangeValue) {
+                return union(context, (RangeValue) other, this, true);
+            }
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.or(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), false);
+        }
+
+        @Override
+        public ValueDesc intersect(ValueDesc other) {
+            if (other instanceof EmptyValue) {
+                return other.intersect(this);
+            }
+            if (other instanceof DiscreteValue) {
+                Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                        ExpressionUtils.and(toExpr, other.toExpr), context);
+                DiscreteValue discreteValue = new DiscreteValue(context, reference, originExpr);
+                discreteValue.values.addAll(((DiscreteValue) other).values);
+                discreteValue.values.retainAll(this.values);
+                if (discreteValue.values.isEmpty()) {
+                    return new EmptyValue(context, reference, originExpr);
+                } else {
+                    return discreteValue;
+                }
+            }
+            if (other instanceof RangeValue) {
+                return intersect(context, (RangeValue) other, this);
+            }
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.and(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), true);
+        }
+
+        @Override
+        public String toString() {
+            return values.toString();
+        }
+    }
+
+    /**
+     * Represents processing result.
+     */
+    public static class UnknownValue extends ValueDesc {
+        private final List<ValueDesc> sourceValues;
+        private final boolean isAnd;
+
+        private UnknownValue(ExpressionRewriteContext context, Expression expr) {
+            super(context, expr, expr);
+            sourceValues = ImmutableList.of();
+            isAnd = false;
+        }
+
+        public UnknownValue(ExpressionRewriteContext context, Expression toExpr,
+                List<ValueDesc> sourceValues, boolean isAnd) {
+            super(context, getReference(sourceValues, toExpr), toExpr);
+            this.sourceValues = ImmutableList.copyOf(sourceValues);
+            this.isAnd = isAnd;
+        }
+
+        // reference is used to simplify multiple ValueDescs.
+        // when ValueDesc A op ValueDesc B, only A and B's references equals,
+        // can reduce them, like A op B = A.
+        // If A and B's reference not equal, A op B will always get UnknownValue(A op B).
+        //
+        // for example:
+        // 1. RangeValue(a < 10, reference=a) union RangeValue(a > 20, reference=a)
+        //    = UnknownValue1(a < 10 or a > 20, reference=a)
+        // 2. RangeValue(a < 10, reference=a) union RangeValue(b > 20, reference=b)
+        //    = UnknownValue2(a < 10 or b > 20, reference=(a < 10 or b > 20))
+        // then given EmptyValue(, reference=a) E,
+        // 1. since E and UnknownValue1's reference equals, then
+        //    E union UnknownValue1 = E.union(UnknownValue1) = UnknownValue1,
+        // 2. since E and UnknownValue2's reference not equals, then
+        //    E union UnknownValue2 = UnknownValue3(E union UnknownValue2, reference=E union UnknownValue2)
+        private static Expression getReference(List<ValueDesc> sourceValues, Expression toExpr) {
+            Expression reference = sourceValues.get(0).reference;
+            for (int i = 1; i < sourceValues.size(); i++) {
+                if (!reference.equals(sourceValues.get(i).reference)) {
+                    return toExpr;
+                }
+            }
+            return reference;
+        }
+
+        public List<ValueDesc> getSourceValues() {
+            return sourceValues;
+        }
+
+        public boolean isAnd() {
+            return this.isAnd;
+        }
+
+        @Override
+        public ValueDesc union(ValueDesc other) {
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.or(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), false);
+        }
+
+        @Override
+        public ValueDesc intersect(ValueDesc other) {
+            Expression originExpr = FoldConstantRuleOnFE.evaluate(
+                    ExpressionUtils.and(toExpr, other.toExpr), context);
+            return new UnknownValue(context, originExpr,
+                    ImmutableList.of(this, other), true);
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
index ca50f7c4d49225..628d94d1dccafd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
@@ -20,8 +20,12 @@
 import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
 import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
 import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.DiscreteValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.EmptyValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.RangeValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.UnknownValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.ValueDesc;
 import org.apache.doris.nereids.trees.expressions.And;
-import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
 import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
@@ -36,37 +40,30 @@
 import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
-import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
 import com.google.common.collect.BoundType;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
 import com.google.common.collect.Range;
-import com.google.common.collect.Sets;
+import org.apache.commons.lang3.NotImplementedException;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.Iterator;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map.Entry;
 import java.util.Set;
-import java.util.function.BinaryOperator;
 import java.util.stream.Collectors;
 
 /**
  * This class implements the function to simplify expression range.
  * for example:
+ *
  * a > 1 and a > 2 => a > 2
  * a > 1 or a > 2 => a > 1
  * a in (1,2,3) and a > 1 => a in (2,3)
  * a in (1,2,3) and a in (3,4,5) => a = 3
  * a in (1,2,3) and a in (4,5,6) => false
+ *
  * The logic is as follows:
  * 1. for `And` expression.
  *    1. extract conjunctions then build `ValueDesc` for each conjunction
@@ -85,479 +82,100 @@ public class SimplifyRange implements ExpressionPatternRuleFactory {
     public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
         return ImmutableList.of(
                 matchesTopType(CompoundPredicate.class)
-                        .thenApply(ctx -> SimplifyRange.rewrite(ctx.expr, ctx.rewriteContext))
+                        .thenApply(ctx -> rewrite(ctx.expr, ctx.rewriteContext))
         );
     }
 
     /** rewrite */
     public static Expression rewrite(CompoundPredicate expr, ExpressionRewriteContext context) {
-        ValueDesc valueDesc = expr.accept(new RangeInference(), context);
-        Expression toExpr = valueDesc.toExpression();
-        if (toExpr == null) {
-            // this mean cannot simplify
-            return valueDesc.toExpr;
-        }
-        return toExpr;
+        ValueDesc valueDesc = (new RangeInference()).getValue(expr, context);
+        return INSTANCE.getExpression(valueDesc);
     }
 
-    private static class RangeInference extends ExpressionVisitor<ValueDesc, ExpressionRewriteContext> {
-
-        @Override
-        public ValueDesc visit(Expression expr, ExpressionRewriteContext context) {
-            return new UnknownValue(context, expr);
-        }
-
-        private ValueDesc buildRange(ExpressionRewriteContext context, ComparisonPredicate predicate) {
-            Expression right = predicate.child(1);
-            if (right.isNullLiteral()) {
-                return new UnknownValue(context, predicate);
-            }
-            // only handle `NumericType` and `DateLikeType`
-            if (right.isLiteral() && (right.getDataType().isNumericType() || right.getDataType().isDateLikeType())) {
-                return ValueDesc.range(context, predicate);
-            }
-            return new UnknownValue(context, predicate);
-        }
-
-        @Override
-        public ValueDesc visitGreaterThan(GreaterThan greaterThan, ExpressionRewriteContext context) {
-            return buildRange(context, greaterThan);
-        }
-
-        @Override
-        public ValueDesc visitGreaterThanEqual(GreaterThanEqual greaterThanEqual, ExpressionRewriteContext context) {
-            return buildRange(context, greaterThanEqual);
-        }
-
-        @Override
-        public ValueDesc visitLessThan(LessThan lessThan, ExpressionRewriteContext context) {
-            return buildRange(context, lessThan);
-        }
-
-        @Override
-        public ValueDesc visitLessThanEqual(LessThanEqual lessThanEqual, ExpressionRewriteContext context) {
-            return buildRange(context, lessThanEqual);
-        }
-
-        @Override
-        public ValueDesc visitEqualTo(EqualTo equalTo, ExpressionRewriteContext context) {
-            return buildRange(context, equalTo);
-        }
-
-        @Override
-        public ValueDesc visitInPredicate(InPredicate inPredicate, ExpressionRewriteContext context) {
-            // only handle `NumericType` and `DateLikeType`
-            if (ExpressionUtils.isAllNonNullLiteral(inPredicate.getOptions())
-                    && (ExpressionUtils.matchNumericType(inPredicate.getOptions())
-                    || ExpressionUtils.matchDateLikeType(inPredicate.getOptions()))) {
-                return ValueDesc.discrete(context, inPredicate);
-            }
-            return new UnknownValue(context, inPredicate);
-        }
-
-        @Override
-        public ValueDesc visitAnd(And and, ExpressionRewriteContext context) {
-            return simplify(context, and, ExpressionUtils.extractConjunction(and),
-                    ValueDesc::intersect, ExpressionUtils::and);
-        }
-
-        @Override
-        public ValueDesc visitOr(Or or, ExpressionRewriteContext context) {
-            return simplify(context, or, ExpressionUtils.extractDisjunction(or),
-                    ValueDesc::union, ExpressionUtils::or);
-        }
-
-        private ValueDesc simplify(ExpressionRewriteContext context,
-                Expression originExpr, List<Expression> predicates,
-                BinaryOperator<ValueDesc> op, BinaryOperator<Expression> exprOp) {
-
-            Multimap<Expression, ValueDesc> groupByReference
-                    = Multimaps.newListMultimap(new LinkedHashMap<>(), ArrayList::new);
-            for (Expression predicate : predicates) {
-                ValueDesc valueDesc = predicate.accept(this, null);
-                List<ValueDesc> valueDescs = (List<ValueDesc>) groupByReference.get(valueDesc.reference);
-                valueDescs.add(valueDesc);
-            }
-
-            List<ValueDesc> valuePerRefs = Lists.newArrayList();
-            for (Entry<Expression, Collection<ValueDesc>> referenceValues : groupByReference.asMap().entrySet()) {
-                List<ValueDesc> valuePerReference = (List) referenceValues.getValue();
-
-                // merge per reference
-                ValueDesc simplifiedValue = valuePerReference.get(0);
-                for (int i = 1; i < valuePerReference.size(); i++) {
-                    simplifiedValue = op.apply(simplifiedValue, valuePerReference.get(i));
-                }
-
-                valuePerRefs.add(simplifiedValue);
-            }
-
-            if (valuePerRefs.size() == 1) {
-                return valuePerRefs.get(0);
-            }
-
-            // use UnknownValue to wrap different references
-            return new UnknownValue(context, originExpr, valuePerRefs, exprOp);
+    private Expression getExpression(ValueDesc value) {
+        if (value instanceof EmptyValue) {
+            return getExpression((EmptyValue) value);
+        } else if (value instanceof DiscreteValue) {
+            return getExpression((DiscreteValue) value);
+        } else if (value instanceof RangeValue) {
+            return getExpression((RangeValue) value);
+        } else if (value instanceof UnknownValue) {
+            return getExpression((UnknownValue) value);
+        } else {
+            throw new NotImplementedException("not implements");
         }
     }
 
-    private abstract static class ValueDesc {
-        ExpressionRewriteContext context;
-        Expression toExpr;
-        Expression reference;
-
-        public ValueDesc(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
-            this.context = context;
-            this.toExpr = toExpr;
-            this.reference = reference;
-        }
-
-        public abstract ValueDesc union(ValueDesc other);
-
-        public static ValueDesc union(ExpressionRewriteContext context,
-                RangeValue range, DiscreteValue discrete, boolean reverseOrder) {
-            long count = discrete.values.stream().filter(x -> range.range.test(x)).count();
-            if (count == discrete.values.size()) {
-                return range;
-            }
-            Expression toExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.or(range.toExpr, discrete.toExpr), context);
-            List<ValueDesc> sourceValues = reverseOrder
-                    ? ImmutableList.of(discrete, range)
-                    : ImmutableList.of(range, discrete);
-            return new UnknownValue(context, toExpr, sourceValues, ExpressionUtils::or);
-        }
-
-        public abstract ValueDesc intersect(ValueDesc other);
-
-        public static ValueDesc intersect(ExpressionRewriteContext context, RangeValue range, DiscreteValue discrete) {
-            DiscreteValue result = new DiscreteValue(context, discrete.reference, discrete.toExpr);
-            discrete.values.stream().filter(x -> range.range.contains(x)).forEach(result.values::add);
-            if (!result.values.isEmpty()) {
-                return result;
-            }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.and(range.toExpr, discrete.toExpr), context);
-            return new EmptyValue(context, range.reference, originExpr);
-        }
-
-        public abstract Expression toExpression();
-
-        public static ValueDesc range(ExpressionRewriteContext context, ComparisonPredicate predicate) {
-            Literal value = (Literal) predicate.right();
-            if (predicate instanceof EqualTo) {
-                return new DiscreteValue(context, predicate.left(), predicate, value);
-            }
-            RangeValue rangeValue = new RangeValue(context, predicate.left(), predicate);
-            if (predicate instanceof GreaterThanEqual) {
-                rangeValue.range = Range.atLeast(value);
-            } else if (predicate instanceof GreaterThan) {
-                rangeValue.range = Range.greaterThan(value);
-            } else if (predicate instanceof LessThanEqual) {
-                rangeValue.range = Range.atMost(value);
-            } else if (predicate instanceof LessThan) {
-                rangeValue.range = Range.lessThan(value);
-            }
-
-            return rangeValue;
-        }
-
-        public static ValueDesc discrete(ExpressionRewriteContext context, InPredicate in) {
-            // Set<Literal> literals = (Set) Utils.fastToImmutableSet(in.getOptions());
-            Set<Literal> literals = in.getOptions().stream().map(Literal.class::cast).collect(Collectors.toSet());
-            return new DiscreteValue(context, in.getCompareExpr(), in, literals);
+    private Expression getExpression(EmptyValue value) {
+        Expression reference = value.getReference();
+        if (reference.nullable()) {
+            return new And(new IsNull(reference), new NullLiteral(BooleanType.INSTANCE));
+        } else {
+            return BooleanLiteral.FALSE;
         }
     }
 
-    private static class EmptyValue extends ValueDesc {
-
-        public EmptyValue(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
-            super(context, reference, toExpr);
-        }
-
-        @Override
-        public ValueDesc union(ValueDesc other) {
-            return other;
-        }
-
-        @Override
-        public ValueDesc intersect(ValueDesc other) {
-            return this;
-        }
-
-        @Override
-        public Expression toExpression() {
-            if (reference.nullable()) {
-                return new And(new IsNull(reference), new NullLiteral(BooleanType.INSTANCE));
+    private Expression getExpression(RangeValue value) {
+        Expression reference = value.getReference();
+        Range<Literal> range = value.getRange();
+        List<Expression> result = Lists.newArrayList();
+        if (range.hasLowerBound()) {
+            if (range.lowerBoundType() == BoundType.CLOSED) {
+                result.add(new GreaterThanEqual(reference, range.lowerEndpoint()));
             } else {
-                return BooleanLiteral.FALSE;
+                result.add(new GreaterThan(reference, range.lowerEndpoint()));
             }
         }
-    }
-
-    /**
-     * use @see com.google.common.collect.Range to wrap `ComparisonPredicate`
-     * for example:
-     * a > 1 => (1...+∞)
-     */
-    private static class RangeValue extends ValueDesc {
-        Range<Literal> range;
-
-        public RangeValue(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
-            super(context, reference, toExpr);
-        }
-
-        @Override
-        public ValueDesc union(ValueDesc other) {
-            if (other instanceof EmptyValue) {
-                return other.union(this);
-            }
-            if (other instanceof RangeValue) {
-                Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.or(toExpr, other.toExpr), context);
-                RangeValue o = (RangeValue) other;
-                if (range.isConnected(o.range)) {
-                    RangeValue rangeValue = new RangeValue(context, reference, originExpr);
-                    rangeValue.range = range.span(o.range);
-                    return rangeValue;
-                }
-                return new UnknownValue(context, originExpr,
-                        ImmutableList.of(this, other), ExpressionUtils::or);
-            }
-            if (other instanceof DiscreteValue) {
-                return union(context, this, (DiscreteValue) other, false);
-            }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.or(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), ExpressionUtils::or);
-        }
-
-        @Override
-        public ValueDesc intersect(ValueDesc other) {
-            if (other instanceof EmptyValue) {
-                return other.intersect(this);
-            }
-            if (other instanceof RangeValue) {
-                Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.and(toExpr, other.toExpr), context);
-                RangeValue o = (RangeValue) other;
-                if (range.isConnected(o.range)) {
-                    RangeValue rangeValue = new RangeValue(context, reference, originExpr);
-                    rangeValue.range = range.intersection(o.range);
-                    return rangeValue;
-                }
-                return new EmptyValue(context, reference, originExpr);
-            }
-            if (other instanceof DiscreteValue) {
-                return intersect(context, this, (DiscreteValue) other);
-            }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.and(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), ExpressionUtils::and);
-        }
-
-        @Override
-        public Expression toExpression() {
-            List<Expression> result = Lists.newArrayList();
-            if (range.hasLowerBound()) {
-                if (range.lowerBoundType() == BoundType.CLOSED) {
-                    result.add(new GreaterThanEqual(reference, range.lowerEndpoint()));
-                } else {
-                    result.add(new GreaterThan(reference, range.lowerEndpoint()));
-                }
-            }
-            if (range.hasUpperBound()) {
-                if (range.upperBoundType() == BoundType.CLOSED) {
-                    result.add(new LessThanEqual(reference, range.upperEndpoint()));
-                } else {
-                    result.add(new LessThan(reference, range.upperEndpoint()));
-                }
-            }
-            if (!result.isEmpty()) {
-                return ExpressionUtils.and(result);
+        if (range.hasUpperBound()) {
+            if (range.upperBoundType() == BoundType.CLOSED) {
+                result.add(new LessThanEqual(reference, range.upperEndpoint()));
             } else {
-                if (reference.nullable()) {
-                    return new Or(new Not(new IsNull(reference)), new NullLiteral(BooleanType.INSTANCE));
-                } else {
-                    return BooleanLiteral.TRUE;
-                }
+                result.add(new LessThan(reference, range.upperEndpoint()));
             }
         }
-
-        @Override
-        public String toString() {
-            return range == null ? "UnknownRange" : range.toString();
-        }
-    }
-
-    /**
-     * use `Set` to wrap `InPredicate`
-     * for example:
-     * a in (1,2,3) => [1,2,3]
-     */
-    private static class DiscreteValue extends ValueDesc {
-        Set<Literal> values;
-
-        public DiscreteValue(ExpressionRewriteContext context,
-                Expression reference, Expression toExpr, Literal... values) {
-            this(context, reference, toExpr, Arrays.asList(values));
-        }
-
-        public DiscreteValue(ExpressionRewriteContext context,
-                Expression reference, Expression toExpr, Collection<Literal> values) {
-            super(context, reference, toExpr);
-            this.values = Sets.newTreeSet(values);
-        }
-
-        @Override
-        public ValueDesc union(ValueDesc other) {
-            if (other instanceof EmptyValue) {
-                return other.union(this);
-            }
-            if (other instanceof DiscreteValue) {
-                Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.or(toExpr, other.toExpr), context);
-                DiscreteValue discreteValue = new DiscreteValue(context, reference, originExpr);
-                discreteValue.values.addAll(((DiscreteValue) other).values);
-                discreteValue.values.addAll(this.values);
-                return discreteValue;
-            }
-            if (other instanceof RangeValue) {
-                return union(context, (RangeValue) other, this, true);
-            }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.or(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), ExpressionUtils::or);
-        }
-
-        @Override
-        public ValueDesc intersect(ValueDesc other) {
-            if (other instanceof EmptyValue) {
-                return other.intersect(this);
-            }
-            if (other instanceof DiscreteValue) {
-                Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.and(toExpr, other.toExpr), context);
-                DiscreteValue discreteValue = new DiscreteValue(context, reference, originExpr);
-                discreteValue.values.addAll(((DiscreteValue) other).values);
-                discreteValue.values.retainAll(this.values);
-                if (discreteValue.values.isEmpty()) {
-                    return new EmptyValue(context, reference, originExpr);
-                } else {
-                    return discreteValue;
-                }
-            }
-            if (other instanceof RangeValue) {
-                return intersect(context, (RangeValue) other, this);
-            }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.and(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), ExpressionUtils::and);
-        }
-
-        @Override
-        public Expression toExpression() {
-            // NOTICE: it's related with `InPredicateToEqualToRule`
-            // They are same processes, so must change synchronously.
-            if (values.size() == 1) {
-                return new EqualTo(reference, values.iterator().next());
-
-                // this condition should as same as OrToIn, or else meet dead loop
-            } else if (values.size() < OrToIn.REWRITE_OR_TO_IN_PREDICATE_THRESHOLD) {
-                Iterator<Literal> iterator = values.iterator();
-                return new Or(new EqualTo(reference, iterator.next()), new EqualTo(reference, iterator.next()));
+        if (!result.isEmpty()) {
+            return ExpressionUtils.and(result);
+        } else {
+            if (reference.nullable()) {
+                return new Or(new Not(new IsNull(reference)), new NullLiteral(BooleanType.INSTANCE));
             } else {
-                return new InPredicate(reference, Lists.newArrayList(values));
+                return BooleanLiteral.TRUE;
             }
         }
-
-        @Override
-        public String toString() {
-            return values.toString();
-        }
     }
 
-    /**
-     * Represents processing result.
-     */
-    private static class UnknownValue extends ValueDesc {
-        private final List<ValueDesc> sourceValues;
-        private final BinaryOperator<Expression> mergeExprOp;
-
-        private UnknownValue(ExpressionRewriteContext context, Expression expr) {
-            super(context, expr, expr);
-            sourceValues = ImmutableList.of();
-            mergeExprOp = null;
-        }
-
-        public UnknownValue(ExpressionRewriteContext context, Expression toExpr,
-                List<ValueDesc> sourceValues, BinaryOperator<Expression> mergeExprOp) {
-            super(context, getReference(sourceValues, toExpr), toExpr);
-            this.sourceValues = ImmutableList.copyOf(sourceValues);
-            this.mergeExprOp = mergeExprOp;
-        }
-
-        // reference is used to simplify multiple ValueDescs.
-        // when ValueDesc A op ValueDesc B, only A and B's references equals,
-        // can reduce them, like A op B = A.
-        // If A and B's reference not equal, A op B will always get UnknownValue(A op B).
-        //
-        // for example:
-        // 1. RangeValue(a < 10, reference=a) union RangeValue(a > 20, reference=a)
-        //    = UnknownValue1(a < 10 or a > 20, reference=a)
-        // 2. RangeValue(a < 10, reference=a) union RangeValue(b > 20, reference=b)
-        //    = UnknownValue2(a < 10 or b > 20, reference=(a < 10 or b > 20))
-        // then given EmptyValue(, reference=a) E,
-        // 1. since E and UnknownValue1's reference equals, then
-        //    E union UnknownValue1 = E.union(UnknownValue1) = UnknownValue1,
-        // 2. since E and UnknownValue2's reference not equals, then
-        //    E union UnknownValue2 = UnknownValue3(E union UnknownValue2, reference=E union UnknownValue2)
-        private static Expression getReference(List<ValueDesc> sourceValues, Expression toExpr) {
-            Expression reference = sourceValues.get(0).reference;
-            for (int i = 1; i < sourceValues.size(); i++) {
-                if (!reference.equals(sourceValues.get(i).reference)) {
-                    return toExpr;
-                }
-            }
-            return reference;
-        }
-
-        @Override
-        public ValueDesc union(ValueDesc other) {
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.or(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), ExpressionUtils::or);
-        }
-
-        @Override
-        public ValueDesc intersect(ValueDesc other) {
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.and(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), ExpressionUtils::and);
+    private Expression getExpression(DiscreteValue value) {
+        Expression reference = value.getReference();
+        Set<Literal> values = value.getValues();
+        // NOTICE: it's related with `InPredicateToEqualToRule`
+        // They are same processes, so must change synchronously.
+        if (values.size() == 1) {
+            return new EqualTo(reference, values.iterator().next());
+
+            // this condition should as same as OrToIn, or else meet dead loop
+        } else if (values.size() < OrToIn.REWRITE_OR_TO_IN_PREDICATE_THRESHOLD) {
+            Iterator<Literal> iterator = values.iterator();
+            return new Or(new EqualTo(reference, iterator.next()), new EqualTo(reference, iterator.next()));
+        } else {
+            return new InPredicate(reference, Lists.newArrayList(values));
         }
+    }
 
-        @Override
-        public Expression toExpression() {
-            if (sourceValues.isEmpty()) {
-                return toExpr;
-            }
-            Expression result = sourceValues.get(0).toExpression();
-            for (int i = 1; i < sourceValues.size(); i++) {
-                result = mergeExprOp.apply(result, sourceValues.get(i).toExpression());
-            }
-            result = FoldConstantRuleOnFE.evaluate(result, context);
-            // ATTN: we must return original expr, because OrToIn is implemented with MutableState,
-            //   newExpr will lose these states leading to dead loop by OrToIn -> SimplifyRange -> FoldConstantByFE
-            if (result.equals(toExpr)) {
-                return toExpr;
-            }
-            return result;
-        }
+    private Expression getExpression(UnknownValue value) {
+        List<ValueDesc> sourceValues = value.getSourceValues();
+        Expression originExpr = value.getOriginExpr();
+        if (sourceValues.isEmpty()) {
+            return originExpr;
+        }
+        List<Expression> sourceExprs = sourceValues.stream().map(sourceValue -> getExpression(sourceValue))
+                .collect(Collectors.toList());
+        Expression result = value.isAnd() ? ExpressionUtils.and(sourceExprs) : ExpressionUtils.or(sourceExprs);
+        result = FoldConstantRuleOnFE.evaluate(result, value.getExpressionRewriteContext());
+        // ATTN: we must return original expr, because OrToIn is implemented with MutableState,
+        //   newExpr will lose these states leading to dead loop by OrToIn -> SimplifyRange -> FoldConstantByFE
+        if (result.equals(originExpr)) {
+            return originExpr;
+        }
+        return result;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 53ab8a50683ecb..22b681a6246d92 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -192,12 +192,34 @@ public static Optional<Expression> optionalAnd(Collection<Expression> collection
         return optionalAnd(ImmutableList.copyOf(collection));
     }
 
+    /**
+     *  AND expression, also remove duplicate expression, boolean literal
+     */
     public static Expression and(Collection<Expression> expressions) {
-        return combineAsLeftDeepTree(And.class, expressions);
+        Set<Expression> distinctExpressions = Sets.newLinkedHashSetWithExpectedSize(expressions.size());
+        for (Expression expression : expressions) {
+            if (expression.equals(BooleanLiteral.FALSE)) {
+                return BooleanLiteral.FALSE;
+            } else if (!expression.equals(BooleanLiteral.TRUE)) {
+                distinctExpressions.add(expression);
+            }
+        }
+
+        List<Expression> exprList = Lists.newArrayList(distinctExpressions);
+        if (exprList.isEmpty()) {
+            return BooleanLiteral.TRUE;
+        } else if (exprList.size() == 1) {
+            return exprList.get(0);
+        } else {
+            return new And(exprList);
+        }
     }
 
+    /**
+     *  AND expression, also remove duplicate expression, boolean literal
+     */
     public static Expression and(Expression... expressions) {
-        return combineAsLeftDeepTree(And.class, Lists.newArrayList(expressions));
+        return and(Lists.newArrayList(expressions));
     }
 
     public static Optional<Expression> optionalOr(List<Expression> expressions) {
@@ -208,12 +230,34 @@ public static Optional<Expression> optionalOr(List<Expression> expressions) {
         }
     }
 
+    /**
+     *  OR expression, also remove duplicate expression, boolean literal
+     */
     public static Expression or(Expression... expressions) {
-        return combineAsLeftDeepTree(Or.class, Lists.newArrayList(expressions));
+        return or(Lists.newArrayList(expressions));
     }
 
+    /**
+     *  OR expression, also remove duplicate expression, boolean literal
+     */
     public static Expression or(Collection<Expression> expressions) {
-        return combineAsLeftDeepTree(Or.class, expressions);
+        Set<Expression> distinctExpressions = Sets.newLinkedHashSetWithExpectedSize(expressions.size());
+        for (Expression expression : expressions) {
+            if (expression.equals(BooleanLiteral.TRUE)) {
+                return BooleanLiteral.TRUE;
+            } else if (!expression.equals(BooleanLiteral.FALSE)) {
+                distinctExpressions.add(expression);
+            }
+        }
+
+        List<Expression> exprList = Lists.newArrayList(distinctExpressions);
+        if (exprList.isEmpty()) {
+            return BooleanLiteral.FALSE;
+        } else if (exprList.size() == 1) {
+            return exprList.get(0);
+        } else {
+            return new Or(exprList);
+        }
     }
 
     /**
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java
index c5f69b1edbbb45..296cae5fe4023b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.doris.nereids.rules.expression;
 
+import org.apache.doris.nereids.rules.expression.rules.AddMinMax;
 import org.apache.doris.nereids.rules.expression.rules.DistinctPredicatesRule;
 import org.apache.doris.nereids.rules.expression.rules.ExtractCommonFactorRule;
 import org.apache.doris.nereids.rules.expression.rules.InPredicateDedup;
@@ -274,4 +275,65 @@ void testDeadLoop() {
 
         assertRewrite("a and (b > 0 and b < 10)", "a and (b > 0 and b < 10)");
     }
+
+    @Test
+    void testOrAddMinMax() {
+        executor = new ExpressionRuleExecutor(ImmutableList.of(
+            bottomUp(
+                AddMinMax.INSTANCE
+            )
+        ));
+
+        assertRewriteAfterTypeCoercion("TA >= 10", "TA >= 10");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20", "TA >= 10 and TA <= 20");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 or TA >= 30",
+                "(TA <= 20 OR TA >= 30) AND (TA >= 10)");
+        assertRewriteAfterTypeCoercion("TA >= 10 and TA <= 20 or TA >= 50 and TA <= 60 or TA >= 100 and TA <= 120",
+                "(TA <= 20 or TA >= 50 and TA <= 60 or TA >= 100) AND TA >= 10 and TA <= 120");
+        assertRewriteAfterTypeCoercion("TA > 10 and TA < 20 or TA > 50 and TA < 60 or TA > 100 and TA < 120",
+                "(TA < 20 or TA > 50 and TA < 60 or TA > 100) AND TA > 10 and TA < 120");
+        // only slot reference add min max
+        assertRewriteAfterTypeCoercion("TA + TB > 10 and TA + TB < 20 or TA + TB > 50 and TA + TB < 60 or TA + TB > 100 and TA + TB < 120",
+                "TA + TB > 10 and TA + TB < 20 or TA + TB > 50 and TA + TB < 60 or TA + TB > 100 and TA + TB < 120");
+        assertRewriteAfterTypeCoercion("ISNULL (TA > 10) and TA > 10 and TA < 20 or TA > 50 and TA < 60 or TA > 100 and TA < 120",
+                "(ISNULL(TA > 10) and TA < 20 or TA > 50 and TA < 60 or TA > 100) AND TA > 10 and TA < 120");
+        assertRewriteAfterTypeCoercion("ISNULL (TA > 10) or TA > 10 and TA < 20 or TA > 50 and TA < 60 or TA > 100 and TA < 120",
+                "ISNULL (TA > 10) or TA > 10 and TA < 20 or TA > 50 and TA < 60 or TA > 100 and TA < 120");
+        assertRewriteAfterTypeCoercion("TA = 4 or (TA > 4 and TB is null)", "(TA = 4 or (TA > 4 and TB is null)) and TA >= 4");
+        assertRewriteAfterTypeCoercion("TA in (10, 50, 100) or TA in (20, 40)", "TA in (10, 50, 100) or TA in (20, 40)");
+        assertRewriteAfterTypeCoercion("TA in (10, 50, 100) or TA >= 70",
+                "(TA in (10, 50, 100) or TA >= 70) AND TA >= 10");
+        assertRewriteAfterTypeCoercion("TA in (10, 50, 100) or TA < 70",
+                "(TA in (10, 50, 100) or TA < 70) AND TA <= 100");
+        assertRewriteAfterTypeCoercion("TA in (10, 50, 100) or TA >= 70 and TA <= 90",
+                "(TA in (10, 50, 100) or TA >= 70 AND TA <= 90) AND TA >= 10 AND TA <= 100");
+        assertRewriteAfterTypeCoercion("TA in (10, 50, 100) or TA >= 70 and TA < 120",
+                "(TA in (10, 50, 100) or TA >= 70) AND TA >= 10 AND TA < 120");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 and TB between 10 and 20", "TA >= 10 and TA <= 20 and TB >= 10 and TB <= 20");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 and TB between 10 and 20 or TA between 100 and 120 and TB between 100 and 120",
+                "(TA <= 20 and TB <= 20 or TA >= 100 and TB >= 100) AND TA >= 10 AND TA <= 120 AND TB >= 10 AND TB <= 120");
+        assertRewriteAfterTypeCoercion("TA >= 10 AND (TA between 12 and 20 and TB between 10 and 20 or TA between 100 and 120 and TB between 100 and 120)",
+                "TA >= 10 and (TA <= 20 and TB <= 20 or TA >= 100 and TB >= 100) AND TA >= 12 AND TA <= 120 AND TB >= 10 AND TB <= 120");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 and TB between 100 and 120 or TA between 100 and 120 and TB between 10 and 20",
+                "(TA <= 20 and TB >= 100 or TA >= 100 and TB <= 20) AND TA >= 10 AND TA <= 120 AND TB >= 10 AND TB <= 120");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 and TB between 10 and 20 AND TC between 10 and 20 or TA between 100 and 120 and TB between 100 and 120",
+                "(TA <= 20 and TB <= 20 and TC >= 10 and TC <= 20 or TA >= 100 and TB >= 100) AND TA >= 10 AND TA <= 120 AND TB >= 10 AND TB <= 120");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 and TB between 10 and 20 AND TC between 10 and 20 or TA between 100 and 120 and TB between 100 and 120 and TC between 100 AND 120",
+                "(TA <= 20 and TB <= 20 and TC <= 20 or TA >= 100 and TB >= 100 and TC >= 100) AND TA >= 10 AND TA <= 120 AND TB >= 10 AND TB <= 120 AND TC >= 10 AND TC <= 120");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 and TB between 100 and 120 OR TB between 10 and 20 AND TC between 100 and 120 OR TA between 100 and 120 AND TC between 10 and 20",
+                "TA >= 10 and TA <= 20 and TB >= 100 AND TB <= 120 OR TB >= 10 AND TB <= 20 AND TC >= 100 AND TC <= 120 OR TA >= 100 AND TA <= 120 AND TC >= 10 AND TC <= 20");
+        assertRewriteAfterTypeCoercion("((TA = 1 AND SC ='1') OR SC = '1212') AND TA =1", "(SC = '1' OR SC = '1212') AND TA =1");
+        assertRewriteAfterTypeCoercion("(TA + TC > 10 and TB > 10) or (TB > 10 and TB > 20)", "(TA + TC > 10 or TB > 20) AND TB > 10");
+        assertRewriteAfterTypeCoercion("((TA + TC > 10 or TA + TC > 5) and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))",
+                "((TA + TC > 10 or TA + TC > 5) or (TB > 20 or TB < 10)) and TB > 10");
+        assertRewriteAfterTypeCoercion("(CA >= date '2024-01-01' and CA <= date '2024-01-03') or (CA > date '2024-01-05' and CA < date '2024-01-07')",
+                "(CA <= date '2024-01-03' or CA > date '2024-01-05') and CA >= date '2024-01-01' and CA < date '2024-01-07')");
+        assertRewriteAfterTypeCoercion("CA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') or CA < date '2024-01-01'",
+                "(CA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') or CA < date '2024-01-01') AND CA <= date '2024-01-03'");
+        assertRewriteAfterTypeCoercion("(AA >= timestamp '2024-01-01 00:00:00' and AA <= timestamp '2024-01-03 00:00:00') or (AA > timestamp '2024-01-05 00:00:00' and AA < timestamp '2024-01-07 00:00:00')",
+                "(AA <= timestamp '2024-01-03 00:00:00' or AA > timestamp '2024-01-05 00:00:00') and AA >= timestamp '2024-01-01 00:00:00' and AA < timestamp '2024-01-07 00:00:00')");
+        assertRewriteAfterTypeCoercion("AA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') or AA < timestamp '2024-01-01 01:00:00'",
+                "(AA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') or AA < timestamp '2024-01-01 01:00:00' ) and AA <= timestamp '2024-01-03 02:00:00'");
+
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java
index ad9db4a8ddafef..1bf2d1432a9f6d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java
@@ -17,18 +17,22 @@
 
 package org.apache.doris.nereids.rules.expression;
 
+import org.apache.doris.catalog.Column;
 import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.analyzer.UnboundRelation;
 import org.apache.doris.nereids.analyzer.UnboundSlot;
 import org.apache.doris.nereids.parser.NereidsParser;
 import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
+import org.apache.doris.nereids.trees.expressions.ExprId;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
 import org.apache.doris.nereids.trees.plans.RelationId;
 import org.apache.doris.nereids.types.BigIntType;
 import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.DateTimeV2Type;
 import org.apache.doris.nereids.types.DateV2Type;
 import org.apache.doris.nereids.types.DecimalV3Type;
 import org.apache.doris.nereids.types.DoubleType;
@@ -104,8 +108,12 @@ protected Expression replaceUnboundSlot(Expression expression, Map<String, Slot>
             children.add(newChild);
         }
         if (expression instanceof UnboundSlot) {
+            ExprId exprId = StatementScopeIdGenerator.newExprId();
             String name = ((UnboundSlot) expression).getName();
-            mem.putIfAbsent(name, SlotReference.of(name, getType(name.charAt(0))));
+            List<String> qualifier = ImmutableList.of();
+            DataType dataType = getType(name.charAt(0));
+            Column column = new Column(name, dataType.toCatalogDataType());
+            mem.putIfAbsent(name, new SlotReference(exprId, name, dataType, true, qualifier, null, column));
             return mem.get(name);
         }
         return hasNewChildren ? expression.withChildren(children) : expression;
@@ -131,6 +139,8 @@ protected DataType getType(char t) {
                 return BooleanType.INSTANCE;
             case 'C':
                 return DateV2Type.INSTANCE;
+            case 'A':
+                return DateTimeV2Type.SYSTEM_DEFAULT;
             case 'M':
                 return DecimalV3Type.SYSTEM_DEFAULT;
             default:
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
index 2848965fb06b7a..bc3367331a451c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
@@ -189,7 +189,7 @@ public void testSimplify() {
         assertRewrite("TA + TC = 1 and TA + TC = 3", "(TA + TC) is null and null");
         assertRewriteNotNull("TA + TC in (1) and TA + TC in (3)", "FALSE");
         assertRewrite("TA + TC in (1) and TA + TC in (3)", "(TA + TC) is null and null");
-        assertRewrite("TA + TC in (1) and TA + TC in (1)", "(TA + TC) IN (1)");
+        assertRewrite("TA + TC in (1) and TA + TC in (1)", "TA + TC in (1)");
         assertRewriteNotNull("(TA + TC > 3 and TA + TC < 1) and TB < 5", "FALSE");
         assertRewrite("(TA + TC > 3 and TA + TC < 1) and TB < 5", "(TA + TC) is null and null and TB < 5");
         assertRewrite("(TA + TC > 3 and TA + TC < 1) or TB < 5", "((TA + TC) is null and null) OR TB < 5");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
index 613965b1238e27..3d88c131c97f15 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
@@ -17,10 +17,13 @@
 
 package org.apache.doris.nereids.sqltest;
 
+import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.PlanChecker;
 
 import org.junit.jupiter.api.Test;
 
+import java.util.stream.Collectors;
+
 public class InferTest extends SqlTestBase {
     @Test
     void testInferNotNullAndInferPredicates() {
@@ -49,9 +52,13 @@ void testInferNotNullFromFilterAndEliminateOuter2() {
                 .printlnTree()
                 .matches(
                     innerLogicalJoin(
-                        logicalOlapScan(),
                         logicalFilter().when(
-                                f -> f.getPredicate().toString().equals("OR[(id#0 = 4),(id#0 > 4)]"))
+                                f -> f.getPredicate().toString().equals("(id#2 >= 4)")),
+                        logicalFilter().when(
+                                f -> ExpressionUtils.and(f.getConjuncts().stream()
+                                        .sorted((a, b) -> a.toString().compareTo(b.toString()))
+                                        .collect(Collectors.toList()))
+                                        .toString().equals("AND[(id#0 >= 4),OR[(id#0 = 4),(id#0 > 4)]]"))
                     )
 
                 );
@@ -69,8 +76,10 @@ void testInferNotNullFromFilterAndEliminateOuter3() {
                         logicalFilter(
                             leftOuterLogicalJoin(
                                 logicalFilter().when(
-                                        f -> f.getPredicate().toString().equals("OR[(id#0 = 4),(id#0 > 4)]")),
-                                logicalOlapScan()
+                                        f -> f.getPredicate().toString().equals("AND[(id#0 >= 4),OR[(id#0 = 4),(id#0 > 4)]]")),
+                                logicalFilter().when(
+                                        f -> f.getPredicate().toString().equals("(id#2 >= 4)")
+                                )
                             )
                         ).when(f -> f.getPredicate().toString()
                                 .equals("OR[(id#0 = 4),AND[(id#0 > 4),score#3 IS NULL]]"))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query13.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query13.out
index 5a2ada8884b3a1..91efc8daefd858 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query13.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query13.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
 ------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query34.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query34.out
index 12e90cc7312c45..c5dc60c2da2bce 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query34.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query34.out
@@ -25,7 +25,7 @@ PhysicalResultSink
 ------------------------------------filter((store.s_county = 'Williamson County'))
 --------------------------------------PhysicalOlapScan[store]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (2000, 2001, 2002))
+--------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (2000, 2001, 2002))
 ----------------------------------PhysicalOlapScan[date_dim]
 --------------------------PhysicalProject
 ----------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query48.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query48.out
index 6b34ba86efb1b0..7e89836305a832 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query48.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query48.out
@@ -9,9 +9,9 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query85.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query85.out
index 2826be96b9c8c8..d12d90bd23015b 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query85.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query85.out
@@ -20,12 +20,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV'))
 ------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out
index 8f366f3d8c0202..bb0cc25e21667d 100644
--- a/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out
+++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query88.out
@@ -24,7 +24,7 @@ PhysicalResultSink
 --------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 ----------------------------------------PhysicalOlapScan[time_dim]
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+----------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((store.s_store_name = 'ese'))
@@ -45,7 +45,7 @@ PhysicalResultSink
 --------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 ----------------------------------------PhysicalOlapScan[time_dim]
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+----------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((store.s_store_name = 'ese'))
@@ -66,7 +66,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -87,7 +87,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -108,7 +108,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -129,7 +129,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -150,7 +150,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -171,7 +171,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_hint_tpch_p0/shape/q19.out b/regression-test/data/nereids_hint_tpch_p0/shape/q19.out
index 5cabdf9f163c56..ebd4f59d682f8f 100644
--- a/regression-test/data/nereids_hint_tpch_p0/shape/q19.out
+++ b/regression-test/data/nereids_hint_tpch_p0/shape/q19.out
@@ -5,12 +5,12 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]])
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]])
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem]
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
 Hint log:
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.out
index 8138a8e20ed7e5..0db3330914c18a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.out
@@ -7,14 +7,14 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
+--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY'))
 --------------------PhysicalOlapScan[customer_address] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out
index 8138a8e20ed7e5..0db3330914c18a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out
@@ -7,14 +7,14 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
+--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY'))
 --------------------PhysicalOlapScan[customer_address] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query34.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query34.out
index ad28c2d46104d0..c75d4fc3e18155 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query34.out
@@ -13,9 +13,9 @@ PhysicalResultSink
 ----------------PhysicalDistribute[DistributionSpecHash]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF0 s_store_sk->[ss_store_sk]
 --------------------------------PhysicalProject
@@ -24,9 +24,9 @@ PhysicalResultSink
 ----------------------------------filter((store.s_county = 'Williamson County'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (2000, 2001, 2002))
---------------------------------PhysicalOlapScan[date_dim]
+------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
+--------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
---------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
-----------------------------PhysicalOlapScan[household_demographics]
+--------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (2000, 2001, 2002))
+----------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out
index 6ed75b1e21eca8..d11dadeae0b923 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out
@@ -9,9 +9,9 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out
index c734d7945f3a56..6f4f258704f239 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF5 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV'))
 ------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out
index d016dca48db832..5da04ad61d3f42 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out
index 9c7d9382777bee..7b5d0d01ba6a17 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query34.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query34.out
index 6c90e3a48d7cba..15e7650bae15c3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
+----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out
index 57618c21f263d3..aef568a41842bb 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out
index 9f12763cec5114..63fc92a6f489d9 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=()
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query88.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query88.out
index a20acf339431ea..ae9b03a84ef7de 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out
index a16b428cbc5159..2cac8d809ad124 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query34.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query34.out
index 9a53073479fb8c..a19fe778a57647 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
+----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out
index fa9006ce24f673..d994b40da36a4a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF0 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out
index 424102e34021a1..62c4b147f59a22 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF2 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query88.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query88.out
index a20acf339431ea..ae9b03a84ef7de 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out
index 4b72ea24b2c114..55cadf71bf5d45 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
 ------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query34.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query34.out
index 489abcd4c2e33c..79c46ccb77bbb8 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query34.out
@@ -15,17 +15,17 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF0 hd_demo_sk->[ss_hdemo_sk]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
-------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
+------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
-------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
---------------------------------PhysicalOlapScan[household_demographics]
+------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000))
+--------------------------------PhysicalOlapScan[date_dim]
 ------------------------PhysicalProject
 --------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
 ----------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out
index a0f0e2107fcda4..bc6273c3821e21 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out
@@ -9,9 +9,9 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out
index 91ce7e962e317d..63b60404936e7e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk]
+----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk]
 ------------------------------------PhysicalProject
 --------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX'))
 ----------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF3
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query88.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query88.out
index 09c630cef443ef..a619efe9def125 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out
index e8a4d92c4f7bf1..dc2239ca3f9701 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
 ------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query34.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query34.out
index 489abcd4c2e33c..79c46ccb77bbb8 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query34.out
@@ -15,17 +15,17 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF0 hd_demo_sk->[ss_hdemo_sk]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
-------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
+------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
-------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
---------------------------------PhysicalOlapScan[household_demographics]
+------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000))
+--------------------------------PhysicalOlapScan[date_dim]
 ------------------------PhysicalProject
 --------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
 ----------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out
index f8e2288f356010..82bba37deb0400 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out
@@ -9,9 +9,9 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out
index 6818ffdb89889a..6aed57ea98194a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk]
+----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk]
 ------------------------------------PhysicalProject
 --------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX'))
 ----------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF3
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query88.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query88.out
index 09c630cef443ef..a619efe9def125 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query13.out
index 862b1f5d64b5f6..8cf5a0dbc062aa 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('MI', 'OK', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('NC', 'OH', 'WA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('FL', 'GA', 'MT'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('MI', 'OK', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('NC', 'OH', 'WA'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('FL', 'GA', 'MT'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('College', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('College', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query34.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query34.out
index 5aa611041037f9..654477132bf75e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query34.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (2000, 2001, 2002))
+----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (2000, 2001, 2002))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Arthur County', 'Halifax County', 'Lunenburg County', 'Oglethorpe County', 'Perry County', 'Salem County', 'Sumner County', 'Terrell County'))
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query48.out
index 2cac05386db12c..3cd8b7c9432955 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('GA', 'MI', 'NH'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('KY', 'SD', 'TX'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('FL', 'NY', 'OH'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('GA', 'MI', 'NH'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('KY', 'SD', 'TX'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('FL', 'NY', 'OH'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF0 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query85.out
index 106cf30dfcdd28..4496c97b3fd7a8 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('CA', 'TX', 'VA'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('AR', 'MO', 'NE'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'WA'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('CA', 'TX', 'VA'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('AR', 'MO', 'NE'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'WA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF2 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query88.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query88.out
index 3d5752485f6b0a..b96bfd94b0dcda 100644
--- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query88.out
+++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
+----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
+------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
+----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
+--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
+------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out
index ad82594176a219..78faf3234691b3 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out
index ad82594176a219..78faf3234691b3 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out
index ad82594176a219..78faf3234691b3 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out
index ad82594176a219..78faf3234691b3 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/new_shapes_p0/hint_tpch/shape/q19.out b/regression-test/data/new_shapes_p0/hint_tpch/shape/q19.out
index 5cabdf9f163c56..ebd4f59d682f8f 100644
--- a/regression-test/data/new_shapes_p0/hint_tpch/shape/q19.out
+++ b/regression-test/data/new_shapes_p0/hint_tpch/shape/q19.out
@@ -5,12 +5,12 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]])
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]])
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem]
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
 Hint log:
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.out
index 9c7d9382777bee..7b5d0d01ba6a17 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.out
index 6c90e3a48d7cba..15e7650bae15c3 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
+----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.out
index 57618c21f263d3..aef568a41842bb 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.out
index 9f12763cec5114..63fc92a6f489d9 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=()
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.out
index a20acf339431ea..ae9b03a84ef7de 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.out
index a16b428cbc5159..2cac8d809ad124 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.out
index 9a53073479fb8c..a19fe778a57647 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
+----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000))
 ------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalProject
 ------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.out
index fa9006ce24f673..d994b40da36a4a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.out
@@ -7,9 +7,9 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF0 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.out
index 424102e34021a1..62c4b147f59a22 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.out
@@ -13,11 +13,11 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->[wr_refunded_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[wr_returning_cdemo_sk];RF5 cd_marital_status->[cd_marital_status];RF6 cd_education_status->[cd_education_status]
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF2 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.out
index a20acf339431ea..ae9b03a84ef7de 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
@@ -40,7 +40,7 @@ PhysicalResultSink
 ----------------------------------PhysicalProject
 ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20
 ----------------------------------PhysicalProject
-------------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------------PhysicalOlapScan[household_demographics]
 ------------------------------PhysicalProject
 --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
@@ -60,7 +60,7 @@ PhysicalResultSink
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
 ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
@@ -80,7 +80,7 @@ PhysicalResultSink
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
@@ -100,7 +100,7 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
@@ -120,7 +120,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
@@ -140,7 +140,7 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
@@ -160,7 +160,7 @@ PhysicalResultSink
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query13.out
index 4b72ea24b2c114..55cadf71bf5d45 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query13.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
 ------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query34.out
index 489abcd4c2e33c..79c46ccb77bbb8 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query34.out
@@ -15,17 +15,17 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF0 hd_demo_sk->[ss_hdemo_sk]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
-------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
+------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
-------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
---------------------------------PhysicalOlapScan[household_demographics]
+------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000))
+--------------------------------PhysicalOlapScan[date_dim]
 ------------------------PhysicalProject
 --------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
 ----------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out
index 3f16a98d674c76..ad78a4f21c50ed 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out
@@ -7,11 +7,11 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out
index d9d74a4dd87f7e..1fed378a9afc1d 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk]
+----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query88.out
index 09c630cef443ef..a619efe9def125 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query13.out
index e8a4d92c4f7bf1..dc2239ca3f9701 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query13.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
 ------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalProject
 ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query34.out
index 489abcd4c2e33c..79c46ccb77bbb8 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query34.out
@@ -15,17 +15,17 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF0 hd_demo_sk->[ss_hdemo_sk]
 --------------------------------PhysicalProject
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 --------------------------------PhysicalProject
-----------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (1998, 1999, 2000))
-------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
+------------------------------------PhysicalOlapScan[household_demographics]
 ----------------------------PhysicalProject
-------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
---------------------------------PhysicalOlapScan[household_demographics]
+------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000))
+--------------------------------PhysicalOlapScan[date_dim]
 ------------------------PhysicalProject
 --------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County'))
 ----------------------------PhysicalOlapScan[store]
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out
index 1bf363b1426199..e7bd6986059878 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out
@@ -7,11 +7,11 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
+--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out
index 332573e1b96303..be579258fa046a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 --------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF5 RF6 RF7
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
+------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->[cd_demo_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W'))
 ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk]
+----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query88.out
index 09c630cef443ef..a619efe9def125 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+----------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count <= 6)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
+--------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.out
index 8138a8e20ed7e5..0db3330914c18a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.out
@@ -7,14 +7,14 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
+--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY'))
 --------------------PhysicalOlapScan[customer_address] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query13.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query13.out
index 8138a8e20ed7e5..0db3330914c18a 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query13.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query13.out
@@ -7,14 +7,14 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 300.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
+--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->[ca_address_sk]
 ----------------PhysicalProject
 ------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY'))
 --------------------PhysicalOlapScan[customer_address] apply RFs: RF3
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query34.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query34.out
index ad28c2d46104d0..c75d4fc3e18155 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query34.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query34.out
@@ -13,9 +13,9 @@ PhysicalResultSink
 ----------------PhysicalDistribute[DistributionSpecHash]
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF0 s_store_sk->[ss_store_sk]
 --------------------------------PhysicalProject
@@ -24,9 +24,9 @@ PhysicalResultSink
 ----------------------------------filter((store.s_county = 'Williamson County'))
 ------------------------------------PhysicalOlapScan[store]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(date_dim.d_dom >= 1),(date_dim.d_dom <= 3)],AND[(date_dim.d_dom >= 25),(date_dim.d_dom <= 28)]] and d_year IN (2000, 2001, 2002))
---------------------------------PhysicalOlapScan[date_dim]
+------------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
+--------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
---------------------------filter((household_demographics.hd_vehicle_count > 0) and (if((hd_vehicle_count > 0), (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)), NULL) > 1.2) and hd_buy_potential IN ('0-500', '1001-5000'))
-----------------------------PhysicalOlapScan[household_demographics]
+--------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (2000, 2001, 2002))
+----------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out
index 6ed75b1e21eca8..d11dadeae0b923 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out
@@ -9,9 +9,9 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit >= 0.00),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00),(store_sales.ss_net_profit <= 25000.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->[ss_addr_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 50.00),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00),(store_sales.ss_sales_price <= 200.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
 --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query85.out
index c734d7945f3a56..6f4f258704f239 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query85.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query85.out
@@ -18,12 +18,12 @@ PhysicalResultSink
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF5 wp_web_page_sk->[ws_web_page_sk]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00),(web_sales.ws_net_profit <= 300.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit >= 50.00),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
+------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->[ca_address_sk]
 --------------------------------PhysicalProject
 ----------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV'))
 ------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price >= 50.00),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00),(web_sales.ws_sales_price <= 200.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
+----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->[wr_refunded_cdemo_sk]
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number]
 ----------------------------------------PhysicalProject
diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out
index d016dca48db832..5da04ad61d3f42 100644
--- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out
+++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query88.out
@@ -23,7 +23,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -43,7 +43,7 @@ PhysicalResultSink
 ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30))
 --------------------------------------PhysicalOlapScan[time_dim]
 ------------------------------PhysicalProject
---------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------------------PhysicalOlapScan[household_demographics]
 --------------------------PhysicalProject
 ----------------------------filter((store.s_store_name = 'ese'))
@@ -63,7 +63,7 @@ PhysicalResultSink
 ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30))
 ------------------------------------PhysicalOlapScan[time_dim]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 --------------------------------PhysicalOlapScan[household_demographics]
 ------------------------PhysicalProject
 --------------------------filter((store.s_store_name = 'ese'))
@@ -83,7 +83,7 @@ PhysicalResultSink
 --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30))
 ----------------------------------PhysicalOlapScan[time_dim]
 --------------------------PhysicalProject
-----------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 ----------------------PhysicalProject
 ------------------------filter((store.s_store_name = 'ese'))
@@ -103,7 +103,7 @@ PhysicalResultSink
 ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30))
 --------------------------------PhysicalOlapScan[time_dim]
 ------------------------PhysicalProject
---------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 --------------------PhysicalProject
 ----------------------filter((store.s_store_name = 'ese'))
@@ -123,7 +123,7 @@ PhysicalResultSink
 ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30))
 ------------------------------PhysicalOlapScan[time_dim]
 ----------------------PhysicalProject
-------------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 --------------------------PhysicalOlapScan[household_demographics]
 ------------------PhysicalProject
 --------------------filter((store.s_store_name = 'ese'))
@@ -143,7 +143,7 @@ PhysicalResultSink
 --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30))
 ----------------------------PhysicalOlapScan[time_dim]
 --------------------PhysicalProject
-----------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ------------------------PhysicalOlapScan[household_demographics]
 ----------------PhysicalProject
 ------------------filter((store.s_store_name = 'ese'))
@@ -163,7 +163,7 @@ PhysicalResultSink
 ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30))
 --------------------------PhysicalOlapScan[time_dim]
 ------------------PhysicalProject
---------------------filter(OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 0, 3))
+--------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3))
 ----------------------PhysicalOlapScan[household_demographics]
 --------------PhysicalProject
 ----------------filter((store.s_store_name = 'ese'))
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out
index ad82594176a219..78faf3234691b3 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q19.out
index ad82594176a219..78faf3234691b3 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q19.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q19.out
index ad82594176a219..78faf3234691b3 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q19.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 
diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out
index ad82594176a219..78faf3234691b3 100644
--- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out
+++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out
@@ -5,11 +5,11 @@ PhysicalResultSink
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity >= 1.00),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00),(lineitem.l_quantity <= 30.00),(part.p_size <= 15)]]) build RFs:RF0 p_partkey->[l_partkey]
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
 --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalProject
---------------filter((part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(part.p_size <= 15)]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
+--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))
 ----------------PhysicalOlapScan[part]
 

From d1e8bebb3c610bc729b0a088fd693f013aeb5336 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Wed, 11 Dec 2024 14:20:41 +0800
Subject: [PATCH 267/399] [refactor](shuffle) (PART I) Unify all hash-partition
 based shuffling (#45256)

---
 .../pipeline/exec/exchange_sink_operator.cpp  | 308 ++++--------------
 be/src/pipeline/exec/exchange_sink_operator.h |  58 +---
 be/src/pipeline/shuffle/writer.cpp            | 114 +++++++
 be/src/pipeline/shuffle/writer.h              |  53 +++
 be/src/vec/runtime/partitioner.h              |  16 +-
 .../scale_writer_partitioning_exchanger.hpp   |  92 ++++--
 .../vec/sink/tablet_sink_hash_partitioner.cpp | 152 +++++++++
 .../vec/sink/tablet_sink_hash_partitioner.h   |  83 +++++
 be/src/vec/sink/vdata_stream_sender.cpp       |  12 +-
 be/src/vec/sink/vdata_stream_sender.h         |   8 +-
 10 files changed, 562 insertions(+), 334 deletions(-)
 create mode 100644 be/src/pipeline/shuffle/writer.cpp
 create mode 100644 be/src/pipeline/shuffle/writer.h
 create mode 100644 be/src/vec/sink/tablet_sink_hash_partitioner.cpp
 create mode 100644 be/src/vec/sink/tablet_sink_hash_partitioner.h

diff --git a/be/src/pipeline/exec/exchange_sink_operator.cpp b/be/src/pipeline/exec/exchange_sink_operator.cpp
index 04b9653e9c8299..aa893fc0a26f2e 100644
--- a/be/src/pipeline/exec/exchange_sink_operator.cpp
+++ b/be/src/pipeline/exec/exchange_sink_operator.cpp
@@ -38,6 +38,7 @@
 #include "util/uid_util.h"
 #include "vec/columns/column_const.h"
 #include "vec/exprs/vexpr.h"
+#include "vec/sink/tablet_sink_hash_partitioner.h"
 
 namespace doris::pipeline {
 #include "common/compile_check_begin.h"
@@ -120,6 +121,57 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf
         _sink_buffer->set_dependency(state->fragment_instance_id().lo, _queue_dependency, this);
     }
 
+    if (_part_type == TPartitionType::HASH_PARTITIONED) {
+        _partition_count = channels.size();
+        _partitioner =
+                std::make_unique<vectorized::Crc32HashPartitioner<vectorized::ShuffleChannelIds>>(
+                        channels.size());
+        RETURN_IF_ERROR(_partitioner->init(p._texprs));
+        RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
+        _profile->add_info_string("Partitioner",
+                                  fmt::format("Crc32HashPartitioner({})", _partition_count));
+    } else if (_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) {
+        _partition_count = channels.size();
+        _partitioner =
+                std::make_unique<vectorized::Crc32HashPartitioner<vectorized::ShuffleChannelIds>>(
+                        channels.size());
+        RETURN_IF_ERROR(_partitioner->init(p._texprs));
+        RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
+        _profile->add_info_string("Partitioner",
+                                  fmt::format("Crc32HashPartitioner({})", _partition_count));
+    } else if (_part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED) {
+        _partition_count = channels.size();
+        _profile->add_info_string("Partitioner",
+                                  fmt::format("Crc32HashPartitioner({})", _partition_count));
+        _partitioner = std::make_unique<vectorized::TabletSinkHashPartitioner>(
+                _partition_count, p._tablet_sink_txn_id, p._tablet_sink_schema,
+                p._tablet_sink_partition, p._tablet_sink_location, p._tablet_sink_tuple_id, this);
+        RETURN_IF_ERROR(_partitioner->init({}));
+        RETURN_IF_ERROR(_partitioner->prepare(state, {}));
+    } else if (_part_type == TPartitionType::TABLE_SINK_HASH_PARTITIONED) {
+        _partition_count =
+                channels.size() * config::table_sink_partition_write_max_partition_nums_per_writer;
+        _partitioner = std::make_unique<vectorized::ScaleWriterPartitioner>(
+                channels.size(), _partition_count, channels.size(), 1,
+                config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold /
+                                        state->task_num() ==
+                                0
+                        ? config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold
+                        : config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold /
+                                  state->task_num(),
+                config::table_sink_partition_write_min_data_processed_rebalance_threshold /
+                                        state->task_num() ==
+                                0
+                        ? config::table_sink_partition_write_min_data_processed_rebalance_threshold
+                        : config::table_sink_partition_write_min_data_processed_rebalance_threshold /
+                                  state->task_num());
+
+        RETURN_IF_ERROR(_partitioner->init(p._texprs));
+        RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
+        _profile->add_info_string("Partitioner",
+                                  fmt::format("Crc32HashPartitioner({})", _partition_count));
+    }
+
     return Status::OK();
 }
 
@@ -145,6 +197,7 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) {
     SCOPED_TIMER(exec_time_counter());
     SCOPED_TIMER(_open_timer);
     RETURN_IF_ERROR(Base::open(state));
+    _writer.reset(new Writer());
     auto& p = _parent->cast<ExchangeSinkOperatorX>();
 
     if (_part_type == TPartitionType::UNPARTITIONED || _part_type == TPartitionType::RANDOM ||
@@ -190,113 +243,16 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) {
             }
         }
     }
-    if (_part_type == TPartitionType::HASH_PARTITIONED) {
-        _partition_count = channels.size();
-        _partitioner =
-                std::make_unique<vectorized::Crc32HashPartitioner<vectorized::ShuffleChannelIds>>(
-                        channels.size());
-        RETURN_IF_ERROR(_partitioner->init(p._texprs));
-        RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
-        _profile->add_info_string("Partitioner",
-                                  fmt::format("Crc32HashPartitioner({})", _partition_count));
-    } else if (_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) {
-        _partition_count = channels.size();
-        _partitioner =
-                std::make_unique<vectorized::Crc32HashPartitioner<vectorized::ShuffleChannelIds>>(
-                        channels.size());
-        RETURN_IF_ERROR(_partitioner->init(p._texprs));
-        RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
-        _profile->add_info_string("Partitioner",
-                                  fmt::format("Crc32HashPartitioner({})", _partition_count));
-    } else if (_part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED) {
-        _partition_count = channels.size();
-        _profile->add_info_string("Partitioner",
-                                  fmt::format("Crc32HashPartitioner({})", _partition_count));
-        _txn_id = p._tablet_sink_txn_id;
-        _schema = std::make_shared<OlapTableSchemaParam>();
-        RETURN_IF_ERROR(_schema->init(p._tablet_sink_schema));
-        _vpartition = std::make_unique<VOlapTablePartitionParam>(_schema, p._tablet_sink_partition);
-        RETURN_IF_ERROR(_vpartition->init());
-        auto find_tablet_mode = vectorized::OlapTabletFinder::FindTabletMode::FIND_TABLET_EVERY_ROW;
-        _tablet_finder =
-                std::make_unique<vectorized::OlapTabletFinder>(_vpartition.get(), find_tablet_mode);
-        _tablet_sink_tuple_desc = _state->desc_tbl().get_tuple_descriptor(p._tablet_sink_tuple_id);
-        _tablet_sink_row_desc = p._pool->add(new RowDescriptor(_tablet_sink_tuple_desc, false));
-        _tablet_sink_expr_ctxs.resize(p._tablet_sink_expr_ctxs.size());
-        for (size_t i = 0; i < _tablet_sink_expr_ctxs.size(); i++) {
-            RETURN_IF_ERROR(p._tablet_sink_expr_ctxs[i]->clone(state, _tablet_sink_expr_ctxs[i]));
-        }
-        // if _part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED, we handle the processing of auto_increment column
-        // on exchange node rather than on TabletWriter
-        _block_convertor =
-                std::make_unique<vectorized::OlapTableBlockConvertor>(_tablet_sink_tuple_desc);
-        _block_convertor->init_autoinc_info(_schema->db_id(), _schema->table_id(),
-                                            _state->batch_size());
-        _location = p._pool->add(new OlapTableLocationParam(p._tablet_sink_location));
-        _row_distribution.init(
-                {.state = _state,
-                 .block_convertor = _block_convertor.get(),
-                 .tablet_finder = _tablet_finder.get(),
-                 .vpartition = _vpartition.get(),
-                 .add_partition_request_timer = _add_partition_request_timer,
-                 .txn_id = _txn_id,
-                 .pool = p._pool.get(),
-                 .location = _location,
-                 .vec_output_expr_ctxs = &_tablet_sink_expr_ctxs,
-                 .schema = _schema,
-                 .caller = (void*)this,
-                 .create_partition_callback = &ExchangeSinkLocalState::empty_callback_function});
-    } else if (_part_type == TPartitionType::TABLE_SINK_HASH_PARTITIONED) {
-        _partition_count =
-                channels.size() * config::table_sink_partition_write_max_partition_nums_per_writer;
-        _partitioner =
-                std::make_unique<vectorized::Crc32HashPartitioner<vectorized::ShuffleChannelIds>>(
-                        _partition_count);
-        _partition_function = std::make_unique<HashPartitionFunction>(_partitioner.get());
-
-        scale_writer_partitioning_exchanger = std::make_unique<
-                vectorized::ScaleWriterPartitioningExchanger<HashPartitionFunction>>(
-                channels.size(), *_partition_function, _partition_count, channels.size(), 1,
-                config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold /
-                                        state->task_num() ==
-                                0
-                        ? config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold
-                        : config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold /
-                                  state->task_num(),
-                config::table_sink_partition_write_min_data_processed_rebalance_threshold /
-                                        state->task_num() ==
-                                0
-                        ? config::table_sink_partition_write_min_data_processed_rebalance_threshold
-                        : config::table_sink_partition_write_min_data_processed_rebalance_threshold /
-                                  state->task_num());
-
-        RETURN_IF_ERROR(_partitioner->init(p._texprs));
-        RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
-        _profile->add_info_string("Partitioner",
-                                  fmt::format("Crc32HashPartitioner({})", _partition_count));
-    }
 
     if (_part_type == TPartitionType::HASH_PARTITIONED ||
         _part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED ||
-        _part_type == TPartitionType::TABLE_SINK_HASH_PARTITIONED) {
+        _part_type == TPartitionType::TABLE_SINK_HASH_PARTITIONED ||
+        _part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED) {
         RETURN_IF_ERROR(_partitioner->open(state));
-    } else if (_part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED) {
-        RETURN_IF_ERROR(_row_distribution.open(_tablet_sink_row_desc));
     }
     return Status::OK();
 }
 
-Status ExchangeSinkLocalState::_send_new_partition_batch(vectorized::Block* input_block) {
-    RETURN_IF_ERROR(_row_distribution.automatic_create_partition());
-    auto& p = _parent->cast<ExchangeSinkOperatorX>();
-    // Recovery back
-    _row_distribution.clear_batching_stats();
-    _row_distribution._batching_block->clear_column_data();
-    _row_distribution._deal_batched = false;
-    RETURN_IF_ERROR(p.sink(_state, input_block, false));
-    return Status::OK();
-}
-
 std::string ExchangeSinkLocalState::name_suffix() {
     std::string name = " (id=" + std::to_string(_parent->node_id());
     auto& p = _parent->cast<ExchangeSinkOperatorX>();
@@ -491,105 +447,10 @@ Status ExchangeSinkOperatorX::sink(RuntimeState* state, vectorized::Block* block
         local_state.current_channel_idx =
                 (local_state.current_channel_idx + 1) % local_state.channels.size();
     } else if (_part_type == TPartitionType::HASH_PARTITIONED ||
-               _part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) {
-        auto rows = block->rows();
-        {
-            SCOPED_TIMER(local_state._split_block_hash_compute_timer);
-            RETURN_IF_ERROR(local_state._partitioner->do_partitioning(state, block));
-        }
-        int64_t old_channel_mem_usage = 0;
-        for (const auto& channel : local_state.channels) {
-            old_channel_mem_usage += channel->mem_usage();
-        }
-        if (_part_type == TPartitionType::HASH_PARTITIONED) {
-            SCOPED_TIMER(local_state._distribute_rows_into_channels_timer);
-            RETURN_IF_ERROR(channel_add_rows(
-                    state, local_state.channels, local_state._partition_count,
-                    local_state._partitioner->get_channel_ids().get<uint32_t>(), rows, block, eos));
-        } else {
-            SCOPED_TIMER(local_state._distribute_rows_into_channels_timer);
-            RETURN_IF_ERROR(channel_add_rows(
-                    state, local_state.channels, local_state._partition_count,
-                    local_state._partitioner->get_channel_ids().get<uint32_t>(), rows, block, eos));
-        }
-        int64_t new_channel_mem_usage = 0;
-        for (const auto& channel : local_state.channels) {
-            new_channel_mem_usage += channel->mem_usage();
-        }
-        COUNTER_UPDATE(local_state.memory_used_counter(),
-                       new_channel_mem_usage - old_channel_mem_usage);
-    } else if (_part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED) {
-        int64_t old_channel_mem_usage = 0;
-        for (const auto& channel : local_state.channels) {
-            old_channel_mem_usage += channel->mem_usage();
-        }
-        // check out of limit
-        std::shared_ptr<vectorized::Block> convert_block = std::make_shared<vectorized::Block>();
-        const auto& num_channels = local_state._partition_count;
-        std::vector<std::vector<uint32>> channel2rows;
-        channel2rows.resize(num_channels);
-        auto input_rows = block->rows();
-
-        if (input_rows > 0) {
-            bool has_filtered_rows = false;
-            int64_t filtered_rows = 0;
-            local_state._number_input_rows += input_rows;
-
-            RETURN_IF_ERROR(local_state._row_distribution.generate_rows_distribution(
-                    *block, convert_block, filtered_rows, has_filtered_rows,
-                    local_state._row_part_tablet_ids, local_state._number_input_rows));
-            if (local_state._row_distribution.batching_rows() > 0) {
-                SCOPED_TIMER(local_state._send_new_partition_timer);
-                RETURN_IF_ERROR(local_state._send_new_partition_batch(block));
-            } else {
-                const auto& row_ids = local_state._row_part_tablet_ids[0].row_ids;
-                const auto& tablet_ids = local_state._row_part_tablet_ids[0].tablet_ids;
-                for (int idx = 0; idx < row_ids.size(); ++idx) {
-                    const auto& row = row_ids[idx];
-                    const auto& tablet_id_hash =
-                            HashUtil::zlib_crc_hash(&tablet_ids[idx], sizeof(int64), 0);
-                    channel2rows[tablet_id_hash % num_channels].emplace_back(row);
-                }
-            }
-        }
-
-        {
-            SCOPED_TIMER(local_state._distribute_rows_into_channels_timer);
-            // the convert_block maybe different with block after execute exprs
-            // when send data we still use block
-            RETURN_IF_ERROR(channel_add_rows_with_idx(state, local_state.channels, num_channels,
-                                                      channel2rows, block, eos));
-        }
-        int64_t new_channel_mem_usage = 0;
-        for (const auto& channel : local_state.channels) {
-            new_channel_mem_usage += channel->mem_usage();
-        }
-        COUNTER_UPDATE(local_state.memory_used_counter(),
-                       new_channel_mem_usage - old_channel_mem_usage);
-    } else if (_part_type == TPartitionType::TABLE_SINK_HASH_PARTITIONED) {
-        int64_t old_channel_mem_usage = 0;
-        for (const auto& channel : local_state.channels) {
-            old_channel_mem_usage += channel->mem_usage();
-        }
-        {
-            SCOPED_TIMER(local_state._split_block_hash_compute_timer);
-            RETURN_IF_ERROR(local_state._partitioner->do_partitioning(state, block));
-        }
-        std::vector<std::vector<uint32>> assignments =
-                local_state.scale_writer_partitioning_exchanger->accept(block);
-        {
-            SCOPED_TIMER(local_state._distribute_rows_into_channels_timer);
-            RETURN_IF_ERROR(channel_add_rows_with_idx(state, local_state.channels,
-                                                      local_state.channels.size(), assignments,
-                                                      block, eos));
-        }
-
-        int64_t new_channel_mem_usage = 0;
-        for (const auto& channel : local_state.channels) {
-            new_channel_mem_usage += channel->mem_usage();
-        }
-        COUNTER_UPDATE(local_state.memory_used_counter(),
-                       new_channel_mem_usage - old_channel_mem_usage);
+               _part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED ||
+               _part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED ||
+               _part_type == TPartitionType::TABLE_SINK_HASH_PARTITIONED) {
+        RETURN_IF_ERROR(local_state._writer->write(&local_state, state, block, eos));
     } else if (_part_type == TPartitionType::TABLE_SINK_RANDOM_PARTITIONED) {
         // Control the number of channels according to the flow, thereby controlling the number of table sink writers.
         // 1. select channel
@@ -641,44 +502,6 @@ void ExchangeSinkLocalState::register_channels(pipeline::ExchangeSinkBuffer* buf
     }
 }
 
-Status ExchangeSinkOperatorX::channel_add_rows(
-        RuntimeState* state, std::vector<std::shared_ptr<vectorized::Channel>>& channels,
-        size_t num_channels, const uint32_t* __restrict channel_ids, size_t rows,
-        vectorized::Block* block, bool eos) {
-    std::vector<std::vector<uint32_t>> channel2rows;
-    channel2rows.resize(num_channels);
-    for (uint32_t i = 0; i < rows; i++) {
-        channel2rows[channel_ids[i]].emplace_back(i);
-    }
-
-    RETURN_IF_ERROR(
-            channel_add_rows_with_idx(state, channels, num_channels, channel2rows, block, eos));
-    return Status::OK();
-}
-
-Status ExchangeSinkOperatorX::channel_add_rows_with_idx(
-        RuntimeState* state, std::vector<std::shared_ptr<vectorized::Channel>>& channels,
-        size_t num_channels, std::vector<std::vector<uint32_t>>& channel2rows,
-        vectorized::Block* block, bool eos) {
-    Status status = Status::OK();
-    for (int i = 0; i < num_channels; ++i) {
-        if (!channels[i]->is_receiver_eof() && !channel2rows[i].empty()) {
-            status = channels[i]->add_rows(block, channel2rows[i], false);
-            HANDLE_CHANNEL_STATUS(state, channels[i], status);
-            channel2rows[i].clear();
-        }
-    }
-    if (eos) {
-        for (int i = 0; i < num_channels; ++i) {
-            if (!channels[i]->is_receiver_eof()) {
-                status = channels[i]->add_rows(block, channel2rows[i], true);
-                HANDLE_CHANNEL_STATUS(state, channels[i], status);
-            }
-        }
-    }
-    return Status::OK();
-}
-
 std::string ExchangeSinkLocalState::debug_string(int indentation_level) const {
     fmt::memory_buffer debug_string_buffer;
     fmt::format_to(debug_string_buffer, "{}", Base::debug_string(indentation_level));
@@ -697,17 +520,10 @@ Status ExchangeSinkLocalState::close(RuntimeState* state, Status exec_status) {
     if (_closed) {
         return Status::OK();
     }
-    if (_part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED &&
-        _block_convertor != nullptr && _tablet_finder != nullptr) {
-        _state->update_num_rows_load_filtered(_block_convertor->num_filtered_rows() +
-                                              _tablet_finder->num_filtered_rows());
-        _state->update_num_rows_load_unselected(
-                _tablet_finder->num_immutable_partition_filtered_rows());
-        // sink won't see those filtered rows, we should compensate here
-        _state->set_num_rows_load_total(_state->num_rows_load_filtered() +
-                                        _state->num_rows_load_unselected());
-    }
     SCOPED_TIMER(exec_time_counter());
+    if (_partitioner) {
+        RETURN_IF_ERROR(_partitioner->close(state));
+    }
     SCOPED_TIMER(_close_timer);
     if (_queue_dependency) {
         COUNTER_UPDATE(_wait_queue_timer, _queue_dependency->watcher_elapse_time());
diff --git a/be/src/pipeline/exec/exchange_sink_operator.h b/be/src/pipeline/exec/exchange_sink_operator.h
index 8d094b43f613f0..e88389b1d7bb5a 100644
--- a/be/src/pipeline/exec/exchange_sink_operator.h
+++ b/be/src/pipeline/exec/exchange_sink_operator.h
@@ -26,6 +26,7 @@
 #include "common/status.h"
 #include "exchange_sink_buffer.h"
 #include "operator.h"
+#include "pipeline/shuffle/writer.h"
 #include "vec/sink/scale_writer_partitioning_exchanger.hpp"
 #include "vec/sink/vdata_stream_sender.h"
 
@@ -39,20 +40,6 @@ class ExchangeSinkLocalState final : public PipelineXSinkLocalState<> {
     ENABLE_FACTORY_CREATOR(ExchangeSinkLocalState);
     using Base = PipelineXSinkLocalState<>;
 
-private:
-    class HashPartitionFunction {
-    public:
-        HashPartitionFunction(vectorized::PartitionerBase* partitioner)
-                : _partitioner(partitioner) {}
-
-        int get_partition(vectorized::Block* block, int position) {
-            return _partitioner->get_channel_ids().get<uint32_t>()[position];
-        }
-
-    private:
-        vectorized::PartitionerBase* _partitioner;
-    };
-
 public:
     ExchangeSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state)
             : Base(parent, state), _serializer(this) {
@@ -106,19 +93,20 @@ class ExchangeSinkLocalState final : public PipelineXSinkLocalState<> {
     std::string name_suffix() override;
     segment_v2::CompressionTypePB compression_type() const;
     std::string debug_string(int indentation_level) const override;
-    static Status empty_callback_function(void* sender, TCreatePartitionResult* result) {
-        return Status::OK();
+    RuntimeProfile::Counter* send_new_partition_timer() { return _send_new_partition_timer; }
+    RuntimeProfile::Counter* add_partition_request_timer() { return _add_partition_request_timer; }
+    RuntimeProfile::Counter* split_block_hash_compute_timer() {
+        return _split_block_hash_compute_timer;
+    }
+    RuntimeProfile::Counter* distribute_rows_into_channels_timer() {
+        return _distribute_rows_into_channels_timer;
     }
-    Status _send_new_partition_batch(vectorized::Block* input_block);
     std::vector<std::shared_ptr<vectorized::Channel>> channels;
     int current_channel_idx {0}; // index of current channel to send to if _random == true
     bool only_local_exchange {false};
 
     void on_channel_finished(InstanceLoId channel_id);
-
-    // for external table sink hash partition
-    std::unique_ptr<vectorized::ScaleWriterPartitioningExchanger<HashPartitionFunction>>
-            scale_writer_partitioning_exchanger;
+    vectorized::PartitionerBase* partitioner() const { return _partitioner.get(); }
 
 private:
     friend class ExchangeSinkOperatorX;
@@ -176,28 +164,16 @@ class ExchangeSinkLocalState final : public PipelineXSinkLocalState<> {
      */
     std::vector<std::shared_ptr<Dependency>> _local_channels_dependency;
     std::unique_ptr<vectorized::PartitionerBase> _partitioner;
+    std::unique_ptr<Writer> _writer;
     size_t _partition_count;
 
     std::shared_ptr<Dependency> _finish_dependency;
 
     // for shuffle data by partition and tablet
-    int64_t _txn_id = -1;
-    vectorized::VExprContextSPtrs _tablet_sink_expr_ctxs;
-    std::unique_ptr<VOlapTablePartitionParam> _vpartition = nullptr;
-    std::unique_ptr<vectorized::OlapTabletFinder> _tablet_finder = nullptr;
-    std::shared_ptr<OlapTableSchemaParam> _schema = nullptr;
-    std::unique_ptr<vectorized::OlapTableBlockConvertor> _block_convertor = nullptr;
-    TupleDescriptor* _tablet_sink_tuple_desc = nullptr;
-    RowDescriptor* _tablet_sink_row_desc = nullptr;
-    OlapTableLocationParam* _location = nullptr;
-    vectorized::VRowDistribution _row_distribution;
+
     RuntimeProfile::Counter* _add_partition_request_timer = nullptr;
-    std::vector<vectorized::RowPartTabletIds> _row_part_tablet_ids;
-    int64_t _number_input_rows = 0;
     TPartitionType::type _part_type;
 
-    // for external table sink hash partition
-    std::unique_ptr<HashPartitionFunction> _partition_function = nullptr;
     std::atomic<bool> _reach_limit = false;
     int _last_local_channel_idx = -1;
 
@@ -230,6 +206,7 @@ class ExchangeSinkOperatorX final : public DataSinkOperatorX<ExchangeSinkLocalSt
     // In a merge sort scenario, there are only n RPCs, so a shared sink buffer is not needed.
     /// TODO: Modify this to let FE handle the judgment instead of BE.
     std::shared_ptr<ExchangeSinkBuffer> get_sink_buffer(InstanceLoId sender_ins_id);
+    vectorized::VExprContextSPtrs& tablet_sink_expr_ctxs() { return _tablet_sink_expr_ctxs; }
 
 private:
     friend class ExchangeSinkLocalState;
@@ -237,17 +214,6 @@ class ExchangeSinkOperatorX final : public DataSinkOperatorX<ExchangeSinkLocalSt
     template <typename ChannelPtrType>
     void _handle_eof_channel(RuntimeState* state, ChannelPtrType channel, Status st);
 
-    Status channel_add_rows(RuntimeState* state,
-                            std::vector<std::shared_ptr<vectorized::Channel>>& channels,
-                            size_t num_channels, const uint32_t* __restrict channel_ids,
-                            size_t rows, vectorized::Block* block, bool eos);
-
-    Status channel_add_rows_with_idx(RuntimeState* state,
-                                     std::vector<std::shared_ptr<vectorized::Channel>>& channels,
-                                     size_t num_channels,
-                                     std::vector<std::vector<uint32_t>>& channel2rows,
-                                     vectorized::Block* block, bool eos);
-
     // Use ExchangeSinkOperatorX to create a sink buffer.
     // The sink buffer can be shared among multiple ExchangeSinkLocalState instances,
     // or each ExchangeSinkLocalState can have its own sink buffer.
diff --git a/be/src/pipeline/shuffle/writer.cpp b/be/src/pipeline/shuffle/writer.cpp
new file mode 100644
index 00000000000000..c27fd9a7aeb731
--- /dev/null
+++ b/be/src/pipeline/shuffle/writer.cpp
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "writer.h"
+
+#include "pipeline/exec/exchange_sink_operator.h"
+#include "vec/core/block.h"
+
+namespace doris::pipeline {
+#include "common/compile_check_begin.h"
+template <typename ChannelPtrType>
+void Writer::_handle_eof_channel(RuntimeState* state, ChannelPtrType channel, Status st) const {
+    channel->set_receiver_eof(st);
+    // Chanel will not send RPC to the downstream when eof, so close chanel by OK status.
+    static_cast<void>(channel->close(state));
+}
+
+Status Writer::write(ExchangeSinkLocalState* local_state, RuntimeState* state,
+                     vectorized::Block* block, bool eos) const {
+    auto rows = block->rows();
+    {
+        SCOPED_TIMER(local_state->split_block_hash_compute_timer());
+        RETURN_IF_ERROR(local_state->partitioner()->do_partitioning(state, block));
+    }
+    int64_t old_channel_mem_usage = 0;
+    for (const auto& channel : local_state->channels) {
+        old_channel_mem_usage += channel->mem_usage();
+    }
+    {
+        SCOPED_TIMER(local_state->distribute_rows_into_channels_timer());
+        const auto& channel_filed = local_state->partitioner()->get_channel_ids();
+        if (channel_filed.len == sizeof(uint32_t)) {
+            RETURN_IF_ERROR(_channel_add_rows(state, local_state->channels,
+                                              local_state->channels.size(),
+                                              channel_filed.get<uint32_t>(), rows, block, eos));
+        } else {
+            RETURN_IF_ERROR(_channel_add_rows(state, local_state->channels,
+                                              local_state->channels.size(),
+                                              channel_filed.get<int64_t>(), rows, block, eos));
+        }
+    }
+    int64_t new_channel_mem_usage = 0;
+    for (const auto& channel : local_state->channels) {
+        new_channel_mem_usage += channel->mem_usage();
+    }
+    COUNTER_UPDATE(local_state->memory_used_counter(),
+                   new_channel_mem_usage - old_channel_mem_usage);
+    return Status::OK();
+}
+
+template <typename ChannelIdType>
+Status Writer::_channel_add_rows(RuntimeState* state,
+                                 std::vector<std::shared_ptr<vectorized::Channel>>& channels,
+                                 size_t partition_count,
+                                 const ChannelIdType* __restrict channel_ids, size_t rows,
+                                 vectorized::Block* block, bool eos) const {
+    std::vector<uint32_t> partition_rows_histogram;
+    auto row_idx = vectorized::PODArray<uint32_t>(rows);
+    {
+        partition_rows_histogram.assign(partition_count + 2, 0);
+        for (size_t i = 0; i < rows; ++i) {
+            partition_rows_histogram[channel_ids[i] + 1]++;
+        }
+        for (size_t i = 1; i <= partition_count + 1; ++i) {
+            partition_rows_histogram[i] += partition_rows_histogram[i - 1];
+        }
+        for (int32_t i = cast_set<int32_t>(rows) - 1; i >= 0; --i) {
+            row_idx[partition_rows_histogram[channel_ids[i] + 1] - 1] = i;
+            partition_rows_histogram[channel_ids[i] + 1]--;
+        }
+    }
+#define HANDLE_CHANNEL_STATUS(state, channel, status)    \
+    do {                                                 \
+        if (status.is<ErrorCode::END_OF_FILE>()) {       \
+            _handle_eof_channel(state, channel, status); \
+        } else {                                         \
+            RETURN_IF_ERROR(status);                     \
+        }                                                \
+    } while (0)
+    Status status = Status::OK();
+    for (size_t i = 0; i < partition_count; ++i) {
+        uint32_t start = partition_rows_histogram[i + 1];
+        uint32_t size = partition_rows_histogram[i + 2] - start;
+        if (!channels[i]->is_receiver_eof() && size > 0) {
+            status = channels[i]->add_rows(block, row_idx.data(), start, size, false);
+            HANDLE_CHANNEL_STATUS(state, channels[i], status);
+        }
+    }
+    if (eos) {
+        for (int i = 0; i < partition_count; ++i) {
+            if (!channels[i]->is_receiver_eof()) {
+                status = channels[i]->add_rows(block, row_idx.data(), 0, 0, true);
+                HANDLE_CHANNEL_STATUS(state, channels[i], status);
+            }
+        }
+    }
+    return Status::OK();
+}
+
+} // namespace doris::pipeline
diff --git a/be/src/pipeline/shuffle/writer.h b/be/src/pipeline/shuffle/writer.h
new file mode 100644
index 00000000000000..0eb772120293e5
--- /dev/null
+++ b/be/src/pipeline/shuffle/writer.h
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "vec/sink/vdata_stream_sender.h"
+
+namespace doris {
+class RuntimeState;
+class Status;
+namespace vectorized {
+class Block;
+class Channel;
+} // namespace vectorized
+namespace pipeline {
+
+#include "common/compile_check_begin.h"
+class ExchangeSinkLocalState;
+
+class Writer {
+public:
+    Writer() = default;
+
+    Status write(ExchangeSinkLocalState* local_state, RuntimeState* state, vectorized::Block* block,
+                 bool eos) const;
+
+private:
+    template <typename ChannelIdType>
+    Status _channel_add_rows(RuntimeState* state,
+                             std::vector<std::shared_ptr<vectorized::Channel>>& channels,
+                             size_t partition_count, const ChannelIdType* __restrict channel_ids,
+                             size_t rows, vectorized::Block* block, bool eos) const;
+
+    template <typename ChannelPtrType>
+    void _handle_eof_channel(RuntimeState* state, ChannelPtrType channel, Status st) const;
+};
+#include "common/compile_check_end.h"
+} // namespace pipeline
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/runtime/partitioner.h b/be/src/vec/runtime/partitioner.h
index d5492c3be87683..53d8b84d09c752 100644
--- a/be/src/vec/runtime/partitioner.h
+++ b/be/src/vec/runtime/partitioner.h
@@ -21,11 +21,8 @@
 #include "vec/exprs/vexpr.h"
 #include "vec/exprs/vexpr_context.h"
 
-namespace doris {
+namespace doris::vectorized {
 #include "common/compile_check_begin.h"
-
-namespace vectorized {
-
 struct ChannelField {
     const void* channel_id;
     const uint32_t len;
@@ -48,12 +45,16 @@ class PartitionerBase {
 
     virtual Status open(RuntimeState* state) = 0;
 
+    virtual Status close(RuntimeState* state) = 0;
+
     virtual Status do_partitioning(RuntimeState* state, Block* block) const = 0;
 
     virtual ChannelField get_channel_ids() const = 0;
 
     virtual Status clone(RuntimeState* state, std::unique_ptr<PartitionerBase>& partitioner) = 0;
 
+    size_t partition_count() const { return _partition_count; }
+
 protected:
     const size_t _partition_count;
 };
@@ -74,6 +75,8 @@ class Crc32HashPartitioner : public PartitionerBase {
 
     Status open(RuntimeState* state) override { return VExpr::open(_partition_expr_ctxs, state); }
 
+    Status close(RuntimeState* state) override { return Status::OK(); }
+
     Status do_partitioning(RuntimeState* state, Block* block) const override;
 
     ChannelField get_channel_ids() const override { return {_hash_vals.data(), sizeof(uint32_t)}; }
@@ -108,8 +111,5 @@ struct SpillPartitionChannelIds {
         return ((l >> 16) | (l << 16)) % r;
     }
 };
-
-} // namespace vectorized
-} // namespace doris
-
 #include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/sink/scale_writer_partitioning_exchanger.hpp b/be/src/vec/sink/scale_writer_partitioning_exchanger.hpp
index f7435249c20f64..92e52af4c67a33 100644
--- a/be/src/vec/sink/scale_writer_partitioning_exchanger.hpp
+++ b/be/src/vec/sink/scale_writer_partitioning_exchanger.hpp
@@ -24,28 +24,51 @@
 
 #include "vec/core/block.h"
 #include "vec/exec/skewed_partition_rebalancer.h"
+#include "vec/runtime/partitioner.h"
 
 namespace doris::vectorized {
 
-template <typename PartitionFunction>
-class ScaleWriterPartitioningExchanger {
+class ScaleWriterPartitioner final : public PartitionerBase {
 public:
-    ScaleWriterPartitioningExchanger(int channel_size, PartitionFunction& partition_function,
-                                     int partition_count, int task_count, int task_bucket_count,
-                                     long min_partition_data_processed_rebalance_threshold,
-                                     long min_data_processed_rebalance_threshold)
-            : _channel_size(channel_size),
-              _partition_function(partition_function),
+    using HashValType = uint32_t;
+    ScaleWriterPartitioner(int channel_size, int partition_count, int task_count,
+                           int task_bucket_count,
+                           long min_partition_data_processed_rebalance_threshold,
+                           long min_data_processed_rebalance_threshold)
+            : PartitionerBase(partition_count),
+              _channel_size(channel_size),
               _partition_rebalancer(partition_count, task_count, task_bucket_count,
                                     min_partition_data_processed_rebalance_threshold,
                                     min_data_processed_rebalance_threshold),
               _partition_row_counts(partition_count, 0),
               _partition_writer_ids(partition_count, -1),
-              _partition_writer_indexes(partition_count, 0) {}
+              _partition_writer_indexes(partition_count, 0),
+              _task_count(task_count),
+              _task_bucket_count(task_bucket_count),
+              _min_partition_data_processed_rebalance_threshold(
+                      min_partition_data_processed_rebalance_threshold),
+              _min_data_processed_rebalance_threshold(min_data_processed_rebalance_threshold) {
+        _crc_partitioner =
+                std::make_unique<vectorized::Crc32HashPartitioner<vectorized::ShuffleChannelIds>>(
+                        _partition_count);
+    }
+
+    ~ScaleWriterPartitioner() override = default;
+
+    Status init(const std::vector<TExpr>& texprs) override {
+        return _crc_partitioner->init(texprs);
+    }
+
+    Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override {
+        return _crc_partitioner->prepare(state, row_desc);
+    }
+
+    Status open(RuntimeState* state) override { return _crc_partitioner->open(state); }
+
+    Status close(RuntimeState* state) override { return _crc_partitioner->close(state); }
 
-    std::vector<std::vector<uint32_t>> accept(Block* block) {
-        std::vector<std::vector<uint32_t>> writerAssignments(_channel_size,
-                                                             std::vector<uint32_t>());
+    Status do_partitioning(RuntimeState* state, Block* block) const override {
+        _hash_vals.resize(block->rows());
         for (int partition_id = 0; partition_id < _partition_row_counts.size(); partition_id++) {
             _partition_row_counts[partition_id] = 0;
             _partition_writer_ids[partition_id] = -1;
@@ -53,40 +76,59 @@ class ScaleWriterPartitioningExchanger {
 
         _partition_rebalancer.rebalance();
 
-        for (int position = 0; position < block->rows(); position++) {
-            int partition_id = _partition_function.get_partition(block, position);
+        RETURN_IF_ERROR(_crc_partitioner->do_partitioning(state, block));
+        const auto* crc_values = _crc_partitioner->get_channel_ids().get<uint32_t>();
+        for (size_t position = 0; position < block->rows(); position++) {
+            int partition_id = crc_values[position];
             _partition_row_counts[partition_id] += 1;
 
             // Get writer id for this partition by looking at the scaling state
             int writer_id = _partition_writer_ids[partition_id];
             if (writer_id == -1) {
-                writer_id = get_next_writer_id(partition_id);
+                writer_id = _get_next_writer_id(partition_id);
                 _partition_writer_ids[partition_id] = writer_id;
             }
-            writerAssignments[writer_id].push_back(position);
+            _hash_vals[position] = writer_id;
         }
 
-        for (int partition_id = 0; partition_id < _partition_row_counts.size(); partition_id++) {
+        for (size_t partition_id = 0; partition_id < _partition_row_counts.size(); partition_id++) {
             _partition_rebalancer.add_partition_row_count(partition_id,
                                                           _partition_row_counts[partition_id]);
         }
         _partition_rebalancer.add_data_processed(block->bytes());
 
-        return writerAssignments;
+        return Status::OK();
+    }
+
+    ChannelField get_channel_ids() const override {
+        return {_hash_vals.data(), sizeof(HashValType)};
     }
 
-    int get_next_writer_id(int partition_id) {
+    Status clone(RuntimeState* state, std::unique_ptr<PartitionerBase>& partitioner) override {
+        partitioner.reset(new ScaleWriterPartitioner(
+                _channel_size, _partition_count, _task_count, _task_bucket_count,
+                _min_partition_data_processed_rebalance_threshold,
+                _min_data_processed_rebalance_threshold));
+        return Status::OK();
+    }
+
+private:
+    int _get_next_writer_id(int partition_id) const {
         return _partition_rebalancer.get_task_id(partition_id,
                                                  _partition_writer_indexes[partition_id]++);
     }
 
-private:
     int _channel_size;
-    PartitionFunction& _partition_function;
-    SkewedPartitionRebalancer _partition_rebalancer;
-    std::vector<int> _partition_row_counts;
-    std::vector<int> _partition_writer_ids;
-    std::vector<int> _partition_writer_indexes;
+    std::unique_ptr<PartitionerBase> _crc_partitioner;
+    mutable SkewedPartitionRebalancer _partition_rebalancer;
+    mutable std::vector<int> _partition_row_counts;
+    mutable std::vector<int> _partition_writer_ids;
+    mutable std::vector<int> _partition_writer_indexes;
+    mutable std::vector<HashValType> _hash_vals;
+    const int _task_count;
+    const int _task_bucket_count;
+    const long _min_partition_data_processed_rebalance_threshold;
+    const long _min_data_processed_rebalance_threshold;
 };
 
 } // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/sink/tablet_sink_hash_partitioner.cpp b/be/src/vec/sink/tablet_sink_hash_partitioner.cpp
new file mode 100644
index 00000000000000..7e854d19addd2b
--- /dev/null
+++ b/be/src/vec/sink/tablet_sink_hash_partitioner.cpp
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/sink/tablet_sink_hash_partitioner.h"
+
+#include "pipeline/exec/operator.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+TabletSinkHashPartitioner::TabletSinkHashPartitioner(
+        size_t partition_count, int64_t txn_id, const TOlapTableSchemaParam& tablet_sink_schema,
+        const TOlapTablePartitionParam& tablet_sink_partition,
+        const TOlapTableLocationParam& tablet_sink_location, const TTupleId& tablet_sink_tuple_id,
+        pipeline::ExchangeSinkLocalState* local_state)
+        : PartitionerBase(partition_count),
+          _txn_id(txn_id),
+          _tablet_sink_schema(tablet_sink_schema),
+          _tablet_sink_partition(tablet_sink_partition),
+          _tablet_sink_location(tablet_sink_location),
+          _tablet_sink_tuple_id(tablet_sink_tuple_id),
+          _local_state(local_state) {}
+
+Status TabletSinkHashPartitioner::init(const std::vector<TExpr>& texprs) {
+    return Status::OK();
+}
+
+Status TabletSinkHashPartitioner::prepare(RuntimeState* state, const RowDescriptor& row_desc) {
+    return Status::OK();
+}
+
+Status TabletSinkHashPartitioner::open(RuntimeState* state) {
+    _schema = std::make_shared<OlapTableSchemaParam>();
+    RETURN_IF_ERROR(_schema->init(_tablet_sink_schema));
+    _vpartition = std::make_unique<VOlapTablePartitionParam>(_schema, _tablet_sink_partition);
+    RETURN_IF_ERROR(_vpartition->init());
+    auto find_tablet_mode = vectorized::OlapTabletFinder::FindTabletMode::FIND_TABLET_EVERY_ROW;
+    _tablet_finder =
+            std::make_unique<vectorized::OlapTabletFinder>(_vpartition.get(), find_tablet_mode);
+    _tablet_sink_tuple_desc = state->desc_tbl().get_tuple_descriptor(_tablet_sink_tuple_id);
+    _tablet_sink_row_desc =
+            state->obj_pool()->add(new RowDescriptor(_tablet_sink_tuple_desc, false));
+    auto& ctxs =
+            _local_state->parent()->cast<pipeline::ExchangeSinkOperatorX>().tablet_sink_expr_ctxs();
+    _tablet_sink_expr_ctxs.resize(ctxs.size());
+    for (size_t i = 0; i < _tablet_sink_expr_ctxs.size(); i++) {
+        RETURN_IF_ERROR(ctxs[i]->clone(state, _tablet_sink_expr_ctxs[i]));
+    }
+    // if _part_type == TPartitionType::TABLET_SINK_SHUFFLE_PARTITIONED, we handle the processing of auto_increment column
+    // on exchange node rather than on TabletWriter
+    _block_convertor =
+            std::make_unique<vectorized::OlapTableBlockConvertor>(_tablet_sink_tuple_desc);
+    _block_convertor->init_autoinc_info(_schema->db_id(), _schema->table_id(), state->batch_size());
+    _location = state->obj_pool()->add(new OlapTableLocationParam(_tablet_sink_location));
+    _row_distribution.init(
+            {.state = state,
+             .block_convertor = _block_convertor.get(),
+             .tablet_finder = _tablet_finder.get(),
+             .vpartition = _vpartition.get(),
+             .add_partition_request_timer = _local_state->add_partition_request_timer(),
+             .txn_id = _txn_id,
+             .pool = state->obj_pool(),
+             .location = _location,
+             .vec_output_expr_ctxs = &_tablet_sink_expr_ctxs,
+             .schema = _schema,
+             .caller = (void*)this,
+             .create_partition_callback = &TabletSinkHashPartitioner::empty_callback_function});
+    RETURN_IF_ERROR(_row_distribution.open(_tablet_sink_row_desc));
+    return Status::OK();
+}
+
+Status TabletSinkHashPartitioner::do_partitioning(RuntimeState* state, Block* block) const {
+    _hash_vals.resize(block->rows());
+    if (block->empty()) {
+        return Status::OK();
+    }
+    std::fill(_hash_vals.begin(), _hash_vals.end(), -1);
+    bool has_filtered_rows = false;
+    int64_t filtered_rows = 0;
+    int64_t number_input_rows = _local_state->rows_input_counter()->value();
+    std::shared_ptr<vectorized::Block> convert_block = std::make_shared<vectorized::Block>();
+    RETURN_IF_ERROR(_row_distribution.generate_rows_distribution(
+            *block, convert_block, filtered_rows, has_filtered_rows, _row_part_tablet_ids,
+            number_input_rows));
+    if (_row_distribution.batching_rows() > 0) {
+        SCOPED_TIMER(_local_state->send_new_partition_timer());
+        RETURN_IF_ERROR(_send_new_partition_batch(state, block));
+    } else {
+        const auto& row_ids = _row_part_tablet_ids[0].row_ids;
+        const auto& tablet_ids = _row_part_tablet_ids[0].tablet_ids;
+        for (int idx = 0; idx < row_ids.size(); ++idx) {
+            const auto& row = row_ids[idx];
+            const auto& tablet_id_hash =
+                    HashUtil::zlib_crc_hash(&tablet_ids[idx], sizeof(HashValType), 0);
+            _hash_vals[row] = tablet_id_hash % _partition_count;
+        }
+    }
+
+    return Status::OK();
+}
+
+ChannelField TabletSinkHashPartitioner::get_channel_ids() const {
+    return {_hash_vals.data(), sizeof(HashValType)};
+}
+
+Status TabletSinkHashPartitioner::clone(RuntimeState* state,
+                                        std::unique_ptr<PartitionerBase>& partitioner) {
+    partitioner.reset(new TabletSinkHashPartitioner(_partition_count, _txn_id, _tablet_sink_schema,
+                                                    _tablet_sink_partition, _tablet_sink_location,
+                                                    _tablet_sink_tuple_id, _local_state));
+    return Status::OK();
+}
+
+Status TabletSinkHashPartitioner::close(RuntimeState* state) {
+    if (_block_convertor != nullptr && _tablet_finder != nullptr) {
+        state->update_num_rows_load_filtered(_block_convertor->num_filtered_rows() +
+                                             _tablet_finder->num_filtered_rows());
+        state->update_num_rows_load_unselected(
+                _tablet_finder->num_immutable_partition_filtered_rows());
+        // sink won't see those filtered rows, we should compensate here
+        state->set_num_rows_load_total(state->num_rows_load_filtered() +
+                                       state->num_rows_load_unselected());
+    }
+    return Status::OK();
+}
+
+Status TabletSinkHashPartitioner::_send_new_partition_batch(RuntimeState* state,
+                                                            vectorized::Block* input_block) const {
+    RETURN_IF_ERROR(_row_distribution.automatic_create_partition());
+    auto& p = _local_state->parent()->cast<pipeline::ExchangeSinkOperatorX>();
+    // Recovery back
+    _row_distribution.clear_batching_stats();
+    _row_distribution._batching_block->clear_column_data();
+    _row_distribution._deal_batched = false;
+    RETURN_IF_ERROR(p.sink(state, input_block, false));
+    return Status::OK();
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/sink/tablet_sink_hash_partitioner.h b/be/src/vec/sink/tablet_sink_hash_partitioner.h
new file mode 100644
index 00000000000000..dbbc7c889267fb
--- /dev/null
+++ b/be/src/vec/sink/tablet_sink_hash_partitioner.h
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exec/tablet_info.h"
+#include "pipeline/exec/exchange_sink_operator.h"
+#include "runtime/runtime_state.h"
+#include "vec/core/block.h"
+#include "vec/runtime/partitioner.h"
+#include "vec/sink/vrow_distribution.h"
+#include "vec/sink/vtablet_block_convertor.h"
+#include "vec/sink/vtablet_finder.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+class TabletSinkHashPartitioner final : public PartitionerBase {
+public:
+    using HashValType = int64_t;
+    TabletSinkHashPartitioner(size_t partition_count, int64_t txn_id,
+                              const TOlapTableSchemaParam& tablet_sink_schema,
+                              const TOlapTablePartitionParam& tablet_sink_partition,
+                              const TOlapTableLocationParam& tablet_sink_location,
+                              const TTupleId& tablet_sink_tuple_id,
+                              pipeline::ExchangeSinkLocalState* local_state);
+
+    ~TabletSinkHashPartitioner() override = default;
+
+    Status init(const std::vector<TExpr>& texprs) override;
+
+    Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override;
+
+    Status open(RuntimeState* state) override;
+
+    Status do_partitioning(RuntimeState* state, Block* block) const override;
+
+    ChannelField get_channel_ids() const override;
+    Status clone(RuntimeState* state, std::unique_ptr<PartitionerBase>& partitioner) override;
+
+    Status close(RuntimeState* state) override;
+
+private:
+    static Status empty_callback_function(void* sender, TCreatePartitionResult* result) {
+        return Status::OK();
+    }
+
+    Status _send_new_partition_batch(RuntimeState* state, vectorized::Block* input_block) const;
+
+    const int64_t _txn_id = -1;
+    const TOlapTableSchemaParam _tablet_sink_schema;
+    const TOlapTablePartitionParam _tablet_sink_partition;
+    const TOlapTableLocationParam _tablet_sink_location;
+    const TTupleId _tablet_sink_tuple_id;
+    mutable pipeline::ExchangeSinkLocalState* _local_state;
+    mutable OlapTableLocationParam* _location = nullptr;
+    mutable vectorized::VRowDistribution _row_distribution;
+    mutable vectorized::VExprContextSPtrs _tablet_sink_expr_ctxs;
+    mutable std::unique_ptr<VOlapTablePartitionParam> _vpartition = nullptr;
+    mutable std::unique_ptr<vectorized::OlapTabletFinder> _tablet_finder = nullptr;
+    mutable std::shared_ptr<OlapTableSchemaParam> _schema = nullptr;
+    mutable std::unique_ptr<vectorized::OlapTableBlockConvertor> _block_convertor = nullptr;
+    mutable TupleDescriptor* _tablet_sink_tuple_desc = nullptr;
+    mutable RowDescriptor* _tablet_sink_row_desc = nullptr;
+    mutable std::vector<vectorized::RowPartTabletIds> _row_part_tablet_ids;
+    mutable std::vector<HashValType> _hash_vals;
+};
+#include "common/compile_check_end.h"
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/sink/vdata_stream_sender.cpp b/be/src/vec/sink/vdata_stream_sender.cpp
index 66aacc59f6c02a..abed61334583b8 100644
--- a/be/src/vec/sink/vdata_stream_sender.cpp
+++ b/be/src/vec/sink/vdata_stream_sender.cpp
@@ -239,18 +239,18 @@ BlockSerializer::BlockSerializer(pipeline::ExchangeSinkLocalState* parent, bool
         : _parent(parent), _is_local(is_local), _batch_size(parent->state()->batch_size()) {}
 
 Status BlockSerializer::next_serialized_block(Block* block, PBlock* dest, size_t num_receivers,
-                                              bool* serialized, bool eos,
-                                              const std::vector<uint32_t>* rows) {
+                                              bool* serialized, bool eos, const uint32_t* data,
+                                              const uint32_t offset, const uint32_t size) {
     if (_mutable_block == nullptr) {
         _mutable_block = MutableBlock::create_unique(block->clone_empty());
     }
 
     {
         SCOPED_TIMER(_parent->merge_block_timer());
-        if (rows) {
-            if (!rows->empty()) {
-                const auto* begin = rows->data();
-                RETURN_IF_ERROR(_mutable_block->add_rows(block, begin, begin + rows->size()));
+        if (data) {
+            if (size > 0) {
+                RETURN_IF_ERROR(
+                        _mutable_block->add_rows(block, data + offset, data + offset + size));
             }
         } else if (!block->empty()) {
             RETURN_IF_ERROR(_mutable_block->merge(*block));
diff --git a/be/src/vec/sink/vdata_stream_sender.h b/be/src/vec/sink/vdata_stream_sender.h
index 16ea49e443c630..0ff1f252d5441f 100644
--- a/be/src/vec/sink/vdata_stream_sender.h
+++ b/be/src/vec/sink/vdata_stream_sender.h
@@ -80,7 +80,8 @@ class BlockSerializer {
     BlockSerializer() : _batch_size(0) {};
 #endif
     Status next_serialized_block(Block* src, PBlock* dest, size_t num_receivers, bool* serialized,
-                                 bool eos, const std::vector<uint32_t>* rows = nullptr);
+                                 bool eos, const uint32_t* data = nullptr,
+                                 const uint32_t offset = 0, const uint32_t size = 0);
     Status serialize_block(PBlock* dest, size_t num_receivers = 1);
     Status serialize_block(const Block* src, PBlock* dest, size_t num_receivers = 1);
 
@@ -150,7 +151,8 @@ class Channel {
     Status send_remote_block(std::unique_ptr<PBlock>&& block, bool eos = false);
     Status send_broadcast_block(std::shared_ptr<BroadcastPBlockHolder>& block, bool eos = false);
 
-    Status add_rows(Block* block, const std::vector<uint32_t>& rows, bool eos) {
+    Status add_rows(Block* block, const uint32_t* data, const uint32_t offset, const uint32_t size,
+                    bool eos) {
         if (_fragment_instance_id.lo == -1) {
             return Status::OK();
         }
@@ -160,7 +162,7 @@ class Channel {
             _pblock = std::make_unique<PBlock>();
         }
         RETURN_IF_ERROR(_serializer.next_serialized_block(block, _pblock.get(), 1, &serialized, eos,
-                                                          &rows));
+                                                          data, offset, size));
         if (serialized) {
             RETURN_IF_ERROR(_send_current_block(eos));
         }

From a9ce09371e4b017a984546a092f09e531180cb3d Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Wed, 11 Dec 2024 12:06:55 +0530
Subject: [PATCH 268/399] [Enhancement] (nereids)implement
 showWaringErrorsCommand in nereids (#44816)

Issue Number: close #42752
---
 .../org/apache/doris/nereids/DorisParser.g4   |  4 +-
 .../nereids/parser/LogicalPlanBuilder.java    | 18 ++++++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/ShowWarningErrorsCommand.java    | 64 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../show/test_show_warning_errors_command.out |  5 ++
 .../test_show_warning_errors_command.groovy   | 27 ++++++++
 7 files changed, 122 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowWarningErrorsCommand.java
 create mode 100644 regression-test/data/nereids_p0/show/test_show_warning_errors_command.out
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_warning_errors_command.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index c833e7d4b629b6..a385e1e5ddb3bb 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -255,7 +255,8 @@ supportedShowStatement
     | SHOW SQL_BLOCK_RULE (FOR ruleName=identifier)?                                #showSqlBlockRule
     | SHOW CREATE VIEW name=multipartIdentifier                                     #showCreateView
     | SHOW CREATE MATERIALIZED VIEW mvName=identifier
-        ON tableName=multipartIdentifier                                            #showCreateMaterializedView   
+        ON tableName=multipartIdentifier                                            #showCreateMaterializedView  
+    | SHOW (WARNINGS | ERRORS) limitClause?                                         #showWarningErrors
     | SHOW BACKENDS                                                                 #showBackends
     | SHOW REPLICA DISTRIBUTION FROM baseTableRef                                   #showReplicaDistribution
     | SHOW FULL? TRIGGERS ((FROM | IN) database=multipartIdentifier)? wildWhere?    #showTriggers    
@@ -327,7 +328,6 @@ unsupportedShowStatement
         ((FROM | IN) database=multipartIdentifier)? wildWhere?                      #showColumns
     | SHOW ((CHAR SET) | CHARSET) wildWhere?                                        #showCharset
     | SHOW COUNT LEFT_PAREN ASTERISK RIGHT_PAREN (WARNINGS | ERRORS)                #showWaringErrorCount
-    | SHOW (WARNINGS | ERRORS) limitClause?                                         #showWaringErrors
     | SHOW LOAD WARNINGS ((((FROM | IN) database=multipartIdentifier)?
         wildWhere? limitClause?) | (ON url=STRING_LITERAL))                         #showLoadWarings
     | SHOW STREAM? LOAD ((FROM | IN) database=multipartIdentifier)? wildWhere?
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index a825f80f4e08e0..12ae62bac10930 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -280,6 +280,7 @@
 import org.apache.doris.nereids.DorisParser.ShowTriggersContext;
 import org.apache.doris.nereids.DorisParser.ShowVariablesContext;
 import org.apache.doris.nereids.DorisParser.ShowViewContext;
+import org.apache.doris.nereids.DorisParser.ShowWarningErrorsContext;
 import org.apache.doris.nereids.DorisParser.ShowWhitelistContext;
 import org.apache.doris.nereids.DorisParser.SimpleColumnDefContext;
 import org.apache.doris.nereids.DorisParser.SimpleColumnDefsContext;
@@ -586,6 +587,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowTriggersCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowVariablesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowWarningErrorsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowWhiteListCommand;
 import org.apache.doris.nereids.trees.plans.commands.SyncCommand;
 import org.apache.doris.nereids.trees.plans.commands.UnsetDefaultStorageVaultCommand;
@@ -4838,6 +4840,22 @@ public LogicalPlan visitRecoverDatabase(RecoverDatabaseContext ctx) {
         return new RecoverDatabaseCommand(dbName, dbId, newDbName);
     }
 
+    @Override
+    public LogicalPlan visitShowWarningErrors(ShowWarningErrorsContext ctx) {
+        boolean isWarning = ctx.WARNINGS() != null;
+
+        // Extract the limit value if present
+        long limit = 0;
+        Optional<LimitClauseContext> limitCtx = Optional.ofNullable(ctx.limitClause());
+        if (ctx.limitClause() != null) {
+            limit = Long.parseLong(limitCtx.get().limit.getText());
+            if (limit < 0) {
+                throw new ParseException("Limit requires non-negative number", limitCtx.get());
+            }
+        }
+        return new ShowWarningErrorsCommand(isWarning, limit);
+    }
+
     @Override
     public RecoverTableCommand visitRecoverTable(RecoverTableContext ctx) {
         List<String> dbTblNameParts = visitMultipartIdentifier(ctx.name);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index b0e475a8161535..9bab39c946b45b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -241,6 +241,7 @@ public enum PlanType {
     SHOW_VARIABLES_COMMAND,
     SHOW_AUTHORS_COMMAND,
     SHOW_VIEW_COMMAND,
+    SHOW_WARNING_ERRORS_COMMAND,
     SHOW_WHITE_LIST_COMMAND,
     SHOW_TABLETS_BELONG_COMMAND,
     SYNC_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowWarningErrorsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowWarningErrorsCommand.java
new file mode 100644
index 00000000000000..09814a34b71f44
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowWarningErrorsCommand.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * Represents the command for SHOW WARNINGS or SHOW ERRORS.
+ */
+public class ShowWarningErrorsCommand extends ShowCommand {
+    private static final ShowResultSetMetaData META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("Level", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Code", ScalarType.createVarchar(10)))
+                    .addColumn(new Column("Message", ScalarType.createVarchar(100)))
+                    .build();
+
+    private final boolean isWarning;
+    private final long limit;
+
+    public ShowWarningErrorsCommand(boolean isWarning, long limit) {
+        super(PlanType.SHOW_WARNING_ERRORS_COMMAND);
+        this.isWarning = isWarning;
+        this.limit = limit;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        List<List<String>> rowSet = Lists.newArrayList();
+        // Only success
+        return new ShowResultSet(META_DATA, rowSet);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowWarningErrorsCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 3634732077b7bd..085b510b095e00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -125,6 +125,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowTriggersCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowVariablesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowViewCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowWarningErrorsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowWhiteListCommand;
 import org.apache.doris.nereids.trees.plans.commands.SyncCommand;
 import org.apache.doris.nereids.trees.plans.commands.UnsetDefaultStorageVaultCommand;
@@ -302,6 +303,10 @@ default R visitShowProcedureStatusCommand(ShowProcedureStatusCommand showProcedu
         return visitCommand(showProcedureStatusCommand, context);
     }
 
+    default R visitShowWarningErrorsCommand(ShowWarningErrorsCommand showWarningErrorsCommand, C context) {
+        return visitCommand(showWarningErrorsCommand, context);
+    }
+
     default R visitShowCreateProcedureCommand(ShowCreateProcedureCommand showCreateProcedureCommand, C context) {
         return visitCommand(showCreateProcedureCommand, context);
     }
diff --git a/regression-test/data/nereids_p0/show/test_show_warning_errors_command.out b/regression-test/data/nereids_p0/show/test_show_warning_errors_command.out
new file mode 100644
index 00000000000000..b62a27211f6c34
--- /dev/null
+++ b/regression-test/data/nereids_p0/show/test_show_warning_errors_command.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !cmd --
+
+-- !cmd --
+
diff --git a/regression-test/suites/nereids_p0/show/test_show_warning_errors_command.groovy b/regression-test/suites/nereids_p0/show/test_show_warning_errors_command.groovy
new file mode 100644
index 00000000000000..b731caee95c2b0
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_warning_errors_command.groovy
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_warning_errors_command", "query,warnings_errors") {
+        // Execute the SHOW WARNINGS command and verify the output
+        checkNereidsExecute("SHOW ERRORS;")
+        qt_cmd("SHOW ERRORS;")
+
+
+        // Execute the SHOW WARNINGS command and verify the output
+        checkNereidsExecute("SHOW WARNINGS LIMIT 5;")
+        qt_cmd("SHOW WARNINGS LIMIT 5;")
+}

From 1341e93c2d26ec7518cbbe234ad2ac721b08148f Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Wed, 11 Dec 2024 15:09:09 +0800
Subject: [PATCH 269/399] [improvement](statistics)Support auto analyze columns
 that haven't been analyzed for a long time. (#42399)

Support auto analyze columns that haven't been analyzed for a long time.
Add a very low priority job queue for auto analyze to process this kind
of columns.

The purpose of this change is to make sure all tables could be auto
analyzed within a certain time. In the earlier Doris versions, users
often encounter this kind of issues:
User load some new data to a large table everyday, but the change rate
(percentage of new data) is very low, because there is a large size of
old data. In this case, auto analyze for this table will not be
triggered for a very long time, because the default trigger threshold of
auto analyze is 40% (more than 40% of the data in a table is changed
since last analyze). This will probably cause a bad plan because
min/max/ndv statistics are outdated.
---
 .../java/org/apache/doris/common/Config.java  |   2 +-
 .../analysis/ShowAutoAnalyzeJobsStmt.java     |   4 +-
 .../doris/analysis/ShowColumnStatsStmt.java   |   2 +
 .../doris/analysis/ShowTableStatsStmt.java    |   9 +-
 .../org/apache/doris/catalog/OlapTable.java   |  11 --
 .../apache/doris/statistics/AnalysisInfo.java |   9 +-
 .../doris/statistics/AnalysisInfoBuilder.java |   9 +-
 .../doris/statistics/AnalysisManager.java     |   7 +-
 .../apache/doris/statistics/ColStatsMeta.java |   9 +-
 .../apache/doris/statistics/JobPriority.java  |   1 +
 .../statistics/StatisticsAutoCollector.java   |  40 ++--
 .../statistics/StatisticsJobAppender.java     |  66 +++++--
 .../statistics/StatisticsRepository.java      |   3 +-
 .../doris/statistics/TableStatsMeta.java      |  10 +-
 .../doris/statistics/util/StatisticsUtil.java |  52 +++++-
 .../statistics/StatisticsJobAppenderTest.java | 175 ++++++++++++++----
 .../statistics/util/StatisticsUtilTest.java   | 168 ++++++++++++++---
 .../suites/statistics/analyze_stats.groovy    |  33 +++-
 18 files changed, 472 insertions(+), 138 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 50d43bffa1b821..c5c7023b20f54b 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2948,7 +2948,7 @@ public class Config extends ConfigBase {
             "Columns that have not been collected within the specified interval will trigger automatic analyze. "
                 + "0 means not trigger."
     })
-    public static long auto_analyze_interval_seconds = 0;
+    public static long auto_analyze_interval_seconds = 86400; // 24 hours.
 
     //==========================================================================
     //                    begin of cloud config
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java
index 9b07796df784d4..7cff1e2b949455 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java
@@ -40,7 +40,7 @@
  *        [TABLE]
  *        [
  *            WHERE
- *            [PRIORITY = ["HIGH"|"MID"|"LOW"]]
+ *            [PRIORITY = ["HIGH"|"MID"|"LOW"|"VERY_LOW"]]
  *        ]
  */
 public class ShowAutoAnalyzeJobsStmt extends ShowStmt implements NotFallbackInParser {
@@ -175,7 +175,7 @@ private void analyzeSubPredicate(Expr subExpr) throws AnalysisException {
 
         if (!valid) {
             throw new AnalysisException("Where clause should looks like: "
-                    + "PRIORITY = \"HIGH|MID|LOW\"");
+                    + "PRIORITY = \"HIGH|MID|LOW|VERY_LOW\"");
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
index 354b57bc55c7d9..d180ef0d807fff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
@@ -70,6 +70,7 @@ public class ShowColumnStatsStmt extends ShowStmt implements NotFallbackInParser
                     .add("updated_time")
                     .add("update_rows")
                     .add("last_analyze_row_count")
+                    .add("last_analyze_version")
                     .build();
 
     private static final ImmutableList<String> PARTITION_COLUMN_TITLE_NAMES =
@@ -185,6 +186,7 @@ public ShowResultSet constructResultSet(List<Pair<Pair<String, String>, ColumnSt
             row.add(String.valueOf(p.second.updatedTime));
             row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.updatedRows));
             row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.rowCount));
+            row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.tableVersion));
             result.add(row);
         });
         return new ShowResultSet(getMetaData(), result);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
index ccffee3086dbea..ea9b96d0afecd4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
@@ -62,6 +62,7 @@ public class ShowTableStatsStmt extends ShowStmt implements NotFallbackInParser
                     .add("new_partition")
                     .add("user_inject")
                     .add("enable_auto_analyze")
+                    .add("last_analyze_time")
                     .build();
 
     private static final ImmutableList<String> PARTITION_TITLE_NAMES =
@@ -230,6 +231,7 @@ public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, Tabl
             row.add("");
             row.add("");
             row.add(String.valueOf(table.autoAnalyzeEnabled()));
+            row.add("");
             result.add(row);
             return new ShowResultSet(getMetaData(), result);
         }
@@ -242,13 +244,16 @@ public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, Tabl
         LocalDateTime dateTime =
                 LocalDateTime.ofInstant(Instant.ofEpochMilli(tableStatistic.updatedTime),
                 java.time.ZoneId.systemDefault());
-        String formattedDateTime = dateTime.format(formatter);
-        row.add(formattedDateTime);
+        LocalDateTime lastAnalyzeTime =
+                LocalDateTime.ofInstant(Instant.ofEpochMilli(tableStatistic.lastAnalyzeTime),
+                    java.time.ZoneId.systemDefault());
+        row.add(dateTime.format(formatter));
         row.add(tableStatistic.analyzeColumns().toString());
         row.add(tableStatistic.jobType.toString());
         row.add(String.valueOf(tableStatistic.partitionChanged.get()));
         row.add(String.valueOf(tableStatistic.userInjected));
         row.add(table == null ? "N/A" : String.valueOf(table.autoAnalyzeEnabled()));
+        row.add(lastAnalyzeTime.format(formatter));
         result.add(row);
         return new ShowResultSet(getMetaData(), result);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index c1d62a4be0d4d5..f3d27908bdd3d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -969,17 +969,6 @@ public Set<Column> getSchemaAllIndexes(boolean full) {
         return columns;
     }
 
-    public List<Column> getMvColumns(boolean full) {
-        List<Column> columns = Lists.newArrayList();
-        for (Long indexId : indexIdToMeta.keySet()) {
-            if (indexId == baseIndexId) {
-                continue;
-            }
-            columns.addAll(getSchemaByIndexId(indexId, full));
-        }
-        return columns;
-    }
-
     public List<Column> getBaseSchemaKeyColumns() {
         return getKeyColumnsByIndexId(baseIndexId);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index 6ec413821ea37b..58b2c3e3d1f8dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -183,6 +183,9 @@ public enum ScheduleType {
     @SerializedName("updateRows")
     public final long updateRows;
 
+    @SerializedName("tv")
+    public final long tableVersion;
+
     public final Map<Long, Long> partitionUpdateRows = new ConcurrentHashMap<>();
 
     @SerializedName("tblUpdateTime")
@@ -206,8 +209,8 @@ public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId,
             long lastExecTimeInMs, long timeCostInMs, AnalysisState state, ScheduleType scheduleType,
             boolean partitionOnly, boolean samplingPartition,
             boolean isAllPartition, long partitionCount, CronExpression cronExpression, boolean forceFull,
-            boolean usingSqlForExternalTable, long tblUpdateTime, long rowCount, boolean userInject,
-            long updateRows, JobPriority priority, Map<Long, Long> partitionUpdateRows, boolean enablePartition) {
+            boolean usingSqlForExternalTable, long tblUpdateTime, long rowCount, boolean userInject, long updateRows,
+            long tableVersion, JobPriority priority, Map<Long, Long> partitionUpdateRows, boolean enablePartition) {
         this.jobId = jobId;
         this.taskId = taskId;
         this.taskIds = taskIds;
@@ -244,6 +247,7 @@ public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId,
         this.rowCount = rowCount;
         this.userInject = userInject;
         this.updateRows = updateRows;
+        this.tableVersion = tableVersion;
         this.priority = priority;
         if (partitionUpdateRows != null) {
             this.partitionUpdateRows.putAll(partitionUpdateRows);
@@ -292,6 +296,7 @@ public String toString() {
         sj.add("rowCount: " + rowCount);
         sj.add("userInject: " + userInject);
         sj.add("updateRows: " + updateRows);
+        sj.add("tableVersion: " + tableVersion);
         sj.add("priority: " + priority.name());
         sj.add("enablePartition: " + enablePartition);
         return sj.toString();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index 73817363ef164f..bbd0d6164954be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -63,6 +63,7 @@ public class AnalysisInfoBuilder {
     private long rowCount;
     private boolean userInject = false;
     private long updateRows;
+    private long tableVersion;
     private JobPriority priority;
     private Map<Long, Long> partitionUpdateRows;
     private boolean enablePartition;
@@ -104,6 +105,7 @@ public AnalysisInfoBuilder(AnalysisInfo info) {
         rowCount = info.rowCount;
         userInject = info.userInject;
         updateRows = info.updateRows;
+        tableVersion = info.tableVersion;
         priority = info.priority;
         partitionUpdateRows = info.partitionUpdateRows;
         enablePartition = info.enablePartition;
@@ -274,6 +276,11 @@ public AnalysisInfoBuilder setUpdateRows(long updateRows) {
         return this;
     }
 
+    public AnalysisInfoBuilder setTableVersion(long tableVersion) {
+        this.tableVersion = tableVersion;
+        return this;
+    }
+
     public AnalysisInfoBuilder setPriority(JobPriority priority) {
         this.priority = priority;
         return this;
@@ -295,7 +302,7 @@ public AnalysisInfo build() {
                 sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType,
                 partitionOnly, samplingPartition, isAllPartition, partitionCount,
                 cronExpression, forceFull, usingSqlForExternalTable, tblUpdateTime, rowCount, userInject, updateRows,
-                priority, partitionUpdateRows, enablePartition);
+                tableVersion, priority, partitionUpdateRows, enablePartition);
     }
 
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index c2b20707f133e8..ece9daf25204c3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -129,6 +129,7 @@ public class AnalysisManager implements Writable {
     public final Map<TableName, Set<Pair<String, String>>> highPriorityJobs = new LinkedHashMap<>();
     public final Map<TableName, Set<Pair<String, String>>> midPriorityJobs = new LinkedHashMap<>();
     public final Map<TableName, Set<Pair<String, String>>> lowPriorityJobs = new LinkedHashMap<>();
+    public final Map<TableName, Set<Pair<String, String>>> veryLowPriorityJobs = new LinkedHashMap<>();
 
     // Tracking running manually submitted async tasks, keep in mem only
     protected final ConcurrentMap<Long, Map<Long, BaseAnalysisTask>> analysisJobIdToTaskMap = new ConcurrentHashMap<>();
@@ -381,7 +382,7 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) {
         }
         infoBuilder.setColName(stringJoiner.toString());
         infoBuilder.setTaskIds(Lists.newArrayList());
-        infoBuilder.setTblUpdateTime(System.currentTimeMillis());
+        infoBuilder.setTblUpdateTime(table.getUpdateTime());
         // Empty table row count is 0. Call fetchRowCount() when getRowCount() returns <= 0,
         // because getRowCount may return <= 0 if cached is not loaded. This is mainly for external table.
         long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0 :
@@ -389,6 +390,7 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) {
         infoBuilder.setRowCount(rowCount);
         TableStatsMeta tableStatsStatus = findTableStatsStatus(table.getId());
         infoBuilder.setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get());
+        infoBuilder.setTableVersion(table instanceof OlapTable ? ((OlapTable) table).getVisibleVersion() : 0);
         infoBuilder.setPriority(JobPriority.MANUAL);
         infoBuilder.setPartitionUpdateRows(tableStatsStatus == null ? null : tableStatsStatus.partitionUpdateRows);
         infoBuilder.setEnablePartition(StatisticsUtil.enablePartitionAnalyze());
@@ -547,12 +549,15 @@ public List<AutoAnalysisPendingJob> showAutoPendingJobs(ShowAutoAnalyzeJobsStmt
             result.addAll(getPendingJobs(highPriorityJobs, JobPriority.HIGH, tblName));
             result.addAll(getPendingJobs(midPriorityJobs, JobPriority.MID, tblName));
             result.addAll(getPendingJobs(lowPriorityJobs, JobPriority.LOW, tblName));
+            result.addAll(getPendingJobs(veryLowPriorityJobs, JobPriority.VERY_LOW, tblName));
         } else if (priority.equals(JobPriority.HIGH.name())) {
             result.addAll(getPendingJobs(highPriorityJobs, JobPriority.HIGH, tblName));
         } else if (priority.equals(JobPriority.MID.name())) {
             result.addAll(getPendingJobs(midPriorityJobs, JobPriority.MID, tblName));
         } else if (priority.equals(JobPriority.LOW.name())) {
             result.addAll(getPendingJobs(lowPriorityJobs, JobPriority.LOW, tblName));
+        } else if (priority.equals(JobPriority.VERY_LOW.name())) {
+            result.addAll(getPendingJobs(veryLowPriorityJobs, JobPriority.VERY_LOW, tblName));
         }
         return result;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
index 6cb2ced9286a65..78f51c2ac0c323 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
@@ -52,11 +52,15 @@ public class ColStatsMeta {
     @SerializedName("rowCount")
     public long rowCount;
 
+    @SerializedName("tv")
+    public long tableVersion;
+
     @SerializedName("pur")
     public ConcurrentMap<Long, Long> partitionUpdateRows = new ConcurrentHashMap<>();
 
-    public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod, AnalysisType analysisType, JobType jobType,
-            long queriedTimes, long rowCount, long updatedRows, Map<Long, Long> partitionUpdateRows) {
+    public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod, AnalysisType analysisType,
+                        JobType jobType, long queriedTimes, long rowCount, long updatedRows,
+                        long tableVersion, Map<Long, Long> partitionUpdateRows) {
         this.updatedTime = updatedTime;
         this.analysisMethod = analysisMethod;
         this.analysisType = analysisType;
@@ -64,6 +68,7 @@ public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod, AnalysisTyp
         this.queriedTimes.addAndGet(queriedTimes);
         this.updatedRows = updatedRows;
         this.rowCount = rowCount;
+        this.tableVersion = tableVersion;
         if (partitionUpdateRows != null) {
             this.partitionUpdateRows.putAll(partitionUpdateRows);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java
index c3656b929279e6..df95b3cbede887 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java
@@ -21,5 +21,6 @@ public enum JobPriority {
     HIGH,
     MID,
     LOW,
+    VERY_LOW,
     MANUAL;
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 38af43bef7316f..6c1fd1ba95951d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -26,7 +26,6 @@
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.Pair;
 import org.apache.doris.common.util.MasterDaemon;
-import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.datasource.hive.HMSExternalTable;
 import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
 import org.apache.doris.statistics.AnalysisInfo.JobType;
@@ -37,7 +36,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.time.LocalTime;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -92,10 +90,11 @@ protected void runAfterCatalogReady() {
     }
 
     protected void collect() {
-        while (canCollect()) {
+        while (StatisticsUtil.canCollect()) {
             Pair<Entry<TableName, Set<Pair<String, String>>>, JobPriority> job = getJob();
             if (job == null) {
                 // No more job to process, break and sleep.
+                LOG.info("No auto analyze jobs to process.");
                 break;
             }
             try {
@@ -112,11 +111,6 @@ protected void collect() {
         }
     }
 
-    protected boolean canCollect() {
-        return StatisticsUtil.enableAutoAnalyze()
-            && StatisticsUtil.inAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()));
-    }
-
     protected Pair<Entry<TableName, Set<Pair<String, String>>>, JobPriority> getJob() {
         AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
         Optional<Entry<TableName, Set<Pair<String, String>>>> job = fetchJobFromMap(manager.highPriorityJobs);
@@ -128,7 +122,11 @@ protected Pair<Entry<TableName, Set<Pair<String, String>>>, JobPriority> getJob(
             return Pair.of(job.get(), JobPriority.MID);
         }
         job = fetchJobFromMap(manager.lowPriorityJobs);
-        return job.map(entry -> Pair.of(entry, JobPriority.LOW)).orElse(null);
+        if (job.isPresent()) {
+            return Pair.of(job.get(), JobPriority.LOW);
+        }
+        job = fetchJobFromMap(manager.veryLowPriorityJobs);
+        return job.map(tableNameSetEntry -> Pair.of(tableNameSetEntry, JobPriority.VERY_LOW)).orElse(null);
     }
 
     protected Optional<Map.Entry<TableName, Set<Pair<String, String>>>> fetchJobFromMap(
@@ -142,9 +140,13 @@ protected Optional<Map.Entry<TableName, Set<Pair<String, String>>>> fetchJobFrom
 
     protected void processOneJob(TableIf table, Set<Pair<String, String>> columns,
             JobPriority priority) throws DdlException {
-        // appendMvColumn(table, columns);
         appendAllColumns(table, columns);
-        columns = columns.stream().filter(c -> StatisticsUtil.needAnalyzeColumn(table, c)).collect(Collectors.toSet());
+        columns = columns.stream().filter(
+                c -> StatisticsUtil.needAnalyzeColumn(table, c) || StatisticsUtil.isLongTimeColumn(table, c))
+            .collect(Collectors.toSet());
+        if (columns.isEmpty()) {
+            return;
+        }
         AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns, priority);
         if (analyzeJob == null) {
             return;
@@ -178,15 +180,6 @@ protected void appendAllColumns(TableIf table, Set<Pair<String, String>> columns
         }
     }
 
-    protected void appendMvColumn(TableIf table, Set<String> columns) {
-        if (!(table instanceof OlapTable)) {
-            return;
-        }
-        OlapTable olapTable = (OlapTable) table;
-        Set<String> mvColumns = olapTable.getMvColumns(false).stream().map(Column::getName).collect(Collectors.toSet());
-        columns.addAll(mvColumns);
-    }
-
     protected boolean supportAutoAnalyze(TableIf tableIf) {
         if (tableIf == null) {
             return false;
@@ -248,9 +241,10 @@ protected AnalysisInfo createAnalyzeJobForTbl(
                 .setTaskIds(new ArrayList<>())
                 .setLastExecTimeInMs(System.currentTimeMillis())
                 .setJobType(JobType.SYSTEM)
-                .setTblUpdateTime(System.currentTimeMillis())
+                .setTblUpdateTime(table.getUpdateTime())
                 .setRowCount(rowCount)
                 .setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get())
+                .setTableVersion(table instanceof OlapTable ? ((OlapTable) table).getVisibleVersion() : 0)
                 .setPriority(priority)
                 .setPartitionUpdateRows(tableStatsStatus == null ? null : tableStatsStatus.partitionUpdateRows)
                 .setEnablePartition(StatisticsUtil.enablePartitionAnalyze())
@@ -275,4 +269,8 @@ protected void executeSystemAnalysisJob(AnalysisInfo jobInfo)
             future.get();
         }
     }
+
+    public boolean isReady() {
+        return waited;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
index b67d1cf947cb4e..4a3e93550fc9c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java
@@ -29,6 +29,7 @@
 import org.apache.doris.datasource.InternalCatalog;
 import org.apache.doris.statistics.util.StatisticsUtil;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -46,7 +47,7 @@ public class StatisticsJobAppender extends MasterDaemon {
     private static final Logger LOG = LogManager.getLogger(StatisticsJobAppender.class);
 
     public static final long INTERVAL = 1000;
-    public static final int JOB_MAP_SIZE = 1000;
+    public static final int JOB_MAP_SIZE = 100;
     public static final int TABLE_BATCH_SIZE = 100;
 
     private long currentDbId = 0;
@@ -70,6 +71,11 @@ protected void runAfterCatalogReady() {
             LOG.info("Stats table not available, skip");
             return;
         }
+        if (Env.getCurrentEnv().getStatisticsAutoCollector() == null
+                || !Env.getCurrentEnv().getStatisticsAutoCollector().isReady()) {
+            LOG.info("Statistics auto collector not ready, skip");
+            return;
+        }
         if (Env.isCheckpointThread()) {
             return;
         }
@@ -81,7 +87,7 @@ protected void appendJobs() {
         appendColumnsToJobs(manager.highPriorityColumns, manager.highPriorityJobs);
         appendColumnsToJobs(manager.midPriorityColumns, manager.midPriorityJobs);
         if (StatisticsUtil.enableAutoAnalyzeInternalCatalog()) {
-            appendToLowJobs(manager.lowPriorityJobs);
+            appendToLowJobs(manager.lowPriorityJobs, manager.veryLowPriorityJobs);
         }
     }
 
@@ -136,7 +142,8 @@ protected void appendColumnsToJobs(Queue<QueryColumn> columnQueue, Map<TableName
         }
     }
 
-    protected void appendToLowJobs(Map<TableName, Set<Pair<String, String>>> jobs) {
+    protected void appendToLowJobs(Map<TableName, Set<Pair<String, String>>> lowPriorityJobs,
+                                   Map<TableName, Set<Pair<String, String>>> veryLowPriorityJobs) {
         if (System.currentTimeMillis() - lastRoundFinishTime < lowJobIntervalMs) {
             return;
         }
@@ -162,27 +169,33 @@ protected void appendToLowJobs(Map<TableName, Set<Pair<String, String>>> jobs) {
                 if (t.getBaseSchema().size() > StatisticsUtil.getAutoAnalyzeTableWidthThreshold()) {
                     continue;
                 }
-                Set<Pair<String, String>> columnIndexPairs = t.getColumnIndexPairs(
-                        t.getSchemaAllIndexes(false).stream()
-                                .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
-                                .map(Column::getName).collect(Collectors.toSet()))
+                Set<String> columns = t.getSchemaAllIndexes(false).stream()
+                        .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
+                        .map(Column::getName).collect(Collectors.toSet());
+                Set<Pair<String, String>> columnIndexPairs = t.getColumnIndexPairs(columns)
                         .stream().filter(p -> StatisticsUtil.needAnalyzeColumn(t, p))
                         .collect(Collectors.toSet());
-                if (columnIndexPairs.isEmpty()) {
-                    continue;
-                }
                 TableName tableName = new TableName(t.getDatabase().getCatalog().getName(),
                         t.getDatabase().getFullName(), t.getName());
-                synchronized (jobs) {
-                    // If job map reach the upper limit, stop adding new jobs.
-                    if (!jobs.containsKey(tableName) && jobs.size() >= JOB_MAP_SIZE) {
-                        LOG.info("Low job map full.");
+                // Append to low job map first.
+                if (!columnIndexPairs.isEmpty()) {
+                    boolean appended = doAppend(lowPriorityJobs, columnIndexPairs, tableName);
+                    // If low job map is full, stop this iteration.
+                    if (!appended) {
+                        LOG.debug("Low Priority job map is full.");
                         return;
                     }
-                    if (jobs.containsKey(tableName)) {
-                        jobs.get(tableName).addAll(columnIndexPairs);
-                    } else {
-                        jobs.put(tableName, columnIndexPairs);
+                } else {
+                    // Append to very low job map.
+                    columnIndexPairs = t.getColumnIndexPairs(columns)
+                        .stream().filter(p -> StatisticsUtil.isLongTimeColumn(t, p))
+                        .collect(Collectors.toSet());
+                    if (!columnIndexPairs.isEmpty()) {
+                        boolean appended = doAppend(veryLowPriorityJobs, columnIndexPairs, tableName);
+                        // If very low job map is full, simply ignore it and go to the next table.
+                        if (!appended) {
+                            LOG.debug("Very low Priority job map is full.");
+                        }
                     }
                 }
                 currentTableId = t.getId();
@@ -200,6 +213,23 @@ protected void appendToLowJobs(Map<TableName, Set<Pair<String, String>>> jobs) {
         lastRoundFinishTime = System.currentTimeMillis();
     }
 
+    @VisibleForTesting
+    public boolean doAppend(Map<TableName, Set<Pair<String, String>>> jobMap,
+                         Set<Pair<String, String>> columnIndexPairs,
+                         TableName tableName) {
+        synchronized (jobMap) {
+            if (!jobMap.containsKey(tableName) && jobMap.size() >= JOB_MAP_SIZE) {
+                return false;
+            }
+            if (jobMap.containsKey(tableName)) {
+                jobMap.get(tableName).addAll(columnIndexPairs);
+            } else {
+                jobMap.put(tableName, columnIndexPairs);
+            }
+        }
+        return true;
+    }
+
     // For unit test only.
     public void setLastRoundFinishTime(long value) {
         lastRoundFinishTime = value;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index ba23ab84dc7a32..ac4704b54c6aae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -379,9 +379,8 @@ public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsSt
                     objects.catalog.getId(), objects.db.getId(), objects.table.getId(), indexId, colName,
                     null, columnStatistic);
             Env.getCurrentEnv().getStatisticsCache().syncColStats(data);
-            long timestamp = System.currentTimeMillis();
             AnalysisInfo mockedJobInfo = new AnalysisInfoBuilder()
-                    .setTblUpdateTime(timestamp)
+                    .setTblUpdateTime(objects.table.getUpdateTime())
                     .setColName("")
                     .setRowCount((long) Double.parseDouble(rowCount))
                     .setJobColumns(Sets.newHashSet())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index 6a7f2933996d88..4ebdb019f27e36 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -80,6 +80,9 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
     @SerializedName("updateTime")
     public long updatedTime;
 
+    @SerializedName("lat")
+    public long lastAnalyzeTime;
+
     @SerializedName("colNameToColStatsMeta")
     private ConcurrentMap<String, ColStatsMeta> deprecatedColNameToColStatsMeta = new ConcurrentHashMap<>();
 
@@ -160,6 +163,7 @@ public Set<Pair<String, String>> analyzeColumns() {
 
     public void update(AnalysisInfo analyzedJob, TableIf tableIf) {
         updatedTime = analyzedJob.tblUpdateTime;
+        lastAnalyzeTime = analyzedJob.createTime;
         if (analyzedJob.userInject) {
             userInjected = true;
         }
@@ -168,14 +172,16 @@ public void update(AnalysisInfo analyzedJob, TableIf tableIf) {
             if (colStatsMeta == null) {
                 colToColStatsMeta.put(colPair, new ColStatsMeta(analyzedJob.createTime, analyzedJob.analysisMethod,
                         analyzedJob.analysisType, analyzedJob.jobType, 0, analyzedJob.rowCount,
-                        analyzedJob.updateRows, analyzedJob.enablePartition ? analyzedJob.partitionUpdateRows : null));
+                        analyzedJob.updateRows, analyzedJob.tableVersion,
+                        analyzedJob.enablePartition ? analyzedJob.partitionUpdateRows : null));
             } else {
-                colStatsMeta.updatedTime = analyzedJob.tblUpdateTime;
+                colStatsMeta.updatedTime = analyzedJob.createTime;
                 colStatsMeta.analysisType = analyzedJob.analysisType;
                 colStatsMeta.analysisMethod = analyzedJob.analysisMethod;
                 colStatsMeta.jobType = analyzedJob.jobType;
                 colStatsMeta.updatedRows = analyzedJob.updateRows;
                 colStatsMeta.rowCount = analyzedJob.rowCount;
+                colStatsMeta.tableVersion = analyzedJob.tableVersion;
                 if (analyzedJob.enablePartition) {
                     if (colStatsMeta.partitionUpdateRows == null) {
                         colStatsMeta.partitionUpdateRows = new ConcurrentHashMap<>();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index a9c1612eb48b9c..dd037617ba6e30 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -53,6 +53,7 @@
 import org.apache.doris.common.FeConstants;
 import org.apache.doris.common.Pair;
 import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.datasource.CatalogIf;
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.InternalCatalog;
@@ -1012,12 +1013,6 @@ public static boolean needAnalyzeColumn(TableIf table, Pair<String, String> colu
         if (columnStatsMeta == null) {
             return true;
         }
-        // Column hasn't been analyzed for longer than config interval.
-        if (Config.auto_analyze_interval_seconds > 0
-                && System.currentTimeMillis() - columnStatsMeta.updatedTime
-                        > Config.auto_analyze_interval_seconds * 1000) {
-            return true;
-        }
         // Partition table partition stats never been collected.
         if (StatisticsUtil.enablePartitionAnalyze() && table.isPartitionedTable()
                 && columnStatsMeta.partitionUpdateRows == null) {
@@ -1072,7 +1067,7 @@ public static boolean needAnalyzeColumn(TableIf table, Pair<String, String> colu
             }
             // External is hard to calculate change rate, use time interval to control analyze frequency.
             return System.currentTimeMillis()
-                    - tableStatsStatus.updatedTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
+                    - tableStatsStatus.lastAnalyzeTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
         }
     }
 
@@ -1127,4 +1122,47 @@ public static boolean needAnalyzePartition(OlapTable table, TableStatsMeta table
         }
         return false;
     }
+
+    // This function return true means the column hasn't been analyzed for longer than the configured time.
+    public static boolean isLongTimeColumn(TableIf table, Pair<String, String> column) {
+        if (column == null) {
+            return false;
+        }
+        if (!table.autoAnalyzeEnabled()) {
+            return false;
+        }
+        if (!(table instanceof OlapTable)) {
+            return false;
+        }
+        AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+        TableStatsMeta tblStats = manager.findTableStatsStatus(table.getId());
+        // Table never been analyzed, skip it for higher priority jobs.
+        if (tblStats == null) {
+            LOG.warn("Table stats is null.");
+            return false;
+        }
+        ColStatsMeta columnStats = tblStats.findColumnStatsMeta(column.first, column.second);
+        if (columnStats == null) {
+            // Column never been analyzed, skip it for higher priority jobs.
+            return false;
+        }
+        // User injected column stats, don't do auto analyze, avoid overwrite user injected stats.
+        if (tblStats.userInjected) {
+            return false;
+        }
+        boolean isLongTime = Config.auto_analyze_interval_seconds > 0
+                && System.currentTimeMillis() - columnStats.updatedTime > Config.auto_analyze_interval_seconds * 1000;
+        if (!isLongTime) {
+            return false;
+        }
+        // For olap table, if the table visible version and row count doesn't change since last analyze,
+        // we don't need to analyze it because its data is not changed.
+        OlapTable olapTable = (OlapTable) table;
+        return olapTable.getVisibleVersion() != columnStats.tableVersion
+                || olapTable.getRowCount() != columnStats.rowCount;
+    }
+
+    public static boolean canCollect() {
+        return enableAutoAnalyze() && inAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()));
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java
index e3255ab23a0381..5b890795f01076 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java
@@ -33,6 +33,8 @@
 import org.apache.doris.statistics.util.StatisticsUtil;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import mockit.Mock;
 import mockit.MockUp;
 import org.junit.jupiter.api.Assertions;
@@ -178,13 +180,21 @@ public Set<Pair<String, String>> getColumnIndexPairs(Set<String> columns) {
             }
         };
 
-        Map<TableName, Set<Pair<String, String>>> testMap = new HashMap<>();
+        new MockUp<StatisticsUtil>() {
+            @Mock
+            public boolean needAnalyzeColumn(TableIf table, Pair<String, String> column) {
+                return true;
+            }
+        };
+
+        Map<TableName, Set<Pair<String, String>>> testLowMap = new HashMap<>();
+        Map<TableName, Set<Pair<String, String>>> testVeryLowMap = new HashMap<>();
         StatisticsJobAppender appender = new StatisticsJobAppender();
-        appender.appendToLowJobs(testMap);
-        Assertions.assertEquals(100, testMap.size());
-        testMap.clear();
-        appender.appendToLowJobs(testMap);
-        Assertions.assertEquals(40, testMap.size());
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
+        Assertions.assertEquals(100, testLowMap.size());
+        testLowMap.clear();
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
+        Assertions.assertEquals(40, testLowMap.size());
 
         for (int i = 0; i < StatisticsJobAppender.JOB_MAP_SIZE; i++) {
             Database db = new Database(id++, "testDb" + i);
@@ -198,38 +208,93 @@ public Set<Pair<String, String>> getColumnIndexPairs(Set<String> columns) {
             db.createTableWithLock(table2, true, false);
         }
 
-        testMap.clear();
-        appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
+        testLowMap.clear();
         appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
         appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
         appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
+        Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE, testLowMap.size());
+    }
+
+    @Test
+    public void testAppendQueryColumnToVeryLowJobMap() throws DdlException {
+        InternalCatalog testCatalog = new InternalCatalog();
+        int id = 10;
+        for (int i = 0; i < 70; i++) {
+            Database db = new Database(id++, "testDb" + i);
+            testCatalog.unprotectCreateDb(db);
+            Column column1 = new Column("placeholder", PrimitiveType.INT);
+            List<Column> schema = new ArrayList<>();
+            schema.add(column1);
+            OlapTable table1 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null);
+            OlapTable table2 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null);
+            db.createTableWithLock(table1, true, false);
+            db.createTableWithLock(table2, true, false);
+        }
+
+        new MockUp<Env>() {
+            @Mock
+            public InternalCatalog getCurrentInternalCatalog() {
+                return testCatalog;
+            }
+        };
+
+        new MockUp<OlapTable>() {
+            @Mock
+            public List<Column> getBaseSchema() {
+                return Lists.newArrayList();
+            }
+
+            @Mock
+            public Set<Pair<String, String>> getColumnIndexPairs(Set<String> columns) {
+                return Collections.singleton(Pair.of("mockIndex", "mockColumn"));
+            }
+        };
+
+        new MockUp<StatisticsUtil>() {
+            @Mock
+            public boolean needAnalyzeColumn(TableIf table, Pair<String, String> column) {
+                return false;
+            }
+
+            @Mock
+            public boolean isLongTimeColumn(TableIf table, Pair<String, String> column) {
+                return true;
+            }
+        };
+
+        Map<TableName, Set<Pair<String, String>>> testLowMap = new HashMap<>();
+        Map<TableName, Set<Pair<String, String>>> testVeryLowMap = new HashMap<>();
+        StatisticsJobAppender appender = new StatisticsJobAppender();
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
+        Assertions.assertEquals(100, testVeryLowMap.size());
+        testVeryLowMap.clear();
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
+        Assertions.assertEquals(40, testVeryLowMap.size());
+
+        for (int i = 0; i < StatisticsJobAppender.JOB_MAP_SIZE; i++) {
+            Database db = new Database(id++, "testDb" + i);
+            testCatalog.unprotectCreateDb(db);
+            Column column1 = new Column("placeholder", PrimitiveType.INT);
+            List<Column> schema = new ArrayList<>();
+            schema.add(column1);
+            OlapTable table1 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null);
+            OlapTable table2 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null);
+            db.createTableWithLock(table1, true, false);
+            db.createTableWithLock(table2, true, false);
+        }
+
+        testLowMap.clear();
         appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
         appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
         appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE, testMap.size());
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
+        Assertions.assertEquals(0, testLowMap.size());
+        Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE, testVeryLowMap.size());
     }
 
     @Test
@@ -270,12 +335,48 @@ public int getAutoAnalyzeTableWidthThreshold() {
                 return thresholds[count++];
             }
         };
-        Map<TableName, Set<Pair<String, String>>> testMap = new HashMap<>();
+        Map<TableName, Set<Pair<String, String>>> testLowMap = new HashMap<>();
+        Map<TableName, Set<Pair<String, String>>> testVeryLowMap = new HashMap<>();
         StatisticsJobAppender appender = new StatisticsJobAppender();
-        appender.appendToLowJobs(testMap);
-        Assertions.assertEquals(0, testMap.size());
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
+        Assertions.assertEquals(0, testLowMap.size());
         appender.setLastRoundFinishTime(0);
-        appender.appendToLowJobs(testMap);
-        Assertions.assertEquals(1, testMap.size());
+        appender.appendToLowJobs(testLowMap, testVeryLowMap);
+        Assertions.assertEquals(1, testLowMap.size());
+    }
+
+    @Test
+    public void testDoAppend() {
+        Map<TableName, Set<Pair<String, String>>> jobMap = Maps.newHashMap();
+        Set<Pair<String, String>> columnIndexPairs1 = Sets.newHashSet();
+        Set<Pair<String, String>> columnIndexPairs2 = Sets.newHashSet();
+        TableName tableName1 = new TableName("catalog1", "db1", "table1");
+        TableName tableName2 = new TableName("catalog2", "db2", "table2");
+        Pair<String, String> pair1 = Pair.of("index1", "col1");
+        columnIndexPairs1.add(pair1);
+
+        StatisticsJobAppender appender = new StatisticsJobAppender();
+        Assertions.assertTrue(appender.doAppend(jobMap, columnIndexPairs1, tableName1));
+        Assertions.assertEquals(1, jobMap.size());
+        Assertions.assertTrue(jobMap.containsKey(tableName1));
+        Assertions.assertEquals(1, jobMap.get(tableName1).size());
+        Assertions.assertTrue(jobMap.get(tableName1).contains(pair1));
+
+        Pair<String, String> pair2 = Pair.of("index2", "col2");
+        columnIndexPairs1.add(pair2);
+        Assertions.assertTrue(appender.doAppend(jobMap, columnIndexPairs1, tableName1));
+        Assertions.assertEquals(1, jobMap.size());
+        Assertions.assertTrue(jobMap.containsKey(tableName1));
+        Assertions.assertEquals(2, jobMap.get(tableName1).size());
+        Assertions.assertTrue(jobMap.get(tableName1).contains(pair1));
+        Assertions.assertTrue(jobMap.get(tableName1).contains(pair2));
+
+        Pair<String, String> pair3 = Pair.of("index3", "col3");
+        columnIndexPairs2.add(pair3);
+        Assertions.assertTrue(appender.doAppend(jobMap, columnIndexPairs2, tableName2));
+        Assertions.assertEquals(2, jobMap.size());
+        Assertions.assertTrue(jobMap.containsKey(tableName2));
+        Assertions.assertEquals(1, jobMap.get(tableName2).size());
+        Assertions.assertTrue(jobMap.get(tableName2).contains(pair3));
     }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
index fbac718e421c29..ef1e9ca02970b1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
@@ -32,6 +32,7 @@
 import org.apache.doris.datasource.hive.HMSExternalCatalog;
 import org.apache.doris.datasource.hive.HMSExternalTable;
 import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
 import org.apache.doris.datasource.jdbc.JdbcExternalCatalog;
 import org.apache.doris.datasource.jdbc.JdbcExternalTable;
 import org.apache.doris.qe.SessionVariable;
@@ -208,12 +209,6 @@ public TableStatsMeta findTableStatsStatus(long tblId) {
         Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
 
         // Test external table auto analyze enabled.
-        new MockUp<AnalysisManager>() {
-            @Mock
-            public TableStatsMeta findTableStatsStatus(long tblId) {
-                return null;
-            }
-        };
         externalCatalog.getCatalogProperty().addProperty(ExternalCatalog.ENABLE_AUTO_ANALYZE, "false");
         HMSExternalTable hmsTable1 = new HMSExternalTable(1, "name", "dbName", externalCatalog);
         externalCatalog.setAutoAnalyzePolicy("dbName", "name", "enable");
@@ -238,27 +233,10 @@ public TableStatsMeta findTableStatsStatus(long tblId) {
         tableMeta.userInjected = false;
         Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
 
-        // Test column hasn't been analyzed for longer than 1 day.
         new MockUp<TableStatsMeta>() {
             @Mock
             public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
-                return new ColStatsMeta(0, null, null, null, 0, 100, 0, null);
-            }
-        };
-        new MockUp<OlapTable>() {
-            @Mock
-            public long getRowCount() {
-                return 100;
-            }
-        };
-        Config.auto_analyze_interval_seconds = 60 * 60 * 24;
-        Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
-        Config.auto_analyze_interval_seconds = 0;
-
-        new MockUp<TableStatsMeta>() {
-            @Mock
-            public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
-                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 0, 0, null);
+                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 0, 0, 0, null);
             }
         };
 
@@ -312,7 +290,7 @@ public long getRowCount() {
         new MockUp<TableStatsMeta>() {
             @Mock
             public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
-                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 100, 0, null);
+                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 100, 0, 0, null);
             }
         };
         tableMeta.partitionChanged.set(false);
@@ -322,7 +300,7 @@ public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
         new MockUp<TableStatsMeta>() {
             @Mock
             public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
-                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 0, 0, null);
+                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 0, 0, 0, null);
             }
         };
         tableMeta.partitionChanged.set(false);
@@ -338,7 +316,7 @@ public long getRowCount() {
         new MockUp<TableStatsMeta>() {
             @Mock
             public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
-                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 500, 0, null);
+                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 500, 0, 0, null);
             }
         };
         tableMeta.partitionChanged.set(false);
@@ -354,7 +332,7 @@ public long getRowCount() {
         new MockUp<TableStatsMeta>() {
             @Mock
             public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
-                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 100, 80, null);
+                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 100, 80, 0, null);
             }
         };
         tableMeta.partitionChanged.set(false);
@@ -382,6 +360,140 @@ public long getRowCount() {
         tableMeta.partitionChanged.set(false);
         tableMeta.updatedRows.set(85);
         Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName())));
+    }
+
+    @Test
+    void testLongTimeNoAnalyze() {
+        Column column = new Column("testColumn", PrimitiveType.INT);
+        List<Column> schema = new ArrayList<>();
+        schema.add(column);
+        OlapTable table = new OlapTable(200, "testTable", schema, null, null, null);
 
+        // Test column is null
+        Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, null));
+
+        // Test table auto analyze is disabled.
+        new MockUp<OlapTable>() {
+            @Mock
+            public boolean autoAnalyzeEnabled() {
+                return false;
+            }
+        };
+        Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName())));
+        new MockUp<OlapTable>() {
+            @Mock
+            public boolean autoAnalyzeEnabled() {
+                return true;
+            }
+        };
+
+        // Test external table
+        new MockUp<ExternalTable>() {
+            @Mock
+            public boolean autoAnalyzeEnabled() {
+                return true;
+            }
+        };
+        IcebergExternalTable icebergTable = new IcebergExternalTable(0, "", "", null);
+        Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(icebergTable, Pair.of("index", column.getName())));
+
+        // Test table stats meta is null.
+        new MockUp<AnalysisManager>() {
+            @Mock
+            public TableStatsMeta findTableStatsStatus(long tblId) {
+                return null;
+            }
+        };
+        Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName())));
+
+        // Test column stats meta is null
+        TableStatsMeta tableMeta = new TableStatsMeta();
+        new MockUp<AnalysisManager>() {
+            @Mock
+            public TableStatsMeta findTableStatsStatus(long tblId) {
+                return tableMeta;
+            }
+        };
+        new MockUp<TableStatsMeta>() {
+            @Mock
+            public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
+                return null;
+            }
+        };
+        Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName())));
+        new MockUp<TableStatsMeta>() {
+            @Mock
+            public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
+                return new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 100, 0, 0, null);
+            }
+        };
+
+        // Test table stats is user injected
+        tableMeta.userInjected = true;
+        Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName())));
+        tableMeta.userInjected = false;
+
+        // Test Config.auto_analyze_interval_seconds == 0
+        Config.auto_analyze_interval_seconds = 0;
+        Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName())));
+
+        // Test column analyzed within the time interval
+        Config.auto_analyze_interval_seconds = 86400;
+        Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName())));
+
+        // Test column hasn't analyzed for longer than time interval, but version and row count doesn't change
+        new MockUp<TableStatsMeta>() {
+            @Mock
+            public ColStatsMeta findColumnStatsMeta(String indexName, String colName) {
+                ColStatsMeta ret = new ColStatsMeta(System.currentTimeMillis(), null, null, null, 0, 100, 20, 10, null);
+                try {
+                    Thread.sleep(1500);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+                return ret;
+            }
+        };
+        new MockUp<OlapTable>() {
+            @Mock
+            public long getVisibleVersion() {
+                return 10;
+            }
+
+            @Mock
+            public long fetchRowCount() {
+                return 100;
+            }
+        };
+        Config.auto_analyze_interval_seconds = 1;
+        Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName())));
+
+        // Test column hasn't analyzed for longer than time interval, and version change
+        new MockUp<OlapTable>() {
+            @Mock
+            public long getVisibleVersion() {
+                return 11;
+            }
+
+            @Mock
+            public long fetchRowCount() {
+                return 100;
+            }
+        };
+        Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName())));
+
+        // Test column hasn't analyzed for longer than time interval, and row count change
+        new MockUp<OlapTable>() {
+            @Mock
+            public long getVisibleVersion() {
+                return 10;
+            }
+
+            @Mock
+            public long fetchRowCount() {
+                return 101;
+            }
+        };
+        Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName())));
     }
 }
diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy
index 55074e995fe68d..69360da6911ba5 100644
--- a/regression-test/suites/statistics/analyze_stats.groovy
+++ b/regression-test/suites/statistics/analyze_stats.groovy
@@ -2903,7 +2903,38 @@ PARTITION `p599` VALUES IN (599)
     assertEquals("521779.0", alter_result[0][5])
     assertEquals("7.142863009760572", alter_result[0][6])
 
-
     sql """DROP DATABASE IF EXISTS trigger"""
+
+    // Test show last analyze table version
+    sql """create database if not exists test_version"""
+    sql """use test_version"""
+    sql """drop table if exists region"""
+    sql """
+        CREATE TABLE region  (
+            r_regionkey      int NOT NULL,
+            r_name       VARCHAR(25) NOT NULL,
+            r_comment    VARCHAR(152)
+        )ENGINE=OLAP
+        DUPLICATE KEY(`r_regionkey`)
+        COMMENT "OLAP"
+        DISTRIBUTED BY HASH(`r_regionkey`) BUCKETS 1
+        PROPERTIES (
+            "replication_num" = "1" 
+        );
+    """
+    sql """analyze table region with sync"""
+    def versionResult = sql """show column stats region"""
+    assertEquals(versionResult[0][16], "1")
+    assertEquals(versionResult[1][16], "1")
+    assertEquals(versionResult[2][16], "1")
+
+    sql """insert into region values (1, "1", "1")"""
+    sql """analyze table region with sync"""
+    versionResult = sql """show column stats region"""
+    assertEquals(versionResult[0][16], "2")
+    assertEquals(versionResult[1][16], "2")
+    assertEquals(versionResult[2][16], "2")
+
+    sql """drop database if exists test_version"""
 }
 

From 1d314867a867497b94d49aead1ce54fbc079586a Mon Sep 17 00:00:00 2001
From: Xin Liao <liaoxin@selectdb.com>
Date: Wed, 11 Dec 2024 16:04:57 +0800
Subject: [PATCH 270/399] [Opt](load) only MoW table need table commit lock
 when commit transaction for cloud mode (#45220)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

table commit lock only used for MoW table， other table modes do not need
this lock.
---
 .../CloudGlobalTransactionMgr.java            | 26 +++++++++++++------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index 3c56f7dc56a180..97d75403ffcb6e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -1097,8 +1097,7 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId,
         try {
             commitTransactionWithSubTxns(db.getId(), tableList, transactionId, subTransactionStates);
         } finally {
-            decreaseWaitingLockCount(tableList);
-            MetaLockUtils.commitUnlockTables(tableList);
+            afterCommitTransaction(tableList);
         }
         return true;
     }
@@ -1153,16 +1152,28 @@ private void beforeCommitTransaction(List<Table> tableList, long transactionId,
                         entry.getValue().get());
             }
         }
-        increaseWaitingLockCount(tableList);
-        if (!MetaLockUtils.tryCommitLockTables(tableList, timeoutMillis, TimeUnit.MILLISECONDS)) {
-            decreaseWaitingLockCount(tableList);
+
+        List<Table> mowTableList = tableList.stream()
+                .filter(table -> table instanceof OlapTable && ((OlapTable) table).getEnableUniqueKeyMergeOnWrite())
+                .collect(Collectors.toList());
+        increaseWaitingLockCount(mowTableList);
+        if (!MetaLockUtils.tryCommitLockTables(mowTableList, timeoutMillis, TimeUnit.MILLISECONDS)) {
+            decreaseWaitingLockCount(mowTableList);
             // DELETE_BITMAP_LOCK_ERR will be retried on be
             throw new UserException(InternalErrorCode.DELETE_BITMAP_LOCK_ERR,
                     "get table cloud commit lock timeout, tableList=("
-                            + StringUtils.join(tableList, ",") + ")");
+                            + StringUtils.join(mowTableList, ",") + ")");
         }
     }
 
+    private void afterCommitTransaction(List<Table> tableList) {
+        List<Table> mowTableList = tableList.stream()
+                .filter(table -> table instanceof OlapTable && ((OlapTable) table).getEnableUniqueKeyMergeOnWrite())
+                .collect(Collectors.toList());
+        decreaseWaitingLockCount(mowTableList);
+        MetaLockUtils.commitUnlockTables(mowTableList);
+    }
+
     @Override
     public boolean commitAndPublishTransaction(DatabaseIf db, List<Table> tableList, long transactionId,
                                                List<TabletCommitInfo> tabletCommitInfos, long timeoutMillis,
@@ -1171,8 +1182,7 @@ public boolean commitAndPublishTransaction(DatabaseIf db, List<Table> tableList,
         try {
             commitTransaction(db.getId(), tableList, transactionId, tabletCommitInfos, txnCommitAttachment);
         } finally {
-            decreaseWaitingLockCount(tableList);
-            MetaLockUtils.commitUnlockTables(tableList);
+            afterCommitTransaction(tableList);
         }
         return true;
     }

From 2acca680387e9de98c54fcb7ece403893a457f7f Mon Sep 17 00:00:00 2001
From: Xin Liao <liaoxin@selectdb.com>
Date: Wed, 11 Dec 2024 16:06:04 +0800
Subject: [PATCH 271/399] [Opt](load) reduce the length of error url to avoid
 exceeding the MySQL protocol length limit for cloud (#45213)

reduce the length of error url to avoid exceeding the MySQL protocol length limit when save error log to s3.
---
 be/src/runtime/runtime_state.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index f3376d06858ec0..ecaf99061a070b 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -325,7 +325,7 @@ Status RuntimeState::create_error_log_file() {
             // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_err_packet.html
             // shorten the path as much as possible to prevent the length of the presigned URL from
             // exceeding the MySQL error packet size limit
-            ss << "error_log/" << _import_label << "_" << std::hex << _fragment_instance_id.hi;
+            ss << "error_log/" << std::hex << _query_id.hi;
             _s3_error_log_file_path = ss.str();
         }
     }

From 502542d37dc034705730ef7702def02a312cb34d Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Wed, 11 Dec 2024 13:53:13 +0530
Subject: [PATCH 272/399] [feat](binlog) Add Support recover binlog (#44818)

---
 .../apache/doris/binlog/BinlogManager.java    | 27 +++++++++++++++
 .../org/apache/doris/binlog/DBBinlog.java     | 33 +++++++++++++------
 .../doris/catalog/CatalogRecycleBin.java      |  6 ++--
 .../doris/datasource/InternalCatalog.java     |  2 +-
 .../org/apache/doris/persist/EditLog.java     |  8 +++--
 .../org/apache/doris/persist/RecoverInfo.java | 22 +++++++++++--
 .../doris/persist/DropAndRecoverInfoTest.java |  2 +-
 gensrc/thrift/FrontendService.thrift          |  4 +--
 8 files changed, 84 insertions(+), 20 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
index 67bb99a8bcdc18..b9eb91cc5f7ea6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
@@ -34,6 +34,7 @@
 import org.apache.doris.persist.DropPartitionInfo;
 import org.apache.doris.persist.ModifyCommentOperationLog;
 import org.apache.doris.persist.ModifyTablePropertyOperationLog;
+import org.apache.doris.persist.RecoverInfo;
 import org.apache.doris.persist.ReplacePartitionOperationLog;
 import org.apache.doris.persist.ReplaceTableOperationLog;
 import org.apache.doris.persist.TableAddOrDropColumnsInfo;
@@ -446,6 +447,32 @@ public void addDropRollup(DropInfo info, long commitSeq) {
         addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false, info);
     }
 
+
+    private boolean supportedRecoverInfo(RecoverInfo info) {
+        //table name and partitionName added together.
+        // recover table case, tablename must exist in newer version
+        // recover partition case also table name must exist.
+        // so checking only table name here.
+        if (StringUtils.isEmpty(info.getTableName())) {
+            LOG.warn("skip recover info binlog, because tableName is empty. info: {}", info);
+            return false;
+        }
+        return true;
+    }
+
+    public void addRecoverTableRecord(RecoverInfo info, long commitSeq) {
+        if (supportedRecoverInfo(info) == false) {
+            return;
+        }
+        long dbId = info.getDbId();
+        List<Long> tableIds = Lists.newArrayList();
+        tableIds.add(info.getTableId());
+        long timestamp = -1;
+        TBinlogType type = TBinlogType.RECOVER_INFO;
+        String data = info.toJson();
+        addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false, info);
+    }
+
     // get binlog by dbId, return first binlog.version > version
     public Pair<TStatus, TBinlog> getBinlog(long dbId, long tableId, long prevCommitSeq) {
         TStatus status = new TStatus(TStatusCode.OK);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
index b78ed389a0fe86..0816564f150be2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
@@ -25,6 +25,7 @@
 import org.apache.doris.persist.BarrierLog;
 import org.apache.doris.persist.DropInfo;
 import org.apache.doris.persist.DropPartitionInfo;
+import org.apache.doris.persist.RecoverInfo;
 import org.apache.doris.persist.ReplaceTableOperationLog;
 import org.apache.doris.thrift.TBinlog;
 import org.apache.doris.thrift.TBinlogType;
@@ -124,7 +125,7 @@ public void recoverBinlog(TBinlog binlog, boolean dbBinlogEnable) {
 
         allBinlogs.add(binlog);
         binlogSize += BinlogUtils.getApproximateMemoryUsage(binlog);
-        recordDroppedResources(binlog);
+        recordDroppedOrRecoveredResources(binlog);
 
         if (tableIds == null) {
             return;
@@ -178,7 +179,7 @@ public void addBinlog(TBinlog binlog, Object raw) {
                 return;
             }
 
-            recordDroppedResources(binlog, raw);
+            recordDroppedOrRecoveredResources(binlog, raw);
 
             switch (binlog.getType()) {
                 case CREATE_TABLE:
@@ -623,16 +624,16 @@ public void getBinlogInfo(BaseProcResult result) {
         }
     }
 
-    private void recordDroppedResources(TBinlog binlog) {
-        recordDroppedResources(binlog, null);
+    private void recordDroppedOrRecoveredResources(TBinlog binlog) {
+        recordDroppedOrRecoveredResources(binlog, null);
     }
 
     // A method to record the dropped tables, indexes, and partitions.
-    private void recordDroppedResources(TBinlog binlog, Object raw) {
-        recordDroppedResources(binlog.getType(), binlog.getCommitSeq(), binlog.getData(), raw);
+    private void recordDroppedOrRecoveredResources(TBinlog binlog, Object raw) {
+        recordDroppedOrRecoveredResources(binlog.getType(), binlog.getCommitSeq(), binlog.getData(), raw);
     }
 
-    private void recordDroppedResources(TBinlogType binlogType, long commitSeq, String data, Object raw) {
+    private void recordDroppedOrRecoveredResources(TBinlogType binlogType, long commitSeq, String data, Object raw) {
         if (raw == null) {
             switch (binlogType) {
                 case DROP_PARTITION:
@@ -656,6 +657,9 @@ private void recordDroppedResources(TBinlogType binlogType, long commitSeq, Stri
                 case BARRIER:
                     raw = BarrierLog.fromJson(data);
                     break;
+                case RECOVER_INFO:
+                    raw = RecoverInfo.fromJson(data);
+                    break;
                 default:
                     break;
             }
@@ -664,10 +668,10 @@ private void recordDroppedResources(TBinlogType binlogType, long commitSeq, Stri
             }
         }
 
-        recordDroppedResources(binlogType, commitSeq, raw);
+        recordDroppedOrRecoveredResources(binlogType, commitSeq, raw);
     }
 
-    private void recordDroppedResources(TBinlogType binlogType, long commitSeq, Object raw) {
+    private void recordDroppedOrRecoveredResources(TBinlogType binlogType, long commitSeq, Object raw) {
         if (binlogType == TBinlogType.DROP_PARTITION && raw instanceof DropPartitionInfo) {
             long partitionId = ((DropPartitionInfo) raw).getPartitionId();
             if (partitionId > 0) {
@@ -706,7 +710,16 @@ private void recordDroppedResources(TBinlogType binlogType, long commitSeq, Obje
             BarrierLog log = (BarrierLog) raw;
             // keep compatible with doris 2.0/2.1
             if (log.hasBinlog()) {
-                recordDroppedResources(log.getBinlogType(), commitSeq, log.getBinlog(), null);
+                recordDroppedOrRecoveredResources(log.getBinlogType(), commitSeq, log.getBinlog(), null);
+            }
+        } else if ((binlogType == TBinlogType.RECOVER_INFO) && (raw instanceof RecoverInfo)) {
+            RecoverInfo recoverInfo = (RecoverInfo) raw;
+            long partitionId = recoverInfo.getPartitionId();
+            long tableId = recoverInfo.getTableId();
+            if (partitionId > 0) {
+                droppedPartitions.removeIf(entry -> (entry.first == partitionId));
+            } else if (tableId > 0) {
+                droppedTables.removeIf(entry -> (entry.first == tableId));
             }
         }
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
index 745c1c8a351686..b5899435343b13 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
@@ -774,7 +774,8 @@ private synchronized boolean innerRecoverTable(Database db, Table table, String
                 LOG.info("replay recover table[{}]", table.getId());
             } else {
                 // log
-                RecoverInfo recoverInfo = new RecoverInfo(db.getId(), table.getId(), -1L, "", newTableName, "");
+                RecoverInfo recoverInfo = new RecoverInfo(db.getId(), table.getId(),
+                                                    -1L, "", table.getName(), newTableName, "", "");
                 Env.getCurrentEnv().getEditLog().logRecoverTable(recoverInfo);
             }
             // Only olap table need recover dynamic partition, other table like jdbc odbc view.. do not need it
@@ -873,7 +874,8 @@ public synchronized void recoverPartition(long dbId, OlapTable table, String par
         idToRecycleTime.remove(partitionId);
 
         // log
-        RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", "", newPartitionName);
+        RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "",
+                                                    table.getName(), "", partitionName, newPartitionName);
         Env.getCurrentEnv().getEditLog().logRecoverPartition(recoverInfo);
         LOG.info("recover partition[{}]", partitionId);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 212ee89fcea84c..7c7ded88f55960 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -657,7 +657,7 @@ public void recoverDatabase(String dbName, long dbId, String newDbName) throws D
             fullNameToDb.put(db.getFullName(), db);
             idToDb.put(db.getId(), db);
             // log
-            RecoverInfo recoverInfo = new RecoverInfo(db.getId(), -1L, -1L, newDbName, "", "");
+            RecoverInfo recoverInfo = new RecoverInfo(db.getId(), -1L, -1L, newDbName, "", "", "", "");
             Env.getCurrentEnv().getEditLog().logRecoverDb(recoverInfo);
             db.unmarkDropped();
         } finally {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index f1377e9daebfc4..73a2c6338da5d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -297,11 +297,13 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) {
                 case OperationType.OP_RECOVER_TABLE: {
                     RecoverInfo info = (RecoverInfo) journal.getData();
                     env.replayRecoverTable(info);
+                    env.getBinlogManager().addRecoverTableRecord(info, logId);
                     break;
                 }
                 case OperationType.OP_RECOVER_PARTITION: {
                     RecoverInfo info = (RecoverInfo) journal.getData();
                     env.replayRecoverPartition(info);
+                    env.getBinlogManager().addRecoverTableRecord(info, logId);
                     break;
                 }
                 case OperationType.OP_RENAME_TABLE: {
@@ -1444,7 +1446,8 @@ public void logErasePartition(long partitionId) {
     }
 
     public void logRecoverPartition(RecoverInfo info) {
-        logEdit(OperationType.OP_RECOVER_PARTITION, info);
+        long logId = logEdit(OperationType.OP_RECOVER_PARTITION, info);
+        Env.getCurrentEnv().getBinlogManager().addRecoverTableRecord(info, logId);
     }
 
     public void logModifyPartition(ModifyPartitionInfo info) {
@@ -1471,7 +1474,8 @@ public void logEraseTable(long tableId) {
     }
 
     public void logRecoverTable(RecoverInfo info) {
-        logEdit(OperationType.OP_RECOVER_TABLE, info);
+        long logId = logEdit(OperationType.OP_RECOVER_TABLE, info);
+        Env.getCurrentEnv().getBinlogManager().addRecoverTableRecord(info, logId);
     }
 
     public void logDropRollup(DropInfo info) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/RecoverInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/RecoverInfo.java
index 15764a99b43b29..eb4af6494e8a37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/RecoverInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/RecoverInfo.java
@@ -38,10 +38,14 @@ public class RecoverInfo implements Writable, GsonPostProcessable {
     private String newDbName;
     @SerializedName(value = "tableId")
     private long tableId;
+    @SerializedName(value = "tableName")
+    private String tableName;                        /// added for table name.
     @SerializedName(value = "newTableName")
     private String newTableName;
     @SerializedName(value = "partitionId")
     private long partitionId;
+    @SerializedName(value = "partitionName")
+    private String partitionName;
     @SerializedName(value = "newPartitionName")
     private String newPartitionName;
 
@@ -49,13 +53,15 @@ private RecoverInfo() {
         // for persist
     }
 
-    public RecoverInfo(long dbId, long tableId, long partitionId, String newDbName, String newTableName,
-                       String newPartitionName) {
+    public RecoverInfo(long dbId, long tableId, long partitionId, String newDbName, String tableName,
+                        String newTableName, String partitionName, String newPartitionName) {
         this.dbId = dbId;
         this.tableId = tableId;
+        this.tableName = tableName;
         this.partitionId = partitionId;
         this.newDbName = newDbName;
         this.newTableName = newTableName;
+        this.partitionName = partitionName;
         this.newPartitionName = newPartitionName;
     }
 
@@ -67,6 +73,10 @@ public long getTableId() {
         return tableId;
     }
 
+    public String getTableName() {
+        return tableName;
+    }
+
     public long getPartitionId() {
         return partitionId;
     }
@@ -109,4 +119,12 @@ private void readFields(DataInput in) throws IOException {
     public void gsonPostProcess() throws IOException {
         newDbName = ClusterNamespace.getNameFromFullName(newDbName);
     }
+
+    public String toJson() {
+        return GsonUtils.GSON.toJson(this);
+    }
+
+    public static RecoverInfo fromJson(String json) {
+        return GsonUtils.GSON.fromJson(json, RecoverInfo.class);
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
index 8c74fba275352d..63afe375548fe0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java
@@ -86,7 +86,7 @@ public void testRecoveryInfoSerialization() throws Exception {
         file.createNewFile();
         DataOutputStream dos = new DataOutputStream(new FileOutputStream(file));
 
-        RecoverInfo info1 = new RecoverInfo(1, 2, 3, "a", "b", "c");
+        RecoverInfo info1 = new RecoverInfo(1, 2, 3, "a", "", "b", "", "c");
         info1.write(dos);
         dos.flush();
         dos.close();
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 916885028ad4a0..c7d1bf264f2f63 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1199,6 +1199,7 @@ enum TBinlogType {
   RENAME_ROLLUP = 21,
   RENAME_PARTITION = 22,
   DROP_ROLLUP = 23,
+  RECOVER_INFO = 24,
 
   // Keep some IDs for allocation so that when new binlog types are added in the
   // future, the changes can be picked back to the old versions without breaking
@@ -1215,8 +1216,7 @@ enum TBinlogType {
   //    MODIFY_XXX = 17,
   //    MIN_UNKNOWN = 18,
   //    UNKNOWN_3 = 19,
-  MIN_UNKNOWN = 24,
-  UNKNOWN_9 = 25,
+  MIN_UNKNOWN = 25,
   UNKNOWN_10 = 26,
   UNKNOWN_11 = 27,
   UNKNOWN_12 = 28,

From f772baa61f74fb9ff2746f69cb95a1d6aad0c25b Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Wed, 11 Dec 2024 16:55:32 +0800
Subject: [PATCH 273/399] [fix](function) fixed some nested type func's param
 type which is not suitable and make result wrong (#44923)

nereids funcs signature for some nested type func's param type defined
not right which will make result wrong such as
```
mysql> SELECT array_position([1,258],257),array_position([2],258);
+-------------------------------+-------------------------------------------+
| array_position([1, 258], 257) | array_position([2], cast(258 as TINYINT)) |
+-------------------------------+-------------------------------------------+
|                             0 |                                         1 |
+-------------------------------+-------------------------------------------+
1 row in set (0.14 sec)

mysql> select array_apply([258], '>' , 257), array_apply([1,2,3], '>', 258);
+------------------------------+---------------------------------------------------+
| array_apply([258], '>', 257) | array_apply([1, 2, 3], '>', cast(258 as TINYINT)) |
+------------------------------+---------------------------------------------------+
| [258]                        | [3]                                               |
+------------------------------+---------------------------------------------------+
1 row in set (0.11 sec)

mysql>  select array_contains([258], 257), array_contains([1,2,3], 258);
+----------------------------+-------------------------------------------------+
| array_contains([258], 257) | array_contains([1, 2, 3], cast(258 as TINYINT)) |
+----------------------------+-------------------------------------------------+
|                          0 |                                               1 |
+----------------------------+-------------------------------------------------+
1 row in set (0.12 sec)

mysql>
mysql> select array_pushfront([258], 257), array_pushfront([1,2,3], 258);
+-----------------------------+--------------------------------------------------+
| array_pushfront([258], 257) | array_pushfront([1, 2, 3], cast(258 as TINYINT)) |
+-----------------------------+--------------------------------------------------+
| [257, 258]                  | [2, 1, 2, 3]                                     |
+-----------------------------+--------------------------------------------------+
1 row in set (0.12 sec)

mysql> select array_pushback([1,258], 257), array_pushback([1,2,3], 258);
+-------------------------------+-------------------------------------------------+
| array_pushback([1, 258], 257) | array_pushback([1, 2, 3], cast(258 as TINYINT)) |
+-------------------------------+-------------------------------------------------+
| [1, 258, 257]                 | [1, 2, 3, 2]                                    |
+-------------------------------+-------------------------------------------------+
1 row in set (0.10 sec)

mysql> select array_remove([1,258], 257), array_remove([1,2,3], 258);
+-----------------------------+-----------------------------------------------+
| array_remove([1, 258], 257) | array_remove([1, 2, 3], cast(258 as TINYINT)) |
+-----------------------------+-----------------------------------------------+
| [1, 258]                    | [1, 3]                                        |
+-----------------------------+-----------------------------------------------+
1 row in set (0.12 sec)
mysql> select map_contains_key(map(1,258), 257), map_contains_key(map(1,2), 258);
+-----------------------------------------------------+---------------------------------------------------+
| map_contains_key(map(1, 258), cast(257 as TINYINT)) | map_contains_key(map(1, 2), cast(258 as TINYINT)) |
+-----------------------------------------------------+---------------------------------------------------+
|                                                   1 |                                                 0 |
+-----------------------------------------------------+---------------------------------------------------+
1 row in set (0.12 sec)

mysql> select map_contains_key(map(1,258), 257), map_contains_key(map(1,3), 258);
+-----------------------------------------------------+---------------------------------------------------+
| map_contains_key(map(1, 258), cast(257 as TINYINT)) | map_contains_key(map(1, 3), cast(258 as TINYINT)) |
+-----------------------------------------------------+---------------------------------------------------+
|                                                   1 |                                                 0 |
+-----------------------------------------------------+---------------------------------------------------+
1 row in set (0.11 sec)

mysql> select map_contains_key(map(1,258), 257), map_contains_key(map(3,1), 258);
+-----------------------------------------------------+---------------------------------------------------+
| map_contains_key(map(1, 258), cast(257 as TINYINT)) | map_contains_key(map(3, 1), cast(258 as TINYINT)) |
+-----------------------------------------------------+---------------------------------------------------+
|                                                   1 |                                                 0 |
+-----------------------------------------------------+---------------------------------------------------+
```
but true result is
```
mysql> SELECT array_position([1,258],257),array_position([2],258);
+-------------------------------+---------------------------------------------------+
| array_position([1, 258], 257) | array_position(cast([2] as ARRAY<SMALLINT>), 258) |
+-------------------------------+---------------------------------------------------+
|                             0 |                                                 0 |
+-------------------------------+---------------------------------------------------+
1 row in set (0.73 sec)

mysql> select array_apply([258], '>' , 257), array_apply([1,2,3], '>', 258);
+------------------------------+-----------------------------------------------------------+
| array_apply([258], '>', 257) | array_apply(cast([1, 2, 3] as ARRAY<SMALLINT>), '>', 258) |
+------------------------------+-----------------------------------------------------------+
| [258]                        | []                                                        |
+------------------------------+-----------------------------------------------------------+
1 row in set (0.10 sec)

mysql> select array_contains([258], 257), array_contains([1,2,3], 258);
+----------------------------+---------------------------------------------------------+
| array_contains([258], 257) | array_contains(cast([1, 2, 3] as ARRAY<SMALLINT>), 258) |
+----------------------------+---------------------------------------------------------+
|                          0 |                                                       0 |
+----------------------------+---------------------------------------------------------+
1 row in set (0.09 sec)

mysql> select array_pushfront([258], 257), array_pushfront([1,2,3], 258);
+-----------------------------+----------------------------------------------------------+
| array_pushfront([258], 257) | array_pushfront(cast([1, 2, 3] as ARRAY<SMALLINT>), 258) |
+-----------------------------+----------------------------------------------------------+
| [257, 258]                  | [258, 1, 2, 3]                                           |
+-----------------------------+----------------------------------------------------------+
1 row in set (0.11 sec)

mysql> select array_remove([1,258], 257), array_remove([1,2,3], 258);
+-----------------------------+-------------------------------------------------------+
| array_remove([1, 258], 257) | array_remove(cast([1, 2, 3] as ARRAY<SMALLINT>), 258) |
+-----------------------------+-------------------------------------------------------+
| [1, 258]                    | [1, 2, 3]                                             |
+-----------------------------+-------------------------------------------------------+
1 row in set (0.09 sec)

mysql> select countequal([1,258], 257), array_count_equal([1,2,3], 258);
ERROR 1105 (HY000): errCode = 2, detailMessage = Can not found function 'array_count_equal'
mysql> select countequal([1,258], 257), countequal([1,2,3], 258);
+---------------------------+-----------------------------------------------------+
| countequal([1, 258], 257) | countequal(cast([1, 2, 3] as ARRAY<SMALLINT>), 258) |
+---------------------------+-----------------------------------------------------+
|                         0 |                                                   0 |
+---------------------------+-----------------------------------------------------+
1 row in set (0.08 sec)

mysql> select map_contains_key(map(1,258), 257), map_contains_key(map(2,1), 258);
+--------------------------------------------------------------------+-----------------------------------------------------------------+
| map_contains_key(cast(map(1, 258) as MAP<SMALLINT,SMALLINT>), 257) | map_contains_key(cast(map(2, 1) as MAP<SMALLINT,TINYINT>), 258) |
+--------------------------------------------------------------------+-----------------------------------------------------------------+
|                                                                  0 |                                                               0 |
+--------------------------------------------------------------------+-----------------------------------------------------------------+
1 row in set (0.09 sec)

mysql> select map_contains_value(map(1,1), 257), map_contains_value(map(1,2), 258);
+-------------------------------------------------------------------+-------------------------------------------------------------------+
| map_contains_value(cast(map(1, 1) as MAP<TINYINT,SMALLINT>), 257) | map_contains_value(cast(map(1, 2) as MAP<TINYINT,SMALLINT>), 258) |
+-------------------------------------------------------------------+-------------------------------------------------------------------+
|                                                                 0 |                                                                 0 |
+--------------------------------------------------
```
---
 .../functions/scalar/ArrayApply.java          | 16 +++++++--
 .../functions/scalar/ArrayContains.java       | 15 ++++++--
 .../functions/scalar/ArrayPosition.java       | 16 +++++++--
 .../functions/scalar/ArrayPushBack.java       | 16 +++++++--
 .../functions/scalar/ArrayPushFront.java      | 16 +++++++--
 .../functions/scalar/ArrayRemove.java         | 16 +++++++--
 .../functions/scalar/CountEqual.java          | 19 ++++++++--
 .../functions/scalar/MapContainsKey.java      | 17 +++++++--
 .../functions/scalar/MapContainsValue.java    | 17 +++++++--
 .../apache/doris/nereids/types/DataType.java  | 35 +++++++++++++++++++
 .../scalar_function/Array.out                 | 27 ++++++++++++++
 .../scalar_function/Array.groovy              | 17 +++++++++
 12 files changed, 209 insertions(+), 18 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java
index 86138b82cac170..fea589a7e14fd7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java
@@ -41,11 +41,16 @@
  */
 public class ArrayApply extends ScalarFunction
         implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable {
-    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+    public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
             FunctionSignature.retArgType(0)
                     .args(ArrayType.of(new AnyDataType(0)), VarcharType.SYSTEM_DEFAULT,
                             new FollowToAnyDataType(0)));
 
+    public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES = ImmutableList.of(
+            FunctionSignature.retArgType(0)
+                    .args(ArrayType.of(new AnyDataType(0)), VarcharType.SYSTEM_DEFAULT,
+                            new AnyDataType(0)));
+
     /**
      * constructor
      */
@@ -93,6 +98,13 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public List<FunctionSignature> getSignatures() {
-        return SIGNATURES;
+        if (getArgument(0).getDataType().isArrayType()
+                &&
+                ((ArrayType) getArgument(0).getDataType()).getItemType()
+                        .isSameTypeForComplexTypeParam(getArgument(2).getDataType())) {
+            // return least common type
+            return MIN_COMMON_TYPE_SIGNATURES;
+        }
+        return FOLLOW_DATATYPE_SIGNATURE;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayContains.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayContains.java
index 494a4a863c528d..4eee7066584da2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayContains.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayContains.java
@@ -38,10 +38,14 @@
 public class ArrayContains extends ScalarFunction
         implements BinaryExpression, ExplicitlyCastableSignature {
 
-    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+    public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
             FunctionSignature.ret(BooleanType.INSTANCE)
                     .args(ArrayType.of(new AnyDataType(0)), new FollowToAnyDataType(0))
     );
+    public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BooleanType.INSTANCE)
+                    .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+    );
 
     /**
      * constructor with 2 arguments.
@@ -71,6 +75,13 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public List<FunctionSignature> getSignatures() {
-        return SIGNATURES;
+        if (getArgument(0).getDataType().isArrayType()
+                &&
+                ((ArrayType) getArgument(0).getDataType()).getItemType()
+                        .isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+            // return least common type
+            return MIN_COMMON_TYPE_SIGNATURES;
+        }
+        return FOLLOW_DATATYPE_SIGNATURE;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPosition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPosition.java
index 3a94e84d51159e..b782a560719c5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPosition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPosition.java
@@ -38,11 +38,16 @@
 public class ArrayPosition extends ScalarFunction
         implements BinaryExpression, ExplicitlyCastableSignature {
 
-    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+    public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
             FunctionSignature.ret(BigIntType.INSTANCE)
                     .args(ArrayType.of(new AnyDataType(0)), new FollowToAnyDataType(0))
     );
 
+    public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BigIntType.INSTANCE)
+                    .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+    );
+
     /**
      * constructor with 2 arguments.
      */
@@ -71,6 +76,13 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public List<FunctionSignature> getSignatures() {
-        return SIGNATURES;
+        if (getArgument(0).getDataType().isArrayType()
+                &&
+                ((ArrayType) getArgument(0).getDataType()).getItemType()
+                        .isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+            // return least common type
+            return MIN_COMMON_TYPE_SIGNATURES;
+        }
+        return FOLLOW_DATATYPE_SIGNATURE;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushBack.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushBack.java
index 932c2446132903..a96c48c4b274a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushBack.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushBack.java
@@ -38,11 +38,16 @@
 public class ArrayPushBack extends ScalarFunction
         implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable {
 
-    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+    public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
             FunctionSignature.retArgType(0)
                     .args(ArrayType.of(new AnyDataType(0)), new FollowToAnyDataType(0))
     );
 
+    public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES = ImmutableList.of(
+            FunctionSignature.retArgType(0)
+                    .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+    );
+
     /**
      * constructor with 1 argument.
      */
@@ -66,6 +71,13 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public List<FunctionSignature> getSignatures() {
-        return SIGNATURES;
+        if (getArgument(0).getDataType().isArrayType()
+                &&
+                ((ArrayType) getArgument(0).getDataType()).getItemType()
+                        .isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+            // return least common type
+            return MIN_COMMON_TYPE_SIGNATURES;
+        }
+        return FOLLOW_DATATYPE_SIGNATURE;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushFront.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushFront.java
index 26e1cdd91e3d21..e20fea5cc20543 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushFront.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushFront.java
@@ -38,11 +38,16 @@
 public class ArrayPushFront extends ScalarFunction
         implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable {
 
-    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+    public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
             FunctionSignature.retArgType(0)
                     .args(ArrayType.of(new AnyDataType(0)), new FollowToAnyDataType(0))
     );
 
+    public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES = ImmutableList.of(
+            FunctionSignature.retArgType(0)
+                    .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+    );
+
     /**
      * constructor with 1 argument.
      */
@@ -66,6 +71,13 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public List<FunctionSignature> getSignatures() {
-        return SIGNATURES;
+        if (getArgument(0).getDataType().isArrayType()
+                &&
+                ((ArrayType) getArgument(0).getDataType()).getItemType()
+                        .isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+            // return least common type
+            return MIN_COMMON_TYPE_SIGNATURES;
+        }
+        return FOLLOW_DATATYPE_SIGNATURE;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayRemove.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayRemove.java
index acf731b8bb46c6..f5996776867dec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayRemove.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayRemove.java
@@ -38,11 +38,16 @@
 public class ArrayRemove extends ScalarFunction
         implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable {
 
-    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+    public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
             FunctionSignature.retArgType(0).args(
                     ArrayType.of(new AnyDataType(0)), new FollowToAnyDataType(0))
     );
 
+    public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES = ImmutableList.of(
+            FunctionSignature.retArgType(0).args(
+                    ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+    );
+
     /**
      * constructor with 2 arguments.
      */
@@ -66,6 +71,13 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public List<FunctionSignature> getSignatures() {
-        return SIGNATURES;
+        if (getArgument(0).getDataType().isArrayType()
+                &&
+                ((ArrayType) getArgument(0).getDataType()).getItemType()
+                        .isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+            // return least common type
+            return MIN_COMMON_TYPE_SIGNATURES;
+        }
+        return FOLLOW_DATATYPE_SIGNATURE;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CountEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CountEqual.java
index 2ba0c9072435a7..bb76aecde31f07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CountEqual.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CountEqual.java
@@ -38,11 +38,16 @@
 public class CountEqual extends ScalarFunction
         implements BinaryExpression, ExplicitlyCastableSignature {
 
-    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+    public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
             FunctionSignature.ret(BigIntType.INSTANCE)
                     .args(ArrayType.of(new AnyDataType(0)), new FollowToAnyDataType(0))
     );
 
+    public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BigIntType.INSTANCE)
+                    .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+    );
+
     /**
      * constructor with 2 arguments.
      */
@@ -71,6 +76,16 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public List<FunctionSignature> getSignatures() {
-        return SIGNATURES;
+        // to find out element type in array vs param type,
+        // if they are different, return first array element type,
+        // else return least common type between element type and param
+        if (getArgument(0).getDataType().isArrayType()
+                &&
+                ((ArrayType) getArgument(0).getDataType()).getItemType()
+                        .isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+            // return least common type
+            return MIN_COMMON_TYPE_SIGNATURES;
+        }
+        return FOLLOW_DATATYPE_SIGNATURE;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
index b489bd47963497..798101e1a2bde5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
@@ -38,12 +38,18 @@
 public class MapContainsKey extends ScalarFunction
         implements BinaryExpression, ExplicitlyCastableSignature {
 
-    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+    public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
             FunctionSignature.ret(BooleanType.INSTANCE)
                     .args(MapType.of(new AnyDataType(0), AnyDataType.INSTANCE_WITHOUT_INDEX),
                             new FollowToAnyDataType(0))
     );
 
+    public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BooleanType.INSTANCE)
+                    .args(MapType.of(new AnyDataType(0), AnyDataType.INSTANCE_WITHOUT_INDEX),
+                            new AnyDataType(0))
+    );
+
     /**
      * constructor with 2 arguments.
      */
@@ -72,6 +78,13 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public List<FunctionSignature> getSignatures() {
-        return SIGNATURES;
+        if (getArgument(0).getDataType().isMapType()
+                &&
+                ((MapType) getArgument(0).getDataType()).getKeyType()
+                        .isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+            // return least common type
+            return MIN_COMMON_TYPE_SIGNATURES;
+        }
+        return FOLLOW_DATATYPE_SIGNATURE;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
index de77386c889902..0a8bcdc49c5e71 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
@@ -38,12 +38,18 @@
 public class MapContainsValue extends ScalarFunction
         implements BinaryExpression, ExplicitlyCastableSignature {
 
-    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+    public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
             FunctionSignature.ret(BooleanType.INSTANCE)
                     .args(MapType.of(AnyDataType.INSTANCE_WITHOUT_INDEX, new AnyDataType(0)),
                             new FollowToAnyDataType(0))
     );
 
+    public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BooleanType.INSTANCE)
+                    .args(MapType.of(AnyDataType.INSTANCE_WITHOUT_INDEX, new AnyDataType(0)),
+                            new AnyDataType(0))
+    );
+
     /**
      * constructor with 2 arguments.
      */
@@ -72,6 +78,13 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
 
     @Override
     public List<FunctionSignature> getSignatures() {
-        return SIGNATURES;
+        if (getArgument(0).getDataType().isMapType()
+                &&
+                ((MapType) getArgument(0).getDataType()).getValueType()
+                        .isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+            // return least common type
+            return MIN_COMMON_TYPE_SIGNATURES;
+        }
+        return FOLLOW_DATATYPE_SIGNATURE;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
index 1d5f69c5366e0d..9b77017f6de8cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
@@ -644,6 +644,41 @@ public DataType promotion() {
         }
     }
 
+    /**
+     * whether the param dataType is same-like type for nested in complex type
+     *  same-like type means: string-like, date-like, number type
+     */
+    public boolean isSameTypeForComplexTypeParam(DataType paramType) {
+        if (this.isArrayType() && paramType.isArrayType()) {
+            return ((ArrayType) this).getItemType()
+                    .isSameTypeForComplexTypeParam(((ArrayType) paramType).getItemType());
+        } else if (this.isMapType() && paramType.isMapType()) {
+            MapType thisMapType = (MapType) this;
+            MapType otherMapType = (MapType) paramType;
+            return thisMapType.getKeyType().isSameTypeForComplexTypeParam(otherMapType.getKeyType())
+                    && thisMapType.getValueType().isSameTypeForComplexTypeParam(otherMapType.getValueType());
+        } else if (this.isStructType() && paramType.isStructType()) {
+            StructType thisStructType = (StructType) this;
+            StructType otherStructType = (StructType) paramType;
+            if (thisStructType.getFields().size() != otherStructType.getFields().size()) {
+                return false;
+            }
+            for (int i = 0; i < thisStructType.getFields().size(); i++) {
+                if (!thisStructType.getFields().get(i).getDataType().isSameTypeForComplexTypeParam(
+                        otherStructType.getFields().get(i).getDataType())) {
+                    return false;
+                }
+            }
+            return true;
+        } else if (this.isStringLikeType() && paramType.isStringLikeType()) {
+            return true;
+        } else if (this.isDateLikeType() && paramType.isDateLikeType()) {
+            return true;
+        } else {
+            return this.isNumericType() && paramType.isNumericType();
+        }
+    }
+
     /** getAllPromotions */
     public List<DataType> getAllPromotions() {
         if (this instanceof ArrayType) {
diff --git a/regression-test/data/nereids_function_p0/scalar_function/Array.out b/regression-test/data/nereids_function_p0/scalar_function/Array.out
index 9570ac178d7ab7..b2e11fab37e47a 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/Array.out
+++ b/regression-test/data/nereids_function_p0/scalar_function/Array.out
@@ -16896,3 +16896,30 @@ false
 -- !sql_array_match_all_18 --
 true
 
+-- !sql --
+0	0
+
+-- !sql --
+[258]	[]
+
+-- !sql --
+false	false
+
+-- !sql --
+[257, 258]	[258, 1, 2, 3]
+
+-- !sql --
+[1, 258, 257]	[1, 2, 3, 258]
+
+-- !sql --
+[1, 258]	[1, 2, 3]
+
+-- !sql --
+0	0
+
+-- !sql --
+false	false
+
+-- !sql --
+false	false
+
diff --git a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
index defa553279c84e..7178ef4436e94b 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
+++ b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
@@ -1407,4 +1407,21 @@ suite("nereids_scalar_fn_Array") {
 	qt_sql_array_match_any_18 "select array_match_any(x->x=2, array())"
 	qt_sql_array_match_all_18 "select array_match_all(x->x=2, array())"
 
+
+    // tests for nereids array functions for number overflow cases
+    qt_sql """ SELECT array_position([1,258],257),array_position([2],258);"""
+    qt_sql """ select array_apply([258], '>' , 257), array_apply([1,2,3], '>', 258);"""
+    qt_sql """ select array_contains([258], 257), array_contains([1,2,3], 258);"""
+    // pushfront and pushback
+    qt_sql """ select array_pushfront([258], 257), array_pushfront([1,2,3], 258);"""
+    qt_sql """ select array_pushback([1,258], 257), array_pushback([1,2,3], 258);"""
+    // array_remove
+    qt_sql """ select array_remove([1,258], 257), array_remove([1,2,3], 258);"""
+    // countequal
+    qt_sql """ select countequal([1,258], 257), countequal([1,2,3], 258);"""
+    // map_contains_key
+    qt_sql """ select map_contains_key(map(1,258), 257), map_contains_key(map(2,1), 258);"""
+    // map_contains_value
+    qt_sql """ select map_contains_value(map(1,1), 257), map_contains_value(map(1,2), 258);"""
+
 }

From 7a45997d9c299bf55230c4f97fae7fadf78a6203 Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Wed, 11 Dec 2024 17:24:48 +0800
Subject: [PATCH 274/399] [enhance](nereids) remove first_value and
 second_value second parameter if the second parameter is false (#45264)

### What problem does this PR solve?
Problem Summary: remove first_value and second_value second parameter if the second parameter is false
---
 .../rules/analysis/WindowFunctionChecker.java | 14 ++++++--
 ...CheckAndStandardizeWindowFunctionTest.java | 26 +++++++++++++++
 .../nereids_syntax_p0/window_function.out     | 32 +++++++++++++++++++
 .../nereids_syntax_p0/window_function.groovy  |  3 ++
 4 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/WindowFunctionChecker.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/WindowFunctionChecker.java
index d6904ae074c7da..2da28269fd711c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/WindowFunctionChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/WindowFunctionChecker.java
@@ -317,9 +317,15 @@ public Lead visitLead(Lead lead, Void ctx) {
     @Override
     public FirstOrLastValue visitFirstValue(FirstValue firstValue, Void ctx) {
         FirstOrLastValue.checkSecondParameter(firstValue);
-        if (2 == firstValue.arity() && firstValue.child(1).equals(BooleanLiteral.TRUE)) {
-            return firstValue;
+        if (2 == firstValue.arity()) {
+            if (firstValue.child(1).equals(BooleanLiteral.TRUE)) {
+                return firstValue;
+            } else {
+                firstValue = (FirstValue) firstValue.withChildren(firstValue.child(0));
+                windowExpression = windowExpression.withFunction(firstValue);
+            }
         }
+
         Optional<WindowFrame> windowFrame = windowExpression.getWindowFrame();
         if (windowFrame.isPresent()) {
             WindowFrame wf = windowFrame.get();
@@ -347,6 +353,10 @@ public FirstOrLastValue visitFirstValue(FirstValue firstValue, Void ctx) {
     @Override
     public FirstOrLastValue visitLastValue(LastValue lastValue, Void ctx) {
         FirstOrLastValue.checkSecondParameter(lastValue);
+        if (2 == lastValue.arity() && lastValue.child(1).equals(BooleanLiteral.FALSE)) {
+            lastValue = (LastValue) lastValue.withChildren(lastValue.child(0));
+            windowExpression = windowExpression.withFunction(lastValue);
+        }
         return lastValue;
     }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CheckAndStandardizeWindowFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CheckAndStandardizeWindowFunctionTest.java
index ffe88af2fea0ba..2f3b133fdeb1ea 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CheckAndStandardizeWindowFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CheckAndStandardizeWindowFunctionTest.java
@@ -28,11 +28,14 @@
 import org.apache.doris.nereids.trees.expressions.WindowFrame.FrameBoundary;
 import org.apache.doris.nereids.trees.expressions.WindowFrame.FrameUnitsType;
 import org.apache.doris.nereids.trees.expressions.functions.window.DenseRank;
+import org.apache.doris.nereids.trees.expressions.functions.window.FirstValue;
 import org.apache.doris.nereids.trees.expressions.functions.window.Lag;
+import org.apache.doris.nereids.trees.expressions.functions.window.LastValue;
 import org.apache.doris.nereids.trees.expressions.functions.window.Lead;
 import org.apache.doris.nereids.trees.expressions.functions.window.Rank;
 import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber;
 import org.apache.doris.nereids.trees.expressions.functions.window.WindowFunction;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
 import org.apache.doris.nereids.trees.plans.Plan;
@@ -240,6 +243,29 @@ public void testCheckWindowFrameBeforeFunc5() {
         forCheckWindowFrameBeforeFunc(windowFrame2, errorMsg2);
     }
 
+    @Test
+    public void testFirstValueRewrite() {
+        age = rStudent.getOutput().get(3).toSlot();
+        WindowExpression window = new WindowExpression(new FirstValue(age, BooleanLiteral.FALSE), partitionKeyList, orderKeyList);
+        Alias windowAlias = new Alias(window, window.toSql());
+        WindowExpression windowLastValue = new WindowExpression(new LastValue(age, BooleanLiteral.FALSE), partitionKeyList, orderKeyList);
+        Alias windowLastValueAlias = new Alias(windowLastValue, windowLastValue.toSql());
+        List<NamedExpression> outputExpressions = Lists.newArrayList(windowAlias, windowLastValueAlias);
+        Plan root = new LogicalWindow<>(outputExpressions, rStudent);
+
+        PlanChecker.from(MemoTestUtils.createConnectContext(), root)
+                .applyTopDown(new ExtractAndNormalizeWindowExpression())
+                .applyTopDown(new CheckAndStandardizeWindowFunctionAndFrame())
+                .matches(
+                        logicalWindow()
+                                .when(logicalWindow -> {
+                                    WindowExpression newWindowFirstValue = (WindowExpression) logicalWindow.getWindowExpressions().get(0).child(0);
+                                    WindowExpression newWindowLastValue = (WindowExpression) logicalWindow.getWindowExpressions().get(0).child(0);
+                                    return newWindowFirstValue.getFunction().arity() == 1 && newWindowLastValue.getFunction().arity() == 1;
+                                })
+                );
+    }
+
     private void forCheckWindowFrameBeforeFunc(WindowFrame windowFrame, String errorMsg) {
         WindowExpression window = new WindowExpression(new Rank(), partitionKeyList, orderKeyList, windowFrame);
         forCheckWindowFrameBeforeFunc(window, errorMsg);
diff --git a/regression-test/data/nereids_syntax_p0/window_function.out b/regression-test/data/nereids_syntax_p0/window_function.out
index 38eba68274e18f..378524125c9474 100644
--- a/regression-test/data/nereids_syntax_p0/window_function.out
+++ b/regression-test/data/nereids_syntax_p0/window_function.out
@@ -567,3 +567,35 @@
 -- !multi_winf2 --
 1	35
 
+-- !first_value_false --
+1
+1
+1
+1
+1
+1
+1
+1
+2
+2
+2
+2
+2
+2
+
+-- !last_value_false --
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+
diff --git a/regression-test/suites/nereids_syntax_p0/window_function.groovy b/regression-test/suites/nereids_syntax_p0/window_function.groovy
index f2ce708f4c372a..bb19aba17c10fc 100644
--- a/regression-test/suites/nereids_syntax_p0/window_function.groovy
+++ b/regression-test/suites/nereids_syntax_p0/window_function.groovy
@@ -314,4 +314,7 @@ suite("window_function") {
     sql "select last_value(c1,false) over() from window_test"
     sql "select first_value(c1,1) over() from window_test"
     sql "select last_value(c1,0) over() from window_test"
+
+    qt_first_value_false "select last_value(c1,false) over(partition by c2 order by c1) from window_test order by 1"
+    qt_last_value_false "select first_value(c1,false) over(partition by c2 order by c1) from window_test order by 1"
 }

From 24eae4b3d0d5bda57664ab1c1e2166cb2b51fbe7 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Wed, 11 Dec 2024 17:47:35 +0800
Subject: [PATCH 275/399] [Job](Fix)Improve Event Publishing with Timeout
 (#45103)

### Summary:
This PR refines the publishEvent method to improve event publishing
reliability by introducing a timeout mechanism and enhanced logging. The
changes allow for a more responsive system when attempting to publish
events to the disruptor, especially in cases where the ring buffer may
not have sufficient capacity at the time.

#### Timeout Implementation:

A 1-second timeout (in nanoseconds) is set, after which the event
publishing attempt will stop if the required capacity is not available.
The timeout is tracked using System.nanoTime() for precise elapsed time
measurement.

#### Remaining Capacity Check:

The method checks if the remainingCapacity() of the ring buffer is
greater than 1 (this can be adjusted based on your capacity
requirements). If enough capacity is available, the event is published;
otherwise, it waits and retries.
---
 .../doris/job/disruptor/ExecuteTaskEvent.java |  5 ++
 .../doris/job/disruptor/TaskDisruptor.java    | 48 +++++++---
 .../executor/DefaultTaskExecutorHandler.java  | 40 +++------
 .../job/executor/DispatchTaskHandler.java     |  7 +-
 .../doris/job/executor/TaskProcessor.java     | 87 +++++++++++++++++++
 .../manager/TaskDisruptorGroupManager.java    | 48 +++-------
 6 files changed, 158 insertions(+), 77 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/job/executor/TaskProcessor.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/ExecuteTaskEvent.java b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/ExecuteTaskEvent.java
index 3d8f9ed15349ae..ac67715d2ca18e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/ExecuteTaskEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/ExecuteTaskEvent.java
@@ -34,4 +34,9 @@ public static <T extends AbstractTask> EventFactory<ExecuteTaskEvent<T>> factory
         return ExecuteTaskEvent::new;
     }
 
+    public void clear() {
+        this.task = null;
+        this.jobConfig = null;
+    }
+
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java
index 2b2e3df0418dd3..9fb9d94e8df70f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java
@@ -19,7 +19,6 @@
 
 import com.lmax.disruptor.EventFactory;
 import com.lmax.disruptor.EventTranslatorVararg;
-import com.lmax.disruptor.RingBuffer;
 import com.lmax.disruptor.WaitStrategy;
 import com.lmax.disruptor.WorkHandler;
 import com.lmax.disruptor.dsl.Disruptor;
@@ -28,6 +27,7 @@
 import org.apache.logging.log4j.Logger;
 
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Utility class for creating and managing a Disruptor instance.
@@ -73,20 +73,42 @@ public void start() {
      */
     public boolean publishEvent(Object... args) {
         try {
-            RingBuffer<T> ringBuffer = disruptor.getRingBuffer();
-            // Check if the RingBuffer has enough capacity to reserve 10 slots for tasks
-            // If there is insufficient capacity (less than 10 slots available)
-            // log a warning and drop the current task
-            if (!ringBuffer.hasAvailableCapacity(10)) {
-                LOG.warn("ring buffer has no available capacity,task will be dropped,"
-                        + "please check the task queue size.");
-                return false;
+            // Set the timeout to 1 second, converted to nanoseconds for precision
+            long timeoutInNanos = TimeUnit.SECONDS.toNanos(1);  // Timeout set to 1 second
+            long startTime = System.nanoTime();  // Record the start time
+
+            // Loop until the timeout is reached
+            while (System.nanoTime() - startTime < timeoutInNanos) {
+                // Check if there is enough remaining capacity in the ring buffer
+                // Adjusting to check if the required capacity is available (instead of hardcoding 1)
+                if (disruptor.getRingBuffer().remainingCapacity() > 1) {
+                    // Publish the event if there is enough capacity
+                    disruptor.getRingBuffer().publishEvent(eventTranslator, args);
+                    if (LOG.isDebugEnabled()) {
+                        LOG.debug("publishEvent success,the remaining buffer size is {}",
+                                disruptor.getRingBuffer().remainingCapacity());
+                    }
+                    return true;
+                }
+
+                // Wait for a short period before retrying
+                try {
+                    Thread.sleep(10);  // Adjust the wait time as needed (maybe increase if not high-frequency)
+                } catch (InterruptedException e) {
+                    // Log the exception and return false if interrupted
+                    Thread.currentThread().interrupt();  // Restore interrupt status
+                    LOG.warn("Thread interrupted while waiting to publish event", e);
+                    return false;
+                }
             }
-            ringBuffer.publishEvent(eventTranslator, args);
-            return true;
+
+            // Timeout reached without publishing the event
+            LOG.warn("Failed to publish event within the specified timeout (1 second)."
+                            + "Queue may be full. the remaining buffer size is {}",
+                    disruptor.getRingBuffer().remainingCapacity());
         } catch (Exception e) {
-            LOG.warn("Failed to publish event", e);
-            // Handle the exception, e.g., retry or alert
+            // Catching general exceptions to handle unexpected errors
+            LOG.warn("Failed to publish event due to an unexpected error", e);
         }
         return false;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java
index befa8cc35fcbcc..cdfe7c0fe08f63 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java
@@ -36,35 +36,23 @@ public class DefaultTaskExecutorHandler<T extends AbstractTask> implements WorkH
 
     @Override
     public void onEvent(ExecuteTaskEvent<T> executeTaskEvent) {
-        T task = executeTaskEvent.getTask();
-        if (null == task) {
-            log.warn("task is null, ignore,maybe task has been canceled");
-            return;
-        }
-        if (task.isCancelled()) {
-            log.info("task is canceled, ignore. task id is {}", task.getTaskId());
-            return;
-        }
-        log.info("start to execute task, task id is {}", task.getTaskId());
-        try {
-            task.runTask();
-        } catch (Exception e) {
-            //if task.onFail() throw exception, we will catch it here
-            log.warn("task before error, task id is {}", task.getTaskId(), e);
-        }
-        //todo we need discuss whether we need to use semaphore to control the concurrent task num
-        /* Semaphore semaphore = null;
-        // get token
         try {
-            int maxConcurrentTaskNum = executeTaskEvent.getJobConfig().getMaxConcurrentTaskNum();
-            semaphore = TaskTokenManager.tryAcquire(task.getJobId(), maxConcurrentTaskNum);
+            T task = executeTaskEvent.getTask();
+            if (null == task) {
+                log.warn("task is null, ignore,maybe task has been canceled");
+                return;
+            }
+            if (task.isCancelled()) {
+                log.info("task is canceled, ignore. task id is {}", task.getTaskId());
+                return;
+            }
+            log.info("start to execute task, task id is {}", task.getTaskId());
             task.runTask();
         } catch (Exception e) {
-            task.onFail();
-            log.error("execute task error, task id is {}", task.getTaskId(), e);
+            log.error("execute task error, task id is {}", executeTaskEvent.getTask().getTaskId(), e);
         } finally {
-            if (null != semaphore) {
-                semaphore.release();
-            }*/
+            executeTaskEvent.clear();
+        }
+
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java
index d93393aa0ef89f..b8f726c4a0c76f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java
@@ -21,7 +21,6 @@
 import org.apache.doris.job.common.JobStatus;
 import org.apache.doris.job.common.JobType;
 import org.apache.doris.job.common.TaskType;
-import org.apache.doris.job.disruptor.TaskDisruptor;
 import org.apache.doris.job.disruptor.TimerJobEvent;
 import org.apache.doris.job.task.AbstractTask;
 
@@ -40,9 +39,9 @@
 @Log4j2
 public class DispatchTaskHandler<T extends AbstractJob> implements WorkHandler<TimerJobEvent<T>> {
 
-    private final Map<JobType, TaskDisruptor<T>> disruptorMap;
+    private final Map<JobType, TaskProcessor> disruptorMap;
 
-    public DispatchTaskHandler(Map<JobType, TaskDisruptor<T>> disruptorMap) {
+    public DispatchTaskHandler(Map<JobType, TaskProcessor> disruptorMap) {
         this.disruptorMap = disruptorMap;
     }
 
@@ -66,7 +65,7 @@ public void onEvent(TimerJobEvent<T> event) {
                 }
                 JobType jobType = event.getJob().getJobType();
                 for (AbstractTask task : tasks) {
-                    if (!disruptorMap.get(jobType).publishEvent(task, event.getJob().getJobConfig())) {
+                    if (!disruptorMap.get(jobType).addTask(task)) {
                         task.cancel();
                         continue;
                     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/TaskProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/TaskProcessor.java
new file mode 100644
index 00000000000000..d9d3f25dcd8a80
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/TaskProcessor.java
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.job.executor;
+
+import org.apache.doris.job.task.AbstractTask;
+
+import lombok.extern.log4j.Log4j2;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+@Log4j2
+public class TaskProcessor {
+    private ExecutorService executor;
+
+    public  TaskProcessor(int numberOfThreads, int queueSize, ThreadFactory threadFactory) {
+        this.executor = new ThreadPoolExecutor(
+                numberOfThreads,
+                numberOfThreads,
+                0L, TimeUnit.MILLISECONDS,
+                new ArrayBlockingQueue<>(queueSize),
+                threadFactory,
+                new ThreadPoolExecutor.AbortPolicy()
+        );
+    }
+
+    public boolean addTask(AbstractTask task) {
+        try {
+            executor.execute(() -> runTask(task));
+            log.info("Add task to executor, task id: {}", task.getTaskId());
+            return true;
+        } catch (RejectedExecutionException e) {
+            log.warn("Failed to add task to executor, task id: {}", task.getTaskId(), e);
+            return false;
+        }
+    }
+
+    public void shutdown() {
+        log.info("Shutting down executor service...");
+        executor.shutdown();
+        try {
+            if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
+                executor.shutdownNow();
+            }
+        } catch (InterruptedException e) {
+            executor.shutdownNow();
+            Thread.currentThread().interrupt();
+        }
+        log.info("Executor service shut down successfully.");
+    }
+
+    private void runTask(AbstractTask task) {
+        try {
+            if (task == null) {
+                log.warn("Task is null, ignore. Maybe it has been canceled.");
+                return;
+            }
+            if (task.isCancelled()) {
+                log.info("Task is canceled, ignore. Task id: {}", task.getTaskId());
+                return;
+            }
+            log.info("Start to execute task, task id: {}", task.getTaskId());
+            task.runTask();
+        } catch (Exception e) {
+            log.warn("Execute task error, task id: {}", task.getTaskId(), e);
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskDisruptorGroupManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskDisruptorGroupManager.java
index cc82b59a36a36f..e77dfbadcb3742 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskDisruptorGroupManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskDisruptorGroupManager.java
@@ -22,13 +22,10 @@
 import org.apache.doris.job.base.AbstractJob;
 import org.apache.doris.job.base.JobExecutionConfiguration;
 import org.apache.doris.job.common.JobType;
-import org.apache.doris.job.disruptor.ExecuteTaskEvent;
 import org.apache.doris.job.disruptor.TaskDisruptor;
 import org.apache.doris.job.disruptor.TimerJobEvent;
-import org.apache.doris.job.executor.DefaultTaskExecutorHandler;
 import org.apache.doris.job.executor.DispatchTaskHandler;
-import org.apache.doris.job.extensions.insert.InsertTask;
-import org.apache.doris.job.extensions.mtmv.MTMVTask;
+import org.apache.doris.job.executor.TaskProcessor;
 import org.apache.doris.job.task.AbstractTask;
 
 import com.lmax.disruptor.EventFactory;
@@ -44,7 +41,7 @@
 
 public class TaskDisruptorGroupManager<T extends AbstractTask> {
 
-    private final Map<JobType, TaskDisruptor<T>> disruptorMap = new EnumMap<>(JobType.class);
+    private final Map<JobType, TaskProcessor> disruptorMap = new EnumMap<>(JobType.class);
 
     @Getter
     private TaskDisruptor<TimerJobEvent<AbstractJob>> dispatchDisruptor;
@@ -92,44 +89,27 @@ private void registerDispatchDisruptor() {
     }
 
     private void registerInsertDisruptor() {
-        EventFactory<ExecuteTaskEvent<InsertTask>> insertEventFactory = ExecuteTaskEvent.factory();
         ThreadFactory insertTaskThreadFactory = new CustomThreadFactory("insert-task-execute");
-        WorkHandler[] insertTaskExecutorHandlers = new WorkHandler[DISPATCH_INSERT_THREAD_NUM];
-        for (int i = 0; i < DISPATCH_INSERT_THREAD_NUM; i++) {
-            insertTaskExecutorHandlers[i] = new DefaultTaskExecutorHandler<InsertTask>();
-        }
-        EventTranslatorVararg<ExecuteTaskEvent<InsertTask>> eventTranslator =
-                (event, sequence, args) -> {
-                    event.setTask((InsertTask) args[0]);
-                    event.setJobConfig((JobExecutionConfiguration) args[1]);
-                };
-        TaskDisruptor insertDisruptor = new TaskDisruptor<>(insertEventFactory, DISPATCH_INSERT_TASK_QUEUE_SIZE,
-                insertTaskThreadFactory, new LiteTimeoutBlockingWaitStrategy(10, TimeUnit.MILLISECONDS),
-                insertTaskExecutorHandlers, eventTranslator);
-        disruptorMap.put(JobType.INSERT, insertDisruptor);
+
+
+        TaskProcessor insertTaskProcessor = new TaskProcessor(DISPATCH_INSERT_THREAD_NUM,
+                DISPATCH_INSERT_TASK_QUEUE_SIZE, insertTaskThreadFactory);
+        disruptorMap.put(JobType.INSERT, insertTaskProcessor);
     }
 
     private void registerMTMVDisruptor() {
-        EventFactory<ExecuteTaskEvent<MTMVTask>> mtmvEventFactory = ExecuteTaskEvent.factory();
+
         ThreadFactory mtmvTaskThreadFactory = new CustomThreadFactory("mtmv-task-execute");
-        WorkHandler[] insertTaskExecutorHandlers = new WorkHandler[DISPATCH_MTMV_THREAD_NUM];
-        for (int i = 0; i < DISPATCH_MTMV_THREAD_NUM; i++) {
-            insertTaskExecutorHandlers[i] = new DefaultTaskExecutorHandler<MTMVTask>();
-        }
-        EventTranslatorVararg<ExecuteTaskEvent<MTMVTask>> eventTranslator =
-                (event, sequence, args) -> {
-                    event.setTask((MTMVTask) args[0]);
-                    event.setJobConfig((JobExecutionConfiguration) args[1]);
-                };
-        TaskDisruptor mtmvDisruptor = new TaskDisruptor<>(mtmvEventFactory, DISPATCH_MTMV_TASK_QUEUE_SIZE,
-                mtmvTaskThreadFactory, new LiteTimeoutBlockingWaitStrategy(10, TimeUnit.MILLISECONDS),
-                insertTaskExecutorHandlers, eventTranslator);
-        disruptorMap.put(JobType.MV, mtmvDisruptor);
+        TaskProcessor mtmvTaskProcessor = new TaskProcessor(DISPATCH_MTMV_THREAD_NUM,
+                DISPATCH_MTMV_TASK_QUEUE_SIZE, mtmvTaskThreadFactory);
+        disruptorMap.put(JobType.MV, mtmvTaskProcessor);
     }
 
     public boolean dispatchInstantTask(AbstractTask task, JobType jobType,
                                        JobExecutionConfiguration jobExecutionConfiguration) {
-        return disruptorMap.get(jobType).publishEvent(task, jobExecutionConfiguration);
+
+
+        return disruptorMap.get(jobType).addTask(task);
     }
 
 

From 47b2dddedcbd026a17442b1a5575f6aafb907ce2 Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Wed, 11 Dec 2024 17:48:28 +0800
Subject: [PATCH 276/399] [feat](mtmv)use real snapshot instead of
 optional.empty() (#45273)

In the previous PR, a snapshot of the table was obtained and stored in
the statementContext at the beginning of the query.
The modification of this PR is to ensure that the same metadata is used
during the query process. When calling the relevant interface, snapshot
needs to be obtained from statementContext as a parameter and passed to
the relevant method

Related PR: #44911 #44673
---
 .../doris/mtmv/MTMVPartitionExprDateTrunc.java |  3 ++-
 .../apache/doris/mtmv/MTMVPartitionInfo.java   |  6 ++++--
 .../apache/doris/mtmv/MTMVPartitionUtil.java   | 18 +++++++++---------
 .../MTMVRelatedPartitionDescInitGenerator.java |  5 +++--
 ...TMVRelatedPartitionDescRollUpGenerator.java |  4 ++--
 .../exploration/mv/MaterializedViewUtils.java  |  6 ++++--
 6 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
index 95a8717e01c4c7..764c87c4b115fe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
@@ -27,6 +27,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic;
 import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform;
@@ -69,7 +70,7 @@ public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException
                     String.format("timeUnit not support: %s, only support: %s", this.timeUnit, timeUnits));
         }
         MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable();
-        PartitionType partitionType = relatedTable.getPartitionType(Optional.empty());
+        PartitionType partitionType = relatedTable.getPartitionType(MvccUtil.getSnapshotFromContext(relatedTable));
         if (partitionType == PartitionType.RANGE) {
             Type partitionColumnType = MTMVPartitionUtil
                     .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
index 7eae44db0af4cc..5a14867c7e1ab3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
@@ -21,11 +21,11 @@
 import org.apache.doris.catalog.Column;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.datasource.CatalogMgr;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 
 import com.google.gson.annotations.SerializedName;
 
 import java.util.List;
-import java.util.Optional;
 
 /**
  * MTMVPartitionInfo
@@ -116,7 +116,9 @@ public int getRelatedColPos() throws AnalysisException {
         if (partitionType == MTMVPartitionType.SELF_MANAGE) {
             throw new AnalysisException("partitionType is: " + partitionType);
         }
-        List<Column> partitionColumns = getRelatedTable().getPartitionColumns(Optional.empty());
+        MTMVRelatedTableIf mtmvRelatedTableIf = getRelatedTable();
+        List<Column> partitionColumns = mtmvRelatedTableIf.getPartitionColumns(
+                MvccUtil.getSnapshotFromContext(mtmvRelatedTableIf));
         for (int i = 0; i < partitionColumns.size(); i++) {
             if (partitionColumns.get(i).getName().equalsIgnoreCase(relatedCol)) {
                 return i;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 55ce86e75706c7..92436b063e0b49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -32,6 +32,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
 import org.apache.doris.rpc.RpcException;
 
@@ -50,7 +51,6 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -324,7 +324,7 @@ public static boolean isSyncWithPartitions(MTMVRefreshContext context, String mt
         }
         for (String relatedPartitionName : relatedPartitionNames) {
             MTMVSnapshotIf relatedPartitionCurrentSnapshot = relatedTable
-                    .getPartitionSnapshot(relatedPartitionName, context, Optional.empty());
+                    .getPartitionSnapshot(relatedPartitionName, context, MvccUtil.getSnapshotFromContext(relatedTable));
             if (!mtmv.getRefreshSnapshot()
                     .equalsWithRelatedPartition(mtmvPartitionName, relatedPartitionName,
                             relatedPartitionCurrentSnapshot)) {
@@ -441,7 +441,8 @@ private static boolean isSyncWithBaseTable(MTMVRefreshContext context, String mt
         if (!baseTable.needAutoRefresh()) {
             return true;
         }
-        MTMVSnapshotIf baseTableCurrentSnapshot = baseTable.getTableSnapshot(context, Optional.empty());
+        MTMVSnapshotIf baseTableCurrentSnapshot = baseTable.getTableSnapshot(context,
+                MvccUtil.getSnapshotFromContext(baseTable));
         return mtmv.getRefreshSnapshot()
                 .equalsWithBaseTable(mtmvPartitionName, new BaseTableInfo(baseTable), baseTableCurrentSnapshot);
     }
@@ -476,10 +477,9 @@ private static MTMVRefreshPartitionSnapshot generatePartitionSnapshot(MTMVRefres
         if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) {
             MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable();
             for (String relatedPartitionName : relatedPartitionNames) {
-                MTMVSnapshotIf partitionSnapshot = relatedTable
-                        .getPartitionSnapshot(relatedPartitionName, context, Optional.empty());
-                refreshPartitionSnapshot.getPartitions()
-                        .put(relatedPartitionName, partitionSnapshot);
+                MTMVSnapshotIf partitionSnapshot = relatedTable.getPartitionSnapshot(relatedPartitionName, context,
+                        MvccUtil.getSnapshotFromContext(relatedTable));
+                refreshPartitionSnapshot.getPartitions().put(relatedPartitionName, partitionSnapshot);
             }
         }
         for (BaseTableInfo baseTableInfo : baseTables) {
@@ -492,13 +492,13 @@ private static MTMVRefreshPartitionSnapshot generatePartitionSnapshot(MTMVRefres
                 continue;
             }
             refreshPartitionSnapshot.addTableSnapshot(baseTableInfo,
-                    ((MTMVRelatedTableIf) table).getTableSnapshot(context, Optional.empty()));
+                    ((MTMVRelatedTableIf) table).getTableSnapshot(context, MvccUtil.getSnapshotFromContext(table)));
         }
         return refreshPartitionSnapshot;
     }
 
     public static Type getPartitionColumnType(MTMVRelatedTableIf relatedTable, String col) throws AnalysisException {
-        List<Column> partitionColumns = relatedTable.getPartitionColumns(Optional.empty());
+        List<Column> partitionColumns = relatedTable.getPartitionColumns(MvccUtil.getSnapshotFromContext(relatedTable));
         for (Column column : partitionColumns) {
             if (column.getName().equals(col)) {
                 return column.getType();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
index c6b4e331184e2a..28900f38e59f62 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
@@ -18,9 +18,9 @@
 package org.apache.doris.mtmv;
 
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 
 import java.util.Map;
-import java.util.Optional;
 
 /**
  * get all related partition descs
@@ -30,6 +30,7 @@ public class MTMVRelatedPartitionDescInitGenerator implements MTMVRelatedPartiti
     @Override
     public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String> mvProperties,
             RelatedPartitionDescResult lastResult) throws AnalysisException {
-        lastResult.setItems(mvPartitionInfo.getRelatedTable().getAndCopyPartitionItems(Optional.empty()));
+        MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable();
+        lastResult.setItems(relatedTable.getAndCopyPartitionItems(MvccUtil.getSnapshotFromContext(relatedTable)));
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
index 325fab819d9a09..71f7fc358f5975 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
@@ -21,6 +21,7 @@
 import org.apache.doris.analysis.PartitionValue;
 import org.apache.doris.catalog.PartitionType;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
 
 import com.google.common.base.Preconditions;
@@ -31,7 +32,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -46,7 +46,7 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String> mvPrope
             return;
         }
         MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable();
-        PartitionType partitionType = relatedTable.getPartitionType(Optional.empty());
+        PartitionType partitionType = relatedTable.getPartitionType(MvccUtil.getSnapshotFromContext(relatedTable));
         if (partitionType == PartitionType.RANGE) {
             lastResult.setDescs(rollUpRange(lastResult.getDescs(), mvPartitionInfo));
         } else if (partitionType == PartitionType.LIST) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 4c5703e27687e4..4ddb93409379e9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -24,6 +24,7 @@
 import org.apache.doris.catalog.PartitionType;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.catalog.constraint.TableIdentifier;
+import org.apache.doris.datasource.mvcc.MvccUtil;
 import org.apache.doris.mtmv.BaseTableInfo;
 import org.apache.doris.mtmv.MTMVCache;
 import org.apache.doris.mtmv.MTMVRelatedTableIf;
@@ -497,13 +498,14 @@ public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerConte
                 return null;
             }
             MTMVRelatedTableIf relatedTable = (MTMVRelatedTableIf) table;
-            PartitionType type = relatedTable.getPartitionType(Optional.empty());
+            PartitionType type = relatedTable.getPartitionType(MvccUtil.getSnapshotFromContext(relatedTable));
             if (PartitionType.UNPARTITIONED.equals(type)) {
                 context.addFailReason(String.format("related base table is not partition table, the table is %s",
                         table.getName()));
                 return null;
             }
-            Set<Column> partitionColumnSet = new HashSet<>(relatedTable.getPartitionColumns(Optional.empty()));
+            Set<Column> partitionColumnSet = new HashSet<>(
+                    relatedTable.getPartitionColumns(MvccUtil.getSnapshotFromContext(relatedTable)));
             Column mvReferenceColumn = contextPartitionColumn.getColumn().get();
             Expr definExpr = mvReferenceColumn.getDefineExpr();
             if (definExpr instanceof SlotRef) {

From ff090ff641ce8e96c3db918de84875632f453623 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Wed, 11 Dec 2024 17:55:56 +0800
Subject: [PATCH 277/399] =?UTF-8?q?[refactor](shuffle)=20(PART=20II)=20Spl?=
 =?UTF-8?q?it=20local=20exchange=20operators=20from=20loc=E2=80=A6=20(#452?=
 =?UTF-8?q?80)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…al exchanger

This is a PR which wants to decouple the local exchanger and local
exchange operators.

Since all type of exchangers are similar as exchange sink does, I plan
to use exchanger to split data in both local exchange operators and
exchange sink operators.
---
 be/src/pipeline/dependency.cpp                |   5 +-
 be/src/pipeline/dependency.h                  |   2 +-
 .../local_exchange_sink_operator.cpp          |   5 +-
 .../local_exchange_sink_operator.h            |   1 -
 .../local_exchange_source_operator.cpp        |   8 +-
 .../local_exchange/local_exchanger.cpp        | 367 +++++++++++-------
 .../pipeline/local_exchange/local_exchanger.h | 127 +++---
 7 files changed, 325 insertions(+), 190 deletions(-)

diff --git a/be/src/pipeline/dependency.cpp b/be/src/pipeline/dependency.cpp
index 5fef018423df25..dcf5c7a0a81d7c 100644
--- a/be/src/pipeline/dependency.cpp
+++ b/be/src/pipeline/dependency.cpp
@@ -179,12 +179,11 @@ void LocalExchangeSharedState::sub_running_sink_operators() {
     }
 }
 
-void LocalExchangeSharedState::sub_running_source_operators(
-        LocalExchangeSourceLocalState& local_state) {
+void LocalExchangeSharedState::sub_running_source_operators() {
     std::unique_lock<std::mutex> lc(le_lock);
     if (exchanger->_running_source_operators.fetch_sub(1) == 1) {
         _set_always_ready();
-        exchanger->finalize(local_state);
+        exchanger->finalize();
     }
 }
 
diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h
index ad018c8b4f8f3d..f1cfe2b02977e1 100644
--- a/be/src/pipeline/dependency.h
+++ b/be/src/pipeline/dependency.h
@@ -758,7 +758,7 @@ struct LocalExchangeSharedState : public BasicSharedState {
         }
     }
     void sub_running_sink_operators();
-    void sub_running_source_operators(LocalExchangeSourceLocalState& local_state);
+    void sub_running_source_operators();
     void _set_always_ready() {
         for (auto& dep : source_deps) {
             DCHECK(dep);
diff --git a/be/src/pipeline/local_exchange/local_exchange_sink_operator.cpp b/be/src/pipeline/local_exchange/local_exchange_sink_operator.cpp
index 22007a4b220348..b22ee9fd77e72f 100644
--- a/be/src/pipeline/local_exchange/local_exchange_sink_operator.cpp
+++ b/be/src/pipeline/local_exchange/local_exchange_sink_operator.cpp
@@ -144,7 +144,10 @@ Status LocalExchangeSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
     auto& local_state = get_local_state(state);
     SCOPED_TIMER(local_state.exec_time_counter());
     COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)in_block->rows());
-    RETURN_IF_ERROR(local_state._exchanger->sink(state, in_block, eos, local_state));
+    RETURN_IF_ERROR(local_state._exchanger->sink(
+            state, in_block, eos,
+            {local_state._compute_hash_value_timer, local_state._distribute_timer, nullptr},
+            {&local_state._channel_id, local_state._partitioner.get(), &local_state}));
 
     // If all exchange sources ended due to limit reached, current task should also finish
     if (local_state._exchanger->_running_source_operators == 0) {
diff --git a/be/src/pipeline/local_exchange/local_exchange_sink_operator.h b/be/src/pipeline/local_exchange/local_exchange_sink_operator.h
index 435f7a410a4ca6..c067f023c8d420 100644
--- a/be/src/pipeline/local_exchange/local_exchange_sink_operator.h
+++ b/be/src/pipeline/local_exchange/local_exchange_sink_operator.h
@@ -65,7 +65,6 @@ class LocalExchangeSinkLocalState final : public PipelineXSinkLocalState<LocalEx
     RuntimeProfile::Counter* _compute_hash_value_timer = nullptr;
     RuntimeProfile::Counter* _distribute_timer = nullptr;
     std::unique_ptr<vectorized::PartitionerBase> _partitioner = nullptr;
-    std::vector<uint32_t> _partition_rows_histogram;
 
     // Used by random passthrough exchanger
     int _channel_id = 0;
diff --git a/be/src/pipeline/local_exchange/local_exchange_source_operator.cpp b/be/src/pipeline/local_exchange/local_exchange_source_operator.cpp
index c4832b9958c00d..63e36cdfdb0c01 100644
--- a/be/src/pipeline/local_exchange/local_exchange_source_operator.cpp
+++ b/be/src/pipeline/local_exchange/local_exchange_source_operator.cpp
@@ -61,10 +61,10 @@ Status LocalExchangeSourceLocalState::close(RuntimeState* state) {
     }
 
     if (_exchanger) {
-        _exchanger->close(*this);
+        _exchanger->close({_channel_id, this});
     }
     if (_shared_state) {
-        _shared_state->sub_running_source_operators(*this);
+        _shared_state->sub_running_source_operators();
     }
 
     std::vector<DependencySPtr> {}.swap(_local_merge_deps);
@@ -116,7 +116,9 @@ Status LocalExchangeSourceOperatorX::get_block(RuntimeState* state, vectorized::
                                                bool* eos) {
     auto& local_state = get_local_state(state);
     SCOPED_TIMER(local_state.exec_time_counter());
-    RETURN_IF_ERROR(local_state._exchanger->get_block(state, block, eos, local_state));
+    RETURN_IF_ERROR(local_state._exchanger->get_block(
+            state, block, eos, {nullptr, nullptr, local_state._copy_data_timer},
+            {local_state._channel_id, &local_state}));
     local_state.reached_limit(block, eos);
     return Status::OK();
 }
diff --git a/be/src/pipeline/local_exchange/local_exchanger.cpp b/be/src/pipeline/local_exchange/local_exchanger.cpp
index 647988f8b794cb..a963de8b684310 100644
--- a/be/src/pipeline/local_exchange/local_exchanger.cpp
+++ b/be/src/pipeline/local_exchange/local_exchanger.cpp
@@ -29,8 +29,12 @@ namespace doris::pipeline {
 #include "common/compile_check_begin.h"
 template <typename BlockType>
 void Exchanger<BlockType>::_enqueue_data_and_set_ready(int channel_id,
-                                                       LocalExchangeSinkLocalState& local_state,
+                                                       LocalExchangeSinkLocalState* local_state,
                                                        BlockType&& block) {
+    if (local_state == nullptr) {
+        _enqueue_data_and_set_ready(channel_id, std::move(block));
+        return;
+    }
     size_t allocated_bytes = 0;
     // PartitionedBlock is used by shuffle exchanger.
     // PartitionedBlock will be push into multiple queues with different row ranges, so it will be
@@ -44,47 +48,47 @@ void Exchanger<BlockType>::_enqueue_data_and_set_ready(int channel_id,
         allocated_bytes = block->data_block.allocated_bytes();
     }
     std::unique_lock l(_m);
-    local_state._shared_state->add_mem_usage(channel_id, allocated_bytes,
-                                             !std::is_same_v<PartitionedBlock, BlockType> &&
-                                                     !std::is_same_v<BroadcastBlock, BlockType>);
+    local_state->_shared_state->add_mem_usage(channel_id, allocated_bytes,
+                                              !std::is_same_v<PartitionedBlock, BlockType> &&
+                                                      !std::is_same_v<BroadcastBlock, BlockType>);
     if (_data_queue[channel_id].enqueue(std::move(block))) {
-        local_state._shared_state->set_ready_to_read(channel_id);
+        local_state->_shared_state->set_ready_to_read(channel_id);
     } else {
-        local_state._shared_state->sub_mem_usage(channel_id, allocated_bytes);
+        local_state->_shared_state->sub_mem_usage(channel_id, allocated_bytes);
         // `enqueue(block)` return false iff this queue's source operator is already closed so we
         // just unref the block.
         if constexpr (std::is_same_v<PartitionedBlock, BlockType> ||
                       std::is_same_v<BroadcastBlock, BlockType>) {
-            block.first->unref(local_state._shared_state, allocated_bytes, channel_id);
+            block.first->unref(local_state->_shared_state, allocated_bytes, channel_id);
         } else {
-            block->unref(local_state._shared_state, allocated_bytes, channel_id);
+            block->unref(local_state->_shared_state, allocated_bytes, channel_id);
             DCHECK_EQ(block->ref_value(), 0);
         }
     }
 }
 
 template <typename BlockType>
-bool Exchanger<BlockType>::_dequeue_data(LocalExchangeSourceLocalState& local_state,
-                                         BlockType& block, bool* eos,
-                                         vectorized::Block* data_block) {
-    return _dequeue_data(local_state, block, eos, data_block, local_state._channel_id);
-}
-
-template <typename BlockType>
-bool Exchanger<BlockType>::_dequeue_data(LocalExchangeSourceLocalState& local_state,
+bool Exchanger<BlockType>::_dequeue_data(LocalExchangeSourceLocalState* local_state,
                                          BlockType& block, bool* eos, vectorized::Block* data_block,
                                          int channel_id) {
+    if (local_state == nullptr) {
+        if (!_dequeue_data(block, eos, data_block, channel_id)) {
+            throw Exception(ErrorCode::INTERNAL_ERROR, "Exchanger has no data: {}",
+                            data_queue_debug_string(channel_id));
+        }
+        return true;
+    }
     bool all_finished = _running_sink_operators == 0;
     if (_data_queue[channel_id].try_dequeue(block)) {
         if constexpr (std::is_same_v<PartitionedBlock, BlockType> ||
                       std::is_same_v<BroadcastBlock, BlockType>) {
-            local_state._shared_state->sub_mem_usage(channel_id,
-                                                     block.first->data_block.allocated_bytes());
+            local_state->_shared_state->sub_mem_usage(channel_id,
+                                                      block.first->data_block.allocated_bytes());
         } else {
-            local_state._shared_state->sub_mem_usage(channel_id,
-                                                     block->data_block.allocated_bytes());
+            local_state->_shared_state->sub_mem_usage(channel_id,
+                                                      block->data_block.allocated_bytes());
             data_block->swap(block->data_block);
-            block->unref(local_state._shared_state, data_block->allocated_bytes(), channel_id);
+            block->unref(local_state->_shared_state, data_block->allocated_bytes(), channel_id);
             DCHECK_EQ(block->ref_value(), 0);
         }
         return true;
@@ -95,54 +99,88 @@ bool Exchanger<BlockType>::_dequeue_data(LocalExchangeSourceLocalState& local_st
         if (_data_queue[channel_id].try_dequeue(block)) {
             if constexpr (std::is_same_v<PartitionedBlock, BlockType> ||
                           std::is_same_v<BroadcastBlock, BlockType>) {
-                local_state._shared_state->sub_mem_usage(channel_id,
-                                                         block.first->data_block.allocated_bytes());
+                local_state->_shared_state->sub_mem_usage(
+                        channel_id, block.first->data_block.allocated_bytes());
             } else {
-                local_state._shared_state->sub_mem_usage(channel_id,
-                                                         block->data_block.allocated_bytes());
+                local_state->_shared_state->sub_mem_usage(channel_id,
+                                                          block->data_block.allocated_bytes());
                 data_block->swap(block->data_block);
-                block->unref(local_state._shared_state, data_block->allocated_bytes(), channel_id);
+                block->unref(local_state->_shared_state, data_block->allocated_bytes(), channel_id);
                 DCHECK_EQ(block->ref_value(), 0);
             }
             return true;
         }
-        COUNTER_UPDATE(local_state._get_block_failed_counter, 1);
-        local_state._dependency->block();
+        COUNTER_UPDATE(local_state->_get_block_failed_counter, 1);
+        local_state->_dependency->block();
+    }
+    return false;
+}
+
+template <typename BlockType>
+void Exchanger<BlockType>::_enqueue_data_and_set_ready(int channel_id, BlockType&& block) {
+    if constexpr (!std::is_same_v<PartitionedBlock, BlockType> &&
+                  !std::is_same_v<BroadcastBlock, BlockType>) {
+        block->ref(1);
+    }
+    if (!_data_queue[channel_id].enqueue(std::move(block))) {
+        if constexpr (std::is_same_v<PartitionedBlock, BlockType> ||
+                      std::is_same_v<BroadcastBlock, BlockType>) {
+            block.first->unref();
+        } else {
+            block->unref();
+            DCHECK_EQ(block->ref_value(), 0);
+        }
+    }
+}
+
+template <typename BlockType>
+bool Exchanger<BlockType>::_dequeue_data(BlockType& block, bool* eos, vectorized::Block* data_block,
+                                         int channel_id) {
+    if (_data_queue[channel_id].try_dequeue(block)) {
+        if constexpr (!std::is_same_v<PartitionedBlock, BlockType> &&
+                      !std::is_same_v<BroadcastBlock, BlockType>) {
+            data_block->swap(block->data_block);
+            block->unref();
+            DCHECK_EQ(block->ref_value(), 0);
+        }
+        return true;
     }
     return false;
 }
 
 Status ShuffleExchanger::sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                              LocalExchangeSinkLocalState& local_state) {
+                              Profile&& profile, SinkInfo&& sink_info) {
     if (in_block->empty()) {
         return Status::OK();
     }
     {
-        SCOPED_TIMER(local_state._compute_hash_value_timer);
-        RETURN_IF_ERROR(local_state._partitioner->do_partitioning(state, in_block));
+        SCOPED_TIMER(profile.compute_hash_value_timer);
+        RETURN_IF_ERROR(sink_info.partitioner->do_partitioning(state, in_block));
     }
     {
-        SCOPED_TIMER(local_state._distribute_timer);
-        RETURN_IF_ERROR(_split_rows(state,
-                                    local_state._partitioner->get_channel_ids().get<uint32_t>(),
-                                    in_block, local_state));
+        SCOPED_TIMER(profile.distribute_timer);
+        RETURN_IF_ERROR(_split_rows(state, sink_info.partitioner->get_channel_ids().get<uint32_t>(),
+                                    in_block, *sink_info.channel_id, sink_info.local_state));
     }
 
     return Status::OK();
 }
 
-void ShuffleExchanger::close(LocalExchangeSourceLocalState& local_state) {
+void ShuffleExchanger::close(SourceInfo&& source_info) {
     PartitionedBlock partitioned_block;
     bool eos;
     vectorized::Block block;
-    _data_queue[local_state._channel_id].set_eos();
-    while (_dequeue_data(local_state, partitioned_block, &eos, &block)) {
-        partitioned_block.first->unref(local_state._shared_state, local_state._channel_id);
+    _data_queue[source_info.channel_id].set_eos();
+    while (_dequeue_data(source_info.local_state, partitioned_block, &eos, &block,
+                         source_info.channel_id)) {
+        partitioned_block.first->unref(
+                source_info.local_state ? source_info.local_state->_shared_state : nullptr,
+                source_info.channel_id);
     }
 }
 
 Status ShuffleExchanger::get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                                   LocalExchangeSourceLocalState& local_state) {
+                                   Profile&& profile, SourceInfo&& source_info) {
     PartitionedBlock partitioned_block;
     vectorized::MutableBlock mutable_block;
 
@@ -153,14 +191,18 @@ Status ShuffleExchanger::get_block(RuntimeState* state, vectorized::Block* block
             auto block_wrapper = partitioned_block.first;
             RETURN_IF_ERROR(mutable_block.add_rows(&block_wrapper->data_block, offset_start,
                                                    offset_start + partitioned_block.second.length));
-            block_wrapper->unref(local_state._shared_state, local_state._channel_id);
+            block_wrapper->unref(
+                    source_info.local_state ? source_info.local_state->_shared_state : nullptr,
+                    source_info.channel_id);
         } while (mutable_block.rows() < state->batch_size() && !*eos &&
-                 _dequeue_data(local_state, partitioned_block, eos, block));
+                 _dequeue_data(source_info.local_state, partitioned_block, eos, block,
+                               source_info.channel_id));
         return Status::OK();
     };
 
-    if (_dequeue_data(local_state, partitioned_block, eos, block)) {
-        SCOPED_TIMER(local_state._copy_data_timer);
+    if (_dequeue_data(source_info.local_state, partitioned_block, eos, block,
+                      source_info.channel_id)) {
+        SCOPED_TIMER(profile.copy_data_timer);
         mutable_block = vectorized::VectorizedUtils::build_mutable_mem_reuse_block(
                 block, partitioned_block.first->data_block);
         RETURN_IF_ERROR(get_data());
@@ -169,22 +211,25 @@ Status ShuffleExchanger::get_block(RuntimeState* state, vectorized::Block* block
 }
 
 Status ShuffleExchanger::_split_rows(RuntimeState* state, const uint32_t* __restrict channel_ids,
-                                     vectorized::Block* block,
-                                     LocalExchangeSinkLocalState& local_state) {
+                                     vectorized::Block* block, int channel_id,
+                                     LocalExchangeSinkLocalState* local_state) {
+    if (local_state == nullptr) {
+        return _split_rows(state, channel_ids, block, channel_id);
+    }
     const auto rows = cast_set<int32_t>(block->rows());
     auto row_idx = std::make_shared<vectorized::PODArray<uint32_t>>(rows);
+    auto& partition_rows_histogram = _partition_rows_histogram[channel_id];
     {
-        local_state._partition_rows_histogram.assign(_num_partitions + 1, 0);
+        partition_rows_histogram.assign(_num_partitions + 1, 0);
         for (int32_t i = 0; i < rows; ++i) {
-            local_state._partition_rows_histogram[channel_ids[i]]++;
+            partition_rows_histogram[channel_ids[i]]++;
         }
         for (int32_t i = 1; i <= _num_partitions; ++i) {
-            local_state._partition_rows_histogram[i] +=
-                    local_state._partition_rows_histogram[i - 1];
+            partition_rows_histogram[i] += partition_rows_histogram[i - 1];
         }
         for (int32_t i = rows - 1; i >= 0; --i) {
-            (*row_idx)[local_state._partition_rows_histogram[channel_ids[i]] - 1] = i;
-            local_state._partition_rows_histogram[channel_ids[i]]--;
+            (*row_idx)[partition_rows_histogram[channel_ids[i]] - 1] = i;
+            partition_rows_histogram[channel_ids[i]]--;
         }
     }
 
@@ -200,10 +245,10 @@ Status ShuffleExchanger::_split_rows(RuntimeState* state, const uint32_t* __rest
     if (new_block_wrapper->data_block.empty()) {
         return Status::OK();
     }
-    local_state._shared_state->add_total_mem_usage(new_block_wrapper->data_block.allocated_bytes(),
-                                                   local_state._channel_id);
+    local_state->_shared_state->add_total_mem_usage(new_block_wrapper->data_block.allocated_bytes(),
+                                                    channel_id);
     auto bucket_seq_to_instance_idx =
-            local_state._parent->cast<LocalExchangeSinkOperatorX>()._bucket_seq_to_instance_idx;
+            local_state->_parent->cast<LocalExchangeSinkOperatorX>()._bucket_seq_to_instance_idx;
     if (get_type() == ExchangeType::HASH_SHUFFLE) {
         /**
          * If type is `HASH_SHUFFLE`, data are hash-shuffled and distributed to all instances of
@@ -211,32 +256,32 @@ Status ShuffleExchanger::_split_rows(RuntimeState* state, const uint32_t* __rest
          * For example, row 1 get a hash value 1 which means we should distribute to instance 1 on
          * BE 1 and row 2 get a hash value 2 which means we should distribute to instance 1 on BE 3.
          */
-        const auto& map = local_state._parent->cast<LocalExchangeSinkOperatorX>()
+        const auto& map = local_state->_parent->cast<LocalExchangeSinkOperatorX>()
                                   ._shuffle_idx_to_instance_idx;
         new_block_wrapper->ref(cast_set<int>(map.size()));
         for (const auto& it : map) {
             DCHECK(it.second >= 0 && it.second < _num_partitions)
                     << it.first << " : " << it.second << " " << _num_partitions;
-            uint32_t start = local_state._partition_rows_histogram[it.first];
-            uint32_t size = local_state._partition_rows_histogram[it.first + 1] - start;
+            uint32_t start = partition_rows_histogram[it.first];
+            uint32_t size = partition_rows_histogram[it.first + 1] - start;
             if (size > 0) {
                 _enqueue_data_and_set_ready(it.second, local_state,
                                             {new_block_wrapper, {row_idx, start, size}});
             } else {
-                new_block_wrapper->unref(local_state._shared_state, local_state._channel_id);
+                new_block_wrapper->unref(local_state->_shared_state, channel_id);
             }
         }
     } else {
         DCHECK(!bucket_seq_to_instance_idx.empty());
         new_block_wrapper->ref(_num_partitions);
         for (int i = 0; i < _num_partitions; i++) {
-            uint32_t start = local_state._partition_rows_histogram[i];
-            uint32_t size = local_state._partition_rows_histogram[i + 1] - start;
+            uint32_t start = partition_rows_histogram[i];
+            uint32_t size = partition_rows_histogram[i + 1] - start;
             if (size > 0) {
                 _enqueue_data_and_set_ready(bucket_seq_to_instance_idx[i], local_state,
                                             {new_block_wrapper, {row_idx, start, size}});
             } else {
-                new_block_wrapper->unref(local_state._shared_state, local_state._channel_id);
+                new_block_wrapper->unref(local_state->_shared_state, channel_id);
             }
         }
     }
@@ -244,8 +289,53 @@ Status ShuffleExchanger::_split_rows(RuntimeState* state, const uint32_t* __rest
     return Status::OK();
 }
 
+Status ShuffleExchanger::_split_rows(RuntimeState* state, const uint32_t* __restrict channel_ids,
+                                     vectorized::Block* block, int channel_id) {
+    const auto rows = cast_set<int32_t>(block->rows());
+    auto row_idx = std::make_shared<vectorized::PODArray<uint32_t>>(rows);
+    auto& partition_rows_histogram = _partition_rows_histogram[channel_id];
+    {
+        partition_rows_histogram.assign(_num_partitions + 1, 0);
+        for (int32_t i = 0; i < rows; ++i) {
+            partition_rows_histogram[channel_ids[i]]++;
+        }
+        for (int32_t i = 1; i <= _num_partitions; ++i) {
+            partition_rows_histogram[i] += partition_rows_histogram[i - 1];
+        }
+        for (int32_t i = rows - 1; i >= 0; --i) {
+            (*row_idx)[partition_rows_histogram[channel_ids[i]] - 1] = i;
+            partition_rows_histogram[channel_ids[i]]--;
+        }
+    }
+
+    vectorized::Block data_block;
+    std::shared_ptr<BlockWrapper> new_block_wrapper;
+    if (_free_blocks.try_dequeue(data_block)) {
+        new_block_wrapper = BlockWrapper::create_shared(std::move(data_block));
+    } else {
+        new_block_wrapper = BlockWrapper::create_shared(block->clone_empty());
+    }
+
+    new_block_wrapper->data_block.swap(*block);
+    if (new_block_wrapper->data_block.empty()) {
+        return Status::OK();
+    }
+    new_block_wrapper->ref(cast_set<int>(_num_partitions));
+    for (int i = 0; i < _num_partitions; i++) {
+        uint32_t start = partition_rows_histogram[i];
+        uint32_t size = partition_rows_histogram[i + 1] - start;
+        if (size > 0) {
+            _enqueue_data_and_set_ready(i, {new_block_wrapper, {row_idx, start, size}});
+        } else {
+            new_block_wrapper->unref();
+        }
+    }
+
+    return Status::OK();
+}
+
 Status PassthroughExchanger::sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                                  LocalExchangeSinkLocalState& local_state) {
+                                  Profile&& profile, SinkInfo&& sink_info) {
     if (in_block->empty()) {
         return Status::OK();
     }
@@ -256,41 +346,43 @@ Status PassthroughExchanger::sink(RuntimeState* state, vectorized::Block* in_blo
     }
     new_block.swap(*in_block);
     wrapper = BlockWrapper::create_shared(std::move(new_block));
-    auto channel_id = (local_state._channel_id++) % _num_partitions;
-    _enqueue_data_and_set_ready(channel_id, local_state, std::move(wrapper));
+    auto channel_id = ((*sink_info.channel_id)++) % _num_partitions;
+    _enqueue_data_and_set_ready(channel_id, sink_info.local_state, std::move(wrapper));
 
     return Status::OK();
 }
 
-void PassthroughExchanger::close(LocalExchangeSourceLocalState& local_state) {
+void PassthroughExchanger::close(SourceInfo&& source_info) {
     vectorized::Block next_block;
     BlockWrapperSPtr wrapper;
     bool eos;
-    _data_queue[local_state._channel_id].set_eos();
-    while (_dequeue_data(local_state, wrapper, &eos, &next_block)) {
+    _data_queue[source_info.channel_id].set_eos();
+    while (_dequeue_data(source_info.local_state, wrapper, &eos, &next_block,
+                         source_info.channel_id)) {
         // do nothing
     }
 }
 
-void PassToOneExchanger::close(LocalExchangeSourceLocalState& local_state) {
+void PassToOneExchanger::close(SourceInfo&& source_info) {
     vectorized::Block next_block;
     BlockWrapperSPtr wrapper;
     bool eos;
-    _data_queue[local_state._channel_id].set_eos();
-    while (_dequeue_data(local_state, wrapper, &eos, &next_block)) {
+    _data_queue[source_info.channel_id].set_eos();
+    while (_dequeue_data(source_info.local_state, wrapper, &eos, &next_block,
+                         source_info.channel_id)) {
         // do nothing
     }
 }
 
 Status PassthroughExchanger::get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                                       LocalExchangeSourceLocalState& local_state) {
+                                       Profile&& profile, SourceInfo&& source_info) {
     BlockWrapperSPtr next_block;
-    _dequeue_data(local_state, next_block, eos, block);
+    _dequeue_data(source_info.local_state, next_block, eos, block, source_info.channel_id);
     return Status::OK();
 }
 
 Status PassToOneExchanger::sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                                LocalExchangeSinkLocalState& local_state) {
+                                Profile&& profile, SinkInfo&& sink_info) {
     if (in_block->empty()) {
         return Status::OK();
     }
@@ -301,70 +393,72 @@ Status PassToOneExchanger::sink(RuntimeState* state, vectorized::Block* in_block
     new_block.swap(*in_block);
 
     BlockWrapperSPtr wrapper = BlockWrapper::create_shared(std::move(new_block));
-    _enqueue_data_and_set_ready(0, local_state, std::move(wrapper));
+    _enqueue_data_and_set_ready(0, sink_info.local_state, std::move(wrapper));
 
     return Status::OK();
 }
 
 Status PassToOneExchanger::get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                                     LocalExchangeSourceLocalState& local_state) {
-    if (local_state._channel_id != 0) {
+                                     Profile&& profile, SourceInfo&& source_info) {
+    if (source_info.channel_id != 0) {
         *eos = true;
         return Status::OK();
     }
     BlockWrapperSPtr next_block;
-    _dequeue_data(local_state, next_block, eos, block);
+    _dequeue_data(source_info.local_state, next_block, eos, block, source_info.channel_id);
     return Status::OK();
 }
 
 Status LocalMergeSortExchanger::sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                                     LocalExchangeSinkLocalState& local_state) {
+                                     Profile&& profile, SinkInfo&& sink_info) {
     if (!in_block->empty()) {
         vectorized::Block new_block;
         if (!_free_blocks.try_dequeue(new_block)) {
             new_block = {in_block->clone_empty()};
         }
-        DCHECK_LE(local_state._channel_id, _data_queue.size());
+        DCHECK_LE(*sink_info.channel_id, _data_queue.size());
 
         new_block.swap(*in_block);
-        _enqueue_data_and_set_ready(local_state._channel_id, local_state,
+        _enqueue_data_and_set_ready(*sink_info.channel_id, sink_info.local_state,
                                     BlockWrapper::create_shared(std::move(new_block)));
     }
-    if (eos) {
-        local_state._shared_state->source_deps[local_state._channel_id]->set_always_ready();
+    if (eos && sink_info.local_state) {
+        sink_info.local_state->_shared_state->source_deps[*sink_info.channel_id]
+                ->set_always_ready();
     }
     return Status::OK();
 }
 
-void ExchangerBase::finalize(LocalExchangeSourceLocalState& local_state) {
+void ExchangerBase::finalize() {
     DCHECK(_running_source_operators == 0);
     vectorized::Block block;
     while (_free_blocks.try_dequeue(block)) {
         // do nothing
     }
 }
-void LocalMergeSortExchanger::finalize(LocalExchangeSourceLocalState& local_state) {
+
+void LocalMergeSortExchanger::finalize() {
     BlockWrapperSPtr next_block;
     vectorized::Block block;
     bool eos;
     int id = 0;
     for (auto& data_queue : _data_queue) {
         data_queue.set_eos();
-        while (_dequeue_data(local_state, next_block, &eos, &block, id)) {
+        while (_dequeue_data(next_block, &eos, &block, id)) {
             block = vectorized::Block();
         }
         id++;
     }
-    ExchangerBase::finalize(local_state);
+    ExchangerBase::finalize();
 }
 
 Status LocalMergeSortExchanger::build_merger(RuntimeState* state,
-                                             LocalExchangeSourceLocalState& local_state) {
-    RETURN_IF_ERROR(_sort_source->build_merger(state, _merger, local_state.profile()));
+                                             LocalExchangeSourceLocalState* local_state) {
+    RETURN_IF_ERROR(_sort_source->build_merger(state, _merger, local_state->profile()));
     std::vector<vectorized::BlockSupplier> child_block_suppliers;
     for (int channel_id = 0; channel_id < _num_partitions; channel_id++) {
-        vectorized::BlockSupplier block_supplier = [&, id = channel_id](vectorized::Block* block,
-                                                                        bool* eos) {
+        vectorized::BlockSupplier block_supplier = [&, local_state, id = channel_id](
+                                                           vectorized::Block* block, bool* eos) {
             BlockWrapperSPtr next_block;
             _dequeue_data(local_state, next_block, eos, block, id);
             return Status::OK();
@@ -388,20 +482,21 @@ now
     sort(8) --> local merge(1) ---> datasink(1) [2] ---->
 */
 Status LocalMergeSortExchanger::get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                                          LocalExchangeSourceLocalState& local_state) {
-    if (local_state._channel_id != 0) {
+                                          Profile&& profile, SourceInfo&& source_info) {
+    if (source_info.channel_id != 0) {
         *eos = true;
         return Status::OK();
     }
     if (!_merger) {
-        RETURN_IF_ERROR(build_merger(state, local_state));
+        DCHECK(source_info.local_state);
+        RETURN_IF_ERROR(build_merger(state, source_info.local_state));
     }
     RETURN_IF_ERROR(_merger->get_next(block, eos));
     return Status::OK();
 }
 
 Status BroadcastExchanger::sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                                LocalExchangeSinkLocalState& local_state) {
+                                Profile&& profile, SinkInfo&& sink_info) {
     if (in_block->empty()) {
         return Status::OK();
     }
@@ -411,32 +506,40 @@ Status BroadcastExchanger::sink(RuntimeState* state, vectorized::Block* in_block
     }
     new_block.swap(*in_block);
     auto wrapper = BlockWrapper::create_shared(std::move(new_block));
-    local_state._shared_state->add_total_mem_usage(wrapper->data_block.allocated_bytes(),
-                                                   local_state._channel_id);
+    if (sink_info.local_state) {
+        sink_info.local_state->_shared_state->add_total_mem_usage(
+                wrapper->data_block.allocated_bytes(), *sink_info.channel_id);
+    }
+
     wrapper->ref(_num_partitions);
     for (int i = 0; i < _num_partitions; i++) {
-        _enqueue_data_and_set_ready(i, local_state, {wrapper, {0, wrapper->data_block.rows()}});
+        _enqueue_data_and_set_ready(i, sink_info.local_state,
+                                    {wrapper, {0, wrapper->data_block.rows()}});
     }
 
     return Status::OK();
 }
 
-void BroadcastExchanger::close(LocalExchangeSourceLocalState& local_state) {
+void BroadcastExchanger::close(SourceInfo&& source_info) {
     BroadcastBlock partitioned_block;
     bool eos;
     vectorized::Block block;
-    _data_queue[local_state._channel_id].set_eos();
-    while (_dequeue_data(local_state, partitioned_block, &eos, &block)) {
-        partitioned_block.first->unref(local_state._shared_state, local_state._channel_id);
+    _data_queue[source_info.channel_id].set_eos();
+    while (_dequeue_data(source_info.local_state, partitioned_block, &eos, &block,
+                         source_info.channel_id)) {
+        partitioned_block.first->unref(
+                source_info.local_state ? source_info.local_state->_shared_state : nullptr,
+                source_info.channel_id);
     }
 }
 
 Status BroadcastExchanger::get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                                     LocalExchangeSourceLocalState& local_state) {
+                                     Profile&& profile, SourceInfo&& source_info) {
     BroadcastBlock partitioned_block;
 
-    if (_dequeue_data(local_state, partitioned_block, eos, block)) {
-        SCOPED_TIMER(local_state._copy_data_timer);
+    if (_dequeue_data(source_info.local_state, partitioned_block, eos, block,
+                      source_info.channel_id)) {
+        SCOPED_TIMER(profile.copy_data_timer);
         vectorized::MutableBlock mutable_block =
                 vectorized::VectorizedUtils::build_mutable_mem_reuse_block(
                         block, partitioned_block.first->data_block);
@@ -444,7 +547,9 @@ Status BroadcastExchanger::get_block(RuntimeState* state, vectorized::Block* blo
         RETURN_IF_ERROR(mutable_block.add_rows(&block_wrapper->data_block,
                                                partitioned_block.second.offset_start,
                                                partitioned_block.second.length));
-        block_wrapper->unref(local_state._shared_state, local_state._channel_id);
+        block_wrapper->unref(
+                source_info.local_state ? source_info.local_state->_shared_state : nullptr,
+                source_info.channel_id);
     }
 
     return Status::OK();
@@ -452,21 +557,21 @@ Status BroadcastExchanger::get_block(RuntimeState* state, vectorized::Block* blo
 
 Status AdaptivePassthroughExchanger::_passthrough_sink(RuntimeState* state,
                                                        vectorized::Block* in_block,
-                                                       LocalExchangeSinkLocalState& local_state) {
+                                                       SinkInfo&& sink_info) {
     vectorized::Block new_block;
     if (!_free_blocks.try_dequeue(new_block)) {
         new_block = {in_block->clone_empty()};
     }
     new_block.swap(*in_block);
-    auto channel_id = (local_state._channel_id++) % _num_partitions;
-    _enqueue_data_and_set_ready(channel_id, local_state,
+    auto channel_id = ((*sink_info.channel_id)++) % _num_partitions;
+    _enqueue_data_and_set_ready(channel_id, sink_info.local_state,
                                 BlockWrapper::create_shared(std::move(new_block)));
 
     return Status::OK();
 }
 
 Status AdaptivePassthroughExchanger::_shuffle_sink(RuntimeState* state, vectorized::Block* block,
-                                                   LocalExchangeSinkLocalState& local_state) {
+                                                   SinkInfo&& sink_info) {
     std::vector<uint32_t> channel_ids;
     const auto num_rows = block->rows();
     channel_ids.resize(num_rows, 0);
@@ -481,40 +586,39 @@ Status AdaptivePassthroughExchanger::_shuffle_sink(RuntimeState* state, vectoriz
             std::iota(channel_ids.begin() + i, channel_ids.end(), 0);
         }
     }
-    return _split_rows(state, channel_ids.data(), block, local_state);
+    return _split_rows(state, channel_ids.data(), block, std::move(sink_info));
 }
 
 Status AdaptivePassthroughExchanger::_split_rows(RuntimeState* state,
                                                  const uint32_t* __restrict channel_ids,
-                                                 vectorized::Block* block,
-                                                 LocalExchangeSinkLocalState& local_state) {
+                                                 vectorized::Block* block, SinkInfo&& sink_info) {
     const auto rows = cast_set<int32_t>(block->rows());
     auto row_idx = std::make_shared<std::vector<uint32_t>>(rows);
+    auto& partition_rows_histogram = _partition_rows_histogram[*sink_info.channel_id];
     {
-        local_state._partition_rows_histogram.assign(_num_partitions + 1, 0);
+        partition_rows_histogram.assign(_num_partitions + 1, 0);
         for (int32_t i = 0; i < rows; ++i) {
-            local_state._partition_rows_histogram[channel_ids[i]]++;
+            partition_rows_histogram[channel_ids[i]]++;
         }
         for (int32_t i = 1; i <= _num_partitions; ++i) {
-            local_state._partition_rows_histogram[i] +=
-                    local_state._partition_rows_histogram[i - 1];
+            partition_rows_histogram[i] += partition_rows_histogram[i - 1];
         }
 
         for (int32_t i = rows - 1; i >= 0; --i) {
-            (*row_idx)[local_state._partition_rows_histogram[channel_ids[i]] - 1] = i;
-            local_state._partition_rows_histogram[channel_ids[i]]--;
+            (*row_idx)[partition_rows_histogram[channel_ids[i]] - 1] = i;
+            partition_rows_histogram[channel_ids[i]]--;
         }
     }
     for (int32_t i = 0; i < _num_partitions; i++) {
-        const size_t start = local_state._partition_rows_histogram[i];
-        const size_t size = local_state._partition_rows_histogram[i + 1] - start;
+        const size_t start = partition_rows_histogram[i];
+        const size_t size = partition_rows_histogram[i + 1] - start;
         if (size > 0) {
             std::unique_ptr<vectorized::MutableBlock> mutable_block =
                     vectorized::MutableBlock::create_unique(block->clone_empty());
             RETURN_IF_ERROR(mutable_block->add_rows(block, start, size));
             auto new_block = mutable_block->to_block();
 
-            _enqueue_data_and_set_ready(i, local_state,
+            _enqueue_data_and_set_ready(i, sink_info.local_state,
                                         BlockWrapper::create_shared(std::move(new_block)));
         }
     }
@@ -522,34 +626,35 @@ Status AdaptivePassthroughExchanger::_split_rows(RuntimeState* state,
 }
 
 Status AdaptivePassthroughExchanger::sink(RuntimeState* state, vectorized::Block* in_block,
-                                          bool eos, LocalExchangeSinkLocalState& local_state) {
+                                          bool eos, Profile&& profile, SinkInfo&& sink_info) {
     if (in_block->empty()) {
         return Status::OK();
     }
     if (_is_pass_through) {
-        return _passthrough_sink(state, in_block, local_state);
+        return _passthrough_sink(state, in_block, std::move(sink_info));
     } else {
         if (_total_block++ > _num_partitions) {
             _is_pass_through = true;
         }
-        return _shuffle_sink(state, in_block, local_state);
+        return _shuffle_sink(state, in_block, std::move(sink_info));
     }
 }
 
 Status AdaptivePassthroughExchanger::get_block(RuntimeState* state, vectorized::Block* block,
-                                               bool* eos,
-                                               LocalExchangeSourceLocalState& local_state) {
+                                               bool* eos, Profile&& profile,
+                                               SourceInfo&& source_info) {
     BlockWrapperSPtr next_block;
-    _dequeue_data(local_state, next_block, eos, block);
+    _dequeue_data(source_info.local_state, next_block, eos, block, source_info.channel_id);
     return Status::OK();
 }
 
-void AdaptivePassthroughExchanger::close(LocalExchangeSourceLocalState& local_state) {
+void AdaptivePassthroughExchanger::close(SourceInfo&& source_info) {
     vectorized::Block next_block;
     bool eos;
     BlockWrapperSPtr wrapper;
-    _data_queue[local_state._channel_id].set_eos();
-    while (_dequeue_data(local_state, wrapper, &eos, &next_block)) {
+    _data_queue[source_info.channel_id].set_eos();
+    while (_dequeue_data(source_info.local_state, wrapper, &eos, &next_block,
+                         source_info.channel_id)) {
         // do nothing
     }
 }
diff --git a/be/src/pipeline/local_exchange/local_exchanger.h b/be/src/pipeline/local_exchange/local_exchanger.h
index 4d699baa52fb8b..d6871b2ba97cc3 100644
--- a/be/src/pipeline/local_exchange/local_exchanger.h
+++ b/be/src/pipeline/local_exchange/local_exchanger.h
@@ -20,14 +20,33 @@
 #include "pipeline/dependency.h"
 #include "pipeline/exec/operator.h"
 
-namespace doris::pipeline {
+namespace doris {
 #include "common/compile_check_begin.h"
-
+namespace vectorized {
+class PartitionerBase;
+}
+namespace pipeline {
 class LocalExchangeSourceLocalState;
 class LocalExchangeSinkLocalState;
 struct BlockWrapper;
 class SortSourceOperatorX;
 
+struct Profile {
+    RuntimeProfile::Counter* compute_hash_value_timer = nullptr;
+    RuntimeProfile::Counter* distribute_timer = nullptr;
+    RuntimeProfile::Counter* copy_data_timer = nullptr;
+};
+
+struct SinkInfo {
+    int* channel_id;
+    vectorized::PartitionerBase* partitioner;
+    LocalExchangeSinkLocalState* local_state;
+};
+
+struct SourceInfo {
+    int channel_id;
+    LocalExchangeSourceLocalState* local_state;
+};
 /**
  * One exchanger is hold by one `LocalExchangeSharedState`. And one `LocalExchangeSharedState` is
  * shared by all local exchange sink operators and source operators with the same id.
@@ -60,15 +79,15 @@ class ExchangerBase {
               _free_block_limit(free_block_limit) {}
     virtual ~ExchangerBase() = default;
     virtual Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                             LocalExchangeSourceLocalState& local_state) = 0;
+                             Profile&& profile, SourceInfo&& source_info) = 0;
     virtual Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                        LocalExchangeSinkLocalState& local_state) = 0;
+                        Profile&& profile, SinkInfo&& sink_info) = 0;
     virtual ExchangeType get_type() const = 0;
     // Called if a local exchanger source operator are closed. Free the unused data block in data_queue.
-    virtual void close(LocalExchangeSourceLocalState& local_state) = 0;
+    virtual void close(SourceInfo&& source_info) = 0;
     // Called if all local exchanger source operators are closed. We free the memory in
     // `_free_blocks` here.
-    virtual void finalize(LocalExchangeSourceLocalState& local_state);
+    virtual void finalize();
 
     virtual std::string data_queue_debug_string(int i) = 0;
 
@@ -155,12 +174,13 @@ class Exchanger : public ExchangerBase {
 
 protected:
     // Enqueue data block and set downstream source operator to read.
-    void _enqueue_data_and_set_ready(int channel_id, LocalExchangeSinkLocalState& local_state,
+    void _enqueue_data_and_set_ready(int channel_id, LocalExchangeSinkLocalState* local_state,
                                      BlockType&& block);
-    bool _dequeue_data(LocalExchangeSourceLocalState& local_state, BlockType& block, bool* eos,
-                       vectorized::Block* data_block);
-    bool _dequeue_data(LocalExchangeSourceLocalState& local_state, BlockType& block, bool* eos,
+    bool _dequeue_data(LocalExchangeSourceLocalState* local_state, BlockType& block, bool* eos,
                        vectorized::Block* data_block, int channel_id);
+
+    void _enqueue_data_and_set_ready(int channel_id, BlockType&& block);
+    bool _dequeue_data(BlockType& block, bool* eos, vectorized::Block* data_block, int channel_id);
     std::vector<BlockQueue<BlockType>> _data_queue;
 
 private:
@@ -186,7 +206,7 @@ struct BlockWrapper {
     ~BlockWrapper() { DCHECK_EQ(ref_count.load(), 0); }
     void ref(int delta) { ref_count += delta; }
     void unref(LocalExchangeSharedState* shared_state, size_t allocated_bytes, int channel_id) {
-        if (ref_count.fetch_sub(1) == 1) {
+        if (ref_count.fetch_sub(1) == 1 && shared_state != nullptr) {
             DCHECK_GT(allocated_bytes, 0);
             shared_state->sub_total_mem_usage(allocated_bytes, channel_id);
             if (shared_state->exchanger->_free_block_limit == 0 ||
@@ -201,7 +221,7 @@ struct BlockWrapper {
         }
     }
 
-    void unref(LocalExchangeSharedState* shared_state, int channel_id) {
+    void unref(LocalExchangeSharedState* shared_state = nullptr, int channel_id = 0) {
         unref(shared_state, data_block.allocated_bytes(), channel_id);
     }
     int ref_value() const { return ref_count.load(); }
@@ -219,19 +239,24 @@ class ShuffleExchanger : public Exchanger<PartitionedBlock> {
         DCHECK_GT(num_partitions, 0);
         DCHECK_GT(num_sources, 0);
         _data_queue.resize(num_sources);
+        _partition_rows_histogram.resize(running_sink_operators);
     }
     ~ShuffleExchanger() override = default;
-    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                LocalExchangeSinkLocalState& local_state) override;
+    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos, Profile&& profile,
+                SinkInfo&& sink_info) override;
 
-    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                     LocalExchangeSourceLocalState& local_state) override;
-    void close(LocalExchangeSourceLocalState& local_state) override;
+    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos, Profile&& profile,
+                     SourceInfo&& source_info) override;
+    void close(SourceInfo&& source_info) override;
     ExchangeType get_type() const override { return ExchangeType::HASH_SHUFFLE; }
 
 protected:
     Status _split_rows(RuntimeState* state, const uint32_t* __restrict channel_ids,
-                       vectorized::Block* block, LocalExchangeSinkLocalState& local_state);
+                       vectorized::Block* block, int channel_id,
+                       LocalExchangeSinkLocalState* local_state);
+    Status _split_rows(RuntimeState* state, const uint32_t* __restrict channel_ids,
+                       vectorized::Block* block, int channel_id);
+    std::vector<std::vector<uint32_t>> _partition_rows_histogram;
 };
 
 class BucketShuffleExchanger final : public ShuffleExchanger {
@@ -255,13 +280,13 @@ class PassthroughExchanger final : public Exchanger<BlockWrapperSPtr> {
         _data_queue.resize(num_partitions);
     }
     ~PassthroughExchanger() override = default;
-    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                LocalExchangeSinkLocalState& local_state) override;
+    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos, Profile&& profile,
+                SinkInfo&& sink_info) override;
 
-    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                     LocalExchangeSourceLocalState& local_state) override;
+    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos, Profile&& profile,
+                     SourceInfo&& source_info) override;
     ExchangeType get_type() const override { return ExchangeType::PASSTHROUGH; }
-    void close(LocalExchangeSourceLocalState& local_state) override;
+    void close(SourceInfo&& source_info) override;
 };
 
 class PassToOneExchanger final : public Exchanger<BlockWrapperSPtr> {
@@ -273,13 +298,13 @@ class PassToOneExchanger final : public Exchanger<BlockWrapperSPtr> {
         _data_queue.resize(num_partitions);
     }
     ~PassToOneExchanger() override = default;
-    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                LocalExchangeSinkLocalState& local_state) override;
+    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos, Profile&& profile,
+                SinkInfo&& sink_info) override;
 
-    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                     LocalExchangeSourceLocalState& local_state) override;
+    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos, Profile&& profile,
+                     SourceInfo&& source_info) override;
     ExchangeType get_type() const override { return ExchangeType::PASS_TO_ONE; }
-    void close(LocalExchangeSourceLocalState& local_state) override;
+    void close(SourceInfo&& source_info) override;
 };
 
 class LocalMergeSortExchanger final : public Exchanger<BlockWrapperSPtr> {
@@ -292,17 +317,17 @@ class LocalMergeSortExchanger final : public Exchanger<BlockWrapperSPtr> {
         _data_queue.resize(num_partitions);
     }
     ~LocalMergeSortExchanger() override = default;
-    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                LocalExchangeSinkLocalState& local_state) override;
+    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos, Profile&& profile,
+                SinkInfo&& sink_info) override;
 
-    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                     LocalExchangeSourceLocalState& local_state) override;
+    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos, Profile&& profile,
+                     SourceInfo&& source_info) override;
     ExchangeType get_type() const override { return ExchangeType::LOCAL_MERGE_SORT; }
 
-    Status build_merger(RuntimeState* statem, LocalExchangeSourceLocalState& local_state);
+    Status build_merger(RuntimeState* statem, LocalExchangeSourceLocalState* local_state);
 
-    void close(LocalExchangeSourceLocalState& local_state) override {}
-    void finalize(LocalExchangeSourceLocalState& local_state) override;
+    void close(SourceInfo&& source_info) override {}
+    void finalize() override;
 
 private:
     std::unique_ptr<vectorized::VSortedRunMerger> _merger;
@@ -318,13 +343,13 @@ class BroadcastExchanger final : public Exchanger<BroadcastBlock> {
         _data_queue.resize(num_partitions);
     }
     ~BroadcastExchanger() override = default;
-    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                LocalExchangeSinkLocalState& local_state) override;
+    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos, Profile&& profile,
+                SinkInfo&& sink_info) override;
 
-    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                     LocalExchangeSourceLocalState& local_state) override;
+    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos, Profile&& profile,
+                     SourceInfo&& source_info) override;
     ExchangeType get_type() const override { return ExchangeType::BROADCAST; }
-    void close(LocalExchangeSourceLocalState& local_state) override;
+    void close(SourceInfo&& source_info) override;
 };
 
 //The code in AdaptivePassthroughExchanger is essentially
@@ -337,26 +362,28 @@ class AdaptivePassthroughExchanger : public Exchanger<BlockWrapperSPtr> {
             : Exchanger<BlockWrapperSPtr>(running_sink_operators, num_partitions,
                                           free_block_limit) {
         _data_queue.resize(num_partitions);
+        _partition_rows_histogram.resize(running_sink_operators);
     }
-    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos,
-                LocalExchangeSinkLocalState& local_state) override;
+    Status sink(RuntimeState* state, vectorized::Block* in_block, bool eos, Profile&& profile,
+                SinkInfo&& sink_info) override;
 
-    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos,
-                     LocalExchangeSourceLocalState& local_state) override;
+    Status get_block(RuntimeState* state, vectorized::Block* block, bool* eos, Profile&& profile,
+                     SourceInfo&& source_info) override;
     ExchangeType get_type() const override { return ExchangeType::ADAPTIVE_PASSTHROUGH; }
 
-    void close(LocalExchangeSourceLocalState& local_state) override;
+    void close(SourceInfo&& source_info) override;
 
 private:
     Status _passthrough_sink(RuntimeState* state, vectorized::Block* in_block,
-                             LocalExchangeSinkLocalState& local_state);
-    Status _shuffle_sink(RuntimeState* state, vectorized::Block* in_block,
-                         LocalExchangeSinkLocalState& local_state);
+                             SinkInfo&& sink_info);
+    Status _shuffle_sink(RuntimeState* state, vectorized::Block* in_block, SinkInfo&& sink_info);
     Status _split_rows(RuntimeState* state, const uint32_t* __restrict channel_ids,
-                       vectorized::Block* block, LocalExchangeSinkLocalState& local_state);
+                       vectorized::Block* block, SinkInfo&& sink_info);
 
     std::atomic_bool _is_pass_through = false;
     std::atomic_int32_t _total_block = 0;
+    std::vector<std::vector<uint32_t>> _partition_rows_histogram;
 };
 #include "common/compile_check_end.h"
-} // namespace doris::pipeline
+} // namespace pipeline
+} // namespace doris
\ No newline at end of file

From e8adcd7b2b3465489f7ffaa57fb1c19ed29a609a Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Wed, 11 Dec 2024 18:49:04 +0800
Subject: [PATCH 278/399] [opt](compaction) Don't check missed rows in cumu
 compaction if input rowsets are not in tablet (#45279)

Problem Summary:

Suppose a heavy schema change process on BE converting tablet A to
tablet B.
1. during schema change double write, new loads write [X-Y] on tablet B.
2. rowsets with version [a],[a+1],...,[b-1],[b] on tablet B are picked
for cumu compaction(X<=a<b<=Y).(cumu compaction on new tablet during
schema change double write is allowed after
https://github.com/apache/doris/pull/16470)
3. schema change remove all rowsets on tablet B before version
Z(b<=Z<=Y) before it begins to convert historical rowsets.
4. schema change finishes.
5. cumu compation begins on new tablet with version [a],...,[b]. If
there are duplicate keys between these rowsets, the compaction check
will fail because these rowsets have skipped to calculate delete bitmap
in commit phase and publish phase because tablet B is in NOT_READY state
when writing.

This PR makes cumu compaction skip to do missed rows check when it finds
out that the input rowsets are not exist in tablet.

Note that the compaction will fail because `Tablet::modify_rowsets` will
check if rowsets in `to_delete` still exist in tablet's
`_rs_version_map`.
---
 be/src/olap/compaction.cpp                    |  25 ++-
 be/src/olap/cumulative_compaction.cpp         |  14 ++
 be/src/olap/cumulative_compaction_policy.cpp  |  16 ++
 be/src/olap/schema_change.cpp                 |   2 +
 be/src/olap/tablet.cpp                        |   9 ++
 be/src/olap/tablet.h                          |   1 +
 .../test_compaction_on_sc_new_tablet.out      |  25 +++
 .../test_compaction_on_sc_new_tablet.groovy   | 149 ++++++++++++++++++
 8 files changed, 240 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/data/fault_injection_p0/test_compaction_on_sc_new_tablet.out
 create mode 100644 regression-test/suites/fault_injection_p0/test_compaction_on_sc_new_tablet.groovy

diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 7b2bb5d4ee3cfb..0d811d9348f1f5 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -1013,8 +1013,31 @@ Status CompactionMixin::modify_rowsets() {
             if (!_tablet->tablet_meta()->tablet_schema()->cluster_key_uids().empty()) {
                 merged_missed_rows_size += _stats.filtered_rows;
             }
+
+            // Suppose a heavy schema change process on BE converting tablet A to tablet B.
+            // 1. during schema change double write, new loads write [X-Y] on tablet B.
+            // 2. rowsets with version [a],[a+1],...,[b-1],[b] on tablet B are picked for cumu compaction(X<=a<b<=Y).(cumu compaction
+            //    on new tablet during schema change double write is allowed after https://github.com/apache/doris/pull/16470)
+            // 3. schema change remove all rowsets on tablet B before version Z(b<=Z<=Y) before it begins to convert historical rowsets.
+            // 4. schema change finishes.
+            // 5. cumu compation begins on new tablet with version [a],...,[b]. If there are duplicate keys between these rowsets,
+            //    the compaction check will fail because these rowsets have skipped to calculate delete bitmap in commit phase and
+            //    publish phase because tablet B is in NOT_READY state when writing.
+
+            // Considering that the cumu compaction will fail finally in this situation because `Tablet::modify_rowsets` will check if rowsets in
+            // `to_delete`(_input_rowsets) still exist in tablet's `_rs_version_map`, we can just skip to check missed rows here.
+            bool need_to_check_missed_rows = true;
+            {
+                std::shared_lock rlock(_tablet->get_header_lock());
+                need_to_check_missed_rows =
+                        std::all_of(_input_rowsets.begin(), _input_rowsets.end(),
+                                    [&](const RowsetSharedPtr& rowset) {
+                                        return tablet()->rowset_exists_unlocked(rowset);
+                                    });
+            }
+
             if (_tablet->tablet_state() == TABLET_RUNNING &&
-                merged_missed_rows_size != missed_rows_size) {
+                merged_missed_rows_size != missed_rows_size && need_to_check_missed_rows) {
                 std::stringstream ss;
                 ss << "cumulative compaction: the merged rows(" << _stats.merged_rows
                    << "), filtered rows(" << _stats.filtered_rows
diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp
index 2dfd30fb86ed9a..a9509a005763f6 100644
--- a/be/src/olap/cumulative_compaction.cpp
+++ b/be/src/olap/cumulative_compaction.cpp
@@ -100,6 +100,20 @@ Status CumulativeCompaction::prepare_compact() {
 }
 
 Status CumulativeCompaction::execute_compact() {
+    DBUG_EXECUTE_IF("CumulativeCompaction::execute_compact.block", {
+        auto target_tablet_id = dp->param<int64_t>("tablet_id", -1);
+        if (target_tablet_id == _tablet->tablet_id()) {
+            LOG(INFO) << "start debug block "
+                      << "CumulativeCompaction::execute_compact.block";
+            while (DebugPoints::instance()->is_enable(
+                    "CumulativeCompaction::execute_compact.block")) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(200));
+            }
+            LOG(INFO) << "end debug block "
+                      << "CumulativeCompaction::execute_compact.block";
+        }
+    })
+
     std::unique_lock<std::mutex> lock(tablet()->get_cumulative_compaction_lock(), std::try_to_lock);
     if (!lock.owns_lock()) {
         return Status::Error<TRY_LOCK_FAILED, false>(
diff --git a/be/src/olap/cumulative_compaction_policy.cpp b/be/src/olap/cumulative_compaction_policy.cpp
index ee7a2b1812a0ae..c812a12b656580 100644
--- a/be/src/olap/cumulative_compaction_policy.cpp
+++ b/be/src/olap/cumulative_compaction_policy.cpp
@@ -28,6 +28,7 @@
 #include "olap/olap_common.h"
 #include "olap/tablet.h"
 #include "olap/tablet_meta.h"
+#include "util/debug_points.h"
 
 namespace doris {
 
@@ -246,6 +247,21 @@ int SizeBasedCumulativeCompactionPolicy::pick_input_rowsets(
         const int64_t max_compaction_score, const int64_t min_compaction_score,
         std::vector<RowsetSharedPtr>* input_rowsets, Version* last_delete_version,
         size_t* compaction_score, bool allow_delete) {
+    DBUG_EXECUTE_IF("SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", {
+        auto target_tablet_id = dp->param<int64_t>("tablet_id", -1);
+        if (target_tablet_id == tablet->tablet_id()) {
+            auto start_version = dp->param<int64_t>("start_version", -1);
+            auto end_version = dp->param<int64_t>("end_version", -1);
+            for (auto& rowset : candidate_rowsets) {
+                if (rowset->start_version() >= start_version &&
+                    rowset->end_version() <= end_version) {
+                    input_rowsets->push_back(rowset);
+                }
+            }
+        }
+        return input_rowsets->size();
+    })
+
     size_t promotion_size = tablet->cumulative_promotion_size();
     auto max_version = tablet->max_version().first;
     int transient_size = 0;
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index cdb637b1c42647..08b9b9a93d90f5 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -867,6 +867,8 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques
         return_columns[i] = i;
     }
 
+    DBUG_EXECUTE_IF("SchemaChangeJob::_do_process_alter_tablet.block", DBUG_BLOCK);
+
     // begin to find deltas to convert from base tablet to new tablet so that
     // obtain base tablet and new tablet's push lock and header write lock to prevent loading data
     {
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 644ca9133eb885..a1a56507ffc67a 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -512,6 +512,15 @@ Status Tablet::add_rowset(RowsetSharedPtr rowset) {
     return Status::OK();
 }
 
+bool Tablet::rowset_exists_unlocked(const RowsetSharedPtr& rowset) {
+    if (auto it = _rs_version_map.find(rowset->version()); it == _rs_version_map.end()) {
+        return false;
+    } else if (rowset->rowset_id() != it->second->rowset_id()) {
+        return false;
+    }
+    return true;
+}
+
 Status Tablet::modify_rowsets(std::vector<RowsetSharedPtr>& to_add,
                               std::vector<RowsetSharedPtr>& to_delete, bool check_delete) {
     // the compaction process allow to compact the single version, eg: version[4-4].
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 0b7d758ab8fd88..40b911d6391b9b 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -173,6 +173,7 @@ class Tablet final : public BaseTablet {
     // MUST hold EXCLUSIVE `_meta_lock`.
     Status modify_rowsets(std::vector<RowsetSharedPtr>& to_add,
                           std::vector<RowsetSharedPtr>& to_delete, bool check_delete = false);
+    bool rowset_exists_unlocked(const RowsetSharedPtr& rowset);
 
     Status add_inc_rowset(const RowsetSharedPtr& rowset);
     /// Delete stale rowset by timing. This delete policy uses now() minutes
diff --git a/regression-test/data/fault_injection_p0/test_compaction_on_sc_new_tablet.out b/regression-test/data/fault_injection_p0/test_compaction_on_sc_new_tablet.out
new file mode 100644
index 00000000000000..e7188943a10a19
--- /dev/null
+++ b/regression-test/data/fault_injection_p0/test_compaction_on_sc_new_tablet.out
@@ -0,0 +1,25 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1	1	1	1
+2	2	2	2
+3	3	3	3
+4	4	4	4
+5	5	5	5
+6	6	6	6
+7	7	7	7
+8	8	8	8
+9	9	9	9
+10	10	10	10
+
+-- !sql --
+1	9	9	9
+2	2	2	2
+3	3	3	3
+4	4	4	4
+5	5	5	5
+6	6	6	6
+7	7	7	7
+8	8	8	8
+9	9	9	9
+10	10	10	10
+
diff --git a/regression-test/suites/fault_injection_p0/test_compaction_on_sc_new_tablet.groovy b/regression-test/suites/fault_injection_p0/test_compaction_on_sc_new_tablet.groovy
new file mode 100644
index 00000000000000..2f3c44ef2dd00b
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_compaction_on_sc_new_tablet.groovy
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+import org.apache.doris.regression.suite.ClusterOptions
+
+suite("test_compaction_on_sc_new_tablet", "docker") {
+    def options = new ClusterOptions()
+    options.setFeNum(1)
+    options.setBeNum(1)
+    options.enableDebugPoints()
+    options.cloudMode = false
+    options.beConfigs += [
+        'enable_java_support=false',
+        'enable_mow_compaction_correctness_check_core=true'
+    ]
+    docker(options) {
+        try {
+            GetDebugPoint().clearDebugPointsForAllFEs()
+            GetDebugPoint().clearDebugPointsForAllBEs()
+            def table1 = "test_compaction_on_sc_new_tablet"
+            sql "DROP TABLE IF EXISTS ${table1} FORCE;"
+            sql """ CREATE TABLE IF NOT EXISTS ${table1} (
+                    `k`  int,
+                    `c1` int,
+                    `c2` int,
+                    `c3` int
+                    ) UNIQUE KEY(k)
+                DISTRIBUTED BY HASH(k) BUCKETS 1
+                PROPERTIES (
+                "disable_auto_compaction" = "true",
+                "replication_num" = "1",
+                "enable_unique_key_merge_on_write" = "true"); """
+
+            // [2-11]
+            for (int i = 1; i <= 10; i++) {
+                sql "insert into ${table1} values($i,$i,$i,$i);"
+            }
+            qt_sql "select * from ${table1} order by k;"
+
+
+            def beNodes = sql_return_maparray("show backends;")
+            def tabletStats = sql_return_maparray("show tablets from ${table1};")
+            logger.info("tabletStats: \n${tabletStats}")
+            def tabletStat = tabletStats.get(0)
+            def tabletBackendId = tabletStat.BackendId
+            def tabletId = tabletStat.TabletId
+            def version = tabletStat.Version
+            def tabletBackend;
+            for (def be : beNodes) {
+                if (be.BackendId == tabletBackendId) {
+                    tabletBackend = be
+                    break;
+                }
+            }
+            logger.info("tablet ${tabletId} is on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}, version=${version}");
+
+            // blocking the schema change process before it gains max version
+            GetDebugPoint().enableDebugPointForAllBEs("SchemaChangeJob::_do_process_alter_tablet.block")
+            Thread.sleep(2000)
+
+            sql "alter table ${table1} modify column c1 varchar(100);"
+
+            Thread.sleep(4000)
+
+            // double write [11-22]
+            for (int i = 20; i <= 30; i++) {
+                sql "insert into ${table1} values(1,9,9,9);"
+            }
+
+            tabletStats = sql_return_maparray("show tablets from ${table1};")
+            logger.info("tabletStats: \n${tabletStats}")
+            assertEquals(2, tabletStats.size())
+
+            def oldTabletStat
+            def newTabletStat
+            for (def stat: tabletStats) {
+                if (!stat.TabletId.equals(tabletId)) {
+                    newTabletStat = stat
+                } else {
+                    oldTabletStat = stat
+                }
+            }
+            logger.info("old tablet=[tablet_id=${oldTabletStat.TabletId}, version=${oldTabletStat.Version}]")
+            logger.info("new tablet=[tablet_id=${newTabletStat.TabletId}, version=${newTabletStat.Version}]")
+
+            
+            // trigger cumu compaction on new tablet
+            int start_version = 15
+            int end_version = 17
+            // block compaction process on new tablet
+            GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction::execute_compact.block", [tablet_id: "${newTabletStat.TabletId}"])
+            // manully set cumu compaction's input rowsets on new tablet
+            GetDebugPoint().enableDebugPointForAllBEs("SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets",
+                    [tablet_id:"${newTabletStat.TabletId}", start_version:"${start_version}", end_version:"${end_version}"])
+
+            Thread.sleep(2000)
+
+            logger.info("trigger compaction [15-17] on new tablet ${newTabletStat.TabletId}")
+            def (code, out, err) = be_run_cumulative_compaction(tabletBackend.Host, tabletBackend.HttpPort, newTabletStat.TabletId)
+            logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+            Assert.assertEquals(code, 0)
+            def compactJson = parseJson(out.trim())
+            Assert.assertEquals("success", compactJson.status.toLowerCase())
+
+            // make the schema change run to complete and wait for it
+            GetDebugPoint().disableDebugPointForAllBEs("SchemaChangeJob::_do_process_alter_tablet.block")
+            waitForSchemaChangeDone {
+                sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${table1}' ORDER BY createtime DESC LIMIT 1 """
+                time 2000
+            }
+
+            Thread.sleep(2000)
+
+            // make the cumu compaction run to complete and wait for it
+            GetDebugPoint().disableDebugPointForAllBEs("CumulativeCompaction::execute_compact.block")
+
+
+            // BE should skip to check merged rows in cumu compaction, otherwise it will cause coredump
+            // becasue [11-22] in new tablet will skip to calc delete bitmap becase tablet is in NOT_READY state
+            Thread.sleep(7000)
+
+            qt_sql "select * from ${table1} order by k;"
+
+        } catch(Exception e) {
+            logger.info(e.getMessage())
+            throw e
+        } finally {
+            GetDebugPoint().clearDebugPointsForAllFEs()
+            GetDebugPoint().clearDebugPointsForAllBEs()
+        }
+    }
+}

From 88b36177e81e7ab83a42ffb18322d990dd0d4c16 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Wed, 11 Dec 2024 21:09:01 +0800
Subject: [PATCH 279/399] [regression-test](fix) two tests using same table
 (#45312)

---
 .../suites/correctness_p0/test_bitmap_intersect.groovy        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/regression-test/suites/correctness_p0/test_bitmap_intersect.groovy b/regression-test/suites/correctness_p0/test_bitmap_intersect.groovy
index 4023e5dac8c85a..6d3866fea86655 100644
--- a/regression-test/suites/correctness_p0/test_bitmap_intersect.groovy
+++ b/regression-test/suites/correctness_p0/test_bitmap_intersect.groovy
@@ -16,7 +16,7 @@
  // under the License.
 
  suite("test_bitmap_intersect") {
-     def tableName = "test_bitmap"
+     def tableName = "test_bitmap_intersect"
 
 
      sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -35,4 +35,4 @@
      select bitmap_to_string(bitmap_intersect(user_ids)) from ( select tag, bitmap_union(user_ids) user_ids 
      from ${tableName} group by tag ) t;  """
 
- }
\ No newline at end of file
+ }

From 279c5b7e50065ce2fadb2abd0c70d5281e98aa6d Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Wed, 11 Dec 2024 21:59:22 +0800
Subject: [PATCH 280/399] [chore](regression) Enable batch_download_file in
 p0/p1 pipeline (#45295)

---
 regression-test/pipeline/p0/conf/be.conf | 4 ++++
 regression-test/pipeline/p1/conf/be.conf | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf
index 802ce4bc5f0f2a..760b8762430bd7 100644
--- a/regression-test/pipeline/p0/conf/be.conf
+++ b/regression-test/pipeline/p0/conf/be.conf
@@ -73,3 +73,7 @@ pipeline_task_leakage_detect_period_sec=1
 crash_in_memory_tracker_inaccurate = true
 enable_table_size_correctness_check=true
 enable_brpc_connection_check=true
+
+# enable download small files in batch, see apache/doris#45061 for details
+enable_batch_download = true
+
diff --git a/regression-test/pipeline/p1/conf/be.conf b/regression-test/pipeline/p1/conf/be.conf
index 70dc4c23e1ac8c..4401bc36498422 100644
--- a/regression-test/pipeline/p1/conf/be.conf
+++ b/regression-test/pipeline/p1/conf/be.conf
@@ -65,3 +65,7 @@ pipeline_task_leakage_detect_period_sec=1
 crash_in_memory_tracker_inaccurate = true
 enable_table_size_correctness_check=true
 enable_brpc_connection_check=true
+
+# enable download small files in batch, see apache/doris#45061 for details
+enable_batch_download = true
+

From 32fa95d28623547395f5cc65ff16a32a1a24df69 Mon Sep 17 00:00:00 2001
From: Uniqueyou <134280716+wyxxxcat@users.noreply.github.com>
Date: Wed, 11 Dec 2024 22:09:06 +0800
Subject: [PATCH 281/399] [Fix] Add column with default value string throw
 exception (#45000)

```
ALTER TABLE t ADD COLUMN create_time VARCHAR(20) DEFAULT "CURRENT_TIMESTAMP";
```
```
errCode = 2, detailMessage = Unexpected exception: Cannot invoke "org.apache.doris.analysis.DefaultValueExprDef.getExprName()" because "this.defaultValueExprDef" is null
```
---
 .../org/apache/doris/analysis/ColumnDef.java  |  3 +-
 ...est_alter_add_column_default_value_str.out | 14 ++++
 ..._alter_add_column_default_value_str.groovy | 72 +++++++++++++++++++
 3 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/data/alter_p0/test_alter_add_column_default_value_str.out
 create mode 100644 regression-test/suites/alter_p0/test_alter_add_column_default_value_str.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index 93bcb92f58062a..c2c55c4ce7c955 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -134,7 +134,8 @@ public static DefaultValue currentTimeStampDefaultValueWithPrecision(Long precis
         }
 
         public boolean isCurrentTimeStamp() {
-            return "CURRENT_TIMESTAMP".equals(value) && NOW.equals(defaultValueExprDef.getExprName());
+            return "CURRENT_TIMESTAMP".equals(value) && defaultValueExprDef != null
+                    && NOW.equals(defaultValueExprDef.getExprName());
         }
 
         public boolean isCurrentTimeStampWithPrecision() {
diff --git a/regression-test/data/alter_p0/test_alter_add_column_default_value_str.out b/regression-test/data/alter_p0/test_alter_add_column_default_value_str.out
new file mode 100644
index 00000000000000..b180902fd16246
--- /dev/null
+++ b/regression-test/data/alter_p0/test_alter_add_column_default_value_str.out
@@ -0,0 +1,14 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select1 --
+1	1	1
+2	2	2
+3	3	3
+
+-- !select4 --
+1	1	1	CURRENT_TIMESTAMP	BITMAP_EMPTY
+2	2	2	CURRENT_TIMESTAMP	BITMAP_EMPTY
+3	3	3	CURRENT_TIMESTAMP	BITMAP_EMPTY
+4	4	4	4	4
+5	5	5	5	5
+6	6	6	6	6
+
diff --git a/regression-test/suites/alter_p0/test_alter_add_column_default_value_str.groovy b/regression-test/suites/alter_p0/test_alter_add_column_default_value_str.groovy
new file mode 100644
index 00000000000000..f3d0386498193b
--- /dev/null
+++ b/regression-test/suites/alter_p0/test_alter_add_column_default_value_str.groovy
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('test_alter_add_column_default_value_str') {
+    def tbl = 'test_alter_add_column_default_value_str_tbl'
+    sql "DROP TABLE IF EXISTS ${tbl} FORCE"
+    sql """
+        CREATE TABLE ${tbl} (
+            `k1` BIGINT NOT NULL,
+            `v1` BIGINT NULL,
+            `v2` INT NULL,
+        ) ENGINE=OLAP
+        UNIQUE KEY(`k1`)
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+        PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+
+    sql """
+        INSERT INTO ${tbl} VALUES (1,1,1),(2,2,2),(3,3,3)
+    """
+
+    sql """ SYNC """
+
+    // Check data before ALTER TABLE
+    qt_select1 """ SELECT * FROM ${tbl} ORDER BY k1 """
+    def tb1 = sql """ show create table ${tbl}"""
+    logger.info("tb1:{}", tb1[0][1])
+    assertFalse(tb1[0][1].contains("varchar(20) NOT NULL DEFAULT \"CURRENT_TIMESTAMP\""))
+    assertFalse(tb1[0][1].contains("varchar(20) NOT NULL DEFAULT \"BITMAP_EMPTY\""))
+
+    sql """
+        ALTER TABLE ${tbl} add column timestamp_default varchar(20) NOT NULL default "CURRENT_TIMESTAMP";
+    """
+
+    sql """
+        ALTER TABLE ${tbl} add column bitmap_default varchar(20) NOT NULL default "BITMAP_EMPTY";
+    """
+
+    waitForSchemaChangeDone {
+        sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tbl}' ORDER BY createtime DESC LIMIT 1 """
+        time 600
+    }
+
+    // Check table structure after ALTER TABLE
+    def tb2 = sql """ show create table ${tbl}"""
+    logger.info("tb2:{}", tb2[0][1])
+    assertTrue(tb2[0][1].contains("varchar(20) NOT NULL DEFAULT \"CURRENT_TIMESTAMP\""))
+    assertTrue(tb2[0][1].contains("varchar(20) NOT NULL DEFAULT \"BITMAP_EMPTY\""))
+    def resultCreate = sql """ SHOW CREATE TABLE ${tbl} """
+    sql """insert into ${tbl} values (4,4,4,"4","4")"""
+    sql """insert into ${tbl} (k1,v1,v2,timestamp_default,bitmap_default) values (5,5,5,5,"5")"""
+    sql """insert into ${tbl} (k1,v1,v2,timestamp_default,bitmap_default) values (6,6,6,6,"6")"""
+    qt_select4 """ SELECT k1,v1,v2,timestamp_default,bitmap_default FROM ${tbl} ORDER BY k1 """
+
+    sql "DROP TABLE IF EXISTS ${tbl} FORCE"
+}

From a4681f3763d1fcb5ee1dea125f286af5da27be9f Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Wed, 11 Dec 2024 22:16:06 +0800
Subject: [PATCH 282/399] [regression-test](opt) add debug log (#45307)

When running stream load in regression tests and loading same file into
different tables, "org.apache.http.ConnectionClosedException: Premature
end of Content-Length delimited message body (expected: 129,961,387;
received: 125,351,592)" maybe met.
This pr add log to try to debug this problem.
---
 .../org/apache/doris/regression/action/StreamLoadAction.groovy   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
index 70a7f41e82b257..97b6d038fa701c 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
@@ -279,6 +279,7 @@ class StreamLoadAction implements SuiteAction {
                     fileName = cacheHttpFile(client, fileName)
                 } else {
                     entity = new InputStreamEntity(httpGetStream(client, fileName))
+                    log.info("http entity length is ${entity.contentLength}")
                     return entity;
                 }
             }

From 1ee8d40617ba6fff14c7cf71ed2c8851126a1c40 Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Wed, 11 Dec 2024 22:23:15 +0800
Subject: [PATCH 283/399] [fix](txn insert) txn insert show error url (#45254)

```
mysql> begin;

mysql> insert into test values('a', 'a', 1);

mysql> commit;
ERROR 1105 (HY000): errCode = 2, detailMessage = [DATA_QUALITY_ERROR]too many filtered rows, url: http://172.21.16.12:9082/api/_load_error_log?file=__shard_110/error_log_insert_stmt_txn_insert_ef05df534f6b4ad7-8782cce460d85b4b_733cffdc9392473f_bd6b2ec40c621119
```
---
 .../runtime/stream_load/stream_load_executor.cpp  |  9 +++++++--
 .../insert_p0/transaction/txn_insert.groovy       | 15 ++++++++++++++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/be/src/runtime/stream_load/stream_load_executor.cpp b/be/src/runtime/stream_load/stream_load_executor.cpp
index 482fadac44e051..ad4d22946f1b83 100644
--- a/be/src/runtime/stream_load/stream_load_executor.cpp
+++ b/be/src/runtime/stream_load/stream_load_executor.cpp
@@ -85,13 +85,18 @@ Status StreamLoadExecutor::execute_plan_fragment(std::shared_ptr<StreamLoadConte
         ctx->number_unselected_rows = state->num_rows_load_unselected();
         ctx->loaded_bytes = state->num_bytes_load_total();
         int64_t num_selected_rows = ctx->number_total_rows - ctx->number_unselected_rows;
+        ctx->error_url = to_load_error_http_path(state->get_error_log_file_path());
         if (!ctx->group_commit && num_selected_rows > 0 &&
             (double)ctx->number_filtered_rows / num_selected_rows > ctx->max_filter_ratio) {
             // NOTE: Do not modify the error message here, for historical reasons,
             // some users may rely on this error message.
-            *status = Status::DataQualityError("too many filtered rows");
+            if (ctx->need_commit_self) {
+                *status =
+                        Status::DataQualityError("too many filtered rows, url: " + ctx->error_url);
+            } else {
+                *status = Status::DataQualityError("too many filtered rows");
+            }
         }
-        ctx->error_url = to_load_error_http_path(state->get_error_log_file_path());
 
         if (status->ok()) {
             DorisMetrics::instance()->stream_receive_bytes_total->increment(ctx->receive_bytes);
diff --git a/regression-test/suites/insert_p0/transaction/txn_insert.groovy b/regression-test/suites/insert_p0/transaction/txn_insert.groovy
index 424a03ee173f5a..f4c8caa35ba991 100644
--- a/regression-test/suites/insert_p0/transaction/txn_insert.groovy
+++ b/regression-test/suites/insert_p0/transaction/txn_insert.groovy
@@ -107,7 +107,7 @@ suite("txn_insert") {
             sql """ DROP TABLE IF EXISTS $tableMV """
             sql """
                 create table $tableMV (
-                    id int default '10', 
+                    id int not null, 
                     c1 int default '10'
                 ) distributed by hash(id, c1) 
                 properties('replication_num'="1");
@@ -122,6 +122,19 @@ suite("txn_insert") {
             order_qt_select5 """select * from $tableMV"""
             order_qt_select6 """select c1 from $tableMV"""
         } while (0);
+        do {
+            try {
+                sql "begin"
+                sql """insert into $tableMV values(9, 2), (10, 4)"""
+                sql """insert into $tableMV values('aa', 6)"""
+                sql "commit"
+            } catch (Exception e) {
+                sql "rollback"
+                logger.info("insert into $tableMV failed: " + e.getMessage())
+                assertTrue(e.getMessage().contains("too many filtered rows"))
+                assertTrue(e.getMessage().contains("url"))
+            }
+        } while (0);
 
         // ------------------- insert into select -------------------
         for (int j = 0; j < 3; j++) {

From 02e8c8c547ee9cb4d893f4cb3befd7b53acc1f28 Mon Sep 17 00:00:00 2001
From: Xin Liao <liaoxin@selectdb.com>
Date: Wed, 11 Dec 2024 22:35:51 +0800
Subject: [PATCH 284/399] [Chore](load) print error url to the log when
 encountering data quality issues during load (#45212)

There has been an issue where load does not print the error URL.
Printing the error URL in the log is convenient for locating the
problem.
---
 be/src/http/action/stream_load.cpp                 |  3 ++-
 be/src/runtime/fragment_mgr.cpp                    | 14 +++++++++++---
 be/src/runtime/runtime_state.cpp                   |  4 +++-
 .../main/java/org/apache/doris/qe/Coordinator.java |  1 +
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp
index 7e71f3eb910053..e8db5cb542fb4b 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -145,7 +145,8 @@ void StreamLoadAction::handle(HttpRequest* req) {
               << ctx->commit_and_publish_txn_cost_nanos / 1000000
               << ", number_total_rows=" << ctx->number_total_rows
               << ", number_loaded_rows=" << ctx->number_loaded_rows
-              << ", receive_bytes=" << ctx->receive_bytes << ", loaded_bytes=" << ctx->loaded_bytes;
+              << ", receive_bytes=" << ctx->receive_bytes << ", loaded_bytes=" << ctx->loaded_bytes
+              << ", error_url=" << ctx->error_url;
 
     // update statistics
     streaming_load_requests_total->increment(1);
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index ce18071fda0d07..19e8f76366c084 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -390,12 +390,20 @@ void FragmentMgr::coordinator_callback(const ReportStatusRequest& req) {
     params.load_counters.emplace(s_unselected_rows, std::to_string(num_rows_load_unselected));
 
     if (!req.runtime_state->get_error_log_file_path().empty()) {
-        params.__set_tracking_url(
-                to_load_error_http_path(req.runtime_state->get_error_log_file_path()));
+        std::string error_log_url =
+                to_load_error_http_path(req.runtime_state->get_error_log_file_path());
+        LOG(INFO) << "error log file path: " << error_log_url
+                  << ", query id: " << print_id(req.query_id)
+                  << ", fragment instance id: " << print_id(req.fragment_instance_id);
+        params.__set_tracking_url(error_log_url);
     } else if (!req.runtime_states.empty()) {
         for (auto* rs : req.runtime_states) {
             if (!rs->get_error_log_file_path().empty()) {
-                params.__set_tracking_url(to_load_error_http_path(rs->get_error_log_file_path()));
+                std::string error_log_url = to_load_error_http_path(rs->get_error_log_file_path());
+                LOG(INFO) << "error log file path: " << error_log_url
+                          << ", query id: " << print_id(req.query_id)
+                          << ", fragment instance id: " << print_id(rs->fragment_instance_id());
+                params.__set_tracking_url(error_log_url);
             }
             if (rs->wal_id() > 0) {
                 params.__set_txn_id(rs->wal_id());
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index ecaf99061a070b..df7c4141691d0b 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -341,7 +341,9 @@ Status RuntimeState::create_error_log_file() {
         LOG(WARNING) << error_msg.str();
         return Status::InternalError(error_msg.str());
     }
-    VLOG_FILE << "create error log file: " << _error_log_file_path;
+    LOG(INFO) << "create error log file: " << _error_log_file_path
+              << ", query id: " << print_id(_query_id)
+              << ", fragment instance id: " << print_id(_fragment_instance_id);
 
     return Status::OK();
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 4905050e6e8cfe..42e2e0a3634870 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -2409,6 +2409,7 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) {
             updateLoadCounters(params.getLoadCounters());
         }
         if (params.isSetTrackingUrl()) {
+            LOG.info("query_id={} tracking_url: {}", DebugUtil.printId(queryId), params.getTrackingUrl());
             trackingUrl = params.getTrackingUrl();
         }
         if (params.isSetTxnId()) {

From 1fafe516f8b1d42015cdb8a446f3a01bb4939162 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Thu, 12 Dec 2024 10:29:48 +0800
Subject: [PATCH 285/399] [chore](checkpoint) add enable_checkpoint config
 (#45301)

Add a config `enable_checkpoint`, to support pause checkpoint, so that
The user can verify the replay of edit logs easily.
---
 .../src/main/java/org/apache/doris/common/Config.java       | 6 ++++++
 .../src/main/java/org/apache/doris/master/Checkpoint.java   | 5 +++++
 2 files changed, 11 insertions(+)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index c5c7023b20f54b..b02305bdbfc316 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2950,6 +2950,12 @@ public class Config extends ConfigBase {
     })
     public static long auto_analyze_interval_seconds = 86400; // 24 hours.
 
+    // A internal config to control whether to enable the checkpoint.
+    //
+    // ATTN: it only used in test environment.
+    @ConfField(mutable = true, masterOnly = true)
+    public static boolean enable_checkpoint = true;
+
     //==========================================================================
     //                    begin of cloud config
     //==========================================================================
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java b/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
index 4934f8fb0b340f..d3e133fda843f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
@@ -86,6 +86,11 @@ protected void runAfterCatalogReady() {
     // public for unit test, so that we can trigger checkpoint manually.
     // DO NOT call it manually outside the unit test.
     public synchronized void doCheckpoint() throws CheckpointException {
+        if (!Config.enable_checkpoint) {
+            LOG.warn("checkpoint is disabled. please enable the config 'enable_checkpoint'.");
+            return;
+        }
+
         if (!Env.getServingEnv().isHttpReady()) {
             LOG.info("Http server is not ready.");
             return;

From d775eb717208aa385d59d419f3177dd004bb6459 Mon Sep 17 00:00:00 2001
From: starocean999 <lichi@selectdb.com>
Date: Thu, 12 Dec 2024 10:32:38 +0800
Subject: [PATCH 286/399] [fix](nereids) SqlBlockRuleCommand should implement
 ForwardWithSync (#45234)

---
 .../doris/nereids/trees/plans/commands/SqlBlockRuleCommand.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SqlBlockRuleCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SqlBlockRuleCommand.java
index 785da1aeb5861e..47be7f96b26b88 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SqlBlockRuleCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SqlBlockRuleCommand.java
@@ -36,7 +36,7 @@
 /**
  * Common class for SqlBlockRule Commands.
  */
-public abstract class SqlBlockRuleCommand extends Command {
+public abstract class SqlBlockRuleCommand extends Command implements ForwardWithSync {
     public static final String SQL_PROPERTY = "sql";
 
     public static final String SQL_HASH_PROPERTY = "sqlHash";

From e70175fc2d5d53ffe5b7b00c7c634ba6f3c3044e Mon Sep 17 00:00:00 2001
From: LiBinfeng <libinfeng@selectdb.com>
Date: Thu, 12 Dec 2024 10:34:29 +0800
Subject: [PATCH 287/399] [fix](Nereids) fix create routine load forward
 strategy (#45253)

---
 .../trees/plans/commands/load/CreateRoutineLoadCommand.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/CreateRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/CreateRoutineLoadCommand.java
index da7ab86c17abac..5632bdc2585ec0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/CreateRoutineLoadCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/load/CreateRoutineLoadCommand.java
@@ -21,7 +21,7 @@
 import org.apache.doris.catalog.Env;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.commands.Command;
-import org.apache.doris.nereids.trees.plans.commands.NoForward;
+import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
 import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
 import org.apache.doris.qe.ConnectContext;
@@ -68,7 +68,7 @@
       type of routine load:
           KAFKA
 */
-public class CreateRoutineLoadCommand extends Command implements NoForward {
+public class CreateRoutineLoadCommand extends Command implements ForwardWithSync {
     CreateRoutineLoadInfo createRoutineLoadInfo;
 
     public CreateRoutineLoadCommand(CreateRoutineLoadInfo createRoutineLoadInfo) {

From 3af5ef661fdd946772493bfe007e86e35153d822 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Thu, 12 Dec 2024 10:36:00 +0800
Subject: [PATCH 288/399] [improve](backup) Add config
 ignore_backup_tmp_partitions (#45240)

To filter tmp partitions, instead of report exception.
---
 .../java/org/apache/doris/common/Config.java  |  9 +++++
 .../apache/doris/backup/BackupHandler.java    |  4 +--
 .../org/apache/doris/backup/BackupJob.java    | 15 +++++---
 .../org/apache/doris/catalog/OlapTable.java   |  8 +++++
 ...ackup_restore_backup_temp_partition.groovy | 34 +++++++++++++++++++
 5 files changed, 63 insertions(+), 7 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index b02305bdbfc316..42a64a2770ff69 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1638,6 +1638,15 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true, masterOnly = true)
     public static boolean ignore_backup_not_support_table_type = false;
 
+    /**
+     * whether to ignore temp partitions when backup, and not report exception.
+     */
+    @ConfField(mutable = true, masterOnly = true, description = {
+        "是否忽略备份临时分区，不报异常",
+        "Whether to ignore temp partitions when backup, and not report exception."
+    })
+    public static boolean ignore_backup_tmp_partitions = false;
+
     /**
      * A internal config, to control the update interval of backup handler. Only used to speed up tests.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 6a12eee3a78cb3..6f88881e3cb2a3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -446,14 +446,14 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws
             OlapTable olapTbl = (OlapTable) tbl;
             tbl.readLock();
             try {
-                if (olapTbl.existTempPartitions()) {
+                if (!Config.ignore_backup_tmp_partitions && olapTbl.existTempPartitions()) {
                     ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
                             "Do not support backup table " + olapTbl.getName() + " with temp partitions");
                 }
 
                 PartitionNames partitionNames = tblRef.getPartitionNames();
                 if (partitionNames != null) {
-                    if (partitionNames.isTemp()) {
+                    if (!Config.ignore_backup_tmp_partitions && partitionNames.isTemp()) {
                         ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
                                 "Do not support backup temp partitions in table " + tblRef.getName());
                     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index d0a957dcd9ae0d..1c897368624f11 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -597,10 +597,15 @@ private void prepareSnapshotTaskForOlapTableWithoutLock(Database db, OlapTable o
         // check backup table again
         if (backupTableRef.getPartitionNames() != null) {
             for (String partName : backupTableRef.getPartitionNames().getPartitionNames()) {
-                Partition partition = olapTable.getPartition(partName);
+                Partition partition = olapTable.getPartition(partName, false); // exclude tmp partitions
                 if (partition == null) {
-                    status = new Status(ErrCode.NOT_FOUND, "partition " + partName
-                            + " does not exist  in table" + backupTableRef.getName().getTbl());
+                    if (olapTable.getPartition(partName, true) != null) {
+                        status = new Status(ErrCode.NOT_FOUND, "backup tmp partition " + partName
+                                + " in table " + backupTableRef.getName().getTbl() + " is not supported");
+                    } else {
+                        status = new Status(ErrCode.NOT_FOUND, "partition " + partName
+                                + " does not exist in table " + backupTableRef.getName().getTbl());
+                    }
                     return;
                 }
             }
@@ -609,10 +614,10 @@ private void prepareSnapshotTaskForOlapTableWithoutLock(Database db, OlapTable o
         // create snapshot tasks
         List<Partition> partitions = Lists.newArrayList();
         if (backupTableRef.getPartitionNames() == null) {
-            partitions.addAll(olapTable.getPartitions());
+            partitions.addAll(olapTable.getPartitions()); // no temp partitions in OlapTable.getPartitions()
         } else {
             for (String partName : backupTableRef.getPartitionNames().getPartitionNames()) {
-                Partition partition = olapTable.getPartition(partName);
+                Partition partition = olapTable.getPartition(partName, false);  // exclude tmp partitions
                 partitions.add(partition);
             }
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index f3d27908bdd3d8..92d9aa4e9c7f82 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2023,6 +2023,14 @@ public OlapTable selectiveCopy(Collection<String> reservedPartitions, IndexExtSt
             }
         }
 
+        if (isForBackup) {
+            // drop all tmp partitions in copied table
+            for (Partition partition : copied.tempPartitions.getAllPartitions()) {
+                copied.partitionInfo.dropPartition(partition.getId());
+            }
+            copied.tempPartitions = new TempPartitions();
+        }
+
         if (reservedPartitions == null || reservedPartitions.isEmpty()) {
             // reserve all
             return copied;
diff --git a/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy b/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
index 61514bedd88215..8f06ff4eb8bfa3 100644
--- a/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
+++ b/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
@@ -62,6 +62,7 @@ suite("test_backup_restore_backup_temp_partition", "backup_restore") {
         ALTER TABLE ${dbName}.${tableName} ADD TEMPORARY PARTITION tp1 VALUES LESS THAN ("70")
         """
 
+    sql "ADMIN SET FRONTEND CONFIG ('ignore_backup_tmp_partitions' = 'false')"
     test {
         sql """
             BACKUP SNAPSHOT ${dbName}.${snapshotName}
@@ -71,6 +72,39 @@ suite("test_backup_restore_backup_temp_partition", "backup_restore") {
         exception "Do not support backup table ${tableName} with temp partitions"
     }
 
+    // ignore the tmp partitions
+    sql "ADMIN SET FRONTEND CONFIG ('ignore_backup_tmp_partitions' = 'true')"
+    sql """
+        BACKUP SNAPSHOT ${dbName}.${snapshotName}
+        TO `${repoName}`
+        ON (${tableName})
+    """
+
+    syncer.waitSnapshotFinish(dbName)
+    def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+    assertTrue(snapshot != null)
+
+    // The restored table has no tmp partitions
+    sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+
+    sql """
+        RESTORE SNAPSHOT ${dbName}.${snapshotName}
+        FROM `${repoName}`
+        ON (
+            `${tableName}` PARTITION (p1, p2, p3)
+        )
+        PROPERTIES
+        (
+            "backup_timestamp" = "${snapshot}",
+            "reserve_replica" = "true"
+        )
+    """
+
+    syncer.waitAllRestoreFinish(dbName)
+    def res = sql "SHOW TEMPORARY PARTITIONS FROM ${dbName}.${tableName}"
+    assertTrue(res.size() == 0);
+
+    sql "ADMIN SET FRONTEND CONFIG ('ignore_backup_tmp_partitions' = 'false')"
     sql "DROP TABLE ${dbName}.${tableName} FORCE"
     sql "DROP DATABASE ${dbName} FORCE"
     sql "DROP REPOSITORY `${repoName}`"

From 5706b6bfe7c2f6115294167fb4cac0185799d061 Mon Sep 17 00:00:00 2001
From: chunping <cece_mail@163.com>
Date: Thu, 12 Dec 2024 10:39:26 +0800
Subject: [PATCH 289/399] [test](show_data) test the correctness of data
 statistics in cloud mode (#44947)

---
 aazcp.tar.gz                                  | Bin 0 -> 4218 bytes
 regression-test/conf/regression-conf.groovy   |   7 +
 regression-test/framework/pom.xml             |   5 +
 regression-test/plugins/aliyunOssSdk.groovy   | 169 ++++++++++++
 .../suites/show_data/ddl/lineitem_delete.sql  |   2 +
 .../suites/show_data/ddl/lineitem_dup.sql     |  25 ++
 .../suites/show_data/ddl/lineitem_mow.sql     |  25 ++
 .../show_data/test_show_mow_data.groovy       | 240 ++++++++++++++++++
 8 files changed, 473 insertions(+)
 create mode 100644 aazcp.tar.gz
 create mode 100644 regression-test/plugins/aliyunOssSdk.groovy
 create mode 100644 regression-test/suites/show_data/ddl/lineitem_delete.sql
 create mode 100644 regression-test/suites/show_data/ddl/lineitem_dup.sql
 create mode 100644 regression-test/suites/show_data/ddl/lineitem_mow.sql
 create mode 100644 regression-test/suites/show_data/test_show_mow_data.groovy

diff --git a/aazcp.tar.gz b/aazcp.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..681acf72cde8599720e938dcd7301612b5ec5f00
GIT binary patch
literal 4218
zcmV-=5QXm_iwFP!000001MNF&j~mC4`n<oQGcc^7G&_9kD_DCKa7nJjh|68`k=T(I
z#O$z1PBfgMr)Na3bqn}IaA)W2I2Ysr9P%MJ<P9fC0>m*48~HQZmB0BfTvhkHIporc
zq|4p00WHbys_J@oRnJJ%-qt7iYtMBQVC-yd@lP1(PyW5Qxl`QUE^U-{Hm((lh3)OF
zYh>&BvL%McT+Jcmnq%AUBDky8|CbJp+rQs8pQ8^fYyZ-Av9z-Z^Tpz3skpNLix`pp
zO~aywOUJ#wH&NM}=Cj)a4ZOX%InVyZjh$l3{+rvS(lt_eb}Y;v|Ih7TY1PV|8tIhx
z8#PitAkCu=sl8rrciJR$h_O8-@){wH-chSsYu&BAL15IIo!aeMi(@pqjfTP@j?1l%
z4=@L!-90{D4AH1JYt8QAJ}k_Ec&%G*cIq8Yq*|-g56ca?xTTbGDY)9}om#V6s~)%N
zmD*y^YQ0@K>NY!9k?fRTC&5?**Q#~8t>!_ae4CKUopMVqrXf(wcBkCww&y{#@6?a0
zXdc`PKa<Oqqr<~`M*w6jw`!GI{aCDLEF<xHv)yTRD;=?aX)B9?!=ox^z~`}$hCwcB
zLT`k8tK8yXo0}2XT5he{J#N%1$OBkjj{Hc8JVuFJLrM4mV1qlh>igYJtxERaAa}~`
zJ24tLvfr)Tt%1XCtR1(Gj%%$>z1HRqD>-yx8oK5hw$-z|u|)QWRFr_BwYC50!22f|
z8TlW={RQX$?d>T4mo|9*-z={3|BD!_{J+Zo&p-cnoAuYatK|PrII`-0YEqXz!z18|
z{9h<QZy4tPLJ9K!7JRPq|BD#aTB8Q3Vc^(f^3o$$J2mM!nU1JK<LUm^9x1%C3Y-6f
zL&QJ)Q-|Q;56_;*9*g52qJC#Hb^lk|*<AJiFJ$ENq+#gPVzf_Oo46xN$`eh8kG4H<
zryAse1KYFu{1_#2xqXl$@RK^kw&)sQ+YTAq0HxcO>lmk=3lpZGNVK6t>6ltBQweEP
z%E?1nfW_FrFgbqTU;+;qm>TW~U<^j4w)21tfRxtn8@RP*62pRtF=qr39XiyUzF`ev
z>&cm83`Z`prxtbC$e6%>9TcT~5HQ08;b2>k_00BsQDRd2%23F+fDnpYQVT=|07R1f
zRVnuh#~y2E#IjxDF&a@P^evrETwoU%8$$`ASvtq|r3?1}9^deZ*r%=r@ER9|*n>D2
z(Oker0)&iQce0z$Pp4B=<NT_&Gt37P=Ap-~HQTjy;Mqs%S|()-boHiZIH2LvGonp+
zSp<omnA#NW$IZme4?IjA(6=>I5Ss-q5I0h!vVcJlJPrU#)3}ACa+}oK64@`e>urU6
zyWY8T)a{UOms>3?-=R)BYLUuOvs&-ekDBo7fRvkWAm-hAv#Jm%N8lg=eNG${0ys5L
zpR^zIrho_Z72is1LUm(c=pddo^t2%*L;E3hEOeC#b;brm!!ls956&?*T<%f~QD<aR
z*KitLN!QSaKvXahupmxMv<Cx&41rVcR0VtIK@FDOKNX+wm(u)LV=i@&P~N2swJTQF
zCyw3sbax$RS)MsMfErdFq_sUq2ZKTl<}c!3&O+Xu=zY#C@;|owAgVBI)piV~u7Qqh
z2SULa0zZ8^(mdwpcGdAL*BDc^esr5p98z~=1F<y%<V;)R(FAc#Alx-lED(fd3R-~b
zu9#)U+mt&!YzE+K%!8s)mXJSe6nkt0e-N@~SUugey}qPKlaiDpkJhk5$mjXwI&e<;
zgfSrU*9=y{*+ZC=a{(w0(;g0~qhjpK5^l?}QErG)&W2}6xe(l;uIE_%=lL4{=+gnr
zPN?S69{4EL#yym@$Fwo!d_r6Bh_ullL&EEyCafqB3NBYk2US4UZ-rZ+r^&c#5Uh&e
z<JAZde%`zp=O2gIwgE7y1sr_KXyci6MqP4Uw@q(s^%$LKjs_X_x<ao1%k_D%V;(O(
z_iRED@b-6RA$g*bB<ZC`L+Z9Sx{fL5&cX49hzZevUN*|JK{0F#RGmFk8HAr(D5%4?
z0e4mu&;WQ14DL@DT9~v=o|=mVHiG~OOn(5{hfO`xRs75e+baZ5WfxTnvliEF+y?Q1
z{u3gSuk8`bGtIOK5^Rnfdm8PP33r}33(n|!Vw;<k+ui4W7s_Yx?Oak_TR0T>awZ{k
z%I`8@caMNY8T}@Fmj!k<g2oti+}fL-W-?h~o(|c8NeU;TwjqN-kh^xno>Hd*Sz_M4
z2{IhF>5WgR)3II6Y}r%hQxvo4v9yoVi>P%RkII=?<Q6HAmtPhnaeDs@(ge=V(n~5l
z`gNX;W$@P?j~18pFb9htGeI2p7LfuNz(79ji4@Tr!vX3zaQ1>HW6mNzQIXigUMTM8
z;MDip;Im`xj>bm2>qUheYwl=QAr0mpXb@7|QFqY+V{?8b!Qmm}Y7lI@{y|>cDXFD`
zy7_f=V|(}Oh0Q`<n;7{q0Ody%)ac|(rR~j~O@*|?Wk$Wf%V)M=YsVOC&e^`EKY$2<
z1&q86C|n>f1Lyv(6)4_ZzBTYufnyF1YHI0c6%4%?x378BIlE)HY`370ifx;HsHLJ!
zkpM@D3ln^gzDB#DPtx7yQR}eWhzO$iKusYtTmyG);ngv!6-Z^8jzKD1e6iZ}5IT<r
z{f01T&*<+3O6>5NK!-5VuyxI>!XeOxQw=3YdxpjZ9~J@?Q2%4wr4i^!<P|r<uMJS{
ztHKL!h5och?w!OB4;kH2EbgLPrc;%Vr$Xf`i|_ZOfN|+0mx>G=!lMJkaQa3*4S_+B
zjwgOz`>KjG$m0nWai~pv!Vpm;Bq)U3_Y4yYCUy$&7M&)i<=iVFJUfMgbpjTbCEasO
zvi>@`UF(q8$sIuiLZ_`C7v2wbV0esW&>o^kNjI~4@L{7xsJ_tO0NGzs>04BX5;jHv
zD=19T0#u!9hI;@ei~y+Pz<R2nQXxro=v%IgI@$LI0}8cwiw88Hs#_D!6-S6a8*BiG
zHOH|Xu?T5A7gM6`xiMaXJ+h^hZiB^Q_GBV}xIlXnxhW*ON#G0tvU~=~^9hiNPrzqH
zJaKgBy!<Mp>#o>g2E<gb#yB{MVjc(6$mdc{r^GCcP`s##=k|P_c@tjn`eex|oPT-U
z8+#_y^$%%&0QoJZiNvWOHWBW%Pa!yICC*KDZ1@fu37_^#v0k_K(RIX+Gzd~o`*#SO
z<@z&w2x%igI8}U()5xM~lldtp<?a<u(8^FY#}Xu3dp6e~+{<yr`;HS$Jj;A=eAJ$o
z5He(4rGka20G%6PzL~BuVeSRx3n-SLGWV5MAY-aoEWW`=7#Es2|L%b|195blcmKL=
zi5nn(QDVRk9V!cG{RnoRywr<P4WUCq*dF+7?1^&J*Ne(B%4P%#%WINLp3JCJeKu&N
zR=3%#H*ZTb!144_9=LHACZbPCpqMshhM@5Qi1r!dotNi0)Z&wehG`m1&duUs^97Re
zF4Wb=6AXlq)J?KQZjkK){3-hF*%lca$=;`L&udTTim+sY%yhU#lywu1n_Uv6rA#oD
zgI%%Bh>xpjEo#}&hZf;NKgqCU!@Cjip;s54D=Fw)cB7i%XV(wv&3gMztvcJz&$g?b
z5oC?N%1lZpa%lLZX!xK9?*Zf)-?_EG^)Q5F5j)szM_hs8668^95u6mjNm;#l!ep^{
z#v7gk!!p<ium1+TpOR(`<d+`<s7;bLC<_|NpbSaXAH0U=e~wPSOLg~FRLLhVff9AR
z4>D8B&ntE0puCZyN+l)N61Z6Ee03}=h@^BM5mSt=>BtckG{jvV`YK-Nor=D#$6VXN
zo7Lqt=3{iQe%s?u3*f|3;d5XNRi5fA{5wC*pjZ*$$BzBbfKxHXZ`JlN>H7fKjo|?k
zxn4#SR_|ZYhRriyfW1UBmN;-?!e*Q=Hk>W#t@3&Olm(}x>EVSj@s%Bjnt<0HETcuB
z+1@BWNcjj`9=Dyw1%4(r-hM7iq2?o`=$v{Wn~OJ|Kxs)*`y3}NOyDSVV7#?7R{~s0
zWh~t^C>ISfj(fydh5wed2MXv+31v?*fWDM0{QJ9+=VT?jxa-~ojx|#R+<}c{a)Vk!
zcO(m5l$fHFl&nyD<uN$((a~1#=jNn8zVFC$tM=52BomuFr5lg>gIPZwFyE%j*x`T!
zrwnSayVN<Qjt$F8(+N;gv<9pTUF*_R5cnCm*8vOND?_pVSrv1F*^>#f0MEdEgEtJ8
z+w;I9d#>F(+>c*%Vp<McWCS@dsM(LF-5^^6;9cy>3Th!?;s^;%dUt=updqH9r1D-u
ze3x-8NAgfwlr|$#2{rMd;0*D@ryzbc)!R?Q@(s?i|8eymoqu?bnXP8rfzz9RITt&I
z@%e=}9QDR$>`n8^i*F9q20FprB|1T4*}HMhfq3wjctuD`5a4r2yAJgmkOG4DVmG7+
zGC*a9R~!NCJqf=_C&3j(y#Gft2rSj)DG^Ok%Ie8bE55P(ed4<ZP%)DRP{kxvmZ}!V
zq8x6mMD!P2(a!9!bnO)wU7HwRq3H6XACM|O;lKl;ODq07Gj-K#So2DvbX)t?n!tGn
zr)Y_G6O@BKDWgGx_Vw}Se}D4h-#+=>r<cF@?Bd;jy?F007k~KKlaGG=_>;eU`Putl
z{NQgFAO7_6=kHwn@CT1S`Q_tJeh;gcfBfqVrG#sW4YPbP7@0n~Gw)&J9N%0oop2EX
z_TCBE`|`7QFMj!(i$8t%<!A5BrI_Kz>%|kHm5X<NborA%J^ARP$Dh8Rp!&g29)JG9
zJgR{B#pfSge(+zH@BWb7-Oq$cyU@+23JvmA9{i<)C&fZKbnw*)42=*ie5Sr9#b0HQ
zQWvIAsoFX^j$VKFUw!w}93r2;l0?8uy8~!LW&ZR$C<pL)I-UWbgLD2iZJYwWN(<0A
z6+xedSjMZlAw3mebJlcPHceT&fB*hV_wS3X@83U{lt>AY=I1=6%6_maXAnSbN3TXe
zV^vh(bc=E+Bod##!Z*9cqo74~+YybJ1g7MPfA`)AG^8RcM!&H__kW!v)0BAa0_+@a
zpG&t<{@WJf(R`6#iz(nixHqAiKH|J2a2CED63MR+BR7)@F=fm<I~f>D$4gZYD(xFN
zqA~G3agwH849Dy>Spu4jfYM8kaBDc{?n#`ebaKv_NhK}J3E%$(&oTf0#czIn@!@|&
z8T!e8e(&<<pI*HCapncWOeSZkW(i0?`QFEn!3e)(B3om_g49c%{NcR;z5MgPU4H*p
z7k_*|Sv2w6sEtB&9W_%^`Cbya$O-SgmpK<hK6CS_2Cj7l)!CblWr#<9EXZG7y~Uy{
zcZH_rI*5jsk>C%;xp-YG9smjnd4j$QUs0LIFrFdj3h}j`1sod02aR{J8Q`I&Ik+^Z
z4v1xW@F)L0vxsLnT?3Bx_GVfp9Er!0bA>BFTA3qEA$E)Nl*A%Y<HRC)VyTFs9R7s=
zdfh*|dwj9DlrVAO=jvrSQgPAeY9%1$8ygR5Ik$Gc`tJm*u^Owf8mqAytFaoZu^Owf
Q8ZU7CH(YMTm;g`!0RA>N00000

literal 0
HcmV?d00001

diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index ab9bb0beb91869..bc001126bceadd 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -259,3 +259,10 @@ lakesoulMinioEndpoint="*******"
 metaServiceToken = "greedisgood9999"
 instanceId = "default_instance_id"
 multiClusterInstance = "default_instance_id"
+
+storageProvider = "oss"
+cbsS3Ak = "*******"
+cbsS3Sk = "*******"
+cbsS3Endpoint = "oss-cn-beijing.aliyuncs.com"
+cbsS3Bucket = "test-bucket"
+cbsS3Prefix = "test-cluster-prefix"
diff --git a/regression-test/framework/pom.xml b/regression-test/framework/pom.xml
index 813659989ae60c..cad25df8ced92d 100644
--- a/regression-test/framework/pom.xml
+++ b/regression-test/framework/pom.xml
@@ -409,6 +409,11 @@ under the License.
             <!--Regression tests need to include this jar-->
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>com.aliyun.oss</groupId>
+            <artifactId>aliyun-sdk-oss</artifactId>
+            <version>3.18.1</version>
+        </dependency>
 
     </dependencies>
 </project>
diff --git a/regression-test/plugins/aliyunOssSdk.groovy b/regression-test/plugins/aliyunOssSdk.groovy
new file mode 100644
index 00000000000000..cbc132a088dffb
--- /dev/null
+++ b/regression-test/plugins/aliyunOssSdk.groovy
@@ -0,0 +1,169 @@
+
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import org.apache.doris.regression.suite.Suite;
+import org.apache.doris.regression.Config;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.aliyun.oss.ClientException;
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.OSSException;
+import com.aliyun.oss.model.DeleteObjectsRequest;
+import com.aliyun.oss.model.DeleteObjectsResult;
+import com.aliyun.oss.model.ListObjectsRequest;
+import com.aliyun.oss.model.OSSObjectSummary;
+import com.aliyun.oss.model.ObjectListing;
+
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import groovy.util.logging.Slf4j
+
+Suite.metaClass.initOssClient = { String accessKeyId, String accessKeySecret, String endpoint ->
+    return new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret)
+}
+
+Suite.metaClass.listOssObjectWithPrefix = { OSS client, String bucketName, String prefix="" ->
+    try {
+        ObjectListing objectListing = null;
+        String nextMarker = null;
+        final int maxKeys = 500;
+        List<OSSObjectSummary> sums = null;
+        
+        if (!client.doesBucketExist(bucketName)) {
+            logger.info("no bucket named ${bucketName} in ${endpoint}")
+            return
+        }
+
+        // Gets all object with specified marker by paging. Each page will have up to 100 entries.
+        logger.info("List all objects with prefix:");
+        nextMarker = null;
+        do {
+            objectListing = client.listObjects(new ListObjectsRequest(bucketName).
+                    withPrefix(prefix).withMarker(nextMarker).withMaxKeys(maxKeys));
+            
+            sums = objectListing.getObjectSummaries();
+            for (OSSObjectSummary s : sums) {
+                logger.info("\t" + s.getKey());
+            }
+            
+            nextMarker = objectListing.getNextMarker();
+            
+        } while (objectListing.isTruncated());
+    } catch (OSSException oe) {
+        logger.error("Caught an OSSException, which means your request made it to OSS, "
+                + "but was rejected with an error response for some reason.");
+        logger.error("Error Message: " + oe.getErrorMessage());
+        logger.error("Error Code:       " + oe.getErrorCode());
+        logger.error("Request ID:      " + oe.getRequestId());
+        logger.error("Host ID:           " + oe.getHostId());
+    } catch (ClientException ce) {
+        logger.error("Caught an ClientException, which means the client encountered "
+                + "a serious internal problem while trying to communicate with OSS, "
+                + "such as not being able to access the network.");
+        logger.error("Error Message: " + ce.getMessage());
+    } finally {
+        /*
+            * Do not forget to shut down the client finally to release all allocated resources.
+            */
+        //client.shutdown();
+        logger.info("Done!")
+    }
+
+}
+
+// get file size in a specific directory
+Suite.metaClass.calculateFolderLength = { OSS client, String bucketName, String folder ->
+    long size = 0L;
+    ObjectListing objectListing = null;
+    do {
+        // The default value for MaxKey is 100, and the maximum value is 1000
+        ListObjectsRequest request = new ListObjectsRequest(bucketName).withPrefix(folder).withMaxKeys(1000);
+        if (objectListing != null) {
+            request.setMarker(objectListing.getNextMarker());
+        }
+        objectListing = client.listObjects(request);
+        List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
+        for (OSSObjectSummary s : sums) {
+            size += s.getSize();
+        }
+        } while (objectListing.isTruncated());
+    return size;
+}
+
+Suite.metaClass.shutDownOssClient = { OSS client ->
+    client.shutdown();
+}
+
+
+
+Suite.metaClass.getOssAllDirSizeWithPrefix = { OSS client, String bucketName, String prefix="" ->
+    try {
+        if (!client.doesBucketExist(bucketName)) {
+            logger.info("no bucket named ${bucketName} in ${endpoint}")
+            return
+        }
+
+        // Gets all object with specified marker by paging. Each page will have up to 100 entries.
+        logger.info("List all objects with prefix:");
+        ObjectListing objectListing = null;
+        do {
+            // By default, list 100 files or directories at a time
+            ListObjectsRequest request = new ListObjectsRequest(bucketName).withDelimiter("/").withPrefix(prefix);
+            if (objectListing != null) {
+                request.setMarker(objectListing.getNextMarker());
+            }
+            objectListing = client.listObjects(request);
+            List<String> folders = objectListing.getCommonPrefixes();
+            for (String folder : folders) {
+                logger.info(folder + " : " + (calculateFolderLength(client, bucketName, folder) / (1024 * 1024 * 1024)) + "GB");
+            }
+            List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
+            for (OSSObjectSummary s : sums) {
+                logger.info(s.getKey() + " : " + (s.getSize() / (1024 * 1024 * 1024)) + "GB");
+            }
+        } while (objectListing.isTruncated());
+        
+    } catch (OSSException oe) {
+        logger.error("Caught an OSSException, which means your request made it to OSS, "
+                + "but was rejected with an error response for some reason.");
+        logger.error("Error Message: " + oe.getErrorMessage());
+        logger.error("Error Code:       " + oe.getErrorCode());
+        logger.error("Request ID:      " + oe.getRequestId());
+        logger.error("Host ID:           " + oe.getHostId());
+    } catch (ClientException ce) {
+        logger.error("Caught an ClientException, which means the client encountered "
+                + "a serious internal problem while trying to communicate with OSS, "
+                + "such as not being able to access the network.");
+        logger.error("Error Message: " + ce.getMessage());
+    } finally {
+        /*
+            * Do not forget to shut down the client finally to release all allocated resources.
+            */
+        //client.shutdown();
+        logger.info("Done!")
+    }
+}
+
+
+
diff --git a/regression-test/suites/show_data/ddl/lineitem_delete.sql b/regression-test/suites/show_data/ddl/lineitem_delete.sql
new file mode 100644
index 00000000000000..df8b04056491e9
--- /dev/null
+++ b/regression-test/suites/show_data/ddl/lineitem_delete.sql
@@ -0,0 +1,2 @@
+DELETE from ${table} where L_ORDERKEY >= 0;
+
diff --git a/regression-test/suites/show_data/ddl/lineitem_dup.sql b/regression-test/suites/show_data/ddl/lineitem_dup.sql
new file mode 100644
index 00000000000000..29ed215d2363fe
--- /dev/null
+++ b/regression-test/suites/show_data/ddl/lineitem_dup.sql
@@ -0,0 +1,25 @@
+CREATE TABLE IF NOT EXISTS lineitem_mow (
+  L_ORDERKEY    INTEGER NOT NULL,
+  L_PARTKEY     INTEGER NOT NULL,
+  L_SUPPKEY     INTEGER NOT NULL,
+  L_LINENUMBER  INTEGER NOT NULL,
+  L_QUANTITY    DECIMAL(15,2) NOT NULL,
+  L_EXTENDEDPRICE  DECIMAL(15,2) NOT NULL,
+  L_DISCOUNT    DECIMAL(15,2) NOT NULL,
+  L_TAX         DECIMAL(15,2) NOT NULL,
+  L_RETURNFLAG  CHAR(1) NOT NULL,
+  L_LINESTATUS  CHAR(1) NOT NULL,
+  L_SHIPDATE    DATE NOT NULL,
+  L_COMMITDATE  DATE NOT NULL,
+  L_RECEIPTDATE DATE NOT NULL,
+  L_SHIPINSTRUCT CHAR(25) NOT NULL,
+  L_SHIPMODE     CHAR(10) NOT NULL,
+  L_COMMENT      VARCHAR(44) NOT NULL
+)
+DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+PROPERTIES (
+  "replication_num" = "1"
+)
+
+
diff --git a/regression-test/suites/show_data/ddl/lineitem_mow.sql b/regression-test/suites/show_data/ddl/lineitem_mow.sql
new file mode 100644
index 00000000000000..1d29f44ac8fb6a
--- /dev/null
+++ b/regression-test/suites/show_data/ddl/lineitem_mow.sql
@@ -0,0 +1,25 @@
+CREATE TABLE IF NOT EXISTS lineitem_mow (
+  L_ORDERKEY    INTEGER NOT NULL,
+  L_PARTKEY     INTEGER NOT NULL,
+  L_SUPPKEY     INTEGER NOT NULL,
+  L_LINENUMBER  INTEGER NOT NULL,
+  L_QUANTITY    DECIMAL(15,2) NOT NULL,
+  L_EXTENDEDPRICE  DECIMAL(15,2) NOT NULL,
+  L_DISCOUNT    DECIMAL(15,2) NOT NULL,
+  L_TAX         DECIMAL(15,2) NOT NULL,
+  L_RETURNFLAG  CHAR(1) NOT NULL,
+  L_LINESTATUS  CHAR(1) NOT NULL,
+  L_SHIPDATE    DATE NOT NULL,
+  L_COMMITDATE  DATE NOT NULL,
+  L_RECEIPTDATE DATE NOT NULL,
+  L_SHIPINSTRUCT CHAR(25) NOT NULL,
+  L_SHIPMODE     CHAR(10) NOT NULL,
+  L_COMMENT      VARCHAR(44) NOT NULL
+)
+UNIQUE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+PROPERTIES (
+  "replication_num" = "1"
+)
+
+
diff --git a/regression-test/suites/show_data/test_show_mow_data.groovy b/regression-test/suites/show_data/test_show_mow_data.groovy
new file mode 100644
index 00000000000000..c94e5786d7efae
--- /dev/null
+++ b/regression-test/suites/show_data/test_show_mow_data.groovy
@@ -0,0 +1,240 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// The cases is copied from https://github.com/trinodb/trino/tree/master
+// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/tpcds
+// and modified by Doris.
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+ // loading one data 10 times, expect data size not rising
+suite("test_mow_show_data_in_cloud","p2") {
+    //cloud-mode
+    if (!isCloudMode()) {
+        logger.info("not cloud mode, not run")
+        return
+    }
+
+    def repeate_stream_load_same_data = { String tableName, int loadTimes ->
+        for (int i = 0; i < loadTimes; i++) {
+            streamLoad {
+                table tableName
+                set 'column_separator', '|'
+                set 'compress_type', 'GZ'
+                file """${getS3Url()}/regression/tpch/sf1/lineitem.csv.split00.gz"""
+                time 10000 // limit inflight 10s
+                check { result, exception, startTime, endTime ->
+                    if (exception != null) {
+                        throw exception
+                    }
+                    log.info("Stream load result: ${result}".toString())
+                    def json = parseJson(result)
+                    assertEquals("success", json.Status.toLowerCase())
+                    assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+                    assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+                }
+            }
+        }
+    }
+
+    def get_tablets_from_table = { String table ->
+        def res = sql_return_maparray """show tablets from  ${table}"""
+        return res 
+    }
+
+    def show_tablet_compaction = { HashMap tablet -> 
+        StringBuilder sb = new StringBuilder();
+        sb.append("curl -X GET ")
+        sb.append(tablet["CompactionStatus"])
+        String command = sb.toString()
+        logger.info(command)
+        process = command.execute()
+        code = process.waitFor()
+        err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+        out = process.getText()
+        logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+        assertEquals(code, 0)
+        return parseJson(out.trim())
+    }
+
+    def trigger_tablet_compaction = { HashMap tablet, String compact_type ->
+        //support trigger base/cumulative/full compaction
+        def tabletStatusBeforeCompaction = show_tablet_compaction(tablet)
+        
+        String tabletInBe = tablet
+        String showCompactionStatus = tablet["CompactionStatus"]
+        String triggerCompactionUrl = showCompactionStatus.split("show")[0] + "run?tablet_id=" + tablet["TabletId"] + "&compact_type=" + compact_type
+        StringBuilder sb = new StringBuilder();
+        sb.append("curl -X POST ")
+        sb.append(triggerCompactionUrl)
+        String command = sb.toString()
+        logger.info(command)
+        process = command.execute()
+        code = process.waitFor()
+        err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+        out = process.getText()
+        def outJson = parseJson(out)
+        logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+        // if code = 0 means compaction happend, need to check
+        // other condition may indicate no suitable compaction condition
+        if ( code == 0 && outJson.status.toLowerCase() == "success" ){
+            def compactionStatus = "RUNNING"
+            def tabletStatusAfterCompaction = null
+            long startTime = System.currentTimeMillis()
+            long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min
+            do {
+                tabletStatusAfterCompaction = show_tablet_compaction(tablet)
+                logger.info("tabletStatusAfterCompaction class: " + tabletStatusAfterCompaction.class)
+                logger.info("hhhhhh: " + tabletStatusAfterCompaction.toString())
+                if (tabletStatusAfterCompaction.rowsets.size() < tabletStatusBeforeCompaction.rowsets.size()){
+                    compactionStatus = 'FINISHED'
+                }
+                Thread.sleep(60 * 1000)
+            } while (timeoutTimestamp > System.currentTimeMillis() && (status != 'FINISHED'))
+
+            if (status != "FINISHED") {
+                logger.info("compaction not Finish or failed")
+                return false
+            }
+        }
+    }
+
+    def trigger_compaction = { List<List<Object>> tablets ->
+        for(def tablet: tablets) {
+            trigger_tablet_compaction(tablet, "cumulative")
+            trigger_tablet_compaction(tablet, "base")
+            trigger_tablet_compaction(tablet, "full")
+        }
+    }
+
+    def caculate_table_data_size_in_backend_storage = { List<List<Object>> tablets ->
+        storageType = context.config.otherConfigs.get("storageProvider")
+        Double storageSize = 0
+
+        List<String> tabletIds = []
+        for(def tablet: tablets) {
+            tabletIds.add(tablet["TabletId"])
+        }
+
+        if (storageType.toLowerCase() == "oss") {
+            //cbs means cluster backend storage
+            ak = context.config.otherConfigs.get("cbsS3Ak")
+            sk = context.config.otherConfigs.get("cbsS3Sk")
+            endpoint = context.config.otherConfigs.get("cbsS3Endpoint")
+            bucketName =  context.config.otherConfigs.get("cbsS3Bucket")
+            storagePrefix = context.config.otherConfigs.get("cbsS3Prefix")
+
+            client = initOssClient(ak, sk, endpoint)
+            for(String tabletId: tabletIds) {
+                storageSize += calculateFolderLength(client, bucketName, storagePrefix + "/data/" + tabletId)
+            }
+            shutDownOssClient(client)
+        }
+
+        if (storageType.toLowerCase() == "hdfs") {
+            fsName = context.config.otherConfigs.get("cbsFsName")
+            isKerberosFs = context.config.otherConfigs.get("cbsFsKerberos")
+            fsUser = context.config.otherConfigs.get("cbsFsUser")
+            storagePrefix = context.config.otherConfigs.get("cbsFsPrefix")
+        }
+
+        return storageSize
+    }
+
+    def translate_different_unit_to_MB = { String size, String unitField ->
+        Double sizeKb = 0.0
+        if (unitField == "KB") {
+            sizeKb = Double.parseDouble(size) / 1024
+        } else if (unitField == "MB") {
+            sizeKb = Double.parseDouble(size)
+        } else if (unitField == "GB") {
+            sizeKb = Double.parseDouble(size) * 1024 * 1024
+        } else if (unitField == "TB") {
+            sizeKb = Double.parseDouble(size) * 1024 * 1024 * 1024
+        }
+        return sizeKb
+    }
+
+    def show_table_data_size_through_mysql = { String table ->
+        def mysqlShowDataSize = 0L
+        def res = sql_return_maparray " show data from ${table}"
+        def tableSizeInfo = res[0]
+        def fields = tableSizeInfo["Size"].split(" ")
+        if (fields.length == 2 ){
+            def sizeField = fields[0]
+            def unitField = fields[1]
+            mysqlShowDataSize = translate_different_unit_to_MB(sizeField, unitField)
+        }
+        return mysqlShowDataSize
+    }
+
+    def caculate_table_data_size_through_api = { List<List<Object>> tablets ->
+        Double apiCaculateSize = 0 
+        for (HashMap tablet in tablets) {
+            def tabletStatus = show_tablet_compaction(tablet)
+            
+            for(String rowset: tabletStatus.rowsets){
+                def fields = rowset.split(" ")
+                if (fields.length == 7) {
+                    def sizeField = fields[-2]  // the last field（size）
+                    def unitField = fields[-1]   // The second to last field（unit）
+                    // 转换成 KB
+                    apiCaculateSize += translate_different_unit_to_MB(sizeField, unitField )
+                }
+            }
+        }
+
+        return apiCaculateSize
+    }
+
+    def main = {
+        tableName="lineitem_mow"
+        sql "DROP TABLE IF EXISTS ${tableName};"
+        sql new File("""${context.file.parent}/ddl/${tableName}.sql""").text
+        sql new File("""${context.file.parent}/ddl/lineitem_delete.sql""").text.replaceAll("\\\$\\{table\\}", tableName)
+        List<String> tablets = get_tablets_from_table(tableName)
+        def loadTimes = [1, 10]
+        Map<String, List> sizeRecords = ["apiSize":[], "mysqlSize":[], "cbsSize":[]]
+        for (int i in loadTimes){
+            // stream load 1 time, record each size
+            repeate_stream_load_same_data(tableName, i)
+            def rows = sql_return_maparray "select count(*) as count from ${tableName};"
+            logger.info("table ${tableName} has ${rows[0]["count"]} rows")
+            // 加一下触发compaction的机制
+            trigger_compaction(tablets)
+
+            // 然后 sleep 5min， 等fe汇报完
+            sleep(300 * 1000)
+
+            sizeRecords["apiSize"].add(caculate_table_data_size_through_api(tablets))
+            sizeRecords["cbsSize"].add(caculate_table_data_size_in_backend_storage(tablets))
+            sizeRecords["mysqlSize"].add(show_table_data_size_through_mysql(tableName))
+            sleep(300 * 1000)
+            logger.info("after ${i} times stream load, mysqlSize is: ${sizeRecords["mysqlSize"][-1]}, apiSize is: ${sizeRecords["apiSize"][-1]}, storageSize is: ${sizeRecords["cbsSize"][-1]}")
+            
+        }
+
+        // expect mysqlSize == apiSize == storageSize
+        assertEquals(sizeRecords["mysqlSize"][0], sizeRecords["apiSize"][0])
+        assertEquals(sizeRecords["mysqlSize"][0], sizeRecords["cbsSize"][0])
+        // expect load 1 times ==  load 10 times
+        assertEquals(sizeRecords["mysqlSize"][0], sizeRecords["mysqlSize"][1])
+        assertEquals(sizeRecords["apiSize"][0], sizeRecords["apiSize"][1])
+        assertEquals(sizeRecords["cbsSize"][0], sizeRecords["cbsSize"][1])
+    }
+
+    main()
+}

From 5fcf945b7edceae331a926f4a1cbd17c6389a507 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Thu, 12 Dec 2024 11:47:49 +0800
Subject: [PATCH 290/399] =?UTF-8?q?[Bug](case)=20fix=20conflict=20table=20?=
 =?UTF-8?q?name=20between=20test=5Fbitmap=5Fintersect=20and=20test=5Fbitma?=
 =?UTF-8?q?p=E2=80=A6=20(#45314)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fix conflict table name between test_bitmap_intersect and test_bitmap_serialize
---
 .../suites/correctness_p0/test_bitmap_serialize.groovy          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/correctness_p0/test_bitmap_serialize.groovy b/regression-test/suites/correctness_p0/test_bitmap_serialize.groovy
index d9d899f1706638..cf3b4b169d8e20 100644
--- a/regression-test/suites/correctness_p0/test_bitmap_serialize.groovy
+++ b/regression-test/suites/correctness_p0/test_bitmap_serialize.groovy
@@ -16,7 +16,7 @@
  // under the License.
 
  suite("test_bitmap_serialize") {
-     def tableName = "test_bitmap"
+     def tableName = "test_bitmap_serialize"
 
      sql """ DROP TABLE IF EXISTS ${tableName} """
      sql """

From e2c24e386ce15474ad18ae7663205439d854112a Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Thu, 12 Dec 2024 12:02:19 +0800
Subject: [PATCH 291/399] [fix](cluster key) fix cluster key schema change
 (#45203)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Problem Summary:
for schema change: add a key in the tail of all keys, may get:

```
F20241209 14:52:09.513403 3166619 segment_writer.cpp:1309] Check failed: key.compare(last_key) >= 0 key is not sorted! current key: ^A^B06olTqiL2rEn48nTpmP8, last key: ^B^@^OЭ^BxsmIHehvUnv
*** Check failure stack trace: ***
    @     0x563674f2dcc6  google::LogMessage::SendToLog()
    @     0x563674f2a710  google::LogMessage::Flush()
    @     0x563674f2e509  google::LogMessageFatal::~LogMessageFatal()
    @     0x56366a8a72c0  doris::segment_v2::SegmentWriter::_generate_short_key_index()
    @     0x56366a8a6032  doris::segment_v2::SegmentWriter::append_block()
    @     0x56366a7624c9  doris::SegmentCreator::add_block()
    @     0x56366a731d35  doris::BaseBetaRowsetWriter::add_block()
    @     0x56366a94a233  doris::VSchemaChangeDirectly::_inner_process()
    @     0x56366a95d558  doris::SchemaChange::process()
    @     0x56366a9511e5  doris::SchemaChangeJob::_convert_historical_rowsets()
    @     0x56366a94e5e2  doris::SchemaChangeJob::_do_process_alter_tablet()
    @     0x56366a94cf22  doris::SchemaChangeJob::process_alter_tablet()
    @     0x563669e9e857  doris::alter_tablet_callback()
```
---
 be/src/cloud/cloud_schema_change_job.cpp      |     9 +
 be/src/olap/memtable.cpp                      |     2 +
 be/src/olap/schema_change.cpp                 |     9 +
 .../test_schema_change_add_key_column.csv     |  8192 +++++++
 .../test_schema_change_add_key_column.out     |    19 +
 .../test_schema_change_add_key_column1.csv    | 20480 ++++++++++++++++
 .../test_schema_change_add_key_column.groovy  |   148 +
 7 files changed, 28859 insertions(+)
 create mode 100644 regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column.csv
 create mode 100644 regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column.out
 create mode 100644 regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column1.csv
 create mode 100644 regression-test/suites/unique_with_mow_c_p0/test_schema_change_add_key_column.groovy

diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp
index 1cc4d052a81d69..b086def3c03ee5 100644
--- a/be/src/cloud/cloud_schema_change_job.cpp
+++ b/be/src/cloud/cloud_schema_change_job.cpp
@@ -169,6 +169,15 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque
     reader_context.batch_size = ALTER_TABLE_BATCH_SIZE;
     reader_context.delete_bitmap = &_base_tablet->tablet_meta()->delete_bitmap();
     reader_context.version = Version(0, start_resp.alter_version());
+    std::vector<uint32_t> cluster_key_idxes;
+    if (!_base_tablet_schema->cluster_key_uids().empty()) {
+        for (const auto& uid : _base_tablet_schema->cluster_key_uids()) {
+            cluster_key_idxes.emplace_back(_base_tablet_schema->field_index(uid));
+        }
+        reader_context.read_orderby_key_columns = &cluster_key_idxes;
+        reader_context.is_unique = false;
+        reader_context.sequence_id_idx = -1;
+    }
 
     for (auto& split : rs_splits) {
         RETURN_IF_ERROR(split.rs_reader->init(&reader_context));
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index 765f67a07c7884..f8cc79b205535f 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -34,6 +34,7 @@
 #include "runtime/descriptors.h"
 #include "runtime/exec_env.h"
 #include "runtime/thread_context.h"
+#include "util/debug_points.h"
 #include "util/runtime_profile.h"
 #include "util/stopwatch.hpp"
 #include "vec/aggregate_functions/aggregate_function_reader.h"
@@ -589,6 +590,7 @@ void MemTable::shrink_memtable_by_agg() {
 }
 
 bool MemTable::need_flush() const {
+    DBUG_EXECUTE_IF("MemTable.need_flush", { return true; });
     auto max_size = config::write_buffer_size;
     if (_partial_update_mode == UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS) {
         auto update_columns_size = _num_columns;
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 08b9b9a93d90f5..7f947612eed4ac 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -866,6 +866,7 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques
     for (int i = 0; i < num_cols; ++i) {
         return_columns[i] = i;
     }
+    std::vector<uint32_t> cluster_key_idxes;
 
     DBUG_EXECUTE_IF("SchemaChangeJob::_do_process_alter_tablet.block", DBUG_BLOCK);
 
@@ -982,6 +983,14 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques
             reader_context.batch_size = ALTER_TABLE_BATCH_SIZE;
             reader_context.delete_bitmap = &_base_tablet->tablet_meta()->delete_bitmap();
             reader_context.version = Version(0, end_version);
+            if (!_base_tablet_schema->cluster_key_uids().empty()) {
+                for (const auto& uid : _base_tablet_schema->cluster_key_uids()) {
+                    cluster_key_idxes.emplace_back(_base_tablet_schema->field_index(uid));
+                }
+                reader_context.read_orderby_key_columns = &cluster_key_idxes;
+                reader_context.is_unique = false;
+                reader_context.sequence_id_idx = -1;
+            }
             for (auto& rs_split : rs_splits) {
                 res = rs_split.rs_reader->init(&reader_context);
                 if (!res) {
diff --git a/regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column.csv b/regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column.csv
new file mode 100644
index 00000000000000..87e7cd5c2d9867
--- /dev/null
+++ b/regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column.csv
@@ -0,0 +1,8192 @@
+13668,20000,31000,40296
+14535,20001,30999,40697
+13905,20002,30998,40396
+13802,20003,30997,40727
+16784,20004,30996,40263
+12632,20005,30995,40595
+15749,20006,30994,40042
+15315,20007,30993,40226
+14984,20008,30992,40647
+15163,20009,30991,40976
+17216,20010,30990,40258
+13628,20011,30989,40024
+11832,20012,30988,40406
+14862,20013,30987,40038
+14799,20014,30986,40060
+17554,20015,30985,40437
+17259,20016,30984,40374
+13468,20017,30983,40268
+10251,20018,30982,40089
+16500,20019,30981,40674
+15808,20020,30980,40697
+10181,20021,30979,40899
+16206,20022,30978,40190
+11794,20023,30977,40643
+13606,20024,30976,40349
+13331,20025,30975,40952
+17016,20026,30974,40145
+17653,20027,30973,40132
+14528,20028,30972,40840
+12356,20029,30971,40271
+11340,20030,30970,40106
+16323,20031,30969,40123
+13513,20032,30968,40004
+16167,20033,30967,40521
+16005,20034,30966,40507
+11039,20035,30965,40145
+15207,20036,30964,40644
+13976,20037,30963,40662
+16340,20038,30962,40096
+13203,20039,30961,40928
+15528,20040,30960,40269
+16644,20041,30959,40971
+13555,20042,30958,40448
+13051,20043,30957,40597
+16595,20044,30956,40885
+15508,20045,30955,40589
+15392,20046,30954,40337
+17534,20047,30953,40391
+17483,20048,30952,40359
+14545,20049,30951,40564
+10667,20050,30950,40431
+13808,20051,30949,40622
+16938,20052,30948,40645
+12383,20053,30947,40168
+16701,20054,30946,40869
+12041,20055,30945,40180
+14781,20056,30944,40540
+13466,20057,30943,40591
+13395,20058,30942,40921
+16613,20059,30941,40215
+17849,20060,30940,40011
+11001,20061,30939,40268
+17222,20062,30938,40756
+13144,20063,30937,40724
+12276,20064,30936,40283
+17894,20065,30935,40113
+14610,20066,30934,40824
+15108,20067,30933,40757
+11162,20068,30932,40768
+11848,20069,30931,40822
+13165,20070,30930,40920
+16842,20071,30929,40268
+13411,20072,30928,40933
+14364,20073,30927,40110
+14137,20074,30926,40906
+17236,20075,30925,40381
+10087,20076,30924,40772
+13528,20077,30923,40961
+16860,20078,30922,40592
+11985,20079,30921,40495
+12828,20080,30920,40724
+10329,20081,30919,40372
+12021,20082,30918,40082
+14103,20083,30917,40763
+15160,20084,30916,40728
+16637,20085,30915,40106
+14893,20086,30914,40173
+11695,20087,30913,40147
+13755,20088,30912,40126
+12231,20089,30911,40686
+11608,20090,30910,40958
+13524,20091,30909,40516
+14629,20092,30908,40789
+15770,20093,30907,40622
+16967,20094,30906,40319
+17174,20095,30905,40052
+16949,20096,30904,40693
+13020,20097,30903,40593
+11880,20098,30902,40489
+16770,20099,30901,40339
+10762,20100,30900,40010
+10633,20101,30899,40921
+17889,20102,30898,40880
+14806,20103,30897,40638
+13501,20104,30896,40529
+16220,20105,30895,40034
+17404,20106,30894,40586
+15934,20107,30893,40767
+17688,20108,30892,40393
+10038,20109,30891,40416
+14309,20110,30890,40135
+17852,20111,30889,40990
+11215,20112,30888,40019
+10001,20113,30887,40028
+11191,20114,30886,40706
+16341,20115,30885,40890
+10681,20116,30884,40442
+12232,20117,30883,40936
+13669,20118,30882,40037
+10924,20119,30881,40944
+13584,20120,30880,40219
+13509,20121,30879,40846
+11433,20122,30878,40761
+13485,20123,30877,40139
+12264,20124,30876,40718
+13820,20125,30875,40972
+15473,20126,30874,40067
+11425,20127,30873,40969
+13034,20128,30872,40498
+11230,20129,30871,40618
+17820,20130,30870,40575
+11422,20131,30869,40500
+13597,20132,30868,40828
+13659,20133,30867,40369
+16185,20134,30866,40705
+15424,20135,30865,40166
+15003,20136,30864,40320
+10559,20137,30863,40445
+16050,20138,30862,40607
+13804,20139,30861,40363
+17691,20140,30860,40622
+11751,20141,30859,40684
+16628,20142,30858,40016
+16415,20143,30857,40954
+15822,20144,30856,40598
+10724,20145,30855,40717
+14457,20146,30854,40559
+10129,20147,30853,40972
+16739,20148,30852,40812
+16165,20149,30851,40977
+15488,20150,30850,40575
+17768,20151,30849,40427
+13876,20152,30848,40652
+13983,20153,30847,40157
+10896,20154,30846,40848
+10112,20155,30845,40084
+18163,20156,30844,40062
+12214,20157,30843,40006
+15597,20158,30842,40304
+16464,20159,30841,40701
+10004,20160,30840,40227
+11671,20161,30839,40266
+12022,20162,30838,40779
+17674,20163,30837,40743
+18030,20164,30836,40909
+14745,20165,30835,40877
+10936,20166,30834,40612
+16725,20167,30833,40532
+14336,20168,30832,40186
+16133,20169,30831,40673
+14350,20170,30830,40152
+18068,20171,30829,40137
+11504,20172,30828,40242
+11069,20173,30827,40260
+11698,20174,30826,40753
+17307,20175,30825,40021
+11134,20176,30824,40036
+12513,20177,30823,40054
+13786,20178,30822,40697
+12250,20179,30821,40560
+14766,20180,30820,40180
+12776,20181,30819,40031
+16904,20182,30818,40066
+16389,20183,30817,40236
+15280,20184,30816,40840
+16317,20185,30815,40317
+13244,20186,30814,40052
+13175,20187,30813,40495
+14254,20188,30812,40205
+10615,20189,30811,40739
+11870,20190,30810,40911
+10377,20191,30809,40591
+15435,20192,30808,40326
+17402,20193,30807,40603
+15056,20194,30806,40860
+17271,20195,30805,40979
+14597,20196,30804,40605
+15832,20197,30803,40653
+15616,20198,30802,40271
+12967,20199,30801,40808
+15656,20200,30800,40056
+17190,20201,30799,40013
+13721,20202,30798,40519
+12482,20203,30797,40657
+15480,20204,30796,40861
+15446,20205,30795,40496
+14571,20206,30794,40452
+12039,20207,30793,40182
+10870,20208,30792,40541
+10552,20209,30791,40566
+15540,20210,30790,40168
+17185,20211,30789,40926
+16930,20212,30788,40860
+15792,20213,30787,40785
+13170,20214,30786,40919
+13190,20215,30785,40740
+16417,20216,30784,40559
+11309,20217,30783,40317
+17955,20218,30782,40782
+15296,20219,30781,40708
+16520,20220,30780,40852
+14591,20221,30779,40598
+13826,20222,30778,40502
+15915,20223,30777,40958
+13441,20224,30776,40957
+10902,20225,30775,40444
+15384,20226,30774,40866
+16883,20227,30773,40122
+14935,20228,30772,40092
+15671,20229,30771,40505
+17277,20230,30770,40708
+14010,20231,30769,40155
+11944,20232,30768,40840
+10294,20233,30767,40577
+10648,20234,30766,40836
+14988,20235,30765,40412
+13780,20236,30764,40511
+11542,20237,30763,40432
+17396,20238,30762,40064
+13121,20239,30761,40565
+15447,20240,30760,40257
+13338,20241,30759,40159
+16924,20242,30758,40396
+16354,20243,30757,40294
+14541,20244,30756,40747
+17330,20245,30755,40432
+12284,20246,30754,40188
+17431,20247,30753,40777
+11518,20248,30752,40336
+10621,20249,30751,40564
+12046,20250,30750,40381
+10324,20251,30749,40028
+12381,20252,30748,40897
+12538,20253,30747,40644
+15543,20254,30746,40263
+13208,20255,30745,40796
+15134,20256,30744,40526
+10918,20257,30743,40797
+10261,20258,30742,40749
+18188,20259,30741,40123
+16907,20260,30740,40795
+13593,20261,30739,40673
+11335,20262,30738,40794
+16272,20263,30737,40779
+15465,20264,30736,40524
+16746,20265,30735,40348
+10209,20266,30734,40341
+11364,20267,30733,40224
+11956,20268,30732,40680
+17030,20269,30731,40892
+14305,20270,30730,40918
+17332,20271,30729,40770
+16351,20272,30728,40552
+13951,20273,30727,40712
+16169,20274,30726,40902
+15425,20275,30725,40096
+12107,20276,30724,40849
+14566,20277,30723,40138
+12611,20278,30722,40362
+16438,20279,30721,40541
+10416,20280,30720,40405
+10597,20281,30719,40933
+11498,20282,30718,40976
+15234,20283,30717,40598
+15956,20284,30716,40806
+10917,20285,30715,40006
+12498,20286,30714,40817
+12483,20287,30713,40505
+15381,20288,30712,40603
+14917,20289,30711,40750
+14652,20290,30710,40709
+16575,20291,30709,40686
+12797,20292,30708,40608
+12029,20293,30707,40109
+13526,20294,30706,40673
+12557,20295,30705,40162
+15791,20296,30704,40978
+10098,20297,30703,40617
+15512,20298,30702,40875
+11966,20299,30701,40399
+11891,20300,30700,40144
+13384,20301,30699,40957
+16577,20302,30698,40916
+13180,20303,30697,40525
+11817,20304,30696,40683
+11762,20305,30695,40073
+10328,20306,30694,40749
+13540,20307,30693,40140
+14197,20308,30692,40670
+11654,20309,30691,40600
+15289,20310,30690,40410
+13248,20311,30689,40198
+12103,20312,30688,40622
+14227,20313,30687,40349
+12993,20314,30686,40777
+11736,20315,30685,40705
+16170,20316,30684,40546
+15721,20317,30683,40580
+11861,20318,30682,40291
+16634,20319,30681,40355
+13347,20320,30680,40683
+14555,20321,30679,40327
+11872,20322,30678,40042
+15821,20323,30677,40540
+12509,20324,30676,40261
+15841,20325,30675,40022
+16066,20326,30674,40321
+15556,20327,30673,40487
+16647,20328,30672,40932
+15882,20329,30671,40797
+10291,20330,30670,40335
+10518,20331,30669,40016
+13174,20332,30668,40711
+14601,20333,30667,40473
+16979,20334,30666,40483
+17751,20335,30665,40198
+14423,20336,30664,40424
+13815,20337,30663,40131
+17752,20338,30662,40904
+10717,20339,30661,40578
+12395,20340,30660,40266
+14442,20341,30659,40283
+14643,20342,30658,40330
+10256,20343,30657,40726
+18170,20344,30656,40833
+12175,20345,30655,40837
+17998,20346,30654,40519
+10373,20347,30653,40127
+11046,20348,30652,40661
+15042,20349,30651,40519
+12009,20350,30650,40719
+10148,20351,30649,40755
+16507,20352,30648,40462
+16380,20353,30647,40720
+12707,20354,30646,40604
+13867,20355,30645,40563
+14511,20356,30644,40953
+12971,20357,30643,40739
+17067,20358,30642,40258
+10033,20359,30641,40394
+10238,20360,30640,40257
+11328,20361,30639,40393
+17352,20362,30638,40335
+10314,20363,30637,40843
+10076,20364,30636,40990
+15460,20365,30635,40410
+16151,20366,30634,40292
+11119,20367,30633,40222
+14332,20368,30632,40801
+17329,20369,30631,40332
+10579,20370,30630,40457
+17200,20371,30629,40011
+14289,20372,30628,40101
+14938,20373,30627,40089
+12037,20374,30626,40732
+14881,20375,30625,40228
+14026,20376,30624,40679
+17628,20377,30623,40173
+11356,20378,30622,40051
+15884,20379,30621,40903
+16056,20380,30620,40215
+17769,20381,30619,40278
+16093,20382,30618,40868
+14794,20383,30617,40834
+13601,20384,30616,40540
+12138,20385,30615,40562
+17754,20386,30614,40484
+16910,20387,30613,40645
+16419,20388,30612,40835
+15478,20389,30611,40075
+13748,20390,30610,40877
+17387,20391,30609,40939
+13739,20392,30608,40761
+12608,20393,30607,40406
+16875,20394,30606,40789
+15778,20395,30605,40808
+13530,20396,30604,40604
+17693,20397,30603,40543
+10485,20398,30602,40789
+13846,20399,30601,40131
+16547,20400,30600,40837
+15897,20401,30599,40582
+14008,20402,30598,40095
+13950,20403,30597,40573
+14480,20404,30596,40040
+12487,20405,30595,40948
+15268,20406,30594,40967
+16385,20407,30593,40240
+17059,20408,30592,40489
+10680,20409,30591,40694
+10128,20410,30590,40715
+10005,20411,30589,40913
+15047,20412,30588,40040
+17956,20413,30587,40760
+13811,20414,30586,40171
+10639,20415,30585,40230
+13414,20416,30584,40926
+15986,20417,30583,40923
+17476,20418,30582,40344
+11669,20419,30581,40576
+16474,20420,30580,40645
+16282,20421,30579,40024
+14698,20422,30578,40680
+17449,20423,30577,40410
+13707,20424,30576,40596
+15453,20425,30575,40946
+16513,20426,30574,40504
+15718,20427,30573,40495
+15260,20428,30572,40136
+14820,20429,30571,40927
+10000,20430,30570,40584
+16993,20431,30569,40111
+11275,20432,30568,40924
+15307,20433,30567,40231
+15165,20434,30566,40906
+11110,20435,30565,40810
+10908,20436,30564,40561
+10430,20437,30563,40139
+17557,20438,30562,40363
+18025,20439,30561,40467
+13731,20440,30560,40198
+17267,20441,30559,40887
+15479,20442,30558,40296
+13691,20443,30557,40773
+13518,20444,30556,40968
+16853,20445,30555,40599
+17548,20446,30554,40082
+14469,20447,30553,40209
+15870,20448,30552,40394
+15847,20449,30551,40821
+15083,20450,30550,40922
+16356,20451,30549,40086
+16555,20452,30548,40247
+18016,20453,30547,40297
+12504,20454,30546,40733
+14466,20455,30545,40871
+14269,20456,30544,40634
+14333,20457,30543,40287
+13232,20458,30542,40880
+16502,20459,30541,40680
+11256,20460,30540,40820
+14734,20461,30539,40116
+14779,20462,30538,40612
+14192,20463,30537,40321
+11341,20464,30536,40638
+11317,20465,30535,40505
+13661,20466,30534,40627
+15196,20467,30533,40753
+10950,20468,30532,40338
+10576,20469,30531,40110
+12305,20470,30530,40352
+13482,20471,30529,40315
+14005,20472,30528,40125
+12085,20473,30527,40079
+14839,20474,30526,40523
+10216,20475,30525,40940
+13902,20476,30524,40440
+13793,20477,30523,40882
+10637,20478,30522,40482
+16533,20479,30521,40274
+17000,20480,30520,40634
+16300,20481,30519,40391
+13326,20482,30518,40489
+17564,20483,30517,40966
+13898,20484,30516,40735
+16055,20485,30515,40838
+14857,20486,30514,40482
+12863,20487,30513,40506
+12516,20488,30512,40756
+12390,20489,30511,40948
+10674,20490,30510,40573
+12886,20491,30509,40652
+15483,20492,30508,40303
+14899,20493,30507,40454
+13806,20494,30506,40697
+17818,20495,30505,40209
+11991,20496,30504,40931
+17912,20497,30503,40075
+13388,20498,30502,40320
+11572,20499,30501,40506
+10773,20500,30500,40274
+13562,20501,30499,40629
+13689,20502,30498,40175
+13224,20503,30497,40809
+17233,20504,30496,40664
+10817,20505,30495,40040
+17655,20506,30494,40003
+17649,20507,30493,40076
+12014,20508,30492,40614
+17824,20509,30491,40442
+16528,20510,30490,40508
+15829,20511,30489,40105
+17140,20512,30488,40077
+10167,20513,30487,40569
+12689,20514,30486,40525
+10640,20515,30485,40872
+17118,20516,30484,40214
+12862,20517,30483,40543
+17065,20518,30482,40625
+17854,20519,30481,40228
+17602,20520,30480,40232
+15716,20521,30479,40865
+16743,20522,30478,40183
+11036,20523,30477,40033
+15328,20524,30476,40273
+12935,20525,30475,40819
+16297,20526,30474,40783
+16336,20527,30473,40903
+10064,20528,30472,40235
+13696,20529,30471,40437
+13599,20530,30470,40451
+12536,20531,30469,40501
+13596,20532,30468,40552
+17375,20533,30467,40481
+17060,20534,30466,40070
+15755,20535,30465,40894
+11602,20536,30464,40582
+16933,20537,30463,40601
+10540,20538,30462,40199
+11228,20539,30461,40039
+16115,20540,30460,40860
+12366,20541,30459,40573
+13769,20542,30458,40183
+14378,20543,30457,40998
+11696,20544,30456,40162
+14012,20545,30455,40387
+13343,20546,30454,40118
+14861,20547,30453,40980
+15992,20548,30452,40373
+17949,20549,30451,40597
+11600,20550,30450,40753
+13377,20551,30449,40252
+13516,20552,30448,40544
+13002,20553,30447,40358
+10039,20554,30446,40039
+14722,20555,30445,40169
+15383,20556,30444,40235
+11482,20557,30443,40044
+13979,20558,30442,40670
+15887,20559,30441,40219
+17621,20560,30440,40663
+17927,20561,30439,40831
+12146,20562,30438,40330
+17395,20563,30437,40258
+12457,20564,30436,40207
+16183,20565,30435,40384
+13917,20566,30434,40975
+15888,20567,30433,40577
+10365,20568,30432,40648
+16142,20569,30431,40435
+17916,20570,30430,40729
+13238,20571,30429,40308
+15470,20572,30428,40337
+15651,20573,30427,40740
+16217,20574,30426,40018
+11709,20575,30425,40921
+14526,20576,30424,40699
+12415,20577,30423,40125
+12032,20578,30422,40996
+13257,20579,30421,40202
+11458,20580,30420,40082
+13730,20581,30419,40366
+11592,20582,30418,40858
+10044,20583,30417,40529
+11132,20584,30416,40106
+16179,20585,30415,40123
+10060,20586,30414,40299
+17829,20587,30413,40123
+13787,20588,30412,40067
+13387,20589,30411,40186
+16636,20590,30410,40887
+13355,20591,30409,40765
+13225,20592,30408,40989
+17648,20593,30407,40793
+10454,20594,30406,40127
+17930,20595,30405,40873
+12812,20596,30404,40988
+16588,20597,30403,40219
+14518,20598,30402,40720
+13361,20599,30401,40410
+12590,20600,30400,40236
+12826,20601,30399,40092
+12683,20602,30398,40976
+17683,20603,30397,40611
+12210,20604,30396,40957
+17536,20605,30395,40712
+13075,20606,30394,40527
+16582,20607,30393,40891
+12386,20608,30392,40303
+14632,20609,30391,40895
+15303,20610,30390,40807
+17496,20611,30389,40483
+18186,20612,30388,40559
+15525,20613,30387,40212
+12639,20614,30386,40637
+11053,20615,30385,40501
+11533,20616,30384,40392
+18017,20617,30383,40377
+13097,20618,30382,40325
+11733,20619,30381,40943
+10347,20620,30380,40978
+13676,20621,30379,40115
+15639,20622,30378,40488
+16699,20623,30377,40070
+11443,20624,30376,40495
+15346,20625,30375,40436
+10631,20626,30374,40785
+11617,20627,30373,40899
+12389,20628,30372,40112
+12118,20629,30371,40771
+10863,20630,30370,40315
+17094,20631,30369,40993
+12269,20632,30368,40243
+12187,20633,30367,40717
+15091,20634,30366,40509
+17905,20635,30365,40772
+14553,20636,30364,40144
+17152,20637,30363,40664
+16424,20638,30362,40293
+14375,20639,30361,40146
+13756,20640,30360,40287
+15542,20641,30359,40858
+11997,20642,30358,40384
+10078,20643,30357,40133
+11874,20644,30356,40971
+10695,20645,30355,40909
+13605,20646,30354,40576
+17568,20647,30353,40351
+17143,20648,30352,40537
+12209,20649,30351,40614
+15601,20650,30350,40252
+17046,20651,30349,40781
+17582,20652,30348,40481
+14245,20653,30347,40617
+13839,20654,30346,40995
+12907,20655,30345,40972
+10832,20656,30344,40899
+15129,20657,30343,40934
+12293,20658,30342,40448
+17365,20659,30341,40542
+15907,20660,30340,40899
+13472,20661,30339,40127
+11404,20662,30338,40375
+14221,20663,30337,40150
+11082,20664,30336,40926
+14186,20665,30335,40559
+15835,20666,30334,40870
+11174,20667,30333,40807
+14179,20668,30332,40970
+15861,20669,30331,40915
+10201,20670,30330,40365
+12734,20671,30329,40704
+18082,20672,30328,40217
+13146,20673,30327,40980
+10413,20674,30326,40438
+14992,20675,30325,40194
+13391,20676,30324,40534
+16849,20677,30323,40635
+16550,20678,30322,40492
+11116,20679,30321,40935
+17354,20680,30320,40045
+10279,20681,30319,40692
+14235,20682,30318,40239
+11565,20683,30317,40037
+12244,20684,30316,40983
+13673,20685,30315,40451
+17567,20686,30314,40239
+10312,20687,30313,40873
+15965,20688,30312,40843
+12334,20689,30311,40633
+11599,20690,30310,40668
+16460,20691,30309,40300
+12165,20692,30308,40510
+12857,20693,30307,40951
+17095,20694,30306,40764
+11167,20695,30305,40553
+11444,20696,30304,40764
+13692,20697,30303,40744
+14042,20698,30302,40116
+16931,20699,30301,40708
+17480,20700,30300,40795
+15642,20701,30299,40971
+16440,20702,30298,40093
+14368,20703,30297,40652
+14764,20704,30296,40926
+11470,20705,30295,40839
+17172,20706,30294,40704
+14651,20707,30293,40113
+14224,20708,30292,40000
+15790,20709,30291,40861
+11200,20710,30290,40978
+14805,20711,30289,40510
+11281,20712,30288,40813
+14684,20713,30287,40353
+12861,20714,30286,40464
+11531,20715,30285,40951
+16147,20716,30284,40420
+16313,20717,30283,40977
+12855,20718,30282,40012
+10234,20719,30281,40596
+15405,20720,30280,40294
+11665,20721,30279,40708
+13382,20722,30278,40246
+17269,20723,30277,40460
+15271,20724,30276,40240
+12953,20725,30275,40651
+16601,20726,30274,40258
+17445,20727,30273,40554
+15193,20728,30272,40348
+15139,20729,30271,40138
+12193,20730,30270,40192
+10978,20731,30269,40251
+13206,20732,30268,40172
+10072,20733,30267,40135
+14726,20734,30266,40702
+17869,20735,30265,40230
+16337,20736,30264,40007
+15546,20737,30263,40738
+14664,20738,30262,40760
+13335,20739,30261,40114
+11627,20740,30260,40465
+13531,20741,30259,40867
+10783,20742,30258,40169
+11702,20743,30257,40482
+10356,20744,30256,40007
+17504,20745,30255,40135
+12311,20746,30254,40599
+15906,20747,30253,40516
+10574,20748,30252,40784
+11048,20749,30251,40747
+13757,20750,30250,40090
+16989,20751,30249,40813
+12317,20752,30248,40797
+12548,20753,30247,40390
+16004,20754,30246,40260
+15411,20755,30245,40262
+16345,20756,30244,40488
+15969,20757,30243,40883
+14129,20758,30242,40362
+10366,20759,30241,40858
+15218,20760,30240,40368
+14315,20761,30239,40476
+17238,20762,30238,40585
+16956,20763,30237,40421
+14405,20764,30236,40307
+10420,20765,30235,40616
+12230,20766,30234,40713
+17511,20767,30233,40654
+18113,20768,30232,40995
+12359,20769,30231,40797
+13626,20770,30230,40693
+11994,20771,30229,40109
+16127,20772,30228,40737
+12699,20773,30227,40758
+12915,20774,30226,40425
+14822,20775,30225,40725
+14185,20776,30224,40620
+10317,20777,30223,40773
+13310,20778,30222,40922
+14209,20779,30221,40002
+14548,20780,30220,40987
+17155,20781,30219,40897
+12827,20782,30218,40705
+16810,20783,30217,40788
+15475,20784,30216,40166
+14614,20785,30215,40809
+16022,20786,30214,40551
+17593,20787,30213,40986
+10425,20788,30212,40584
+12177,20789,30211,40839
+10979,20790,30210,40794
+15024,20791,30209,40471
+15984,20792,30208,40855
+16731,20793,30207,40750
+13842,20794,30206,40470
+16617,20795,30205,40872
+16221,20796,30204,40498
+13981,20797,30203,40724
+13357,20798,30202,40221
+12072,20799,30201,40668
+10582,20800,30200,40874
+10561,20801,30199,40192
+16693,20802,30198,40503
+12052,20803,30197,40594
+13675,20804,30196,40729
+15820,20805,30195,40556
+15922,20806,30194,40162
+13047,20807,30193,40741
+12613,20808,30192,40525
+13228,20809,30191,40931
+12553,20810,30190,40978
+11897,20811,30189,40403
+12401,20812,30188,40045
+13186,20813,30187,40268
+12531,20814,30186,40097
+11224,20815,30185,40569
+16572,20816,30184,40802
+13890,20817,30183,40093
+10595,20818,30182,40056
+15754,20819,30181,40880
+13643,20820,30180,40927
+10204,20821,30179,40928
+17825,20822,30178,40836
+17573,20823,30177,40443
+14762,20824,30176,40810
+13157,20825,30175,40928
+15663,20826,30174,40361
+12576,20827,30173,40702
+11334,20828,30172,40472
+12756,20829,30171,40692
+14869,20830,30170,40906
+16425,20831,30169,40248
+11006,20832,30168,40604
+17193,20833,30167,40715
+12642,20834,30166,40782
+10401,20835,30165,40329
+12033,20836,30164,40536
+12852,20837,30163,40638
+15200,20838,30162,40412
+10689,20839,30161,40156
+17624,20840,30160,40441
+16135,20841,30159,40730
+12296,20842,30158,40624
+12806,20843,30157,40318
+15774,20844,30156,40935
+13287,20845,30155,40322
+15735,20846,30154,40458
+12987,20847,30153,40612
+12082,20848,30152,40914
+11636,20849,30151,40976
+16329,20850,30150,40222
+14389,20851,30149,40425
+13378,20852,30148,40933
+12741,20853,30147,40884
+14493,20854,30146,40582
+16671,20855,30145,40397
+10928,20856,30144,40975
+14198,20857,30143,40347
+14331,20858,30142,40609
+14122,20859,30141,40974
+13177,20860,30140,40941
+15175,20861,30139,40217
+11802,20862,30138,40784
+16894,20863,30137,40866
+13048,20864,30136,40393
+13055,20865,30135,40247
+14302,20866,30134,40699
+17638,20867,30133,40466
+16126,20868,30132,40061
+17989,20869,30131,40301
+14109,20870,30130,40127
+12153,20871,30129,40113
+16614,20872,30128,40280
+12630,20873,30127,40000
+15952,20874,30126,40690
+16884,20875,30125,40627
+16819,20876,30124,40895
+11312,20877,30123,40445
+11813,20878,30122,40661
+14660,20879,30121,40446
+10545,20880,30120,40708
+13340,20881,30119,40491
+10708,20882,30118,40225
+10252,20883,30117,40379
+12932,20884,30116,40003
+10070,20885,30115,40389
+13624,20886,30114,40170
+12382,20887,30113,40093
+14386,20888,30112,40823
+16393,20889,30111,40421
+13013,20890,30110,40935
+15436,20891,30109,40312
+16098,20892,30108,40065
+15648,20893,30107,40338
+13358,20894,30106,40039
+16666,20895,30105,40483
+10460,20896,30104,40641
+11990,20897,30103,40786
+13030,20898,30102,40556
+14472,20899,30101,40350
+15564,20900,30100,40274
+12444,20901,30099,40536
+17144,20902,30098,40323
+14783,20903,30097,40828
+15032,20904,30096,40530
+15760,20905,30095,40694
+14380,20906,30094,40266
+17490,20907,30093,40962
+11398,20908,30092,40055
+14534,20909,30091,40276
+11290,20910,30090,40137
+17851,20911,30089,40117
+13329,20912,30088,40738
+11699,20913,30087,40343
+17790,20914,30086,40745
+12343,20915,30085,40336
+11124,20916,30084,40695
+15431,20917,30083,40186
+14213,20918,30082,40207
+13185,20919,30081,40843
+10287,20920,30080,40267
+12811,20921,30079,40512
+12870,20922,30078,40105
+18031,20923,30077,40364
+15107,20924,30076,40588
+16568,20925,30075,40102
+13315,20926,30074,40303
+17830,20927,30073,40633
+10467,20928,30072,40204
+14803,20929,30071,40046
+16653,20930,30070,40139
+14504,20931,30069,40524
+15018,20932,30068,40542
+16918,20933,30067,40903
+12164,20934,30066,40801
+15731,20935,30065,40221
+16247,20936,30064,40775
+13844,20937,30063,40909
+17945,20938,30062,40887
+12751,20939,30061,40739
+10854,20940,30060,40965
+11576,20941,30059,40182
+10441,20942,30058,40557
+14942,20943,30057,40443
+17083,20944,30056,40859
+14401,20945,30055,40714
+12074,20946,30054,40822
+14699,20947,30053,40830
+17878,20948,30052,40467
+12997,20949,30051,40931
+15030,20950,30050,40117
+17068,20951,30049,40783
+10231,20952,30048,40114
+10449,20953,30047,40642
+10478,20954,30046,40936
+13535,20955,30045,40754
+14519,20956,30044,40422
+11044,20957,30043,40383
+11246,20958,30042,40312
+14866,20959,30041,40838
+10810,20960,30040,40403
+16401,20961,30039,40713
+15689,20962,30038,40614
+11876,20963,30037,40299
+16874,20964,30036,40937
+16402,20965,30035,40890
+17275,20966,30034,40407
+11613,20967,30033,40701
+13421,20968,30032,40607
+17941,20969,30031,40468
+13812,20970,30030,40641
+11515,20971,30029,40214
+18062,20972,30028,40755
+13504,20973,30027,40589
+13094,20974,30026,40355
+14693,20975,30025,40815
+16161,20976,30024,40215
+13783,20977,30023,40638
+17634,20978,30022,40104
+11545,20979,30021,40423
+11152,20980,30020,40373
+12984,20981,30019,40840
+14275,20982,30018,40452
+17972,20983,30017,40873
+15550,20984,30016,40603
+18172,20985,30015,40413
+16453,20986,30014,40853
+14915,20987,30013,40835
+12387,20988,30012,40960
+14440,20989,30011,40660
+10364,20990,30010,40659
+14140,20991,30009,40625
+15143,20992,30008,40108
+12706,20993,30007,40165
+16873,20994,30006,40591
+13512,20995,30005,40293
+14895,20996,30004,40931
+16574,20997,30003,40313
+17890,20998,30002,40455
+16587,20999,30001,40450
+14542,21000,30000,40322
+17335,21001,31000,40123
+12246,21002,30999,40832
+17440,21003,30998,40340
+16733,21004,30997,40432
+17964,21005,30996,40457
+11065,21006,30995,40282
+17718,21007,30994,40385
+12794,21008,30993,40318
+17741,21009,30992,40810
+12895,21010,30991,40416
+15301,21011,30990,40338
+13127,21012,30989,40687
+13874,21013,30988,40361
+10635,21014,30987,40356
+12946,21015,30986,40346
+10345,21016,30985,40392
+12710,21017,30984,40784
+14459,21018,30983,40711
+11018,21019,30982,40701
+12546,21020,30981,40184
+10318,21021,30980,40662
+14381,21022,30979,40622
+13653,21023,30978,40002
+17673,21024,30977,40337
+15583,21025,30976,40257
+15839,21026,30975,40493
+16772,21027,30974,40521
+15519,21028,30973,40173
+10385,21029,30972,40092
+18125,21030,30971,40584
+11638,21031,30970,40854
+14874,21032,30969,40305
+15750,21033,30968,40018
+14617,21034,30967,40260
+14537,21035,30966,40291
+11789,21036,30965,40564
+18078,21037,30964,40432
+10638,21038,30963,40290
+12752,21039,30962,40049
+10989,21040,30961,40801
+17710,21041,30960,40542
+17974,21042,30959,40644
+16795,21043,30958,40311
+12591,21044,30957,40664
+10067,21045,30956,40998
+14088,21046,30955,40462
+13217,21047,30954,40473
+10427,21048,30953,40560
+12378,21049,30952,40459
+10477,21050,30951,40125
+13771,21051,30950,40882
+12676,21052,30949,40752
+16570,21053,30948,40971
+12473,21054,30947,40803
+17608,21055,30946,40065
+11351,21056,30945,40753
+10887,21057,30944,40917
+10105,21058,30943,40608
+15866,21059,30942,40759
+13978,21060,30941,40413
+12628,21061,30940,40587
+15330,21062,30939,40170
+12198,21063,30938,40607
+14413,21064,30937,40491
+14925,21065,30936,40657
+16723,21066,30935,40062
+12679,21067,30934,40205
+13339,21068,30933,40408
+15970,21069,30932,40563
+10391,21070,30931,40732
+10587,21071,30930,40751
+13713,21072,30929,40374
+12527,21073,30928,40754
+16146,21074,30927,40782
+13265,21075,30926,40758
+14488,21076,30925,40040
+11038,21077,30924,40751
+10009,21078,30923,40416
+15329,21079,30922,40597
+15007,21080,30921,40771
+17799,21081,30920,40254
+11418,21082,30919,40529
+10970,21083,30918,40768
+12674,21084,30917,40983
+16239,21085,30916,40900
+15582,21086,30915,40073
+15366,21087,30914,40294
+18128,21088,30913,40436
+17961,21089,30912,40297
+12539,21090,30911,40825
+11642,21091,30910,40112
+14101,21092,30909,40461
+14153,21093,30908,40791
+16672,21094,30907,40957
+10630,21095,30906,40911
+15650,21096,30905,40743
+10714,21097,30904,40192
+15046,21098,30903,40812
+13936,21099,30902,40825
+11908,21100,30901,40058
+11333,21101,30900,40683
+13758,21102,30899,40598
+13207,21103,30898,40939
+12084,21104,30897,40758
+11310,21105,30896,40990
+10968,21106,30895,40285
+14568,21107,30894,40560
+12947,21108,30893,40031
+10736,21109,30892,40821
+11468,21110,30891,40660
+13538,21111,30890,40672
+15232,21112,30889,40754
+11179,21113,30888,40546
+13071,21114,30887,40322
+11501,21115,30886,40537
+13886,21116,30885,40469
+14665,21117,30884,40866
+17090,21118,30883,40736
+11161,21119,30882,40578
+11452,21120,30881,40266
+13988,21121,30880,40654
+12367,21122,30879,40854
+16944,21123,30878,40546
+14845,21124,30877,40580
+10614,21125,30876,40838
+13539,21126,30875,40641
+13658,21127,30874,40109
+16492,21128,30873,40383
+11145,21129,30872,40975
+17513,21130,30871,40544
+11759,21131,30870,40767
+15520,21132,30869,40868
+14998,21133,30868,40126
+13138,21134,30867,40306
+16546,21135,30866,40398
+14216,21136,30865,40170
+16482,21137,30864,40699
+16277,21138,30863,40937
+13215,21139,30862,40235
+12445,21140,30861,40246
+11606,21141,30860,40575
+16705,21142,30859,40185
+14884,21143,30858,40169
+10670,21144,30857,40395
+16489,21145,30856,40573
+13649,21146,30855,40753
+11867,21147,30854,40348
+16539,21148,30853,40834
+12011,21149,30852,40732
+13007,21150,30851,40896
+11957,21151,30850,40795
+11777,21152,30849,40515
+10995,21153,30848,40322
+12739,21154,30847,40552
+15401,21155,30846,40343
+18074,21156,30845,40265
+14810,21157,30844,40138
+16862,21158,30843,40383
+15698,21159,30842,40113
+16278,21160,30841,40034
+15231,21161,30840,40615
+17816,21162,30839,40377
+10151,21163,30838,40639
+17803,21164,30837,40308
+18147,21165,30836,40054
+10330,21166,30835,40166
+17181,21167,30834,40286
+11889,21168,30833,40961
+16683,21169,30832,40663
+13619,21170,30831,40386
+17195,21171,30830,40257
+17977,21172,30829,40859
+15840,21173,30828,40372
+14897,21174,30827,40967
+11330,21175,30826,40736
+15382,21176,30825,40936
+14035,21177,30824,40656
+17122,21178,30823,40244
+14919,21179,30822,40793
+15066,21180,30821,40870
+17727,21181,30820,40121
+12769,21182,30819,40257
+14920,21183,30818,40907
+16007,21184,30817,40703
+15554,21185,30816,40719
+12986,21186,30815,40856
+11106,21187,30814,40045
+10957,21188,30813,40174
+12158,21189,30812,40538
+13435,21190,30811,40101
+12618,21191,30810,40107
+15490,21192,30809,40384
+17689,21193,30808,40998
+15727,21194,30807,40654
+15253,21195,30806,40833
+17282,21196,30805,40508
+14037,21197,30804,40346
+13954,21198,30803,40180
+17968,21199,30802,40037
+14700,21200,30801,40395
+11780,21201,30800,40220
+13128,21202,30799,40240
+12563,21203,30798,40160
+14110,21204,30797,40760
+13212,21205,30796,40710
+16076,21206,30795,40070
+10456,21207,30794,40933
+14067,21208,30793,40239
+12917,21209,30792,40676
+11346,21210,30791,40748
+12823,21211,30790,40329
+13760,21212,30789,40474
+17750,21213,30788,40883
+10369,21214,30787,40907
+12026,21215,30786,40121
+16016,21216,30785,40015
+17983,21217,30784,40491
+14532,21218,30783,40389
+14621,21219,30782,40790
+11520,21220,30781,40018
+17408,21221,30780,40446
+16455,21222,30779,40629
+13458,21223,30778,40147
+17508,21224,30777,40857
+17173,21225,30776,40286
+10651,21226,30775,40697
+11723,21227,30774,40173
+13502,21228,30773,40811
+17957,21229,30772,40986
+17003,21230,30771,40443
+17163,21231,30770,40218
+14552,21232,30769,40464
+16299,21233,30768,40401
+10245,21234,30767,40963
+12842,21235,30766,40330
+13742,21236,30765,40787
+13375,21237,30764,40924
+13943,21238,30763,40550
+13061,21239,30762,40825
+16597,21240,30761,40274
+16070,21241,30760,40168
+12221,21242,30759,40250
+11257,21243,30758,40082
+12876,21244,30757,40252
+10955,21245,30756,40873
+17251,21246,30755,40591
+12673,21247,30754,40749
+17091,21248,30753,40681
+10438,21249,30752,40344
+15784,21250,30751,40871
+14341,21251,30750,40433
+14116,21252,30749,40078
+14102,21253,30748,40766
+12588,21254,30747,40774
+13462,21255,30746,40506
+17104,21256,30745,40846
+15069,21257,30744,40350
+16854,21258,30743,40578
+15647,21259,30742,40124
+11151,21260,30741,40321
+10277,21261,30740,40598
+17537,21262,30739,40763
+13525,21263,30738,40661
+15008,21264,30737,40376
+12594,21265,30736,40678
+13885,21266,30735,40907
+12371,21267,30734,40822
+13294,21268,30733,40462
+16406,21269,30732,40102
+15111,21270,30731,40271
+11047,21271,30730,40426
+11562,21272,30729,40744
+13948,21273,30728,40572
+16156,21274,30727,40997
+13417,21275,30726,40926
+13102,21276,30725,40694
+12627,21277,30724,40892
+16512,21278,30723,40427
+14013,21279,30722,40551
+16253,21280,30721,40927
+11604,21281,30720,40125
+17531,21282,30719,40142
+15891,21283,30718,40054
+15929,21284,30717,40512
+17993,21285,30716,40138
+13394,21286,30715,40647
+12959,21287,30714,40278
+10823,21288,30713,40557
+10800,21289,30712,40590
+16717,21290,30711,40287
+17748,21291,30710,40316
+13091,21292,30709,40286
+17156,21293,30708,40894
+11927,21294,30707,40892
+10502,21295,30706,40502
+11626,21296,30705,40307
+16252,21297,30704,40043
+15775,21298,30703,40441
+14240,21299,30702,40084
+13745,21300,30701,40696
+12537,21301,30700,40613
+11442,21302,30699,40802
+13568,21303,30698,40530
+10392,21304,30697,40449
+13861,21305,30696,40504
+17007,21306,30695,40124
+17417,21307,30694,40347
+15205,21308,30693,40190
+14134,21309,30692,40106
+14701,21310,30691,40897
+11089,21311,30690,40823
+12874,21312,30689,40120
+14061,21313,30688,40010
+11616,21314,30687,40506
+17206,21315,30686,40557
+11373,21316,30685,40402
+13604,21317,30684,40810
+12024,21318,30683,40850
+17489,21319,30682,40905
+15589,21320,30681,40776
+10190,21321,30680,40606
+13461,21322,30679,40281
+14210,21323,30678,40287
+10393,21324,30677,40634
+13651,21325,30676,40091
+13611,21326,30675,40966
+13367,21327,30674,40373
+12981,21328,30673,40362
+16381,21329,30672,40464
+10222,21330,30671,40988
+14464,21331,30670,40660
+11220,21332,30669,40993
+12566,21333,30668,40951
+14286,21334,30667,40941
+12109,21335,30666,40328
+11512,21336,30665,40501
+14590,21337,30664,40532
+15576,21338,30663,40065
+10083,21339,30662,40755
+14596,21340,30661,40438
+15625,21341,30660,40348
+15903,21342,30659,40440
+10869,21343,30658,40157
+10802,21344,30657,40244
+11770,21345,30656,40119
+16878,21346,30655,40171
+15684,21347,30654,40646
+11090,21348,30653,40586
+11958,21349,30652,40295
+12159,21350,30651,40678
+13359,21351,30650,40362
+13402,21352,30649,40830
+15441,21353,30648,40838
+13299,21354,30647,40641
+13766,21355,30646,40476
+11910,21356,30645,40014
+14749,21357,30644,40827
+17383,21358,30643,40562
+15571,21359,30642,40026
+13889,21360,30641,40147
+14768,21361,30640,40141
+16469,21362,30639,40035
+13523,21363,30638,40271
+17170,21364,30637,40745
+11633,21365,30636,40334
+10710,21366,30635,40146
+16955,21367,30634,40003
+10998,21368,30633,40341
+11771,21369,30632,40624
+16289,21370,30631,40995
+13235,21371,30630,40173
+16896,21372,30629,40933
+11239,21373,30628,40363
+15968,21374,30627,40376
+12620,21375,30626,40705
+13672,21376,30625,40227
+15533,21377,30624,40637
+11005,21378,30623,40825
+17285,21379,30622,40658
+17744,21380,30621,40122
+16592,21381,30620,40823
+15757,21382,30619,40188
+13824,21383,30618,40209
+13105,21384,30617,40879
+16390,21385,30616,40359
+17175,21386,30615,40973
+14007,21387,30614,40082
+17943,21388,30613,40208
+12785,21389,30612,40112
+15988,21390,30611,40662
+17620,21391,30610,40406
+14371,21392,30609,40912
+15719,21393,30608,40898
+16113,21394,30607,40760
+13308,21395,30606,40507
+13585,21396,30605,40496
+12708,21397,30604,40072
+13496,21398,30603,40992
+13582,21399,30602,40589
+11552,21400,30601,40295
+17743,21401,30600,40749
+10660,21402,30599,40067
+14260,21403,30598,40313
+16505,21404,30597,40491
+14829,21405,30596,40446
+17258,21406,30595,40161
+11854,21407,30594,40524
+15222,21408,30593,40941
+15162,21409,30592,40138
+17088,21410,30591,40901
+15152,21411,30590,40007
+16889,21412,30589,40761
+13985,21413,30588,40716
+16388,21414,30587,40522
+11353,21415,30586,40438
+15355,21416,30585,40059
+12757,21417,30584,40983
+13881,21418,30583,40807
+14607,21419,30582,40004
+17603,21420,30581,40466
+16779,21421,30580,40712
+14074,21422,30579,40540
+11304,21423,30578,40631
+11823,21424,30577,40648
+11758,21425,30576,40504
+12282,21426,30575,40802
+12844,21427,30574,40274
+16517,21428,30573,40119
+14449,21429,30572,40047
+11781,21430,30571,40607
+11567,21431,30570,40325
+12438,21432,30569,40013
+12681,21433,30568,40441
+13569,21434,30567,40087
+14383,21435,30566,40207
+14796,21436,30565,40032
+12992,21437,30564,40996
+16053,21438,30563,40595
+11590,21439,30562,40900
+14512,21440,30561,40947
+10065,21441,30560,40592
+13219,21442,30559,40269
+14787,21443,30558,40495
+15148,21444,30557,40823
+16291,21445,30556,40773
+12941,21446,30555,40042
+16936,21447,30554,40513
+12397,21448,30553,40594
+11979,21449,30552,40000
+14156,21450,30551,40670
+18108,21451,30550,40807
+17685,21452,30549,40571
+18181,21453,30548,40981
+17250,21454,30547,40251
+17443,21455,30546,40007
+16839,21456,30545,40994
+15541,21457,30544,40590
+10284,21458,30543,40010
+11830,21459,30542,40011
+12236,21460,30541,40956
+10969,21461,30540,40146
+13801,21462,30539,40071
+11850,21463,30538,40452
+11197,21464,30537,40845
+17635,21465,30536,40990
+13941,21466,30535,40872
+15020,21467,30534,40751
+17386,21468,30533,40461
+12351,21469,30532,40039
+14609,21470,30531,40550
+17979,21471,30530,40579
+13761,21472,30529,40371
+15439,21473,30528,40565
+12754,21474,30527,40387
+15302,21475,30526,40985
+17523,21476,30525,40092
+12420,21477,30524,40762
+15572,21478,30523,40346
+16338,21479,30522,40394
+13106,21480,30521,40412
+13083,21481,30520,40752
+16404,21482,30519,40250
+16019,21483,30518,40436
+12515,21484,30517,40106
+10131,21485,30516,40014
+15895,21486,30515,40283
+13493,21487,30514,40829
+13541,21488,30513,40872
+12561,21489,30512,40851
+16168,21490,30511,40206
+15335,21491,30510,40421
+13302,21492,30509,40065
+16131,21493,30508,40673
+14445,21494,30507,40320
+14489,21495,30506,40336
+15025,21496,30505,40161
+11187,21497,30504,40067
+10159,21498,30503,40399
+11277,21499,30502,40049
+15398,21500,30501,40384
+14774,21501,30500,40062
+12478,21502,30499,40299
+15747,21503,30498,40320
+12271,21504,30497,40535
+10920,21505,30496,40995
+10082,21506,30495,40814
+13176,21507,30494,40477
+11071,21508,30493,40328
+10650,21509,30492,40987
+13424,21510,30491,40116
+16657,21511,30490,40234
+14348,21512,30489,40433
+16503,21513,30488,40489
+15185,21514,30487,40059
+17549,21515,30486,40419
+12169,21516,30485,40353
+10428,21517,30484,40003
+14009,21518,30483,40580
+12462,21519,30482,40865
+18066,21520,30481,40266
+14252,21521,30480,40681
+14458,21522,30479,40239
+12229,21523,30478,40706
+16843,21524,30477,40237
+13704,21525,30476,40964
+18042,21526,30475,40018
+18075,21527,30474,40387
+17459,21528,30473,40792
+17861,21529,30472,40144
+12440,21530,30471,40331
+17468,21531,30470,40302
+10150,21532,30469,40819
+12398,21533,30468,40546
+14694,21534,30467,40176
+16541,21535,30466,40075
+11196,21536,30465,40597
+16834,21537,30464,40865
+16600,21538,30463,40182
+13892,21539,30462,40963
+13578,21540,30461,40339
+14076,21541,30460,40008
+14785,21542,30459,40255
+15621,21543,30458,40447
+17245,21544,30457,40122
+12112,21545,30456,40770
+16616,21546,30455,40657
+17427,21547,30454,40749
+10161,21548,30453,40672
+16137,21549,30452,40629
+13650,21550,30451,40366
+17908,21551,30450,40635
+13715,21552,30449,40270
+15850,21553,30448,40711
+14050,21554,30447,40686
+14960,21555,30446,40225
+13344,21556,30445,40824
+11492,21557,30444,40507
+15732,21558,30443,40694
+11190,21559,30442,40411
+15780,21560,30441,40690
+17800,21561,30440,40005
+13271,21562,30439,40755
+14748,21563,30438,40565
+11959,21564,30437,40324
+10841,21565,30436,40681
+18144,21566,30435,40271
+10778,21567,30434,40004
+14223,21568,30433,40711
+16736,21569,30432,40911
+13982,21570,30431,40539
+13986,21571,30430,40083
+11057,21572,30429,40206
+14738,21573,30428,40218
+16827,21574,30427,40427
+11314,21575,30426,40123
+13489,21576,30425,40077
+13193,21577,30424,40195
+12835,21578,30423,40309
+11536,21579,30422,40265
+16296,21580,30421,40981
+14451,21581,30420,40122
+16015,21582,30419,40345
+14407,21583,30418,40475
+16738,21584,30417,40461
+17590,21585,30416,40805
+10019,21586,30415,40129
+14967,21587,30414,40851
+17471,21588,30413,40122
+14169,21589,30412,40791
+16327,21590,30411,40082
+17982,21591,30410,40706
+13260,21592,30409,40735
+10084,21593,30408,40226
+11074,21594,30407,40777
+14529,21595,30406,40757
+12350,21596,30405,40645
+17113,21597,30404,40836
+15619,21598,30403,40727
+12211,21599,30402,40956
+14842,21600,30401,40140
+12224,21601,30400,40421
+13139,21602,30399,40845
+16504,21603,30398,40341
+15456,21604,30397,40175
+17284,21605,30396,40847
+14658,21606,30395,40504
+14754,21607,30394,40269
+16857,21608,30393,40228
+15407,21609,30392,40989
+15457,21610,30391,40213
+10091,21611,30390,40654
+11659,21612,30389,40913
+17973,21613,30388,40244
+14436,21614,30387,40053
+17951,21615,30386,40937
+16410,21616,30385,40712
+15044,21617,30384,40780
+16408,21618,30383,40522
+11776,21619,30382,40806
+15668,21620,30381,40427
+13416,21621,30380,40654
+15595,21622,30379,40032
+18171,21623,30378,40239
+13831,21624,30377,40568
+13120,21625,30376,40215
+16148,21626,30375,40780
+10367,21627,30374,40712
+16898,21628,30373,40613
+15693,21629,30372,40284
+14708,21630,30371,40200
+16058,21631,30370,40657
+11847,21632,30369,40940
+17506,21633,30368,40708
+14500,21634,30367,40788
+16712,21635,30366,40345
+13851,21636,30365,40936
+14311,21637,30364,40777
+16475,21638,30363,40982
+16293,21639,30362,40804
+13529,21640,30361,40302
+10735,21641,30360,40198
+14995,21642,30359,40043
+11234,21643,30358,40310
+12623,21644,30357,40469
+13933,21645,30356,40465
+17132,21646,30355,40771
+16818,21647,30354,40678
+17413,21648,30353,40437
+16859,21649,30352,40978
+17862,21650,30351,40808
+16885,21651,30350,40843
+12451,21652,30349,40299
+10959,21653,30348,40906
+13911,21654,30347,40431
+13910,21655,30346,40506
+10048,21656,30345,40923
+15708,21657,30344,40277
+10063,21658,30343,40901
+16925,21659,30342,40893
+10687,21660,30341,40316
+12793,21661,30340,40600
+15758,21662,30339,40930
+12489,21663,30338,40836
+10198,21664,30337,40607
+11755,21665,30336,40194
+17262,21666,30335,40809
+10972,21667,30334,40200
+17909,21668,30333,40112
+12303,21669,30332,40198
+17775,21670,30331,40309
+13033,21671,30330,40818
+11730,21672,30329,40323
+11884,21673,30328,40807
+14756,21674,30327,40406
+13409,21675,30326,40912
+14691,21676,30325,40991
+11915,21677,30324,40827
+15746,21678,30323,40513
+11408,21679,30322,40384
+11440,21680,30321,40886
+17958,21681,30320,40113
+11285,21682,30319,40643
+10659,21683,30318,40259
+12748,21684,30317,40387
+17574,21685,30316,40794
+11690,21686,30315,40287
+15677,21687,30314,40947
+12686,21688,30313,40698
+17466,21689,30312,40362
+13481,21690,30311,40058
+16069,21691,30310,40676
+13781,21692,30309,40474
+13356,21693,30308,40215
+11488,21694,30307,40189
+14527,21695,30306,40718
+17791,21696,30305,40526
+10307,21697,30304,40657
+11289,21698,30303,40739
+17123,21699,30302,40921
+17435,21700,30301,40767
+14840,21701,30300,40304
+17150,21702,30299,40126
+10303,21703,30298,40199
+15953,21704,30297,40427
+17871,21705,30296,40001
+14742,21706,30295,40093
+17433,21707,30294,40109
+14530,21708,30293,40001
+11389,21709,30292,40297
+14830,21710,30291,40361
+11563,21711,30290,40785
+12610,21712,30289,40210
+10819,21713,30288,40311
+10280,21714,30287,40307
+13410,21715,30286,40301
+13213,21716,30285,40161
+16092,21717,30284,40910
+15531,21718,30283,40072
+18157,21719,30282,40626
+15022,21720,30281,40347
+10473,21721,30280,40016
+12508,21722,30279,40956
+15680,21723,30278,40597
+15548,21724,30277,40136
+10088,21725,30276,40939
+10140,21726,30275,40714
+15612,21727,30274,40782
+11586,21728,30273,40439
+13546,21729,30272,40080
+12781,21730,30271,40297
+14055,21731,30270,40243
+11456,21732,30269,40189
+17228,21733,30268,40590
+12733,21734,30267,40064
+12136,21735,30266,40009
+16846,21736,30265,40830
+16335,21737,30264,40563
+13953,21738,30263,40890
+14266,21739,30262,40867
+15246,21740,30261,40402
+16328,21741,30260,40395
+12315,21742,30259,40978
+12646,21743,30258,40104
+13251,21744,30257,40633
+17728,21745,30256,40486
+14126,21746,30255,40193
+12944,21747,30254,40897
+15599,21748,30253,40327
+13828,21749,30252,40726
+15930,21750,30251,40682
+16071,21751,30250,40175
+17231,21752,30249,40424
+15666,21753,30248,40710
+11376,21754,30247,40292
+17358,21755,30246,40697
+16646,21756,30245,40580
+17214,21757,30244,40035
+14876,21758,30243,40327
+11265,21759,30242,40436
+15281,21760,30241,40540
+13594,21761,30240,40830
+12191,21762,30239,40909
+15893,21763,30238,40920
+16478,21764,30237,40050
+12448,21765,30236,40608
+16806,21766,30235,40514
+13330,21767,30234,40593
+13480,21768,30233,40357
+14896,21769,30232,40317
+11772,21770,30231,40125
+17696,21771,30230,40387
+11952,21772,30229,40026
+12841,21773,30228,40729
+10812,21774,30227,40820
+16369,21775,30226,40245
+15137,21776,30225,40291
+15097,21777,30224,40909
+16046,21778,30223,40187
+15258,21779,30222,40295
+14100,21780,30221,40380
+12486,21781,30220,40026
+10880,21782,30219,40309
+14032,21783,30218,40243
+13392,21784,30217,40320
+12672,21785,30216,40796
+13816,21786,30215,40929
+10620,21787,30214,40646
+10052,21788,30213,40784
+10949,21789,30212,40272
+17836,21790,30211,40366
+10117,21791,30210,40488
+16606,21792,30209,40948
+17297,21793,30208,40693
+16691,21794,30207,40430
+18132,21795,30206,40407
+10011,21796,30205,40775
+10089,21797,30204,40334
+15918,21798,30203,40902
+16373,21799,30202,40888
+10302,21800,30201,40046
+11370,21801,30200,40887
+17237,21802,30199,40412
+15881,21803,30198,40222
+10058,21804,30197,40425
+11412,21805,30196,40147
+13819,21806,30195,40663
+11546,21807,30194,40776
+11352,21808,30193,40985
+10890,21809,30192,40644
+13336,21810,30191,40394
+12906,21811,30190,40263
+12335,21812,30189,40080
+15900,21813,30188,40322
+12913,21814,30187,40873
+14873,21815,30186,40260
+13776,21816,30185,40459
+13119,21817,30184,40778
+12571,21818,30183,40578
+16648,21819,30182,40030
+16250,21820,30181,40498
+12184,21821,30180,40472
+15080,21822,30179,40901
+13178,21823,30178,40402
+12544,21824,30177,40678
+18143,21825,30176,40108
+13267,21826,30175,40028
+15385,21827,30174,40632
+13226,21828,30173,40522
+14217,21829,30172,40617
+10926,21830,30171,40979
+16604,21831,30170,40533
+16361,21832,30169,40155
+15463,21833,30168,40129
+10753,21834,30167,40736
+11925,21835,30166,40067
+12327,21836,30165,40591
+17530,21837,30164,40858
+11532,21838,30163,40212
+14237,21839,30162,40529
+14971,21840,30161,40433
+11689,21841,30160,40763
+12467,21842,30159,40991
+10618,21843,30158,40961
+16534,21844,30157,40651
+17390,21845,30156,40351
+16488,21846,30155,40367
+14798,21847,30154,40180
+11198,21848,30153,40273
+12983,21849,30152,40792
+16100,21850,30151,40449
+10860,21851,30150,40832
+16906,21852,30149,40300
+12649,21853,30148,40036
+11369,21854,30147,40806
+11434,21855,30146,40298
+12883,21856,30145,40652
+13240,21857,30144,40140
+15077,21858,30143,40495
+14073,21859,30142,40156
+10538,21860,30141,40789
+10295,21861,30140,40959
+17553,21862,30139,40217
+17166,21863,30138,40767
+12437,21864,30137,40235
+13660,21865,30136,40955
+14358,21866,30135,40962
+11694,21867,30134,40057
+11153,21868,30133,40517
+11731,21869,30132,40884
+14293,21870,30131,40929
+12871,21871,30130,40520
+16935,21872,30129,40205
+17334,21873,30128,40898
+14923,21874,30127,40769
+17618,21875,30126,40613
+12279,21876,30125,40373
+12010,21877,30124,40479
+17784,21878,30123,40814
+12000,21879,30122,40306
+15987,21880,30121,40275
+13741,21881,30120,40646
+15233,21882,30119,40861
+14730,21883,30118,40244
+14441,21884,30117,40029
+11865,21885,30116,40669
+12202,21886,30115,40504
+17204,21887,30114,40272
+11657,21888,30113,40133
+15828,21889,30112,40131
+11514,21890,30111,40896
+17756,21891,30110,40057
+14093,21892,30109,40455
+12702,21893,30108,40738
+14688,21894,30107,40984
+11306,21895,30106,40128
+13293,21896,30105,40605
+14058,21897,30104,40941
+16409,21898,30103,40088
+14961,21899,30102,40386
+11478,21900,30101,40089
+11049,21901,30100,40827
+18091,21902,30099,40879
+12119,21903,30098,40262
+16280,21904,30097,40907
+14211,21905,30096,40170
+14450,21906,30095,40168
+10205,21907,30094,40796
+11648,21908,30093,40142
+17498,21909,30092,40508
+11252,21910,30091,40790
+13192,21911,30090,40502
+15654,21912,30089,40447
+15014,21913,30088,40124
+10176,21914,30087,40311
+16912,21915,30086,40034
+17366,21916,30085,40125
+13701,21917,30084,40482
+17474,21918,30083,40099
+16543,21919,30082,40333
+18121,21920,30081,40970
+15789,21921,30080,40554
+12059,21922,30079,40109
+15694,21923,30078,40089
+13129,21924,30077,40207
+12181,21925,30076,40273
+14321,21926,30075,40465
+10429,21927,30074,40999
+13947,21928,30073,40611
+16639,21929,30072,40088
+16315,21930,30071,40292
+14191,21931,30070,40421
+14172,21932,30069,40885
+17467,21933,30068,40845
+16804,21934,30067,40439
+10405,21935,30066,40913
+12357,21936,30065,40829
+12696,21937,30064,40081
+10990,21938,30063,40292
+10253,21939,30062,40044
+13799,21940,30061,40462
+13303,21941,30060,40716
+11692,21942,30059,40910
+13040,21943,30058,40608
+15507,21944,30057,40199
+16461,21945,30056,40203
+16793,21946,30055,40205
+16790,21947,30054,40530
+17422,21948,30053,40188
+10077,21949,30052,40490
+14962,21950,30051,40359
+12653,21951,30050,40544
+14637,21952,30049,40745
+17857,21953,30048,40194
+11933,21954,30047,40600
+13908,21955,30046,40922
+11421,21956,30045,40879
+10534,21957,30044,40403
+11171,21958,30043,40250
+15248,21959,30042,40596
+16612,21960,30041,40381
+14218,21961,30040,40082
+12130,21962,30039,40357
+15298,21963,30038,40091
+12255,21964,30037,40888
+16826,21965,30036,40076
+13290,21966,30035,40256
+17327,21967,30034,40531
+17086,21968,30033,40658
+15085,21969,30032,40865
+14147,21970,30031,40173
+10683,21971,30030,40615
+10838,21972,30029,40070
+13044,21973,30028,40193
+12277,21974,30027,40472
+14157,21975,30026,40472
+17666,21976,30025,40225
+13268,21977,30024,40938
+15226,21978,30023,40920
+12481,21979,30022,40404
+10932,21980,30021,40139
+12598,21981,30020,40487
+17601,21982,30019,40965
+14835,21983,30018,40024
+14544,21984,30017,40955
+14416,21985,30016,40410
+12253,21986,30015,40432
+18051,21987,30014,40830
+10645,21988,30013,40990
+11622,21989,30012,40554
+11929,21990,30011,40141
+12643,21991,30010,40696
+14715,21992,30009,40737
+11238,21993,30008,40701
+10792,21994,30007,40483
+10313,21995,30006,40642
+10596,21996,30005,40346
+13008,21997,30004,40015
+12758,21998,30003,40786
+10358,21999,30002,40875
+10137,22000,30001,40650
+16901,22001,30000,40849
+10288,22002,31000,40611
+17448,22003,30999,40916
+16782,22004,30998,40637
+17081,22005,30997,40538
+10611,22006,30996,40356
+16041,22007,30995,40487
+13967,22008,30994,40991
+16905,22009,30993,40097
+11678,22010,30992,40249
+13269,22011,30991,40888
+18184,22012,30990,40304
+15347,22013,30989,40824
+13039,22014,30988,40723
+10335,22015,30987,40031
+12275,22016,30986,40281
+12418,22017,30985,40905
+13963,22018,30984,40324
+12725,22019,30983,40516
+10173,22020,30982,40797
+12027,22021,30981,40725
+16003,22022,30980,40561
+10182,22023,30979,40033
+14491,22024,30978,40544
+12436,22025,30977,40248
+17663,22026,30976,40654
+17346,22027,30975,40795
+13100,22028,30974,40075
+14214,22029,30973,40942
+12831,22030,30972,40429
+12428,22031,30971,40537
+11247,22032,30970,40525
+11100,22033,30969,40384
+11030,22034,30968,40151
+13161,22035,30967,40091
+11154,22036,30966,40090
+10698,22037,30965,40184
+10541,22038,30964,40008
+18086,22039,30963,40795
+17796,22040,30962,40472
+14409,22041,30961,40167
+15824,22042,30960,40631
+15910,22043,30959,40087
+12890,22044,30958,40349
+14663,22045,30957,40350
+10481,22046,30956,40939
+14968,22047,30955,40780
+10414,22048,30954,40290
+16954,22049,30953,40761
+12808,22050,30952,40382
+12803,22051,30951,40156
+14615,22052,30950,40474
+16264,22053,30949,40621
+14760,22054,30948,40886
+15562,22055,30947,40265
+15159,22056,30946,40147
+12442,22057,30945,40315
+11307,22058,30944,40158
+12495,22059,30943,40919
+18104,22060,30942,40195
+11142,22061,30941,40482
+16786,22062,30940,40297
+16391,22063,30939,40629
+11127,22064,30938,40459
+13429,22065,30937,40632
+17045,22066,30936,40911
+17971,22067,30935,40980
+10361,22068,30934,40735
+17617,22069,30933,40921
+13324,22070,30932,40821
+13810,22071,30931,40710
+12430,22072,30930,40348
+14815,22073,30929,40982
+18013,22074,30928,40481
+13682,22075,30927,40903
+12691,22076,30926,40000
+12152,22077,30925,40127
+11079,22078,30924,40155
+12259,22079,30923,40647
+10085,22080,30922,40534
+12549,22081,30921,40081
+13749,22082,30920,40325
+17755,22083,30919,40814
+14097,22084,30918,40694
+16040,22085,30917,40688
+11467,22086,30916,40628
+17252,22087,30915,40486
+12500,22088,30914,40184
+13882,22089,30913,40214
+16985,22090,30912,40188
+17160,22091,30911,40468
+17661,22092,30910,40498
+17682,22093,30909,40257
+16332,22094,30908,40006
+10733,22095,30907,40143
+13325,22096,30906,40101
+13230,22097,30905,40017
+14330,22098,30904,40981
+11382,22099,30903,40175
+16596,22100,30902,40395
+12218,22101,30901,40286
+11349,22102,30900,40420
+12226,22103,30899,40334
+16128,22104,30898,40627
+12163,22105,30897,40438
+14175,22106,30896,40193
+18127,22107,30895,40994
+15561,22108,30894,40283
+14135,22109,30893,40788
+13365,22110,30892,40349
+11739,22111,30891,40521
+15095,22112,30890,40021
+11767,22113,30889,40554
+14906,22114,30888,40098
+15217,22115,30887,40010
+12380,22116,30886,40466
+15300,22117,30885,40312
+11343,22118,30884,40699
+12911,22119,30883,40778
+10765,22120,30882,40153
+15114,22121,30881,40097
+16620,22122,30880,40238
+14045,22123,30879,40636
+10162,22124,30878,40360
+11417,22125,30877,40925
+10451,22126,30876,40197
+17870,22127,30875,40999
+15706,22128,30874,40089
+10673,22129,30873,40537
+16244,22130,30872,40776
+15536,22131,30871,40156
+14337,22132,30870,40359
+11186,22133,30869,40024
+14125,22134,30868,40084
+17249,22135,30867,40917
+11556,22136,30866,40774
+11313,22137,30865,40237
+18033,22138,30864,40694
+16619,22139,30863,40815
+12182,22140,30862,40237
+10338,22141,30861,40782
+16271,22142,30860,40088
+10301,22143,30859,40176
+18007,22144,30858,40599
+15017,22145,30857,40810
+16565,22146,30856,40979
+10707,22147,30855,40553
+10825,22148,30854,40701
+14800,22149,30853,40267
+17901,22150,30852,40659
+18048,22151,30851,40140
+14563,22152,30850,40453
+13087,22153,30849,40677
+17412,22154,30848,40126
+12866,22155,30847,40558
+17856,22156,30846,40398
+12950,22157,30845,40004
+12042,22158,30844,40628
+10795,22159,30843,40878
+10223,22160,30842,40601
+11051,22161,30841,40697
+10967,22162,30840,40904
+13814,22163,30839,40228
+10281,22164,30838,40697
+13015,22165,30837,40589
+11258,22166,30836,40156
+16454,22167,30835,40594
+10135,22168,30834,40043
+15402,22169,30833,40956
+14412,22170,30832,40273
+11704,22171,30831,40342
+13925,22172,30830,40622
+16088,22173,30829,40251
+11436,22174,30828,40716
+13026,22175,30827,40101
+12530,22176,30826,40548
+16194,22177,30825,40570
+11582,22178,30824,40905
+13775,22179,30823,40689
+11982,22180,30822,40175
+14392,22181,30821,40851
+17207,22182,30820,40508
+17379,22183,30819,40508
+15863,22184,30818,40772
+13247,22185,30817,40168
+11251,22186,30816,40619
+12626,22187,30815,40194
+10975,22188,30814,40387
+15286,22189,30813,40032
+12759,22190,30812,40229
+13859,22191,30811,40667
+14892,22192,30810,40692
+11491,22193,30809,40346
+13734,22194,30808,40053
+15696,22195,30807,40013
+15062,22196,30806,40738
+15254,22197,30805,40087
+16990,22198,30804,40238
+13058,22199,30803,40157
+16470,22200,30802,40140
+10613,22201,30801,40148
+16688,22202,30800,40256
+16837,22203,30799,40351
+10200,22204,30798,40054
+14424,22205,30797,40701
+16942,22206,30796,40929
+12602,22207,30795,40697
+10240,22208,30794,40028
+16999,22209,30793,40235
+14231,22210,30792,40461
+10336,22211,30791,40060
+12227,22212,30790,40568
+12670,22213,30789,40511
+17642,22214,30788,40519
+17255,22215,30787,40978
+11283,22216,30786,40606
+11511,22217,30785,40020
+14633,22218,30784,40868
+10594,22219,30783,40712
+17630,22220,30782,40854
+10165,22221,30781,40920
+11260,22222,30780,40478
+12692,22223,30779,40245
+15834,22224,30778,40666
+12522,22225,30777,40722
+11391,22226,30776,40031
+12141,22227,30775,40089
+17211,22228,30774,40563
+13130,22229,30773,40061
+13939,22230,30772,40121
+11899,22231,30771,40439
+16376,22232,30770,40023
+15551,22233,30769,40580
+17032,22234,30768,40364
+14702,22235,30767,40314
+12015,22236,30766,40646
+17312,22237,30765,40811
+13214,22238,30764,40728
+11923,22239,30763,40035
+12868,22240,30762,40664
+16448,22241,30761,40388
+16350,22242,30760,40078
+10996,22243,30759,40321
+17315,22244,30758,40304
+16309,22245,30757,40095
+11170,22246,30756,40034
+12045,22247,30755,40590
+10239,22248,30754,40171
+13380,22249,30753,40199
+15673,22250,30752,40275
+14631,22251,30751,40841
+11981,22252,30750,40473
+17501,22253,30749,40636
+13172,22254,30748,40465
+13727,22255,30747,40814
+14603,22256,30746,40243
+14871,22257,30745,40123
+17641,22258,30744,40422
+12721,22259,30743,40839
+12018,22260,30742,40890
+12004,22261,30741,40439
+17279,22262,30740,40510
+15728,22263,30739,40198
+11263,22264,30738,40502
+11037,22265,30737,40219
+15711,22266,30736,40566
+17295,22267,30735,40152
+11204,22268,30734,40556
+12362,22269,30733,40221
+10348,22270,30732,40631
+13833,22271,30731,40419
+13503,22272,30730,40758
+17051,22273,30729,40973
+11999,22274,30728,40500
+17099,22275,30727,40328
+12873,22276,30726,40359
+16913,22277,30725,40659
+18041,22278,30724,40943
+16861,22279,30723,40377
+15223,22280,30722,40396
+13790,22281,30721,40395
+15966,22282,30720,40448
+11661,22283,30719,40019
+13381,22284,30718,40771
+14640,22285,30717,40740
+16091,22286,30716,40853
+10940,22287,30715,40557
+14879,22288,30714,40399
+14521,22289,30713,40884
+12637,22290,30712,40858
+16835,22291,30711,40911
+15333,22292,30710,40513
+16063,22293,30709,40557
+14533,22294,30708,40779
+12073,22295,30707,40627
+10352,22296,30706,40160
+10605,22297,30705,40299
+17939,22298,30704,40605
+10642,22299,30703,40599
+15171,22300,30702,40866
+12755,22301,30701,40819
+13753,22302,30700,40048
+10195,22303,30699,40377
+11114,22304,30698,40648
+10897,22305,30697,40974
+15624,22306,30696,40807
+16321,22307,30695,40881
+15943,22308,30694,40037
+11894,22309,30693,40187
+14395,22310,30692,40245
+15118,22311,30691,40969
+13872,22312,30690,40655
+11646,22313,30689,40071
+14168,22314,30688,40833
+11987,22315,30687,40688
+16992,22316,30686,40460
+17969,22317,30685,40480
+10850,22318,30684,40819
+13346,22319,30683,40738
+11547,22320,30682,40118
+16525,22321,30681,40109
+10913,22322,30680,40500
+12459,22323,30679,40065
+15275,22324,30678,40599
+13574,22325,30677,40336
+14875,22326,30676,40817
+10855,22327,30675,40767
+15332,22328,30674,40661
+12511,22329,30673,40475
+16108,22330,30672,40680
+11320,22331,30671,40545
+14176,22332,30670,40599
+13968,22333,30669,40406
+10785,22334,30668,40610
+13014,22335,30667,40749
+16926,22336,30666,40853
+12490,22337,30665,40007
+10965,22338,30664,40999
+11904,22339,30663,40788
+13729,22340,30662,40547
+15187,22341,30661,40631
+18060,22342,30660,40404
+16101,22343,30659,40472
+18165,22344,30658,40350
+17647,22345,30657,40195
+16987,22346,30656,40855
+10221,22347,30655,40614
+14716,22348,30654,40917
+15935,22349,30653,40208
+15991,22350,30652,40729
+10757,22351,30651,40653
+14997,22352,30650,40000
+15263,22353,30649,40534
+14689,22354,30648,40131
+14036,22355,30647,40933
+15101,22356,30646,40004
+12892,22357,30645,40718
+13576,22358,30644,40369
+16662,22359,30643,40185
+17374,22360,30642,40373
+15195,22361,30641,40955
+16036,22362,30640,40393
+12403,22363,30639,40813
+10230,22364,30638,40418
+10153,22365,30637,40508
+12763,22366,30636,40828
+16006,22367,30635,40447
+16755,22368,30634,40197
+12348,22369,30633,40288
+14248,22370,30632,40604
+11769,22371,30631,40081
+11269,22372,30630,40191
+13561,22373,30629,40456
+13712,22374,30628,40332
+15454,22375,30627,40301
+15511,22376,30626,40055
+16414,22377,30625,40120
+14584,22378,30624,40075
+16964,22379,30623,40599
+11868,22380,30622,40628
+10508,22381,30621,40709
+17012,22382,30620,40312
+17804,22383,30619,40735
+12731,22384,30618,40838
+15086,22385,30617,40079
+13312,22386,30616,40814
+16079,22387,30615,40803
+14902,22388,30614,40104
+11350,22389,30613,40377
+12372,22390,30612,40289
+11003,22391,30611,40921
+13938,22392,30610,40385
+16917,22393,30609,40004
+17551,22394,30608,40063
+10407,22395,30607,40026
+17347,22396,30606,40533
+11526,22397,30605,40055
+10466,22398,30604,40848
+12614,22399,30603,40423
+10549,22400,30602,40840
+14244,22401,30601,40641
+14628,22402,30600,40790
+11554,22403,30599,40496
+16483,22404,30598,40587
+15314,22405,30597,40740
+17202,22406,30596,40621
+17730,22407,30595,40232
+11799,22408,30594,40572
+12836,22409,30593,40188
+17676,22410,30592,40147
+17213,22411,30591,40978
+16445,22412,30590,40610
+17341,22413,30589,40326
+11177,22414,30588,40865
+10487,22415,30587,40438
+11383,22416,30586,40455
+17542,22417,30585,40185
+12454,22418,30584,40759
+11535,22419,30583,40204
+15871,22420,30582,40820
+11635,22421,30581,40975
+10840,22422,30580,40771
+12332,22423,30579,40583
+13341,22424,30578,40303
+15428,22425,30577,40132
+12778,22426,30576,40443
+15798,22427,30575,40922
+11610,22428,30574,40455
+16698,22429,30573,40237
+10701,22430,30572,40629
+13663,22431,30571,40254
+13012,22432,30570,40870
+17880,22433,30569,40291
+11837,22434,30568,40982
+15857,22435,30567,40034
+11393,22436,30566,40030
+13122,22437,30565,40365
+11476,22438,30564,40112
+14431,22439,30563,40953
+11728,22440,30562,40364
+14083,22441,30561,40088
+15667,22442,30560,40809
+10888,22443,30559,40207
+12240,22444,30558,40624
+16951,22445,30557,40522
+11240,22446,30556,40186
+15345,22447,30555,40524
+10236,22448,30554,40754
+11577,22449,30553,40242
+10981,22450,30552,40557
+11862,22451,30551,40215
+11463,22452,30550,40790
+11998,22453,30549,40698
+13450,22454,30548,40126
+13286,22455,30547,40523
+17925,22456,30546,40504
+17604,22457,30545,40725
+15723,22458,30544,40211
+11960,22459,30543,40332
+10154,22460,30542,40841
+17367,22461,30541,40248
+16838,22462,30540,40172
+13140,22463,30539,40931
+11163,22464,30538,40807
+13092,22465,30537,40100
+12322,22466,30536,40258
+10432,22467,30535,40502
+11668,22468,30534,40424
+17733,22469,30533,40882
+11339,22470,30532,40483
+13680,22471,30531,40344
+16589,22472,30530,40023
+13042,22473,30529,40581
+15430,22474,30528,40012
+14028,22475,30527,40894
+11173,22476,30526,40765
+17887,22477,30525,40243
+14206,22478,30524,40373
+14276,22479,30523,40655
+14678,22480,30522,40681
+12744,22481,30521,40404
+13088,22482,30520,40577
+13897,22483,30519,40467
+15409,22484,30518,40760
+15499,22485,30517,40393
+17418,22486,30516,40426
+10575,22487,30515,40109
+17438,22488,30514,40349
+15359,22489,30513,40461
+10034,22490,30512,40808
+17338,22491,30511,40025
+10513,22492,30510,40425
+12038,22493,30509,40207
+12612,22494,30508,40579
+13735,22495,30507,40325
+16428,22496,30506,40076
+15273,22497,30505,40134
+17503,22498,30504,40852
+14347,22499,30503,40843
+14581,22500,30502,40881
+15826,22501,30501,40523
+10922,22502,30500,40704
+14182,22503,30499,40613
+14031,22504,30498,40886
+17063,22505,30497,40434
+12905,22506,30496,40084
+15105,22507,30495,40096
+11651,22508,30494,40561
+10721,22509,30493,40184
+17186,22510,30492,40169
+12468,22511,30491,40969
+16697,22512,30490,40795
+12592,22513,30489,40542
+13817,22514,30488,40913
+10357,22515,30487,40338
+14586,22516,30486,40280
+12155,22517,30485,40572
+15604,22518,30484,40167
+17420,22519,30483,40021
+17822,22520,30482,40957
+17747,22521,30481,40470
+17057,22522,30480,40163
+14195,22523,30479,40215
+10934,22524,30478,40928
+12338,22525,30477,40946
+10492,22526,30476,40755
+11121,22527,30475,40046
+11575,22528,30474,40331
+13870,22529,30473,40480
+11916,22530,30472,40203
+13484,22531,30471,40971
+12880,22532,30470,40769
+13418,22533,30469,40248
+16481,22534,30468,40501
+12922,22535,30467,40937
+16498,22536,30466,40967
+11726,22537,30465,40215
+15978,22538,30464,40337
+17697,22539,30463,40987
+14727,22540,30462,40276
+12396,22541,30461,40033
+15773,22542,30460,40638
+11677,22543,30459,40553
+17538,22544,30458,40760
+15563,22545,30457,40824
+15257,22546,30456,40014
+11623,22547,30455,40827
+17287,22548,30454,40976
+15686,22549,30453,40379
+10822,22550,30452,40086
+14943,22551,30451,40404
+11481,22552,30450,40304
+11667,22553,30449,40871
+10217,22554,30448,40643
+11839,22555,30447,40528
+10498,22556,30446,40871
+11464,22557,30445,40945
+16658,22558,30444,40589
+11969,22559,30443,40485
+13028,22560,30442,40110
+13141,22561,30441,40874
+12453,22562,30440,40799
+12194,22563,30439,40565
+11914,22564,30438,40601
+10504,22565,30437,40225
+12316,22566,30436,40932
+10644,22567,30435,40826
+13522,22568,30434,40164
+15981,22569,30433,40868
+16934,22570,30432,40051
+18052,22571,30431,40577
+16400,22572,30430,40998
+13486,22573,30429,40215
+12178,22574,30428,40988
+15942,22575,30427,40319
+14162,22576,30426,40745
+11763,22577,30425,40906
+10738,22578,30424,40809
+13494,22579,30423,40166
+12942,22580,30422,40719
+16757,22581,30421,40118
+11713,22582,30420,40349
+18087,22583,30419,40928
+17306,22584,30418,40118
+11358,22585,30417,40752
+13420,22586,30416,40311
+12497,22587,30415,40978
+13852,22588,30414,40709
+12661,22589,30413,40349
+15297,22590,30412,40936
+14124,22591,30411,40498
+15422,22592,30410,40276
+17643,22593,30409,40404
+15867,22594,30408,40648
+17256,22595,30407,40377
+14316,22596,30406,40143
+13353,22597,30405,40814
+15172,22598,30404,40380
+15364,22599,30403,40002
+17179,22600,30402,40603
+14600,22601,30401,40217
+12113,22602,30400,40315
+17929,22603,30399,40848
+14668,22604,30398,40007
+11857,22605,30397,40416
+17301,22606,30396,40771
+11402,22607,30395,40475
+14947,22608,30394,40732
+15573,22609,30393,40684
+12814,22610,30392,40504
+10014,22611,30391,40283
+10816,22612,30390,40717
+13270,22613,30389,40638
+17525,22614,30388,40142
+12425,22615,30387,40827
+15931,22616,30386,40447
+14376,22617,30385,40241
+14557,22618,30384,40409
+17073,22619,30383,40167
+10786,22620,30382,40417
+14419,22621,30381,40020
+17814,22622,30380,40867
+10963,22623,30379,40926
+11413,22624,30378,40549
+17981,22625,30377,40662
+10505,22626,30376,40942
+18085,22627,30375,40800
+10625,22628,30374,40781
+17827,22629,30373,40112
+10289,22630,30372,40301
+12369,22631,30371,40120
+13017,22632,30370,40374
+15188,22633,30369,40070
+14813,22634,30368,40127
+18099,22635,30367,40410
+15442,22636,30366,40687
+14065,22637,30365,40045
+14752,22638,30364,40884
+17645,22639,30363,40870
+12400,22640,30362,40691
+10804,22641,30361,40606
+17463,22642,30360,40837
+15707,22643,30359,40475
+14570,22644,30358,40866
+17897,22645,30357,40145
+17296,22646,30356,40327
+15210,22647,30355,40836
+11905,22648,30354,40139
+17176,22649,30353,40307
+13018,22650,30352,40672
+16975,22651,30351,40790
+11653,22652,30350,40749
+12172,22653,30349,40440
+11020,22654,30348,40184
+12648,22655,30347,40151
+15450,22656,30346,40425
+15767,22657,30345,40920
+11081,22658,30344,40409
+15600,22659,30343,40160
+10497,22660,30342,40913
+11324,22661,30341,40110
+12768,22662,30340,40936
+11601,22663,30339,40580
+12117,22664,30338,40404
+17771,22665,30337,40955
+12288,22666,30336,40417
+15530,22667,30335,40406
+16395,22668,30334,40831
+16681,22669,30333,40935
+14771,22670,30332,40181
+17672,22671,30331,40230
+14319,22672,30330,40524
+16622,22673,30329,40544
+15117,22674,30328,40399
+17776,22675,30327,40804
+10397,22676,30326,40014
+16976,22677,30325,40361
+15836,22678,30324,40026
+16497,22679,30323,40995
+11756,22680,30322,40531
+10815,22681,30321,40356
+17137,22682,30320,40453
+17898,22683,30319,40895
+13877,22684,30318,40752
+14717,22685,30317,40287
+11977,22686,30316,40487
+18050,22687,30315,40039
+14690,22688,30314,40232
+17591,22689,30313,40107
+12698,22690,30312,40746
+13256,22691,30311,40707
+11084,22692,30310,40029
+16346,22693,30309,40415
+10096,22694,30308,40395
+15092,22695,30307,40726
+14398,22696,30306,40013
+12575,22697,30305,40585
+14226,22698,30304,40264
+16802,22699,30303,40988
+12047,22700,30302,40262
+16506,22701,30301,40325
+16263,22702,30300,40120
+14987,22703,30299,40445
+15700,22704,30298,40009
+10693,22705,30297,40747
+11270,22706,30296,40414
+14481,22707,30295,40896
+11378,22708,30294,40786
+14843,22709,30293,40157
+16412,22710,30292,40934
+14819,22711,30291,40360
+13750,22712,30290,40397
+14656,22713,30289,40102
+14516,22714,30288,40398
+15769,22715,30287,40922
+13931,22716,30286,40851
+11397,22717,30285,40347
+16122,22718,30284,40862
+12204,22719,30283,40076
+14944,22720,30282,40227
+17984,22721,30281,40805
+14577,22722,30280,40256
+15349,22723,30279,40967
+11181,22724,30278,40688
+17042,22725,30277,40349
+13089,22726,30276,40495
+11484,22727,30275,40478
+15136,22728,30274,40617
+13227,22729,30273,40934
+11367,22730,30272,40500
+17831,22731,30271,40756
+15168,22732,30270,40371
+16841,22733,30269,40039
+10879,22734,30268,40788
+15121,22735,30267,40217
+17316,22736,30266,40809
+12289,22737,30265,40126
+16800,22738,30264,40834
+12792,22739,30263,40453
+14149,22740,30262,40020
+17757,22741,30261,40941
+17667,22742,30260,40187
+14789,22743,30259,40443
+11479,22744,30258,40523
+18049,22745,30257,40447
+16180,22746,30256,40156
+11480,22747,30255,40543
+10527,22748,30254,40428
+15388,22749,30253,40139
+17704,22750,30252,40838
+13446,22751,30251,40488
+13784,22752,30250,40712
+11863,22753,30249,40816
+11439,22754,30248,40642
+14474,22755,30247,40782
+14958,22756,30246,40010
+10134,22757,30245,40319
+12028,22758,30244,40557
+10779,22759,30243,40708
+13813,22760,30242,40344
+18146,22761,30241,40772
+10643,22762,30240,40405
+14075,22763,30239,40407
+13665,22764,30238,40568
+16384,22765,30237,40109
+12715,22766,30236,40961
+15521,22767,30235,40876
+15010,22768,30234,40052
+10177,22769,30233,40680
+17960,22770,30232,40700
+15913,22771,30231,40341
+14215,22772,30230,40478
+17736,22773,30229,40657
+16578,22774,30228,40097
+10371,22775,30227,40396
+15251,22776,30226,40124
+17218,22777,30225,40177
+16953,22778,30224,40584
+13737,22779,30223,40118
+13275,22780,30222,40844
+17033,22781,30221,40183
+13977,22782,30220,40840
+17926,22783,30219,40442
+13298,22784,30218,40425
+15704,22785,30217,40515
+11035,22786,30216,40178
+13899,22787,30215,40023
+17520,22788,30214,40948
+13767,22789,30213,40149
+10472,22790,30212,40876
+13617,22791,30211,40400
+17191,22792,30210,40786
+11912,22793,30209,40858
+16367,22794,30208,40723
+13888,22795,30207,40291
+11273,22796,30206,40988
+12270,22797,30205,40551
+14482,22798,30204,40127
+12197,22799,30203,40205
+11271,22800,30202,40818
+11566,22801,30201,40667
+12101,22802,30200,40298
+17817,22803,30199,40167
+11534,22804,30198,40399
+11208,22805,30197,40161
+13965,22806,30196,40897
+15229,22807,30195,40868
+12541,22808,30194,40944
+11446,22809,30193,40907
+15147,22810,30192,40513
+15419,22811,30191,40712
+16706,22812,30190,40492
+13916,22813,30189,40159
+12241,22814,30188,40962
+13354,22815,30187,40042
+17031,22816,30186,40476
+14575,22817,30185,40054
+10867,22818,30184,40431
+11550,22819,30183,40098
+14758,22820,30182,40484
+14573,22821,30181,40095
+16775,22822,30180,40292
+13850,22823,30179,40147
+17240,22824,30178,40976
+13586,22825,30177,40408
+17401,22826,30176,40350
+14400,22827,30175,40946
+13245,22828,30174,40063
+13919,22829,30173,40277
+14138,22830,30172,40366
+18028,22831,30171,40780
+11156,22832,30170,40891
+12154,22833,30169,40846
+15626,22834,30168,40757
+16728,22835,30167,40891
+12300,22836,30166,40679
+16094,22837,30165,40989
+12201,22838,30164,40649
+17766,22839,30163,40066
+17535,22840,30162,40799
+17225,22841,30161,40357
+11274,22842,30160,40542
+15552,22843,30159,40428
+10893,22844,30158,40579
+13984,22845,30157,40404
+18175,22846,30156,40958
+16099,22847,30155,40120
+11683,22848,30154,40507
+12645,22849,30153,40785
+16668,22850,30152,40201
+16920,22851,30151,40255
+14234,22852,30150,40139
+10619,22853,30149,40774
+13369,22854,30148,40541
+11883,22855,30147,40428
+17039,22856,30146,40257
+16609,22857,30145,40766
+11262,22858,30144,40414
+17018,22859,30143,40122
+13969,22860,30142,40466
+11210,22861,30141,40751
+17942,22862,30140,40382
+11853,22863,30139,40161
+10488,22864,30138,40608
+14989,22865,30137,40177
+17891,22866,30136,40382
+12465,22867,30135,40881
+16742,22868,30134,40662
+10632,22869,30133,40557
+10720,22870,30132,40115
+16411,22871,30131,40141
+16900,22872,30130,40553
+18118,22873,30129,40479
+16124,22874,30128,40281
+17882,22875,30127,40841
+14905,22876,30126,40824
+17077,22877,30125,40428
+13487,22878,30124,40243
+12036,22879,30123,40940
+15609,22880,30122,40805
+16783,22881,30121,40340
+18159,22882,30120,40312
+11203,22883,30119,40558
+16077,22884,30118,40346
+15274,22885,30117,40872
+18083,22886,30116,40591
+15379,22887,30115,40802
+13670,22888,30114,40604
+18029,22889,30113,40988
+14506,22890,30112,40732
+11287,22891,30111,40493
+12364,22892,30110,40147
+11245,22893,30109,40022
+12957,22894,30108,40319
+13706,22895,30107,40277
+15063,22896,30106,40303
+12809,22897,30105,40485
+12720,22898,30104,40420
+10410,22899,30103,40106
+11093,22900,30102,40492
+11543,22901,30101,40754
+17787,22902,30100,40245
+14791,22903,30099,40727
+13137,22904,30098,40486
+13397,22905,30097,40849
+16516,22906,30096,40961
+16809,22907,30095,40569
+15041,22908,30094,40957
+15495,22909,30093,40409
+16033,22910,30092,40945
+10423,22911,30091,40927
+13671,22912,30090,40986
+14811,22913,30089,40935
+13862,22914,30088,40893
+10097,22915,30087,40072
+11792,22916,30086,40368
+12523,22917,30085,40439
+13674,22918,30084,40818
+17588,22919,30083,40791
+11975,22920,30082,40155
+11487,22921,30081,40747
+17999,22922,30080,40526
+17139,22923,30079,40210
+18071,22924,30078,40756
+16735,22925,30077,40849
+15905,22926,30076,40730
+16140,22927,30075,40622
+13768,22928,30074,40578
+10422,22929,30073,40736
+10719,22930,30072,40463
+15068,22931,30071,40956
+13992,22932,30070,40617
+12302,22933,30069,40667
+14146,22934,30068,40057
+17509,22935,30067,40143
+16048,22936,30066,40743
+14894,22937,30065,40972
+12570,22938,30064,40207
+18173,22939,30063,40931
+14635,22940,30062,40937
+17837,22941,30061,40417
+14755,22942,30060,40088
+15119,22943,30059,40135
+14228,22944,30058,40994
+18111,22945,30057,40107
+10542,22946,30056,40934
+17381,22947,30055,40547
+11428,22948,30054,40965
+13401,22949,30053,40473
+11530,22950,30052,40630
+15830,22951,30051,40650
+10093,22952,30050,40405
+11643,22953,30049,40033
+15590,22954,30048,40710
+14303,22955,30047,40673
+13279,22956,30046,40555
+10208,22957,30045,40559
+10737,22958,30044,40342
+11073,22959,30043,40526
+16010,22960,30042,40160
+14594,22961,30041,40828
+12927,22962,30040,40151
+11722,22963,30039,40265
+10283,22964,30038,40078
+17319,22965,30037,40999
+17745,22966,30036,40174
+15797,22967,30035,40813
+17084,22968,30034,40578
+10522,22969,30033,40494
+15491,22970,30032,40769
+13900,22971,30031,40527
+15051,22972,30030,40629
+18180,22973,30029,40790
+12162,22974,30028,40224
+11788,22975,30027,40750
+11992,22976,30026,40198
+16605,22977,30025,40766
+16850,22978,30024,40637
+10723,22979,30023,40841
+14292,22980,30022,40377
+16754,22981,30021,40504
+15620,22982,30020,40708
+15793,22983,30019,40892
+17416,22984,30018,40026
+16096,22985,30017,40888
+16675,22986,30016,40315
+12299,22987,30015,40082
+11920,22988,30014,40666
+15862,22989,30013,40114
+14498,22990,30012,40666
+16258,22991,30011,40875
+10360,22992,30010,40217
+14291,22993,30009,40743
+15713,22994,30008,40452
+13777,22995,30007,40656
+14547,22996,30006,40241
+18120,22997,30005,40624
+14996,22998,30004,40356
+16727,22999,30003,40889
+13101,23000,30002,40577
+12470,23001,30001,40745
+14056,23002,30000,40880
+17778,23003,31000,40782
+17075,23004,30999,40551
+13080,23005,30998,40591
+15082,23006,30997,40033
+13566,23007,30996,40898
+10776,23008,30995,40860
+12737,23009,30994,40242
+14524,23010,30993,40665
+12143,23011,30992,40185
+15131,23012,30991,40572
+10991,23013,30990,40131
+15323,23014,30989,40154
+12900,23015,30988,40011
+13922,23016,30987,40075
+17002,23017,30986,40630
+10010,23018,30985,40005
+14979,23019,30984,40621
+10768,23020,30983,40752
+18097,23021,30982,40025
+16887,23022,30981,40296
+16215,23023,30980,40447
+10747,23024,30979,40821
+15899,23025,30978,40107
+13603,23026,30977,40966
+13592,23027,30976,40384
+12921,23028,30975,40100
+10584,23029,30974,40525
+11507,23030,30973,40275
+11666,23031,30972,40900
+11973,23032,30971,40131
+13211,23033,30970,40068
+12212,23034,30969,40053
+14034,23035,30968,40158
+10510,23036,30967,40024
+11095,23037,30966,40340
+13311,23038,30965,40188
+13514,23039,30964,40213
+15138,23040,30963,40878
+10241,23041,30962,40566
+16741,23042,30961,40793
+14494,23043,30960,40315
+17377,23044,30959,40646
+11299,23045,30958,40890
+13560,23046,30957,40506
+14382,23047,30956,40592
+15432,23048,30955,40686
+17353,23049,30954,40117
+15566,23050,30953,40388
+11297,23051,30952,40427
+11400,23052,30951,40845
+17151,23053,30950,40244
+12237,23054,30949,40519
+14827,23055,30948,40977
+13459,23056,30947,40548
+15438,23057,30946,40871
+12713,23058,30945,40662
+13980,23059,30944,40867
+15628,23060,30943,40886
+11165,23061,30942,40822
+17161,23062,30941,40923
+16876,23063,30940,40900
+12205,23064,30939,40776
+14158,23065,30938,40127
+16678,23066,30937,40959
+17556,23067,30936,40008
+12782,23068,30935,40041
+16807,23069,30934,40413
+10387,23070,30933,40015
+11818,23071,30932,40118
+12732,23072,30931,40963
+17966,23073,30930,40746
+10671,23074,30929,40751
+17044,23075,30928,40623
+15306,23076,30927,40303
+15367,23077,30926,40326
+14365,23078,30925,40523
+18169,23079,30924,40916
+16822,23080,30923,40460
+17273,23081,30922,40444
+12342,23082,30921,40944
+12345,23083,30920,40410
+18014,23084,30919,40857
+11102,23085,30918,40724
+17423,23086,30917,40959
+16021,23087,30916,40201
+14788,23088,30915,40766
+16816,23089,30914,40099
+15638,23090,30913,40538
+12484,23091,30912,40696
+13857,23092,30911,40303
+14123,23093,30910,40540
+11655,23094,30909,40453
+17792,23095,30908,40539
+11580,23096,30907,40200
+14268,23097,30906,40530
+10906,23098,30905,40461
+15261,23099,30904,40077
+14298,23100,30903,40049
+14522,23101,30902,40947
+10654,23102,30901,40652
+11785,23103,30900,40917
+11059,23104,30899,40610
+16531,23105,30898,40062
+10028,23106,30897,40154
+10075,23107,30896,40757
+10495,23108,30895,40376
+12339,23109,30894,40269
+16043,23110,30893,40261
+11724,23111,30892,40181
+11390,23112,30891,40390
+15412,23113,30890,40323
+17772,23114,30889,40274
+12582,23115,30888,40348
+15919,23116,30887,40524
+14411,23117,30886,40311
+12499,23118,30885,40257
+17739,23119,30884,40047
+13284,23120,30883,40707
+12854,23121,30882,40719
+15035,23122,30881,40780
+10966,23123,30880,40992
+10242,23124,30879,40855
+12173,23125,30878,40805
+13937,23126,30877,40519
+13705,23127,30876,40088
+11217,23128,30875,40755
+15766,23129,30874,40253
+14592,23130,30873,40969
+10556,23131,30872,40876
+12634,23132,30871,40480
+12463,23133,30870,40300
+13374,23134,30869,40826
+10040,23135,30868,40611
+15262,23136,30867,40757
+15898,23137,30866,40330
+12207,23138,30865,40947
+10828,23139,30864,40005
+10260,23140,30863,40831
+13363,23141,30862,40144
+14272,23142,30861,40336
+10415,23143,30860,40941
+14937,23144,30859,40307
+16008,23145,30858,40315
+17363,23146,30857,40792
+10480,23147,30856,40880
+14338,23148,30855,40151
+15113,23149,30854,40838
+10590,23150,30853,40883
+11607,23151,30852,40260
+10862,23152,30851,40636
+17157,23153,30850,40187
+12667,23154,30849,40115
+12365,23155,30848,40833
+14468,23156,30847,40310
+11761,23157,30846,40499
+13425,23158,30845,40539
+12567,23159,30844,40330
+17841,23160,30843,40937
+18006,23161,30842,40032
+10956,23162,30841,40234
+12464,23163,30840,40141
+13791,23164,30839,40821
+18038,23165,30838,40898
+14082,23166,30837,40129
+15574,23167,30836,40806
+16660,23168,30835,40867
+17052,23169,30834,40966
+15317,23170,30833,40624
+17272,23171,30832,40015
+15029,23172,30831,40526
+15009,23173,30830,40368
+12088,23174,30829,40284
+12129,23175,30828,40203
+10484,23176,30827,40339
+11826,23177,30826,40786
+16212,23178,30825,40396
+17317,23179,30824,40662
+13644,23180,30823,40912
+11995,23181,30822,40954
+17959,23182,30821,40055
+10188,23183,30820,40822
+10062,23184,30819,40392
+12948,23185,30818,40064
+16392,23186,30817,40623
+10022,23187,30816,40252
+15800,23188,30815,40500
+14280,23189,30814,40572
+14072,23190,30813,40560
+11909,23191,30812,40123
+17493,23192,30811,40723
+17486,23193,30810,40724
+16398,23194,30809,40585
+14520,23195,30808,40664
+11185,23196,30807,40267
+15725,23197,30806,40163
+13519,23198,30805,40033
+17657,23199,30804,40304
+13198,23200,30803,40877
+13974,23201,30802,40075
+15198,23202,30801,40554
+11784,23203,30800,40356
+18158,23204,30799,40516
+15777,23205,30798,40180
+11589,23206,30797,40242
+17578,23207,30796,40335
+12669,23208,30795,40294
+11887,23209,30794,40857
+13313,23210,30793,40501
+17382,23211,30792,40358
+16171,23212,30791,40297
+13292,23213,30790,40274
+14599,23214,30789,40259
+10439,23215,30788,40943
+12654,23216,30787,40678
+11278,23217,30786,40204
+13478,23218,30785,40032
+11679,23219,30784,40138
+16057,23220,30783,40962
+12729,23221,30782,40522
+12851,23222,30781,40102
+14686,23223,30780,40754
+16855,23224,30779,40575
+14539,23225,30778,40167
+11010,23226,30777,40604
+13166,23227,30776,40109
+10285,23228,30775,40959
+12903,23229,30774,40833
+14540,23230,30773,40282
+17340,23231,30772,40295
+11529,23232,30771,40993
+13773,23233,30770,40335
+16847,23234,30769,40970
+12705,23235,30768,40059
+14444,23236,30767,40505
+17049,23237,30766,40220
+11445,23238,30765,40480
+12622,23239,30764,40937
+15580,23240,30763,40716
+12377,23241,30762,40593
+16344,23242,30761,40123
+17895,23243,30760,40912
+16932,23244,30759,40473
+11937,23245,30758,40446
+17305,23246,30757,40161
+12363,23247,30756,40301
+15752,23248,30755,40840
+14105,23249,30754,40809
+12180,23250,30753,40702
+15568,23251,30752,40290
+12556,23252,30751,40558
+13656,23253,30750,40236
+15690,23254,30749,40741
+11176,23255,30748,40272
+16303,23256,30747,40311
+14359,23257,30746,40611
+17899,23258,30745,40780
+11094,23259,30744,40612
+15434,23260,30743,40170
+18161,23261,30742,40067
+17583,23262,30741,40140
+17472,23263,30740,40365
+12278,23264,30739,40520
+11741,23265,30738,40620
+16086,23266,30737,40759
+10770,23267,30736,40299
+13181,23268,30735,40629
+11753,23269,30734,40184
+13035,23270,30733,40208
+14193,23271,30732,40386
+14471,23272,30731,40669
+11466,23273,30730,40769
+16813,23274,30729,40334
+10304,23275,30728,40548
+16844,23276,30727,40495
+17260,23277,30726,40070
+11055,23278,30725,40488
+11166,23279,30724,40378
+17324,23280,30723,40677
+15295,23281,30722,40438
+18134,23282,30721,40633
+15744,23283,30720,40556
+11918,23284,30719,40946
+14801,23285,30718,40938
+15859,23286,30717,40773
+13652,23287,30716,40835
+17717,23288,30715,40818
+14002,23289,30714,40215
+14274,23290,30713,40066
+16203,23291,30712,40181
+17100,23292,30711,40669
+13437,23293,30710,40641
+12019,23294,30709,40848
+12424,23295,30708,40457
+16716,23296,30707,40539
+16923,23297,30706,40761
+15493,23298,30705,40041
+11259,23299,30704,40286
+12399,23300,30703,40616
+15529,23301,30702,40872
+13490,23302,30701,40689
+15227,23303,30700,40120
+17576,23304,30699,40913
+13827,23305,30698,40537
+11840,23306,30697,40686
+12257,23307,30696,40443
+12532,23308,30695,40669
+17623,23309,30694,40938
+13156,23310,30693,40112
+15361,23311,30692,40413
+12013,23312,30691,40581
+17835,23313,30690,40586
+16074,23314,30689,40089
+13296,23315,30688,40471
+10696,23316,30687,40133
+14888,23317,30686,40212
+17187,23318,30685,40438
+10079,23319,30684,40461
+11214,23320,30683,40264
+17428,23321,30682,40947
+10255,23322,30681,40349
+16195,23323,30680,40177
+14183,23324,30679,40774
+12108,23325,30678,40531
+11631,23326,30677,40452
+11140,23327,30676,40867
+15801,23328,30675,40497
+11141,23329,30674,40230
+10690,23330,30673,40432
+18089,23331,30672,40891
+14428,23332,30671,40155
+18178,23333,30670,40668
+13427,23334,30669,40922
+16359,23335,30668,40309
+12940,23336,30667,40753
+16965,23337,30666,40425
+12238,23338,30665,40675
+17500,23339,30664,40491
+10206,23340,30663,40467
+14844,23341,30662,40834
+17111,23342,30661,40929
+14776,23343,30660,40502
+10374,23344,30659,40697
+14495,23345,30658,40527
+11684,23346,30657,40375
+12321,23347,30656,40406
+12286,23348,30655,40302
+12171,23349,30654,40178
+11980,23350,30653,40649
+15319,23351,30652,40436
+13789,23352,30651,40177
+17872,23353,30650,40151
+12064,23354,30649,40188
+13093,23355,30648,40422
+14954,23356,30647,40143
+16611,23357,30646,40522
+12931,23358,30645,40794
+11754,23359,30644,40331
+15176,23360,30643,40418
+10296,23361,30642,40787
+10653,23362,30641,40036
+11760,23363,30640,40474
+14247,23364,30639,40932
+14549,23365,30638,40876
+14023,23366,30637,40421
+17539,23367,30636,40775
+13616,23368,30635,40739
+16664,23369,30634,40065
+15819,23370,30633,40749
+17021,23371,30632,40342
+12248,23372,30631,40359
+10656,23373,30630,40302
+15094,23374,30629,40243
+14264,23375,30628,40493
+14655,23376,30627,40767
+16548,23377,30626,40061
+13637,23378,30625,40634
+12879,23379,30624,40782
+11548,23380,30623,40199
+10043,23381,30622,40581
+15241,23382,30621,40684
+12324,23383,30620,40410
+10300,23384,30619,40533
+17120,23385,30618,40435
+13864,23386,30617,40465
+17391,23387,30616,40918
+16444,23388,30615,40759
+15807,23389,30614,40076
+16916,23390,30613,40697
+10337,23391,30612,40081
+13607,23392,30611,40965
+13342,23393,30610,40984
+16603,23394,30609,40997
+14757,23395,30608,40692
+10921,23396,30607,40125
+16261,23397,30606,40209
+13131,23398,30605,40474
+14564,23399,30604,40672
+13634,23400,30603,40105
+18117,23401,30602,40129
+16223,23402,30601,40245
+16828,23403,30600,40803
+10544,23404,30599,40730
+16467,23405,30598,40863
+15267,23406,30597,40231
+11135,23407,30596,40611
+12961,23408,30595,40473
+11637,23409,30594,40474
+14743,23410,30593,40449
+12087,23411,30592,40752
+18139,23412,30591,40296
+11115,23413,30590,40195
+12554,23414,30589,40209
+16545,23415,30588,40222
+13657,23416,30587,40364
+11822,23417,30586,40247
+17294,23418,30585,40531
+16527,23419,30584,40184
+16023,23420,30583,40486
+10868,23421,30582,40123
+16456,23422,30581,40241
+17350,23423,30580,40870
+16085,23424,30579,40477
+17158,23425,30578,40694
+17154,23426,30577,40259
+10340,23427,30576,40087
+15691,23428,30575,40500
+12615,23429,30574,40353
+10446,23430,30573,40855
+13581,23431,30572,40991
+10474,23432,30571,40290
+12414,23433,30570,40881
+17612,23434,30569,40217
+16439,23435,30568,40698
+15756,23436,30567,40773
+13463,23437,30566,40956
+16281,23438,30565,40851
+11041,23439,30564,40414
+10187,23440,30563,40509
+12427,23441,30562,40527
+12597,23442,30561,40470
+14980,23443,30560,40189
+17351,23444,30559,40183
+10202,23445,30558,40686
+13272,23446,30557,40327
+15831,23447,30556,40112
+16748,23448,30555,40263
+17457,23449,30554,40700
+13537,23450,30553,40499
+12213,23451,30552,40332
+10272,23452,30551,40588
+12133,23453,30550,40285
+14323,23454,30549,40074
+17867,23455,30548,40792
+12985,23456,30547,40311
+11509,23457,30546,40083
+12494,23458,30545,40295
+11783,23459,30544,40189
+17006,23460,30543,40603
+10796,23461,30542,40990
+10682,23462,30541,40453
+15878,23463,30540,40412
+17384,23464,30539,40438
+17242,23465,30538,40826
+16064,23466,30537,40963
+16899,23467,30536,40043
+11708,23468,30535,40166
+18070,23469,30534,40332
+17378,23470,30533,40785
+12170,23471,30532,40952
+11748,23472,30531,40045
+15920,23473,30530,40237
+10183,23474,30529,40888
+10593,23475,30528,40587
+12789,23476,30527,40565
+10006,23477,30526,40597
+10164,23478,30525,40534
+16768,23479,30524,40488
+12116,23480,30523,40628
+17555,23481,30522,40320
+13301,23482,30521,40159
+17512,23483,30520,40290
+14931,23484,30519,40809
+13621,23485,30518,40604
+16988,23486,30517,40564
+14964,23487,30516,40545
+15074,23488,30515,40596
+15578,23489,30514,40265
+11131,23490,30513,40039
+12048,23491,30512,40812
+10799,23492,30511,40921
+11540,23493,30510,40831
+10125,23494,30509,40213
+10791,23495,30508,40639
+10464,23496,30507,40453
+14473,23497,30506,40250
+13645,23498,30505,40506
+14052,23499,30504,40838
+14284,23500,30503,40911
+17289,23501,30502,40233
+13923,23502,30501,40816
+10945,23503,30500,40655
+11972,23504,30499,40976
+17491,23505,30498,40922
+11150,23506,30497,40536
+14826,23507,30496,40828
+14624,23508,30495,40291
+10254,23509,30494,40384
+14703,23510,30493,40894
+13854,23511,30492,40902
+11080,23512,30491,40181
+10157,23513,30490,40589
+10452,23514,30489,40336
+17561,23515,30488,40458
+14780,23516,30487,40309
+14334,23517,30486,40943
+11811,23518,30485,40231
+12147,23519,30484,40424
+11303,23520,30483,40405
+14579,23521,30482,40150
+14940,23522,30481,40797
+15423,23523,30480,40686
+14903,23524,30479,40486
+10833,23525,30478,40812
+13579,23526,30477,40877
+12488,23527,30476,40424
+13451,23528,30475,40431
+16285,23529,30474,40291
+10509,23530,30473,40462
+10571,23531,30472,40603
+11000,23532,30471,40831
+14804,23533,30470,40250
+14956,23534,30469,40944
+11123,23535,30468,40501
+11461,23536,30467,40854
+16740,23537,30466,40280
+10528,23538,30465,40188
+16945,23539,30464,40959
+12820,23540,30463,40760
+17770,23541,30462,40415
+17575,23542,30461,40815
+14165,23543,30460,40642
+13258,23544,30459,40145
+10319,23545,30458,40682
+11117,23546,30457,40799
+17168,23547,30456,40053
+11494,23548,30455,40625
+17782,23549,30454,40182
+17571,23550,30453,40372
+10953,23551,30452,40644
+10663,23552,30451,40376
+12603,23553,30450,40276
+16405,23554,30449,40296
+13866,23555,30448,40282
+10024,23556,30447,40387
+15923,23557,30446,40605
+12104,23558,30445,40815
+12790,23559,30444,40616
+14171,23560,30443,40837
+11344,23561,30442,40948
+14415,23562,30441,40647
+10842,23563,30440,40755
+15156,23564,30439,40230
+10739,23565,30438,40215
+13062,23566,30437,40353
+16552,23567,30436,40489
+16558,23568,30435,40023
+16576,23569,30434,40958
+10375,23570,30433,40485
+17934,23571,30432,40428
+17626,23572,30431,40723
+12810,23573,30430,40994
+13236,23574,30429,40105
+10948,23575,30428,40941
+15133,23576,30427,40232
+17370,23577,30426,40423
+12035,23578,30425,40731
+13145,23579,30424,40529
+16640,23580,30423,40131
+16759,23581,30422,40801
+13554,23582,30421,40425
+10697,23583,30420,40575
+14108,23584,30419,40071
+13740,23585,30418,40482
+11381,23586,30417,40472
+11301,23587,30416,40968
+15391,23588,30415,40430
+12847,23589,30414,40042
+10894,23590,30413,40290
+16139,23591,30412,40877
+12542,23592,30411,40645
+12837,23593,30410,40085
+17779,23594,30409,40357
+10229,23595,30408,40668
+16031,23596,30407,40812
+10207,23597,30406,40408
+11706,23598,30405,40587
+15979,23599,30404,40331
+10395,23600,30403,40371
+10781,23601,30402,40988
+15630,23602,30401,40116
+12124,23603,30400,40691
+14496,23604,30399,40278
+11235,23605,30398,40537
+15304,23606,30397,40869
+13920,23607,30396,40629
+13045,23608,30395,40354
+16186,23609,30394,40500
+17587,23610,30393,40115
+18088,23611,30392,40160
+12307,23612,30391,40477
+11766,23613,30390,40489
+15204,23614,30389,40960
+13738,23615,30388,40881
+11932,23616,30387,40121
+18162,23617,30386,40856
+17226,23618,30385,40997
+12092,23619,30384,40904
+16652,23620,30383,40128
+17853,23621,30382,40691
+13413,23622,30381,40702
+12128,23623,30380,40153
+11454,23624,30379,40161
+10149,23625,30378,40510
+14099,23626,30377,40209
+15631,23627,30376,40176
+10763,23628,30375,40329
+11976,23629,30374,40214
+11810,23630,30373,40990
+15842,23631,30372,40699
+15075,23632,30371,40291
+13959,23633,30370,40242
+12062,23634,30369,40855
+13317,23635,30368,40028
+17415,23636,30367,40840
+16721,23637,30366,40309
+15124,23638,30365,40441
+17665,23639,30364,40284
+14505,23640,30363,40727
+17369,23641,30362,40945
+11508,23642,30361,40329
+12179,23643,30360,40929
+11207,23644,30359,40870
+17985,23645,30358,40364
+12234,23646,30357,40032
+17707,23647,30356,40336
+11472,23648,30355,40646
+15005,23649,30354,40350
+10055,23650,30353,40943
+15055,23651,30352,40130
+15164,23652,30351,40594
+12416,23653,30350,40865
+14208,23654,30349,40091
+11681,23655,30348,40803
+15178,23656,30347,40688
+17097,23657,30346,40207
+13223,23658,30345,40649
+12426,23659,30344,40806
+17167,23660,30343,40862
+11787,23661,30342,40391
+10808,23662,30341,40874
+14550,23663,30340,40794
+16298,23664,30339,40901
+17380,23665,30338,40800
+11241,23666,30337,40928
+13132,23667,30336,40863
+10080,23668,30335,40905
+15228,23669,30334,40476
+10634,23670,30333,40703
+16771,23671,30332,40001
+16950,23672,30331,40055
+15505,23673,30330,40934
+17774,23674,30329,40164
+16787,23675,30328,40987
+11063,23676,30327,40038
+16308,23677,30326,40060
+17546,23678,30325,40529
+18039,23679,30324,40333
+17223,23680,30323,40286
+14982,23681,30322,40437
+13447,23682,30321,40431
+14020,23683,30320,40935
+13255,23684,30319,40347
+10380,23685,30318,40370
+16305,23686,30317,40497
+14921,23687,30316,40003
+14370,23688,30315,40002
+10074,23689,30314,40889
+11820,23690,30313,40511
+14177,23691,30312,40750
+11160,23692,30311,40345
+15201,23693,30310,40077
+10244,23694,30309,40024
+17950,23695,30308,40414
+11085,23696,30307,40539
+11291,23697,30306,40119
+10409,23698,30305,40574
+14786,23699,30304,40517
+15865,23700,30303,40962
+15738,23701,30302,40432
+17565,23702,30301,40302
+13796,23703,30300,40716
+17900,23704,30299,40571
+14991,23705,30298,40979
+10748,23706,30297,40781
+13183,23707,30296,40376
+14558,23708,30295,40637
+14188,23709,30294,40655
+11087,23710,30293,40115
+15627,23711,30292,40873
+11268,23712,30291,40849
+16711,23713,30290,40955
+13708,23714,30289,40763
+17215,23715,30288,40709
+16713,23716,30287,40035
+16788,23717,30286,40584
+17533,23718,30285,40640
+10725,23719,30284,40057
+11938,23720,30283,40367
+11011,23721,30282,40050
+13506,23722,30281,40050
+16663,23723,30280,40336
+11276,23724,30279,40732
+10604,23725,30278,40767
+14922,23726,30277,40279
+10598,23727,30276,40297
+16487,23728,30275,40156
+10589,23729,30274,40277
+10376,23730,30273,40881
+14144,23731,30272,40292
+17110,23732,30271,40609
+12001,23733,30270,40264
+10901,23734,30269,40030
+14908,23735,30268,40453
+15937,23736,30267,40337
+15141,23737,30266,40905
+11585,23738,30265,40962
+17368,23739,30264,40896
+14092,23740,30263,40739
+15825,23741,30262,40399
+10047,23742,30261,40046
+15523,23743,30260,40058
+15838,23744,30259,40111
+15167,23745,30258,40298
+10475,23746,30257,40013
+14814,23747,30256,40958
+13930,23748,30255,40010
+10066,23749,30254,40842
+15954,23750,30253,40513
+10555,23751,30252,40213
+10876,23752,30251,40127
+14229,23753,30250,40052
+11282,23754,30249,40082
+12965,23755,30248,40764
+13622,23756,30247,40286
+10215,23757,30246,40493
+11798,23758,30245,40030
+14882,23759,30244,40121
+11144,23760,30243,40632
+11852,23761,30242,40972
+11058,23762,30241,40529
+12665,23763,30240,40702
+11261,23764,30239,40056
+15659,23765,30238,40547
+16371,23766,30237,40575
+17652,23767,30236,40820
+16856,23768,30235,40763
+16166,23769,30234,40975
+12604,23770,30233,40948
+17392,23771,30232,40409
+11596,23772,30231,40120
+16530,23773,30230,40411
+11521,23774,30229,40939
+10732,23775,30228,40450
+16274,23776,30227,40289
+15872,23777,30226,40024
+12261,23778,30225,40867
+14559,23779,30224,40457
+11490,23780,30223,40336
+13266,23781,30222,40576
+10685,23782,30221,40827
+13788,23783,30220,40207
+13010,23784,30219,40730
+10419,23785,30218,40294
+10976,23786,30217,40439
+14270,23787,30216,40458
+15354,23788,30215,40623
+11222,23789,30214,40391
+16879,23790,30213,40279
+12730,23791,30212,40024
+16919,23792,30211,40351
+17874,23793,30210,40291
+12963,23794,30209,40799
+10118,23795,30208,40546
+15209,23796,30207,40117
+13973,23797,30206,40051
+15498,23798,30205,40777
+17056,23799,30204,40194
+11804,23800,30203,40376
+12127,23801,30202,40932
+16937,23802,30201,40700
+11886,23803,30200,40188
+14053,23804,30199,40869
+14387,23805,30198,40227
+15658,23806,30197,40464
+15636,23807,30196,40032
+17712,23808,30195,40365
+15206,23809,30194,40927
+15858,23810,30193,40151
+12740,23811,30192,40958
+15169,23812,30191,40781
+16974,23813,30190,40530
+12745,23814,30189,40077
+16083,23815,30188,40904
+16306,23816,30187,40776
+12656,23817,30186,40879
+17883,23818,30185,40633
+14429,23819,30184,40275
+15504,23820,30183,40775
+15004,23821,30182,40941
+15736,23822,30181,40193
+13640,23823,30180,40613
+10342,23824,30179,40633
+10898,23825,30178,40178
+16591,23826,30177,40433
+10489,23827,30176,40792
+16602,23828,30175,40778
+11394,23829,30174,40352
+10124,23830,30173,40796
+14985,23831,30172,40165
+17970,23832,30171,40272
+15120,23833,30170,40010
+15202,23834,30169,40777
+17789,23835,30168,40042
+11835,23836,30167,40440
+15506,23837,30166,40383
+16136,23838,30165,40535
+17559,23839,30164,40556
+15514,23840,30163,40037
+16959,23841,30162,40766
+12805,23842,30161,40144
+17371,23843,30160,40765
+13173,23844,30159,40353
+12058,23845,30158,40184
+16232,23846,30157,40225
+14724,23847,30156,40958
+14342,23848,30155,40698
+10406,23849,30154,40561
+10692,23850,30153,40432
+15264,23851,30152,40029
+14164,23852,30151,40462
+12994,23853,30150,40415
+12063,23854,30149,40692
+10191,23855,30148,40120
+13153,23856,30147,40894
+13627,23857,30146,40385
+17788,23858,30145,40233
+11561,23859,30144,40536
+10546,23860,30143,40718
+15011,23861,30142,40064
+12599,23862,30141,40305
+17598,23863,30140,40071
+17975,23864,30139,40715
+12319,23865,30138,40144
+11438,23866,30137,40258
+16160,23867,30136,40593
+12864,23868,30135,40203
+14222,23869,30134,40080
+12923,23870,30133,40230
+13334,23871,30132,40518
+16824,23872,30131,40976
+16030,23873,30130,40565
+12120,23874,30129,40518
+16051,23875,30128,40854
+14104,23876,30127,40469
+17662,23877,30126,40331
+16310,23878,30125,40286
+17560,23879,30124,40043
+16707,23880,30123,40870
+16312,23881,30122,40875
+14625,23882,30121,40460
+14886,23883,30120,40234
+13770,23884,30119,40332
+15477,23885,30118,40293
+15990,23886,30117,40649
+16246,23887,30116,40945
+17008,23888,30115,40847
+13110,23889,30114,40408
+16446,23890,30113,40333
+12818,23891,30112,40114
+14585,23892,30111,40105
+10308,23893,30110,40172
+16815,23894,30109,40422
+17323,23895,30108,40966
+15739,23896,30107,40593
+13476,23897,30106,40751
+15701,23898,30105,40857
+17600,23899,30104,40715
+10871,23900,30103,40669
+12031,23901,30102,40292
+12956,23902,30101,40455
+12856,23903,30100,40382
+14408,23904,30099,40430
+12973,23905,30098,40468
+10941,23906,30097,40593
+18122,23907,30096,40089
+15109,23908,30095,40255
+15374,23909,30094,40081
+13695,23910,30093,40724
+10193,23911,30092,40278
+14287,23912,30091,40398
+16370,23913,30090,40022
+14318,23914,30089,40542
+16982,23915,30088,40393
+12434,23916,30087,40001
+15587,23917,30086,40832
+12796,23918,30085,40478
+10310,23919,30084,40817
+10156,23920,30083,40130
+12100,23921,30082,40319
+10547,23922,30081,40679
+10171,23923,30080,40830
+12186,23924,30079,40891
+15904,23925,30078,40197
+17336,23926,30077,40325
+17699,23927,30076,40108
+18102,23928,30075,40010
+17205,23929,30074,40525
+16544,23930,30073,40717
+11615,23931,30072,40447
+11067,23932,30071,40454
+17773,23933,30070,40909
+15064,23934,30069,40449
+11569,23935,30068,40493
+12409,23936,30067,40127
+13276,23937,30066,40564
+11431,23938,30065,40450
+15061,23939,30064,40814
+12853,23940,30063,40657
+13364,23941,30062,40916
+14930,23942,30061,40628
+10443,23943,30060,40926
+12225,23944,30059,40881
+11896,23945,30058,40712
+14425,23946,30057,40420
+10655,23947,30056,40618
+15501,23948,30055,40174
+15845,23949,30054,40062
+11963,23950,30053,40868
+16654,23951,30052,40622
+15549,23952,30051,40449
+15544,23953,30050,40526
+12773,23954,30049,40830
+17248,23955,30048,40597
+10115,23956,30047,40781
+12573,23957,30046,40105
+10267,23958,30045,40877
+18141,23959,30044,40624
+10042,23960,30043,40919
+10563,23961,30042,40339
+17429,23962,30041,40239
+14084,23963,30040,40253
+11881,23964,30039,40686
+11964,23965,30038,40721
+13072,23966,30037,40804
+14620,23967,30036,40953
+16290,23968,30035,40699
+15123,23969,30034,40507
+17505,23970,30033,40951
+10958,23971,30032,40838
+12078,23972,30031,40506
+12263,23973,30030,40162
+12638,23974,30029,40276
+16903,23975,30028,40766
+14929,23976,30027,40023
+15249,23977,30026,40987
+13614,23978,30025,40392
+12514,23979,30024,40101
+11710,23980,30023,40153
+16316,23981,30022,40129
+11902,23982,30021,40252
+10412,23983,30020,40098
+18148,23984,30019,40618
+12144,23985,30018,40078
+11111,23986,30017,40146
+11052,23987,30016,40837
+13383,23988,30015,40053
+11386,23989,30014,40730
+13305,23990,30013,40653
+14772,23991,30012,40818
+17809,23992,30011,40422
+18018,23993,30010,40866
+12795,23994,30009,40221
+11066,23995,30008,40061
+15481,23996,30007,40197
+12323,23997,30006,40386
+11587,23998,30005,40208
+11243,23999,30004,40828
+17545,24000,30003,40299
+17705,24001,30002,40579
+16914,24002,30001,40330
+14220,24003,30000,40200
+17349,24004,31000,40450
+14041,24005,30999,40981
+13022,24006,30998,40437
+13099,24007,30997,40899
+16848,24008,30996,40280
+10309,24009,30995,40533
+10315,24010,30994,40089
+12368,24011,30993,40850
+16387,24012,30992,40140
+17243,24013,30991,40760
+16349,24014,30990,40357
+14913,24015,30989,40371
+17706,24016,30988,40231
+16991,24017,30987,40598
+16580,24018,30986,40235
+14420,24019,30985,40600
+17128,24020,30984,40301
+14910,24021,30983,40727
+11216,24022,30982,40288
+10003,24023,30981,40851
+14018,24024,30980,40993
+18109,24025,30979,40352
+16068,24026,30978,40344
+17860,24027,30977,40399
+12662,24028,30976,40403
+15494,24029,30975,40535
+15994,24030,30974,40638
+16286,24031,30973,40493
+11424,24032,30972,40013
+17938,24033,30971,40834
+17485,24034,30970,40442
+10857,24035,30969,40387
+13234,24036,30968,40257
+11451,24037,30967,40852
+15158,24038,30966,40168
+10665,24039,30965,40542
+13031,24040,30964,40996
+14593,24041,30963,40272
+15877,24042,30962,40287
+17477,24043,30961,40041
+13751,24044,30960,40974
+15588,24045,30959,40187
+14174,24046,30958,40855
+16465,24047,30957,40939
+13632,24048,30956,40879
+13510,24049,30955,40765
+14453,24050,30954,40905
+10232,24051,30953,40933
+12578,24052,30952,40539
+11793,24053,30951,40112
+15527,24054,30950,40483
+14613,24055,30949,40737
+14085,24056,30948,40895
+13063,24057,30947,40719
+12770,24058,30946,40897
+17278,24059,30945,40908
+15318,24060,30944,40321
+14462,24061,30943,40560
+13246,24062,30942,40276
+16318,24063,30941,40011
+17212,24064,30940,40547
+11574,24065,30939,40930
+17828,24066,30938,40230
+11388,24067,30937,40521
+14912,24068,30936,40511
+17014,24069,30935,40551
+12374,24070,30934,40820
+17758,24071,30933,40831
+17918,24072,30932,40876
+12285,24073,30931,40555
+13237,24074,30930,40922
+16011,24075,30929,40227
+10440,24076,30928,40495
+14697,24077,30927,40090
+14777,24078,30926,40330
+14959,24079,30925,40625
+10829,24080,30924,40939
+13723,24081,30923,40781
+18002,24082,30922,40141
+10269,24083,30921,40580
+13635,24084,30920,40305
+17906,24085,30919,40341
+14741,24086,30918,40831
+17613,24087,30917,40389
+15598,24088,30916,40336
+11387,24089,30915,40158
+10646,24090,30914,40783
+10169,24091,30913,40946
+10626,24092,30912,40269
+13095,24093,30911,40661
+11812,24094,30910,40532
+14793,24095,30909,40608
+15559,24096,30908,40515
+10744,24097,30907,40098
+14836,24098,30906,40048
+14391,24099,30905,40857
+17786,24100,30904,40741
+15269,24101,30903,40702
+16722,24102,30902,40481
+15448,24103,30901,40319
+15150,24104,30900,40830
+13946,24105,30899,40334
+17013,24106,30898,40018
+16451,24107,30897,40566
+15418,24108,30896,40815
+14853,24109,30895,40989
+13629,24110,30894,40862
+17424,24111,30893,40733
+16062,24112,30892,40729
+15290,24113,30891,40261
+10095,24114,30890,40173
+13515,24115,30889,40181
+10881,24116,30888,40350
+17944,24117,30887,40727
+14159,24118,30886,40244
+11860,24119,30885,40848
+14605,24120,30884,40956
+15287,24121,30883,40053
+15743,24122,30882,40665
+12565,24123,30881,40987
+15462,24124,30880,40713
+11430,24125,30879,40047
+10760,24126,30878,40428
+16175,24127,30877,40276
+12016,24128,30876,40951
+10057,24129,30875,40850
+11159,24130,30874,40647
+10435,24131,30873,40331
+12605,24132,30872,40049
+14695,24133,30871,40930
+15023,24134,30870,40455
+11571,24135,30869,40605
+12012,24136,30868,40500
+11775,24137,30867,40721
+10899,24138,30866,40176
+13926,24139,30865,40452
+14212,24140,30864,40207
+16176,24141,30863,40013
+17298,24142,30862,40467
+11029,24143,30861,40239
+15012,24144,30860,40362
+12727,24145,30859,40637
+14646,24146,30858,40635
+13520,24147,30857,40680
+17478,24148,30856,40914
+14301,24149,30855,40932
+14329,24150,30854,40707
+11360,24151,30853,40145
+17127,24152,30852,40099
+13408,24153,30851,40460
+10273,24154,30850,40561
+13681,24155,30849,40749
+10123,24156,30848,40196
+12057,24157,30847,40156
+17010,24158,30846,40150
+10486,24159,30845,40069
+13423,24160,30844,40718
+14325,24161,30843,40778
+13532,24162,30842,40028
+10306,24163,30841,40296
+14024,24164,30840,40605
+17400,24165,30839,40345
+16792,24166,30838,40766
+18177,24167,30837,40472
+15130,24168,30836,40405
+15742,24169,30835,40684
+12970,24170,30834,40082
+14765,24171,30833,40620
+16832,24172,30832,40901
+12304,24173,30831,40464
+13406,24174,30830,40154
+12775,24175,30829,40008
+15605,24176,30828,40079
+14907,24177,30827,40822
+11183,24178,30826,40510
+17517,24179,30825,40166
+17656,24180,30824,40837
+15557,24181,30823,40382
+14426,24182,30822,40123
+16870,24183,30821,40360
+17469,24184,30820,40293
+13154,24185,30819,40796
+17047,24186,30818,40454
+17848,24187,30817,40010
+16877,24188,30816,40246
+18126,24189,30815,40038
+14262,24190,30814,40905
+13558,24191,30813,40780
+13445,24192,30812,40600
+16266,24193,30811,40083
+13491,24194,30810,40901
+13201,24195,30809,40942
+12587,24196,30808,40207
+13150,24197,30807,40005
+17855,24198,30806,40870
+11984,24199,30805,40530
+18092,24200,30804,40400
+15817,24201,30803,40895
+10384,24202,30802,40418
+17138,24203,30801,40301
+12200,24204,30800,40042
+10339,24205,30799,40176
+15052,24206,30798,40185
+11206,24207,30797,40270
+10706,24208,30796,40622
+17320,24209,30795,40782
+10026,24210,30794,40110
+15489,24211,30793,40397
+16114,24212,30792,40767
+17406,24213,30791,40467
+17290,24214,30790,40299
+16961,24215,30789,40013
+12767,24216,30788,40291
+15294,24217,30787,40547
+18040,24218,30786,40009
+11122,24219,30785,40617
+15510,24220,30784,40247
+10514,24221,30783,40747
+11893,24222,30782,40349
+16684,24223,30781,40226
+14003,24224,30780,40837
+15515,24225,30779,40195
+17393,24226,30778,40419
+15191,24227,30777,40103
+12095,24228,30776,40249
+15983,24229,30775,40093
+17767,24230,30774,40039
+18138,24231,30773,40226
+13830,24232,30772,40007
+16710,24233,30771,40787
+17798,24234,30770,40895
+15594,24235,30769,40810
+15088,24236,30768,40423
+15737,24237,30767,40844
+15661,24238,30766,40370
+16747,24239,30765,40259
+16501,24240,30764,40457
+15337,24241,30763,40031
+14863,24242,30762,40443
+11528,24243,30761,40119
+15469,24244,30760,40689
+14345,24245,30759,40725
+11486,24246,30758,40917
+13316,24247,30757,40184
+16216,24248,30756,40611
+16155,24249,30755,40139
+11672,24250,30754,40416
+16213,24251,30753,40826
+10884,24252,30752,40167
+14439,24253,30751,40901
+13291,24254,30750,40627
+14288,24255,30749,40629
+11583,24256,30748,40334
+17629,24257,30747,40188
+12655,24258,30746,40407
+12945,24259,30745,40257
+12023,24260,30744,40754
+13809,24261,30743,40792
+15373,24262,30742,40546
+15208,24263,30741,40072
+18045,24264,30740,40930
+13577,24265,30739,40278
+13837,24266,30738,40041
+11951,24267,30737,40512
+17026,24268,30736,40555
+15031,24269,30735,40092
+13550,24270,30734,40444
+16729,24271,30733,40157
+11396,24272,30732,40911
+14418,24273,30731,40727
+11676,24274,30730,40477
+15726,24275,30729,40762
+15570,24276,30728,40713
+10332,24277,30727,40739
+11061,24278,30726,40040
+13379,24279,30725,40249
+10608,24280,30724,40519
+10471,24281,30723,40706
+17914,24282,30722,40228
+14294,24283,30721,40734
+10606,24284,30720,40142
+13613,24285,30719,40195
+16386,24286,30718,40209
+10320,24287,30717,40439
+11549,24288,30716,40129
+12461,24289,30715,40813
+13887,24290,30714,40264
+14414,24291,30713,40102
+16267,24292,30712,40570
+10264,24293,30711,40592
+12924,24294,30710,40165
+15420,24295,30709,40725
+11719,24296,30708,40375
+10585,24297,30707,40010
+17009,24298,30706,40656
+12714,24299,30705,40888
+15417,24300,30704,40230
+14094,24301,30703,40856
+16255,24302,30702,40182
+17742,24303,30701,40374
+15593,24304,30700,40864
+10008,24305,30699,40696
+10562,24306,30698,40628
+11658,24307,30697,40936
+16656,24308,30696,40377
+12405,24309,30695,40381
+10503,24310,30694,40261
+11827,24311,30693,40008
+13855,24312,30692,40150
+14308,24313,30691,40194
+14653,24314,30690,40086
+11652,24315,30689,40843
+11703,24316,30688,40141
+10270,24317,30687,40485
+11384,24318,30686,40112
+15181,24319,30685,40999
+14770,24320,30684,40376
+11906,24321,30683,40141
+13838,24322,30682,40782
+14828,24323,30681,40141
+11473,24324,30680,40474
+14981,24325,30679,40092
+17023,24326,30678,40890
+17414,24327,30677,40383
+10731,24328,30676,40583
+16610,24329,30675,40894
+14606,24330,30674,40476
+11284,24331,30673,40794
+14750,24332,30672,40120
+17024,24333,30671,40216
+11825,24334,30670,40666
+12330,24335,30669,40501
+11764,24336,30668,40180
+15553,24337,30667,40945
+14098,24338,30666,40440
+13250,24339,30665,40712
+16571,24340,30664,40904
+10859,24341,30663,40483
+13107,24342,30662,40626
+13994,24343,30661,40654
+17740,24344,30660,40087
+17177,24345,30659,40658
+17447,24346,30658,40669
+12161,24347,30657,40026
+16495,24348,30656,40426
+12071,24349,30655,40483
+12869,24350,30654,40025
+15073,24351,30653,40317
+12749,24352,30652,40985
+15006,24353,30651,40134
+10942,24354,30650,40465
+17439,24355,30649,40374
+15033,24356,30648,40648
+16796,24357,30647,40077
+10298,24358,30646,40608
+16177,24359,30645,40808
+14435,24360,30644,40083
+14671,24361,30643,40874
+15362,24362,30642,40509
+14608,24363,30641,40879
+13551,24364,30640,40161
+13073,24365,30639,40492
+15524,24366,30638,40808
+14029,24367,30637,40114
+14142,24368,30636,40755
+17986,24369,30635,40562
+16694,24370,30634,40362
+15340,24371,30633,40122
+16902,24372,30632,40438
+13457,24373,30631,40365
+16089,24374,30630,40776
+11506,24375,30629,40425
+16110,24376,30628,40189
+12920,24377,30627,40538
+15305,24378,30626,40952
+13278,24379,30625,40860
+14314,24380,30624,40287
+10426,24381,30623,40662
+15854,24382,30622,40587
+13059,24383,30621,40321
+16044,24384,30620,40430
+15142,24385,30619,40991
+14487,24386,30618,40169
+12765,24387,30617,40009
+16029,24388,30616,40501
+10997,24389,30615,40145
+12684,24390,30614,40991
+11375,24391,30613,40430
+11878,24392,30612,40337
+17701,24393,30611,40598
+13440,24394,30610,40411
+13719,24395,30609,40726
+11934,24396,30608,40329
+13349,24397,30607,40012
+11331,24398,30606,40198
+17876,24399,30605,40725
+14251,24400,30604,40182
+11227,24401,30603,40763
+12877,24402,30602,40897
+16633,24403,30601,40326
+18000,24404,30600,40075
+10713,24405,30599,40220
+11824,24406,30598,40880
+11363,24407,30597,40995
+10688,24408,30596,40277
+11869,24409,30595,40026
+11594,24410,30594,40985
+18124,24411,30593,40408
+15127,24412,30592,40675
+15946,24413,30591,40124
+14403,24414,30590,40341
+11012,24415,30589,40619
+15902,24416,30588,40918
+15485,24417,30587,40009
+10145,24418,30586,40654
+13507,24419,30585,40030
+10274,24420,30584,40082
+13210,24421,30583,40314
+11513,24422,30582,40999
+10742,24423,30581,40195
+12579,24424,30580,40531
+10539,24425,30579,40706
+10368,24426,30578,40759
+11941,24427,30577,40150
+11448,24428,30576,40144
+13254,24429,30575,40089
+13404,24430,30574,40813
+15912,24431,30573,40535
+13800,24432,30572,40675
+18020,24433,30571,40852
+10994,24434,30570,40500
+13464,24435,30569,40397
+10875,24436,30568,40567
+17266,24437,30567,40574
+16715,24438,30566,40001
+12354,24439,30565,40707
+11225,24440,30564,40032
+14823,24441,30563,40571
+17035,24442,30562,40615
+13556,24443,30561,40790
+11539,24444,30560,40166
+11133,24445,30559,40687
+17729,24446,30558,40897
+11953,24447,30557,40708
+13927,24448,30556,40665
+11244,24449,30555,40050
+14250,24450,30554,40696
+13511,24451,30553,40086
+12872,24452,30552,40366
+17702,24453,30551,40174
+11182,24454,30550,40592
+15090,24455,30549,40310
+15685,24456,30548,40040
+16374,24457,30547,40679
+18009,24458,30546,40599
+15852,24459,30545,40761
+10938,24460,30544,40327
+15985,24461,30543,40553
+15039,24462,30542,40278
+17405,24463,30541,40670
+14477,24464,30540,40156
+14081,24465,30539,40262
+16080,24466,30538,40557
+17937,24467,30537,40493
+17274,24468,30536,40938
+17599,24469,30535,40501
+10864,24470,30534,40562
+17446,24471,30533,40433
+14713,24472,30532,40030
+13714,24473,30531,40123
+12283,24474,30530,40203
+12249,24475,30529,40790
+12054,24476,30528,40072
+13725,24477,30527,40869
+10108,24478,30526,40054
+13295,24479,30525,40258
+11523,24480,30524,40600
+18008,24481,30523,40579
+14721,24482,30522,40408
+15184,24483,30521,40935
+12061,24484,30520,40089
+17785,24485,30519,40911
+13575,24486,30518,40355
+13123,24487,30517,40503
+10580,24488,30516,40688
+13559,24489,30515,40947
+13407,24490,30514,40789
+13000,24491,30513,40542
+12419,24492,30512,40477
+10404,24493,30511,40150
+13779,24494,30510,40095
+15225,24495,30509,40116
+12079,24496,30508,40056
+10521,24497,30507,40575
+12431,24498,30506,40563
+16560,24499,30505,40584
+15989,24500,30504,40605
+10712,24501,30503,40496
+13860,24502,30502,40189
+12938,24503,30501,40285
+11970,24504,30500,40344
+15375,24505,30499,40060
+13069,24506,30498,40355
+15378,24507,30497,40840
+10073,24508,30496,40873
+10276,24509,30495,40800
+17994,24510,30494,40733
+12410,24511,30493,40319
+11009,24512,30492,40457
+14990,24513,30491,40605
+10037,24514,30490,40636
+18057,24515,30489,40699
+15393,24516,30488,40675
+10824,24517,30487,40692
+17670,24518,30486,40494
+14936,24519,30485,40190
+14352,24520,30484,40099
+13118,24521,30483,40864
+14091,24522,30482,40128
+10186,24523,30481,40422
+10982,24524,30480,40201
+12308,24525,30479,40063
+15002,24526,30478,40794
+14598,24527,30477,40664
+14351,24528,30476,40871
+13085,24529,30475,40689
+16422,24530,30474,40463
+14934,24531,30473,40092
+11738,24532,30472,40222
+15040,24533,30471,40623
+12329,24534,30470,40993
+17055,24535,30469,40351
+16397,24536,30468,40109
+17203,24537,30467,40660
+15917,24538,30466,40632
+12761,24539,30465,40564
+11673,24540,30464,40505
+17846,24541,30463,40495
+12577,24542,30462,40980
+15976,24543,30461,40258
+12089,24544,30460,40253
+13918,24545,30459,40239
+11773,24546,30458,40054
+16138,24547,30457,40241
+14205,24548,30456,40132
+11720,24549,30455,40861
+11453,24550,30454,40235
+15674,24551,30453,40724
+13956,24552,30452,40721
+10323,24553,30451,40693
+16649,24554,30450,40520
+16508,24555,30449,40495
+15049,24556,30448,40917
+11420,24557,30447,40780
+11126,24558,30446,40278
+17529,24559,30445,40833
+12753,24560,30444,40497
+10197,24561,30443,40123
+11380,24562,30442,40711
+14841,24563,30441,40387
+15535,24564,30440,40927
+13686,24565,30439,40162
+16038,24566,30438,40886
+14115,24567,30437,40175
+14709,24568,30436,40245
+14446,24569,30435,40797
+16035,24570,30434,40347
+15699,24571,30433,40882
+16173,24572,30432,40673
+11578,24573,30431,40474
+11137,24574,30430,40076
+11716,24575,30429,40899
+14437,24576,30428,40021
+14044,24577,30427,40464
+13688,24578,30426,40677
+14353,24579,30425,40593
+17037,24580,30424,40156
+16962,24581,30423,40802
+15724,24582,30422,40232
+12657,24583,30421,40327
+16343,24584,30420,40819
+14180,24585,30419,40457
+16292,24586,30418,40229
+17903,24587,30417,40714
+16573,24588,30416,40616
+14859,24589,30415,40981
+10228,24590,30414,40203
+15276,24591,30413,40889
+12636,24592,30412,40958
+11921,24593,30411,40596
+17610,24594,30410,40844
+12096,24595,30409,40716
+12272,24596,30408,40711
+12989,24597,30407,40873
+17407,24598,30406,40157
+14720,24599,30405,40078
+14054,24600,30404,40959
+16680,24601,30403,40998
+12233,24602,30402,40739
+10120,24603,30401,40731
+17965,24604,30400,40255
+12999,24605,30399,40865
+14463,24606,30398,40366
+14095,24607,30397,40290
+14851,24608,30396,40992
+14148,24609,30395,40226
+12280,24610,30394,40241
+16553,24611,30393,40107
+16756,24612,30392,40611
+11842,24613,30391,40603
+17441,24614,30390,40607
+17308,24615,30389,40408
+18135,24616,30388,40799
+15799,24617,30387,40350
+17877,24618,30386,40271
+11015,24619,30385,40176
+16026,24620,30384,40378
+17826,24621,30383,40394
+15426,24622,30382,40041
+16458,24623,30381,40991
+17257,24624,30380,40214
+12736,24625,30379,40222
+10766,24626,30378,40903
+12292,24627,30377,40463
+17411,24628,30376,40486
+14460,24629,30375,40636
+13499,24630,30374,40140
+15070,24631,30373,40755
+14406,24632,30372,40237
+16969,24633,30371,40551
+16302,24634,30370,40415
+13148,24635,30369,40989
+17106,24636,30368,40741
+15999,24637,30367,40013
+12391,24638,30366,40185
+18106,24639,30365,40903
+16764,24640,30364,40509
+17915,24641,30363,40964
+16319,24642,30362,40786
+15993,24643,30361,40111
+15837,24644,30360,40050
+14849,24645,30359,40800
+11500,24646,30358,40479
+10782,24647,30357,40840
+14623,24648,30356,40495
+11629,24649,30355,40108
+17625,24650,30354,40495
+17096,24651,30353,40208
+17103,24652,30352,40964
+14049,24653,30351,40804
+12139,24654,30350,40392
+17586,24655,30349,40628
+16471,24656,30348,40010
+12070,24657,30347,40553
+16259,24658,30346,40286
+10224,24659,30345,40557
+16864,24660,30344,40932
+17141,24661,30343,40427
+10848,24662,30342,40325
+17995,24663,30341,40717
+13703,24664,30340,40871
+13436,24665,30339,40448
+11517,24666,30338,40945
+18115,24667,30337,40674
+10882,24668,30336,40626
+16434,24669,30335,40884
+14680,24670,30334,40461
+17093,24671,30333,40394
+14747,24672,30332,40150
+10437,24673,30331,40935
+10100,24674,30330,40856
+10178,24675,30329,40533
+16032,24676,30328,40663
+18003,24677,30327,40866
+15444,24678,30326,40120
+10132,24679,30325,40686
+11372,24680,30324,40017
+12131,24681,30323,40593
+17074,24682,30322,40035
+12777,24683,30321,40257
+14636,24684,30320,40908
+17698,24685,30319,40025
+16692,24686,30318,40873
+14433,24687,30317,40998
+12007,24688,30316,40968
+12816,24689,30315,40595
+11955,24690,30314,40355
+12441,24691,30313,40900
+16830,24692,30312,40145
+17310,24693,30311,40783
+11032,24694,30310,40958
+11377,24695,30309,40182
+11236,24696,30308,40715
+11072,24697,30307,40313
+12435,24698,30306,40064
+13847,24699,30305,40801
+12916,24700,30304,40206
+16599,24701,30303,40264
+13971,24702,30302,40985
+18131,24703,30301,40515
+16630,24704,30300,40699
+12291,24705,30299,40865
+17309,24706,30298,40668
+14173,24707,30297,40325
+11831,24708,30296,40740
+14634,24709,30295,40775
+11718,24710,30294,40667
+16608,24711,30293,40250
+12030,24712,30292,40751
+10946,24713,30291,40998
+10056,24714,30290,40389
+12747,24715,30289,40565
+17169,24716,30288,40588
+17675,24717,30287,40443
+12373,24718,30286,40546
+18027,24719,30285,40270
+16635,24720,30284,40210
+15908,24721,30283,40037
+12558,24722,30282,40349
+16915,24723,30281,40365
+16334,24724,30280,40197
+16045,24725,30279,40711
+18164,24726,30278,40865
+17362,24727,30277,40573
+16000,24728,30276,40602
+18055,24729,30275,40171
+13970,24730,30274,40205
+12086,24731,30273,40938
+11968,24732,30272,40449
+10554,24733,30271,40687
+11621,24734,30270,40672
+13025,24735,30269,40459
+10515,24736,30268,40738
+17318,24737,30267,40803
+13929,24738,30266,40780
+14349,24739,30265,40728
+15336,24740,30264,40736
+10641,24741,30263,40673
+14356,24742,30262,40502
+11560,24743,30261,40092
+17217,24744,30260,40144
+16322,24745,30259,40956
+15048,24746,30258,40932
+17759,24747,30257,40283
+14486,24748,30256,40786
+17254,24749,30255,40513
+17935,24750,30254,40693
+18160,24751,30253,40789
+17581,24752,30252,40863
+10577,24753,30251,40496
+14761,24754,30250,40516
+17025,24755,30249,40641
+11838,24756,30248,40720
+16939,24757,30247,40162
+16720,24758,30246,40581
+17813,24759,30245,40586
+16189,24760,30244,40533
+17722,24761,30243,40604
+13792,24762,30242,40174
+11778,24763,30241,40005
+13348,24764,30240,40427
+15190,24765,30239,40107
+10116,24766,30238,40055
+13027,24767,30237,40536
+14190,24768,30236,40620
+14448,24769,30235,40654
+11742,24770,30234,40386
+12742,24771,30233,40109
+17461,24772,30232,40203
+16201,24773,30231,40911
+12043,24774,30230,40442
+14047,24775,30229,40844
+12423,24776,30228,40524
+13934,24777,30227,40857
+16676,24778,30226,40520
+13736,24779,30225,40564
+12281,24780,30224,40936
+17746,24781,30223,40770
+12203,24782,30222,40657
+10390,24783,30221,40154
+14654,24784,30220,40022
+17130,24785,30219,40450
+15687,24786,30218,40968
+17954,24787,30217,40047
+16181,24788,30216,40939
+11864,24789,30215,40586
+16237,24790,30214,40402
+15980,24791,30213,40925
+17263,24792,30212,40835
+16773,24793,30211,40534
+14040,24794,30210,40860
+15975,24795,30209,40002
+12503,24796,30208,40308
+11844,24797,30207,40003
+11950,24798,30206,40431
+18129,24799,30205,40348
+13188,24800,30204,40066
+13966,24801,30203,40033
+12411,24802,30202,40156
+17376,24803,30201,40643
+13772,24804,30200,40685
+13444,24805,30199,40945
+11801,24806,30198,40240
+13570,24807,30197,40698
+16751,24808,30196,40660
+15238,24809,30195,40143
+13077,24810,30194,40481
+10362,24811,30193,40910
+16059,24812,30192,40861
+15503,24813,30191,40512
+12800,24814,30190,40606
+12912,24815,30189,40604
+10027,24816,30188,40977
+15516,24817,30187,40373
+13054,24818,30186,40874
+13280,24819,30185,40966
+10235,24820,30184,40021
+12352,24821,30183,40222
+12904,24822,30182,40640
+15940,24823,30181,40487
+11796,24824,30180,40403
+13818,24825,30179,40720
+13220,24826,30178,40680
+16295,24827,30177,40170
+13505,24828,30176,40501
+15440,24829,30175,40519
+16378,24830,30174,40777
+13350,24831,30173,40398
+13765,24832,30172,40362
+14711,24833,30171,40796
+14953,24834,30170,40468
+10386,24835,30169,40204
+12318,24836,30168,40934
+16983,24837,30167,40439
+17040,24838,30166,40295
+16304,24839,30165,40455
+16157,24840,30164,40158
+14322,24841,30163,40501
+12914,24842,30162,40391
+14986,24843,30161,40330
+15471,24844,30160,40489
+13371,24845,30159,40627
+15146,24846,30158,40092
+12091,24847,30157,40333
+13955,24848,30156,40992
+12801,24849,30155,40952
+11070,24850,30154,40136
+16785,24851,30153,40728
+15702,24852,30152,40749
+12220,24853,30151,40071
+12624,24854,30150,40144
+10237,24855,30149,40862
+11075,24856,30148,40039
+14320,24857,30147,40658
+11898,24858,30146,40865
+10699,24859,30145,40059
+17708,24860,30144,40636
+14249,24861,30143,40751
+10119,24862,30142,40763
+12564,24863,30141,40974
+10142,24864,30140,40142
+11685,24865,30139,40070
+10152,24866,30138,40257
+14928,24867,30137,40047
+13732,24868,30136,40815
+13370,24869,30135,40097
+14682,24870,30134,40335
+14230,24871,30133,40833
+15614,24872,30132,40796
+13103,24873,30131,40275
+17886,24874,30130,40002
+16886,24875,30129,40273
+18112,24876,30128,40532
+13752,24877,30127,40449
+14150,24878,30126,40931
+17840,24879,30125,40172
+17866,24880,30124,40409
+12952,24881,30123,40171
+10399,24882,30122,40468
+13679,24883,30121,40565
+17664,24884,30120,40885
+12412,24885,30119,40067
+17162,24886,30118,40728
+11105,24887,30117,40164
+16529,24888,30116,40342
+17765,24889,30115,40444
+17815,24890,30114,40678
+17528,24891,30113,40759
+12341,24892,30112,40483
+14078,24893,30111,40412
+12126,24894,30110,40151
+15692,24895,30109,40700
+13764,24896,30108,40349
+12002,24897,30107,40794
+12693,24898,30106,40179
+13717,24899,30105,40172
+16416,24900,30104,40102
+14027,24901,30103,40887
+10814,24902,30102,40304
+10987,24903,30101,40642
+10036,24904,30100,40740
+16858,24905,30099,40841
+11503,24906,30098,40378
+14141,24907,30097,40305
+16082,24908,30096,40910
+10764,24909,30095,40419
+13438,24910,30094,40051
+13096,24911,30093,40306
+15311,24912,30092,40317
+14630,24913,30091,40526
+16116,24914,30090,40596
+15945,24915,30089,40779
+16871,24916,30088,40772
+17197,24917,30087,40227
+13452,24918,30086,40578
+12678,24919,30085,40296
+11385,24920,30084,40689
+17987,24921,30083,40718
+12600,24922,30082,40585
+13942,24923,30081,40141
+17619,24924,30080,40753
+15876,24925,30079,40292
+13222,24926,30078,40740
+10586,24927,30077,40480
+13011,24928,30076,40303
+15950,24929,30075,40677
+15464,24930,30074,40642
+17357,24931,30073,40300
+10400,24932,30072,40213
+10136,24933,30071,40346
+17082,24934,30070,40866
+17276,24935,30069,40617
+13557,24936,30068,40816
+14265,24937,30067,40191
+13273,24938,30066,40000
+17686,24939,30065,40468
+16661,24940,30064,40256
+12569,24941,30063,40762
+17516,24942,30062,40548
+14878,24943,30061,40373
+16690,24944,30060,40365
+11188,24945,30059,40225
+15240,24946,30058,40810
+14733,24947,30057,40715
+13283,24948,30056,40523
+17244,24949,30055,40296
+10704,24950,30054,40852
+13880,24951,30053,40897
+18167,24952,30052,40598
+16368,24953,30051,40536
+17716,24954,30050,40641
+13321,24955,30049,40964
+14285,24956,30048,40190
+11674,24957,30047,40345
+17558,24958,30046,40122
+12711,24959,30045,40283
+11625,24960,30044,40084
+12432,24961,30043,40624
+14189,24962,30042,40383
+18065,24963,30041,40997
+11426,24964,30040,40474
+16655,24965,30039,40385
+12589,24966,30038,40765
+13082,24967,30037,40414
+12123,24968,30036,40143
+14945,24969,30035,40781
+16121,24970,30034,40482
+16679,24971,30033,40833
+13869,24972,30032,40272
+10551,24973,30031,40020
+11462,24974,30030,40895
+13006,24975,30029,40598
+15765,24976,30028,40303
+10007,24977,30027,40525
+16413,24978,30026,40581
+15998,24979,30025,40102
+12617,24980,30024,40392
+16703,24981,30023,40416
+10675,24982,30022,40152
+14523,24983,30021,40849
+10874,24984,30020,40830
+11489,24985,30019,40899
+14328,24986,30018,40727
+15021,24987,30017,40246
+13434,24988,30016,40202
+15613,24989,30015,40729
+11803,24990,30014,40161
+16144,24991,30013,40259
+11841,24992,30012,40068
+12840,24993,30011,40116
+13043,24994,30010,40728
+10292,24995,30009,40129
+11017,24996,30008,40087
+10059,24997,30007,40307
+15071,24998,30006,40209
+15369,24999,30005,40977
+10507,25000,30004,40521
+16256,25001,30003,40700
+16243,25002,30002,40506
+10350,25003,30001,40976
+11757,25004,30000,40712
+11223,25005,31000,40211
+11172,25006,30999,40227
+15285,25007,30998,40126
+15102,25008,30997,40389
+16659,25009,30996,40216
+13906,25010,30995,40045
+10612,25011,30994,40954
+12393,25012,30993,40548
+10797,25013,30992,40309
+15957,25014,30991,40061
+10468,25015,30990,40335
+12832,25016,30989,40042
+14427,25017,30988,40093
+11597,25018,30987,40281
+14802,25019,30986,40539
+12361,25020,30985,40181
+10045,25021,30984,40718
+17947,25022,30983,40625
+15106,25023,30982,40002
+11675,25024,30981,40839
+17092,25025,30980,40151
+12076,25026,30979,40999
+12252,25027,30978,40918
+12320,25028,30977,40515
+15675,25029,30976,40258
+12110,25030,30975,40703
+16061,25031,30974,40354
+13259,25032,30973,40358
+12865,25033,30972,40579
+17484,25034,30971,40655
+18152,25035,30970,40473
+12671,25036,30969,40854
+11019,25037,30968,40478
+15350,25038,30967,40839
+16235,25039,30966,40380
+13960,25040,30965,40342
+15221,25041,30964,40676
+15099,25042,30963,40450
+17875,25043,30962,40745
+14911,25044,30961,40568
+17700,25045,30960,40654
+10925,25046,30959,40731
+13399,25047,30958,40228
+16339,25048,30957,40537
+17005,25049,30956,40696
+12581,25050,30955,40192
+11495,25051,30954,40769
+12268,25052,30953,40496
+17996,25053,30952,40215
+16522,25054,30951,40546
+11109,25055,30950,40839
+15697,25056,30949,40310
+12051,25057,30948,40163
+14000,25058,30947,40947
+11435,25059,30946,40462
+17737,25060,30945,40566
+16593,25061,30944,40301
+14014,25062,30943,40549
+16260,25063,30942,40185
+11570,25064,30941,40174
+16518,25065,30940,40585
+12476,25066,30939,40113
+12651,25067,30938,40141
+14068,25068,30937,40012
+15089,25069,30936,40477
+10268,25070,30935,40781
+11640,25071,30934,40111
+16709,25072,30933,40570
+11033,25073,30932,40528
+10160,25074,30931,40460
+14300,25075,30930,40666
+12621,25076,30929,40378
+10588,25077,30928,40259
+10789,25078,30927,40250
+13896,25079,30926,40270
+10262,25080,30925,40562
+12625,25081,30924,40147
+17658,25082,30923,40117
+17637,25083,30922,40457
+10293,25084,30921,40097
+12760,25085,30920,40484
+17499,25086,30919,40565
+13996,25087,30918,40210
+10448,25088,30917,40343
+17552,25089,30916,40079
+11649,25090,30915,40627
+15695,25091,30914,40968
+11510,25092,30913,40368
+12846,25093,30912,40692
+15459,25094,30911,40325
+13067,25095,30910,40751
+16650,25096,30909,40593
+15177,25097,30908,40680
+11750,25098,30907,40848
+15327,25099,30906,40946
+10114,25100,30905,40833
+10774,25101,30904,40208
+12528,25102,30903,40636
+11027,25103,30902,40345
+13320,25104,30901,40657
+10275,25105,30900,40924
+17115,25106,30899,40076
+17322,25107,30898,40789
+13944,25108,30897,40580
+16047,25109,30896,40860
+10851,25110,30895,40459
+11900,25111,30894,40695
+10476,25112,30893,40535
+14307,25113,30892,40814
+15964,25114,30891,40895
+15065,25115,30890,40620
+17268,25116,30889,40042
+16598,25117,30888,40197
+14914,25118,30887,40168
+16521,25119,30886,40547
+13600,25120,30885,40756
+13439,25121,30884,40707
+15722,25122,30883,40017
+14017,25123,30882,40148
+10325,25124,30881,40626
+15896,25125,30880,40247
+13542,25126,30879,40742
+13373,25127,30878,40578
+17731,25128,30877,40192
+10299,25129,30876,40207
+12834,25130,30875,40372
+16911,25131,30874,40890
+16908,25132,30873,40630
+17102,25133,30872,40815
+14867,25134,30871,40940
+14327,25135,30870,40601
+17125,25136,30869,40764
+16766,25137,30868,40736
+13306,25138,30867,40760
+15122,25139,30866,40746
+13187,25140,30865,40917
+18043,25141,30864,40706
+15394,25142,30863,40380
+12055,25143,30862,40821
+13743,25144,30861,40103
+15250,25145,30860,40561
+15974,25146,30859,40637
+16511,25147,30858,40808
+16890,25148,30857,40457
+17692,25149,30856,40986
+11441,25150,30855,40621
+16238,25151,30854,40474
+10977,25152,30853,40468
+10494,25153,30852,40791
+13264,25154,30851,40356
+11371,25155,30850,40282
+14509,25156,30849,40954
+15087,25157,30848,40623
+10388,25158,30847,40000
+17442,25159,30846,40372
+17823,25160,30845,40906
+17607,25161,30844,40086
+12526,25162,30843,40694
+10143,25163,30842,40387
+16353,25164,30841,40561
+10535,25165,30840,40003
+17615,25166,30839,40382
+17687,25167,30838,40848
+17990,25168,30837,40937
+10506,25169,30836,40844
+11088,25170,30835,40594
+17482,25171,30834,40417
+13879,25172,30833,40013
+12772,25173,30832,40403
+13952,25174,30831,40622
+16407,25175,30830,40202
+15353,25176,30829,40603
+12784,25177,30828,40172
+15660,25178,30827,40025
+14891,25179,30826,40303
+10297,25180,30825,40648
+15001,25181,30824,40372
+11815,25182,30823,40211
+11062,25183,30822,40103
+15745,25184,30821,40975
+16880,25185,30820,40699
+14618,25186,30819,40349
+11004,25187,30818,40434
+15748,25188,30817,40482
+10583,25189,30816,40322
+11895,25190,30815,40179
+16977,25191,30814,40740
+13935,25192,30813,40294
+14388,25193,30812,40693
+17669,25194,30811,40716
+18005,25195,30810,40271
+16102,25196,30809,40787
+17980,25197,30808,40954
+13932,25198,30807,40504
+14775,25199,30806,40476
+16452,25200,30805,40337
+11928,25201,30804,40320
+17348,25202,30803,40873
+15182,25203,30802,40786
+11483,25204,30801,40188
+16152,25205,30800,40160
+15814,25206,30799,40634
+16106,25207,30798,40453
+12898,25208,30797,40945
+16562,25209,30796,40095
+16357,25210,30795,40162
+12559,25211,30794,40864
+14281,25212,30793,40238
+15252,25213,30792,40025
+17885,25214,30791,40466
+10852,25215,30790,40114
+18054,25216,30789,40325
+14111,25217,30788,40063
+13064,25218,30787,40626
+15565,25219,30786,40989
+11936,25220,30785,40249
+17264,25221,30784,40394
+16758,25222,30783,40063
+10754,25223,30782,40593
+12297,25224,30781,40238
+11630,25225,30780,40653
+12762,25226,30779,40256
+13639,25227,30778,40997
+12704,25228,30777,40069
+16584,25229,30776,40926
+15288,25230,30775,40676
+11697,25231,30774,40910
+13631,25232,30773,40077
+13822,25233,30772,40836
+12439,25234,30771,40677
+16645,25235,30770,40398
+11347,25236,30769,40499
+12344,25237,30768,40162
+10030,25238,30767,40575
+17967,25239,30766,40166
+13385,25240,30765,40250
+12893,25241,30764,40803
+12859,25242,30763,40161
+15272,25243,30762,40451
+15703,25244,30761,40099
+17219,25245,30760,40635
+17342,25246,30759,40874
+18114,25247,30758,40580
+10573,25248,30757,40414
+10943,25249,30756,40700
+12093,25250,30755,40068
+12456,25251,30754,40393
+16251,25252,30753,40982
+16973,25253,30752,40261
+13152,25254,30751,40357
+10101,25255,30750,40165
+13117,25256,30749,40025
+17426,25257,30748,40356
+11879,25258,30747,40321
+14824,25259,30746,40173
+13053,25260,30745,40501
+11721,25261,30744,40046
+12460,25262,30743,40616
+17911,25263,30742,40463
+13972,25264,30741,40564
+16665,25265,30740,40265
+12115,25266,30739,40110
+16952,25267,30738,40147
+12083,25268,30737,40136
+15717,25269,30736,40392
+15868,25270,30735,40045
+10676,25271,30734,40861
+13323,25272,30733,40855
+11961,25273,30732,40246
+13989,25274,30731,40060
+14580,25275,30730,40856
+17337,25276,30729,40671
+13545,25277,30728,40697
+17760,25278,30727,40275
+15308,25279,30726,40220
+11112,25280,30725,40194
+13904,25281,30724,40312
+10447,25282,30723,40796
+10032,25283,30722,40576
+14868,25284,30721,40379
+10210,25285,30720,40653
+15762,25286,30719,40331
+12449,25287,30718,40472
+16642,25288,30717,40815
+15715,25289,30716,40284
+12195,25290,30715,40303
+10558,25291,30714,40295
+11250,25292,30713,40645
+11846,25293,30712,40969
+13070,25294,30711,40542
+10500,25295,30710,40118
+10536,25296,30709,40222
+17066,25297,30708,40726
+17208,25298,30707,40988
+15284,25299,30706,40803
+11888,25300,30705,40701
+10885,25301,30704,40426
+13202,25302,30703,40188
+14432,25303,30702,40186
+16831,25304,30701,40471
+16581,25305,30700,40802
+11155,25306,30699,40588
+16084,25307,30698,40191
+14089,25308,30697,40955
+15751,25309,30696,40387
+11129,25310,30695,40053
+17304,25311,30694,40993
+13762,25312,30693,40738
+11611,25313,30692,40941
+13483,25314,30691,40721
+14374,25315,30690,40753
+10915,25316,30689,40648
+11139,25317,30688,40905
+13636,25318,30687,40979
+15015,25319,30686,40504
+10865,25320,30685,40961
+10457,25321,30684,40236
+17131,25322,30683,40674
+14737,25323,30682,40100
+17343,25324,30681,40620
+15376,25325,30680,40617
+12991,25326,30679,40221
+12631,25327,30678,40506
+11931,25328,30677,40307
+18151,25329,30676,40954
+12926,25330,30675,40143
+10172,25331,30674,40085
+10846,25332,30673,40960
+17585,25333,30672,40432
+12417,25334,30671,40596
+12156,25335,30670,40341
+11209,25336,30669,40369
+12480,25337,30668,40902
+12020,25338,30667,40610
+10483,25339,30666,40753
+10434,25340,30665,40173
+10185,25341,30664,40049
+10827,25342,30663,40001
+11715,25343,30662,40533
+14877,25344,30661,40973
+15084,25345,30660,40546
+10023,25346,30659,40307
+10146,25347,30658,40076
+16869,25348,30657,40315
+17753,25349,30656,40361
+17519,25350,30655,40020
+13297,25351,30654,40222
+11986,25352,30653,40533
+16928,25353,30652,40235
+14087,25354,30651,40762
+16968,25355,30650,40418
+16394,25356,30649,40578
+12799,25357,30648,40654
+14946,25358,30647,40941
+13309,25359,30646,40981
+11403,25360,30645,40099
+11650,25361,30644,40471
+12125,25362,30643,40104
+16820,25363,30642,40545
+15802,25364,30641,40707
+11189,25365,30640,40490
+11342,25366,30639,40125
+17036,25367,30638,40639
+14219,25368,30637,40367
+15322,25369,30636,40337
+17356,25370,30635,40709
+16119,25371,30634,40628
+16866,25372,30633,40831
+12562,25373,30632,40779
+16805,25374,30631,40093
+17359,25375,30630,40967
+14641,25376,30629,40218
+11068,25377,30628,40223
+17171,25378,30627,40298
+13194,25379,30626,40969
+10482,25380,30625,40949
+16702,25381,30624,40558
+13548,25382,30623,40285
+12779,25383,30622,40923
+16363,25384,30621,40932
+11988,25385,30620,40995
+13189,25386,30619,40327
+13277,25387,30618,40355
+11028,25388,30617,40159
+16719,25389,30616,40934
+13433,25390,30615,40686
+16623,25391,30614,40598
+17043,25392,30613,40654
+15761,25393,30612,40532
+14363,25394,30611,40109
+11829,25395,30610,40231
+13907,25396,30609,40397
+15468,25397,30608,40768
+12545,25398,30607,40088
+12413,25399,30606,40918
+16193,25400,30605,40832
+14645,25401,30604,40589
+10372,25402,30603,40516
+15053,25403,30602,40738
+15334,25404,30601,40495
+17109,25405,30600,40089
+11125,25406,30599,40123
+15038,25407,30598,40119
+17134,25408,30597,40155
+14499,25409,30596,40316
+16673,25410,30595,40598
+12601,25411,30594,40488
+15054,25412,30593,40847
+13432,25413,30592,40390
+10603,25414,30591,40552
+11989,25415,30590,40350
+15646,25416,30589,40887
+15532,25417,30588,40828
+14901,25418,30587,40697
+14718,25419,30586,40725
+15352,25420,30585,40195
+11288,25421,30584,40978
+10041,25422,30583,40984
+14818,25423,30582,40774
+10346,25424,30581,40413
+14838,25425,30580,40713
+14817,25426,30579,40024
+17821,25427,30578,40743
+11455,25428,30577,40070
+17907,25429,30576,40611
+13993,25430,30575,40595
+10086,25431,30574,40932
+15342,25432,30573,40196
+13218,25433,30572,40194
+17430,25434,30571,40657
+16897,25435,30570,40617
+12190,25436,30569,40143
+12584,25437,30568,40020
+11680,25438,30567,40002
+10046,25439,30566,40566
+14854,25440,30565,40127
+10147,25441,30564,40161
+11628,25442,30563,40935
+14847,25443,30562,40297
+15873,25444,30561,40463
+14385,25445,30560,40178
+12867,25446,30559,40805
+10566,25447,30558,40942
+10015,25448,30557,40829
+10560,25449,30556,40340
+12422,25450,30555,40225
+15427,25451,30554,40706
+13961,25452,30553,40343
+12447,25453,30552,40483
+13595,25454,30551,40908
+13419,25455,30550,40147
+16984,25456,30549,40070
+12075,25457,30548,40095
+15787,25458,30547,40175
+17129,25459,30546,40665
+17726,25460,30545,40870
+10104,25461,30544,40649
+14753,25462,30543,40112
+17108,25463,30542,40176
+12980,25464,30541,40788
+12216,25465,30540,40190
+15709,25466,30539,40771
+13962,25467,30538,40283
+13300,25468,30537,40654
+12222,25469,30536,40274
+16510,25470,30535,40922
+14304,25471,30534,40250
+11221,25472,30533,40934
+15586,25473,30532,40966
+10184,25474,30531,40861
+13090,25475,30530,40500
+10790,25476,30529,40687
+12647,25477,30528,40067
+10383,25478,30527,40482
+11296,25479,30526,40069
+17182,25480,30525,40774
+13572,25481,30524,40932
+13700,25482,30523,40357
+10354,25483,30522,40745
+12333,25484,30521,40553
+10470,25485,30520,40671
+12496,25486,30519,40392
+15443,25487,30518,40950
+15377,25488,30517,40827
+18098,25489,30516,40372
+13884,25490,30515,40388
+10572,25491,30514,40206
+18072,25492,30513,40493
+16763,25493,30512,40308
+11357,25494,30511,40030
+12056,25495,30510,40450
+14069,25496,30509,40814
+16494,25497,30508,40074
+14021,25498,30507,40467
+15963,25499,30506,40895
+11834,25500,30505,40444
+16228,25501,30504,40269
+15569,25502,30503,40723
+13351,25503,30502,40772
+16347,25504,30501,40368
+17922,25505,30500,40492
+12328,25506,30499,40241
+10722,25507,30498,40386
+15879,25508,30497,40798
+10749,25509,30496,40448
+14576,25510,30495,40272
+18063,25511,30494,40154
+16744,25512,30493,40226
+17159,25513,30492,40853
+14225,25514,30491,40159
+10600,25515,30490,40978
+11327,25516,30489,40643
+10751,25517,30488,40336
+14367,25518,30487,40174
+15072,25519,30486,40160
+13052,25520,30485,40323
+14048,25521,30484,40402
+10745,25522,30483,40054
+16103,25523,30482,40044
+16284,25524,30481,40772
+17952,25525,30480,40091
+16111,25526,30479,40453
+15321,25527,30478,40626
+13667,25528,30477,40020
+14279,25529,30476,40423
+14993,25530,30475,40575
+13995,25531,30474,40333
+16081,25532,30473,40025
+11192,25533,30472,40276
+15194,25534,30471,40989
+11729,25535,30470,40125
+17991,25536,30469,40126
+15623,25537,30468,40641
+14232,25538,30467,40721
+13678,25539,30466,40730
+14622,25540,30465,40804
+17510,25541,30464,40241
+10985,25542,30463,40944
+12694,25543,30462,40085
+16225,25544,30461,40825
+15149,25545,30460,40629
+17902,25546,30459,40206
+17793,25547,30458,40133
+12954,25548,30457,40476
+12404,25549,30456,40549
+15173,25550,30455,40821
+16037,25551,30454,40632
+12142,25552,30453,40710
+13116,25553,30452,40303
+14143,25554,30451,40638
+15855,25555,30450,40205
+12664,25556,30449,40817
+16803,25557,30448,40966
+12050,25558,30447,40837
+13508,25559,30446,40920
+11205,25560,30445,40900
+16812,25561,30444,40260
+16362,25562,30443,40928
+11474,25563,30442,40502
+16348,25564,30441,40814
+11497,25565,30440,40351
+13654,25566,30439,40888
+11632,25567,30438,40860
+13159,25568,30437,40870
+17098,25569,30436,40375
+15714,25570,30435,40103
+17843,25571,30434,40531
+10265,25572,30433,40337
+16112,25573,30432,40254
+17777,25574,30431,40496
+15455,25575,30430,40407
+11361,25576,30429,40779
+11800,25577,30428,40950
+14246,25578,30427,40986
+12663,25579,30426,40363
+14317,25580,30425,40820
+12998,25581,30424,40046
+13702,25582,30423,40493
+16845,25583,30422,40373
+17781,25584,30421,40876
+16556,25585,30420,40069
+16638,25586,30419,40570
+14673,25587,30418,40376
+13032,25588,30417,40910
+16564,25589,30416,40773
+14679,25590,30415,40006
+15100,25591,30414,40893
+17451,25592,30413,40184
+12996,25593,30412,40208
+15585,25594,30411,40439
+17946,25595,30410,40320
+11505,25596,30409,40977
+12385,25597,30408,40766
+16882,25598,30407,40093
+11553,25599,30406,40649
+15212,25600,30405,40970
+10599,25601,30404,40829
+12309,25602,30403,40986
+16621,25603,30402,40383
+16714,25604,30401,40109
+17858,25605,30400,40823
+15256,25606,30399,40930
+14497,25607,30398,40366
+16172,25608,30397,40392
+11663,25609,30396,40863
+12137,25610,30395,40971
+11138,25611,30394,40914
+17678,25612,30393,40243
+17455,25613,30392,40707
+14152,25614,30391,40262
+11457,25615,30390,40321
+14692,25616,30389,40173
+13231,25617,30388,40547
+10952,25618,30387,40579
+17810,25619,30386,40957
+16268,25620,30385,40170
+16762,25621,30384,40784
+12939,25622,30383,40968
+15526,25623,30382,40543
+17011,25624,30381,40815
+14236,25625,30380,40445
+10909,25626,30379,40107
+16145,25627,30378,40557
+12433,25628,30377,40252
+15309,25629,30376,40297
+11040,25630,30375,40443
+14751,25631,30374,40924
+16449,25632,30373,40328
+16538,25633,30372,40347
+14071,25634,30371,40554
+17845,25635,30370,40973
+16257,25636,30369,40632
+11786,25637,30368,40052
+12982,25638,30367,40869
+11471,25639,30366,40876
+16927,25640,30365,40265
+11765,25641,30364,40888
+15856,25642,30363,40489
+17487,25643,30362,40604
+16218,25644,30361,40905
+10662,25645,30360,40708
+13036,25646,30359,40288
+15019,25647,30358,40971
+15371,25648,30357,40281
+13875,25649,30356,40225
+12394,25650,30355,40466
+11866,25651,30354,40852
+12843,25652,30353,40733
+16134,25653,30352,40639
+15936,25654,30351,40348
+18140,25655,30350,40136
+12701,25656,30349,40609
+11199,25657,30348,40914
+14744,25658,30347,40811
+14443,25659,30346,40699
+10411,25660,30345,40669
+12958,25661,30344,40184
+15916,25662,30343,40041
+13390,25663,30342,40613
+10666,25664,30341,40058
+15413,25665,30340,40073
+14657,25666,30339,40863
+13405,25667,30338,40093
+13865,25668,30337,40499
+15547,25669,30336,40389
+18067,25670,30335,40801
+13200,25671,30334,40327
+15037,25672,30333,40996
+17481,25673,30332,40041
+11219,25674,30331,40269
+12455,25675,30330,40551
+18105,25676,30329,40278
+17409,25677,30328,40530
+13393,25678,30327,40315
+11449,25679,30326,40388
+13386,25680,30325,40704
+17124,25681,30324,40335
+16781,25682,30323,40901
+18076,25683,30322,40341
+17027,25684,30321,40839
+16468,25685,30320,40378
+15214,25686,30319,40784
+17962,25687,30318,40118
+12882,25688,30317,40470
+14554,25689,30316,40159
+11045,25690,30315,40257
+12080,25691,30314,40796
+13285,25692,30313,40606
+14677,25693,30312,40300
+13567,25694,30311,40801
+11477,25695,30310,40604
+13664,25696,30309,40487
+17069,25697,30308,40215
+14278,25698,30307,40337
+17924,25699,30306,40208
+11169,25700,30305,40685
+12176,25701,30304,40171
+17453,25702,30303,40582
+14880,25703,30302,40841
+18133,25704,30301,40563
+10455,25705,30300,40362
+16226,25706,30299,40893
+14160,25707,30298,40477
+16052,25708,30297,40002
+12219,25709,30296,40395
+14963,25710,30295,40375
+11705,25711,30294,40714
+15389,25712,30293,40163
+10905,25713,30292,40165
+17344,25714,30291,40030
+16624,25715,30290,40161
+15678,25716,30289,40184
+14025,25717,30288,40554
+17235,25718,30287,40190
+12160,25719,30286,40475
+15781,25720,30285,40871
+17460,25721,30284,40013
+13196,25722,30283,40884
+10069,25723,30282,40319
+12466,25724,30281,40630
+18011,25725,30280,40227
+15219,25726,30279,40250
+13891,25727,30278,40713
+16929,25728,30277,40839
+17372,25729,30276,40461
+12506,25730,30275,40233
+12918,25731,30274,40692
+10166,25732,30273,40245
+15833,25733,30272,40188
+16696,25734,30271,40796
+10591,25735,30270,40849
+14957,25736,30269,40253
+15135,25737,30268,40579
+17597,25738,30267,40115
+15076,25739,30266,40669
+13443,25740,30265,40190
+10403,25741,30264,40289
+10029,25742,30263,40053
+11315,25743,30262,40542
+10194,25744,30261,40617
+14746,25745,30260,40636
+14649,25746,30259,40388
+14132,25747,30258,40086
+10463,25748,30257,40128
+10530,25749,30256,40086
+11359,25750,30255,40917
+15265,25751,30254,40470
+17004,25752,30253,40834
+11712,25753,30252,40306
+10389,25754,30251,40989
+12168,25755,30250,40180
+15911,25756,30249,40373
+13360,25757,30248,40663
+10929,25758,30247,40364
+16107,25759,30246,40213
+13396,25760,30245,40587
+13253,25761,30244,40716
+12510,25762,30243,40365
+12607,25763,30242,40523
+15093,25764,30241,40093
+15813,25765,30240,40002
+14578,25766,30239,40788
+13549,25767,30238,40167
+14121,25768,30237,40106
+10844,25769,30236,40053
+10759,25770,30235,40782
+11401,25771,30234,40807
+14659,25772,30233,40160
+10225,25773,30232,40923
+14043,25774,30231,40772
+12838,25775,30230,40417
+16018,25776,30229,40017
+10526,25777,30228,40842
+15316,25778,30227,40143
+10716,25779,30226,40134
+14638,25780,30225,40632
+13084,25781,30224,40549
+13677,25782,30223,40949
+12370,25783,30222,40471
+17806,25784,30221,40945
+13191,25785,30220,40328
+17724,25786,30219,40458
+10479,25787,30218,40307
+13517,25788,30217,40907
+15128,25789,30216,40559
+11664,25790,30215,40847
+14062,25791,30214,40735
+10421,25792,30213,40841
+16769,25793,30212,40690
+10780,25794,30211,40399
+14394,25795,30210,40339
+12771,25796,30209,40628
+14046,25797,30208,40684
+13903,25798,30207,40900
+15192,25799,30206,40459
+16262,25800,30205,40026
+17749,25801,30204,40263
+12306,25802,30203,40086
+17184,25803,30202,40291
+12450,25804,30201,40162
+11319,25805,30200,40040
+11911,25806,30199,40531
+13521,25807,30198,40266
+16109,25808,30197,40325
+11255,25809,30196,40105
+17311,25810,30195,40431
+15416,25811,30194,40135
+11168,25812,30193,40286
+14932,25813,30192,40209
+16462,25814,30191,40838
+16686,25815,30190,40927
+13046,25816,30189,40858
+12551,25817,30188,40413
+10179,25818,30187,40772
+10775,25819,30186,40188
+14667,25820,30185,40385
+12298,25821,30184,40107
+17034,25822,30183,40084
+15406,25823,30182,40214
+18130,25824,30181,40642
+17148,25825,30180,40113
+15013,25826,30179,40341
+16677,25827,30178,40366
+11971,25828,30177,40634
+10278,25829,30176,40010
+13843,25830,30175,40870
+12376,25831,30174,40077
+12507,25832,30173,40805
+16466,25833,30172,40336
+11229,25834,30171,40259
+10973,25835,30170,40353
+16200,25836,30169,40579
+14948,25837,30168,40394
+18064,25838,30167,40612
+14096,25839,30166,40081
+13233,25840,30165,40732
+13038,25841,30164,40632
+13957,25842,30163,40661
+14238,25843,30162,40355
+12550,25844,30161,40964
+14616,25845,30160,40481
+12786,25846,30159,40664
+12682,25847,30158,40252
+11407,25848,30157,40135
+10837,25849,30156,40283
+14924,25850,30155,40508
+16326,25851,30154,40735
+14790,25852,30153,40997
+10951,25853,30152,40707
+11395,25854,30151,40297
+11338,25855,30150,40972
+12188,25856,30149,40119
+16311,25857,30148,40256
+15539,25858,30147,40713
+14241,25859,30146,40947
+13288,25860,30145,40031
+17562,25861,30144,40916
+11701,25862,30143,40056
+15395,25863,30142,40406
+15415,25864,30141,40356
+13608,25865,30140,40194
+16355,25866,30139,40589
+11295,25867,30138,40506
+10726,25868,30137,40939
+16963,25869,30136,40336
+10931,25870,30135,40950
+11819,25871,30134,40229
+17654,25872,30133,40798
+11734,25873,30132,40446
+14131,25874,30131,40543
+10012,25875,30130,40185
+14639,25876,30129,40193
+14951,25877,30128,40381
+17570,25878,30127,40963
+11557,25879,30126,40391
+13527,25880,30125,40575
+11749,25881,30124,40345
+12909,25882,30123,40217
+15618,25883,30122,40837
+15683,25884,30121,40737
+12295,25885,30120,40367
+17518,25886,30119,40328
+17888,25887,30118,40592
+13322,25888,30117,40895
+10961,25889,30116,40503
+15928,25890,30115,40218
+17524,25891,30114,40586
+11026,25892,30113,40004
+15151,25893,30112,40330
+17328,25894,30111,40900
+13113,25895,30110,40228
+11157,25896,30109,40205
+12346,25897,30108,40476
+15115,25898,30107,40756
+13243,25899,30106,40180
+15189,25900,30105,40186
+16236,25901,30104,40211
+14355,25902,30103,40961
+15145,25903,30102,40257
+17547,25904,30101,40148
+14890,25905,30100,40525
+16994,25906,30099,40463
+11406,25907,30098,40345
+16829,25908,30097,40869
+12633,25909,30096,40574
+11226,25910,30095,40400
+11833,25911,30094,40881
+10886,25912,30093,40874
+15617,25913,30092,40263
+15768,25914,30091,40698
+12256,25915,30090,40805
+12974,25916,30089,40587
+17923,25917,30088,40663
+14604,25918,30087,40911
+15144,25919,30086,40362
+10564,25920,30085,40241
+15941,25921,30084,40462
+15344,25922,30083,40736
+15112,25923,30082,40659
+15058,25924,30081,40279
+17762,25925,30080,40044
+15313,25926,30079,40681
+17928,25927,30078,40137
+15615,25928,30077,40554
+12697,25929,30076,40694
+15414,25930,30075,40968
+13497,25931,30074,40649
+17234,25932,30073,40216
+14569,25933,30072,40769
+10718,25934,30071,40630
+15729,25935,30070,40074
+18056,25936,30069,40188
+12934,25937,30068,40605
+17149,25938,30067,40275
+16893,25939,30066,40051
+17811,25940,30065,40808
+17280,25941,30064,40050
+14255,25942,30063,40925
+16958,25943,30062,40177
+15909,25944,30061,40784
+12174,25945,30060,40789
+10111,25946,30059,40817
+15408,25947,30058,40806
+13821,25948,30057,40645
+16892,25949,30056,40148
+12185,25950,30055,40688
+17644,25951,30054,40739
+11836,25952,30053,40143
+17019,25953,30052,40061
+15170,25954,30051,40467
+14739,25955,30050,40038
+18153,25956,30049,40266
+16017,25957,30048,40075
+11034,25958,30047,40207
+16229,25959,30046,40188
+16383,25960,30045,40706
+17640,25961,30044,40336
+16563,25962,30043,40362
+12555,25963,30042,40136
+16265,25964,30041,40030
+16851,25965,30040,40556
+10752,25966,30039,40027
+12703,25967,30038,40233
+11096,25968,30037,40928
+13179,25969,30036,40084
+14199,25970,30035,40021
+11856,25971,30034,40040
+12902,25972,30033,40634
+16463,25973,30032,40483
+13565,25974,30031,40924
+15199,25975,30030,40895
+12529,25976,30029,40298
+15846,25977,30028,40156
+17715,25978,30027,40893
+14773,25979,30026,40970
+11254,25980,30025,40598
+15545,25981,30024,40540
+15995,25982,30023,40658
+16314,25983,30022,40257
+14837,25984,30021,40032
+16324,25985,30020,40372
+17566,25986,30019,40100
+15236,25987,30018,40572
+11411,25988,30017,40324
+14417,25989,30016,40357
+17690,25990,30015,40583
+11544,25991,30014,40077
+17497,25992,30013,40189
+11634,25993,30012,40603
+17107,25994,30011,40776
+15672,25995,30010,40025
+10031,25996,30009,40682
+18077,25997,30008,40626
+12375,25998,30007,40762
+12493,25999,30006,40942
+14397,26000,30005,40222
+14267,26001,30004,40067
+14434,26002,30003,40732
+18061,26003,30002,40854
+14455,26004,30001,40155
+15180,26005,30000,40543
+16269,26006,31000,40788
+13050,26007,30999,40723
+13655,26008,30998,40686
+10657,26009,30997,40026
+14372,26010,30996,40287
+11711,26011,30995,40864
+17631,26012,30994,40022
+16162,26013,30993,40369
+15629,26014,30992,40605
+13728,26015,30991,40821
+16774,26016,30990,40733
+10092,26017,30989,40996
+17514,26018,30988,40833
+11475,26019,30987,40179
+17550,26020,30986,40372
+12976,26021,30985,40580
+17145,26022,30984,40887
+14151,26023,30983,40916
+15437,26024,30982,40363
+12090,26025,30981,40573
+15370,26026,30980,40924
+10727,26027,30979,40980
+18036,26028,30978,40318
+12936,26029,30977,40813
+15848,26030,30976,40025
+15277,26031,30975,40810
+14154,26032,30974,40712
+13079,26033,30973,40091
+11892,26034,30972,40672
+18107,26035,30971,40807
+10622,26036,30970,40780
+12978,26037,30969,40432
+17495,26038,30968,40757
+13573,26039,30967,40077
+16585,26040,30966,40750
+11147,26041,30965,40361
+15059,26042,30964,40725
+10843,26043,30963,40664
+15555,26044,30962,40221
+14887,26045,30961,40522
+15961,26046,30960,40785
+17432,26047,30959,40848
+13893,26048,30958,40260
+10453,26049,30957,40356
+15509,26050,30956,40020
+14196,26051,30955,40917
+14324,26052,30954,40636
+15197,26053,30953,40463
+17632,26054,30952,40064
+12813,26055,30951,40697
+14001,26056,30950,40476
+14361,26057,30949,40880
+15324,26058,30948,40637
+16279,26059,30947,40441
+12097,26060,30946,40353
+18191,26061,30945,40436
+11942,26062,30944,40978
+13333,26063,30943,40148
+17611,26064,30942,40245
+17569,26065,30941,40874
+17261,26066,30940,40377
+10189,26067,30939,40737
+13794,26068,30938,40011
+10793,26069,30937,40613
+12262,26070,30936,40855
+17805,26071,30935,40133
+17220,26072,30934,40384
+13829,26073,30933,40359
+11940,26074,30932,40304
+12901,26075,30931,40402
+17058,26076,30930,40868
+10984,26077,30929,40823
+10624,26078,30928,40823
+11768,26079,30927,40265
+18021,26080,30926,40647
+12502,26081,30925,40203
+16674,26082,30924,40012
+12659,26083,30923,40489
+12728,26084,30922,40724
+16998,26085,30921,40358
+16372,26086,30920,40371
+11943,26087,30919,40132
+12025,26088,30918,40928
+15403,26089,30917,40291
+10602,26090,30916,40267
+11143,26091,30915,40160
+17976,26092,30914,40163
+10021,26093,30913,40815
+17053,26094,30912,40100
+12887,26095,30911,40724
+10798,26096,30910,40397
+14767,26097,30909,40098
+16209,26098,30908,40367
+16001,26099,30907,40038
+13602,26100,30906,40746
+18156,26101,30905,40926
+17209,26102,30904,40202
+11164,26103,30903,40827
+16358,26104,30902,40358
+12121,26105,30901,40038
+17978,26106,30900,40401
+10529,26107,30899,40775
+13318,26108,30898,40905
+11913,26109,30897,40760
+12314,26110,30896,40925
+15000,26111,30895,40296
+16105,26112,30894,40268
+14030,26113,30893,40443
+14377,26114,30892,40568
+10548,26115,30891,40135
+17434,26116,30890,40898
+15815,26117,30889,40353
+11231,26118,30888,40847
+13086,26119,30887,40350
+16881,26120,30886,40319
+14687,26121,30885,40262
+15712,26122,30884,40702
+12850,26123,30883,40226
+11097,26124,30882,40612
+10491,26125,30881,40326
+12471,26126,30880,40567
+10883,26127,30879,40353
+13997,26128,30878,40256
+15843,26129,30877,40810
+11195,26130,30876,40659
+12353,26131,30875,40349
+18001,26132,30874,40592
+13078,26133,30873,40443
+11316,26134,30872,40534
+10444,26135,30871,40544
+15140,26136,30870,40407
+13345,26137,30869,40941
+15688,26138,30868,40791
+18168,26139,30867,40805
+17594,26140,30866,40366
+13307,26141,30865,40125
+12764,26142,30864,40605
+16542,26143,30863,40877
+11737,26144,30862,40493
+15211,26145,30861,40811
+11948,26146,30860,40698
+10081,26147,30859,40135
+10168,26148,30858,40039
+15853,26149,30857,40752
+16540,26150,30856,40588
+16549,26151,30855,40205
+14163,26152,30854,40482
+12065,26153,30853,40992
+11735,26154,30852,40702
+17679,26155,30851,40533
+15399,26156,30850,40860
+16294,26157,30849,40385
+12518,26158,30848,40075
+14461,26159,30847,40774
+12540,26160,30846,40878
+10158,26161,30845,40434
+14421,26162,30844,40299
+16360,26163,30843,40110
+16190,26164,30842,40340
+14128,26165,30841,40700
+12008,26166,30840,40691
+10567,26167,30839,40346
+10218,26168,30838,40131
+10669,26169,30837,40356
+13221,26170,30836,40329
+10910,26171,30835,40126
+12925,26172,30834,40746
+12017,26173,30833,40030
+14860,26174,30832,40365
+11499,26175,30831,40560
+14422,26176,30830,40129
+11337,26177,30829,40227
+13124,26178,30828,40777
+16643,26179,30827,40705
+17085,26180,30826,40684
+16943,26181,30825,40948
+16947,26182,30824,40634
+10219,26183,30823,40747
+11021,26184,30822,40647
+12068,26185,30821,40508
+13835,26186,30820,40628
+11791,26187,30819,40115
+14975,26188,30818,40955
+15786,26189,30817,40273
+17303,26190,30816,40336
+13823,26191,30815,40352
+16537,26192,30814,40544
+15771,26193,30813,40410
+11983,26194,30812,40465
+16366,26195,30811,40322
+13470,26196,30810,40000
+12735,26197,30809,40842
+10445,26198,30808,40258
+10127,26199,30807,40899
+18069,26200,30806,40210
+13167,26201,30805,40499
+15216,26202,30804,40713
+14118,26203,30803,40225
+14161,26204,30802,40108
+18073,26205,30801,40316
+16583,26206,30800,40951
+12157,26207,30799,40217
+15161,26208,30798,40248
+17028,26209,30797,40891
+10933,26210,30796,40214
+12897,26211,30795,40621
+11025,26212,30794,40159
+10784,26213,30793,40483
+14669,26214,30792,40825
+10992,26215,30791,40463
+12485,26216,30790,40472
+14187,26217,30789,40183
+17116,26218,30788,40950
+17079,26219,30787,40899
+14795,26220,30786,40281
+14326,26221,30785,40052
+13848,26222,30784,40431
+13683,26223,30783,40248
+11740,26224,30782,40388
+16872,26225,30781,40506
+10907,26226,30780,40060
+17022,26227,30779,40623
+12166,26228,30778,40964
+17015,26229,30777,40267
+11178,26230,30776,40522
+13019,26231,30775,40410
+16808,26232,30774,40293
+12746,26233,30773,40850
+13999,26234,30772,40222
+11855,26235,30771,40846
+17563,26236,30770,40878
+12443,26237,30769,40037
+13281,26238,30768,40858
+18022,26239,30767,40417
+14257,26240,30766,40218
+12199,26241,30765,40164
+15607,26242,30764,40207
+14184,26243,30763,40811
+14470,26244,30762,40506
+15591,26245,30761,40047
+14736,26246,30760,40217
+14456,26247,30759,40273
+17920,26248,30758,40156
+15400,26249,30757,40354
+15759,26250,30756,40974
+16450,26251,30755,40917
+16948,26252,30754,40919
+14856,26253,30753,40404
+15579,26254,30752,40774
+15116,26255,30751,40486
+11965,26256,30750,40132
+18166,26257,30749,40859
+10912,26258,30748,40044
+14133,26259,30747,40934
+15763,26260,30746,40806
+17997,26261,30745,40953
+17270,26262,30744,40642
+18012,26263,30743,40135
+10581,26264,30742,40514
+16198,26265,30741,40810
+14723,26266,30740,40657
+13056,26267,30739,40413
+10519,26268,30738,40007
+10729,26269,30737,40107
+10266,26270,30736,40110
+17425,26271,30735,40190
+15644,26272,30734,40239
+11248,26273,30733,40948
+10524,26274,30732,40718
+15927,26275,30731,40044
+17241,26276,30730,40028
+14543,26277,30729,40654
+15157,26278,30728,40650
+17071,26279,30727,40785
+11568,26280,30726,40107
+15682,26281,30725,40670
+13901,26282,30724,40299
+11064,26283,30723,40777
+10249,26284,30722,40620
+16141,26285,30721,40239
+10813,26286,30720,40407
+10212,26287,30719,40403
+14763,26288,30718,40887
+15057,26289,30717,40347
+14732,26290,30716,40153
+16420,26291,30715,40064
+14090,26292,30714,40815
+16817,26293,30713,40923
+11882,26294,30712,40844
+15610,26295,30711,40146
+11875,26296,30710,40675
+12512,26297,30709,40969
+16426,26298,30708,40664
+15584,26299,30707,40254
+14384,26300,30706,40760
+12690,26301,30705,40253
+12629,26302,30704,40540
+10398,26303,30703,40346
+14233,26304,30702,40311
+14833,26305,30701,40877
+16027,26306,30700,40513
+17720,26307,30699,40286
+13711,26308,30698,40149
+17884,26309,30697,40930
+17543,26310,30696,40749
+16090,26311,30695,40927
+18015,26312,30694,40589
+16342,26313,30693,40817
+12718,26314,30692,40181
+11967,26315,30691,40399
+16129,26316,30690,40583
+11366,26317,30689,40247
+17794,26318,30688,40969
+12265,26319,30687,40134
+12908,26320,30686,40765
+14369,26321,30685,40700
+14502,26322,30684,40459
+15230,26323,30683,40412
+17522,26324,30682,40488
+13274,26325,30681,40801
+15104,26326,30680,40351
+18119,26327,30679,40603
+12798,26328,30678,40934
+15951,26329,30677,40185
+12475,26330,30676,40036
+17614,26331,30675,40497
+17321,26332,30674,40407
+13798,26333,30673,40187
+13004,26334,30672,40442
+12650,26335,30671,40355
+11118,26336,30670,40891
+13733,26337,30669,40981
+14296,26338,30668,40314
+14648,26339,30667,40979
+15451,26340,30666,40733
+11527,26341,30665,40354
+10353,26342,30664,40368
+17714,26343,30663,40619
+17101,26344,30662,40746
+14011,26345,30661,40109
+15948,26346,30660,40820
+12235,26347,30659,40423
+14207,26348,30658,40018
+13108,26349,30657,40627
+12724,26350,30656,40275
+16996,26351,30655,40239
+10257,26352,30654,40750
+13242,26353,30653,40565
+15043,26354,30652,40562
+16554,26355,30651,40753
+12962,26356,30650,40120
+13098,26357,30649,40699
+12312,26358,30648,40700
+15676,26359,30647,40106
+10550,26360,30646,40405
+12609,26361,30645,40729
+10462,26362,30644,40326
+12700,26363,30643,40526
+15449,26364,30642,40669
+11555,26365,30641,40685
+15939,26366,30640,40737
+12336,26367,30639,40195
+10233,26368,30638,40329
+14704,26369,30637,40054
+10750,26370,30636,40608
+15567,26371,30635,40922
+15458,26372,30634,40764
+16490,26373,30633,40718
+14490,26374,30632,40605
+16476,26375,30631,40637
+13498,26376,30630,40539
+15634,26377,30629,40737
+12896,26378,30628,40080
+13263,26379,30627,40010
+16174,26380,30626,40796
+16073,26381,30625,40979
+14728,26382,30624,40767
+16039,26383,30623,40422
+17070,26384,30622,40594
+17580,26385,30621,40326
+10891,26386,30620,40567
+13690,26387,30619,40529
+14525,26388,30618,40774
+13455,26389,30617,40297
+15245,26390,30616,40961
+15410,26391,30615,40144
+10944,26392,30614,40431
+12535,26393,30613,40833
+13991,26394,30612,40909
+12787,26395,30611,40702
+13109,26396,30610,40995
+15467,26397,30609,40579
+14283,26398,30608,40338
+15500,26399,30607,40113
+14402,26400,30606,40059
+13029,26401,30605,40735
+13500,26402,30604,40323
+17020,26403,30603,40259
+13037,26404,30602,40546
+15947,26405,30601,40902
+10983,26406,30600,40235
+11076,26407,30599,40462
+11264,26408,30598,40886
+17694,26409,30597,40069
+10847,26410,30596,40177
+14740,26411,30595,40718
+13832,26412,30594,40021
+11175,26413,30593,40048
+10520,26414,30592,40501
+16227,26415,30591,40275
+17627,26416,30590,40457
+17038,26417,30589,40351
+16184,26418,30588,40912
+17865,26419,30587,40099
+15078,26420,30586,40164
+13149,26421,30585,40259
+13469,26422,30584,40119
+16382,26423,30583,40804
+14978,26424,30582,40596
+11502,26425,30581,40313
+17527,26426,30580,40480
+16230,26427,30579,40935
+10094,26428,30578,40584
+14970,26429,30577,40365
+14410,26430,30576,40540
+17904,26431,30575,40773
+17838,26432,30574,40208
+16579,26433,30573,40752
+14019,26434,30572,40171
+12501,26435,30571,40654
+12208,26436,30570,40504
+12331,26437,30569,40211
+17577,26438,30568,40194
+13759,26439,30567,40501
+14572,26440,30566,40077
+14759,26441,30565,40489
+17797,26442,30564,40055
+10986,26443,30563,40301
+11593,26444,30562,40622
+10771,26445,30561,40488
+18026,26446,30560,40291
+13553,26447,30559,40118
+17711,26448,30558,40275
+12003,26449,30557,40336
+11232,26450,30556,40767
+12223,26451,30555,40152
+14360,26452,30554,40033
+12675,26453,30553,40786
+14263,26454,30552,40823
+11237,26455,30551,40565
+17713,26456,30550,40940
+12053,26457,30549,40049
+13873,26458,30548,40889
+17164,26459,30547,40883
+13709,26460,30546,40379
+10243,26461,30545,40428
+18190,26462,30544,40974
+11249,26463,30543,40776
+11806,26464,30542,40016
+16551,26465,30541,40241
+11746,26466,30540,40792
+11410,26467,30539,40396
+17850,26468,30538,40612
+10192,26469,30537,40859
+13774,26470,30536,40730
+10138,26471,30535,40281
+15622,26472,30534,40495
+17526,26473,30533,40221
+16618,26474,30532,40251
+13171,26475,30531,40238
+14681,26476,30530,40149
+10343,26477,30529,40141
+17940,26478,30528,40306
+13352,26479,30527,40327
+14848,26480,30526,40633
+16241,26481,30525,40729
+12802,26482,30524,40034
+10705,26483,30523,40190
+10051,26484,30522,40453
+15809,26485,30521,40556
+15028,26486,30520,40992
+13209,26487,30519,40725
+10418,26488,30518,40384
+10334,26489,30517,40025
+13114,26490,30516,40511
+11202,26491,30515,40161
+10964,26492,30514,40326
+16631,26493,30513,40710
+14565,26494,30512,40010
+16301,26495,30511,40160
+18187,26496,30510,40459
+15016,26497,30509,40699
+16150,26498,30508,40570
+17465,26499,30507,40339
+16789,26500,30506,40264
+10316,26501,30505,40248
+12260,26502,30504,40875
+16778,26503,30503,40444
+10873,26504,30502,40147
+11946,26505,30501,40882
+17572,26506,30500,40555
+17458,26507,30499,40465
+16276,26508,30498,40732
+15390,26509,30497,40565
+16561,26510,30496,40851
+10113,26511,30495,40295
+10617,26512,30494,40250
+13197,26513,30493,40842
+15387,26514,30492,40275
+10835,26515,30491,40358
+17893,26516,30490,40905
+11042,26517,30489,40403
+13327,26518,30488,40633
+11083,26519,30487,40419
+14253,26520,30486,40470
+12677,26521,30485,40282
+12595,26522,30484,40963
+16330,26523,30483,40249
+17333,26524,30482,40832
+15179,26525,30481,40117
+17475,26526,30480,40935
+10809,26527,30479,40751
+13229,26528,30478,40601
+10061,26529,30477,40532
+10923,26530,30476,40362
+16966,26531,30475,40653
+15827,26532,30474,40987
+15224,26533,30473,40921
+16191,26534,30472,40662
+13687,26535,30471,40701
+13662,26536,30470,40811
+13694,26537,30469,40294
+12929,26538,30468,40593
+17879,26539,30467,40382
+13913,26540,30466,40401
+16486,26541,30465,40546
+14170,26542,30464,40132
+15237,26543,30463,40284
+11747,26544,30462,40733
+10282,26545,30461,40156
+17783,26546,30460,40833
+10807,26547,30459,40161
+12977,26548,30458,40283
+15962,26549,30457,40430
+13143,26550,30456,40684
+16205,26551,30455,40519
+11336,26552,30454,40616
+17677,26553,30453,40890
+15795,26554,30452,40187
+16430,26555,30451,40179
+17734,26556,30450,40202
+18034,26557,30449,40416
+17419,26558,30448,40294
+14467,26559,30447,40891
+18174,26560,30446,40394
+12134,26561,30445,40542
+11233,26562,30444,40374
+16836,26563,30443,40824
+15331,26564,30442,40373
+10311,26565,30441,40957
+13492,26566,30440,40121
+18176,26567,30439,40413
+11598,26568,30438,40551
+16708,26569,30437,40107
+10805,26570,30436,40364
+17681,26571,30435,40701
+10636,26572,30434,40630
+14850,26573,30433,40507
+13125,26574,30432,40463
+14683,26575,30431,40864
+14531,26576,30430,40836
+11849,26577,30429,40662
+18080,26578,30428,40637
+15972,26579,30427,40166
+14343,26580,30426,40749
+14582,26581,30425,40916
+17224,26582,30424,40592
+11086,26583,30423,40129
+11949,26584,30422,40380
+14916,26585,30421,40323
+16248,26586,30420,40752
+12098,26587,30419,40783
+15497,26588,30418,40978
+13479,26589,30417,40759
+12821,26590,30416,40077
+16163,26591,30415,40910
+16214,26592,30414,40151
+15045,26593,30413,40535
+14390,26594,30412,40776
+10684,26595,30411,40934
+13074,26596,30410,40326
+16509,26597,30409,40870
+13987,26598,30408,40382
+11591,26599,30407,40018
+16002,26600,30406,40925
+10993,26601,30405,40079
+11522,26602,30404,40996
+16651,26603,30403,40094
+18096,26604,30402,40482
+16242,26605,30401,40672
+13716,26606,30400,40907
+15657,26607,30399,40234
+10394,26608,30398,40438
+11890,26609,30397,40175
+13975,26610,30396,40445
+16825,26611,30395,40733
+16130,26612,30394,40781
+12888,26613,30393,40559
+10801,26614,30392,40093
+10900,26615,30391,40418
+11101,26616,30390,40892
+14735,26617,30389,40956
+16972,26618,30388,40411
+17863,26619,30387,40686
+13883,26620,30386,40089
+10290,26621,30385,40762
+15292,26622,30384,40293
+17540,26623,30383,40383
+10678,26624,30382,40222
+17291,26625,30381,40801
+13216,26626,30380,40807
+13623,26627,30379,40486
+15487,26628,30378,40222
+12215,26629,30377,40059
+18093,26630,30376,40083
+12543,26631,30375,40789
+13845,26632,30374,40390
+11871,26633,30373,40988
+17064,26634,30372,40761
+14976,26635,30371,40050
+11007,26636,30370,40565
+15110,26637,30369,40473
+16995,26638,30368,40008
+11279,26639,30367,40946
+10121,26640,30366,40571
+12933,26641,30365,40321
+10512,26642,30364,40882
+13807,26643,30363,40358
+17361,26644,30362,40629
+11016,26645,30361,40495
+13618,26646,30360,40268
+17873,26647,30359,40120
+16034,26648,30358,40535
+10246,26649,30357,40613
+14016,26650,30356,40627
+15154,26651,30355,40782
+14846,26652,30354,40128
+10954,26653,30353,40201
+10370,26654,30352,40335
+16840,26655,30351,40422
+12928,26656,30350,40593
+17695,26657,30349,40731
+11687,26658,30348,40803
+12889,26659,30347,40504
+13928,26660,30346,40574
+11901,26661,30345,40237
+13453,26662,30344,40706
+13262,26663,30343,40087
+16245,26664,30342,40923
+14808,26665,30341,40329
+12849,26666,30340,40355
+15705,26667,30339,40751
+14858,26668,30338,40655
+12824,26669,30337,40830
+14650,26670,30336,40558
+17253,26671,30335,40592
+15581,26672,30334,40284
+15710,26673,30333,40963
+13825,26674,30332,40805
+15166,26675,30331,40521
+16095,26676,30330,40004
+17192,26677,30329,40418
+13205,26678,30328,40675
+11098,26679,30327,40661
+18101,26680,30326,40253
+14670,26681,30325,40894
+13763,26682,30324,40199
+17462,26683,30323,40817
+12247,26684,30322,40022
+12583,26685,30321,40916
+16153,26686,30320,40722
+15649,26687,30319,40411
+17605,26688,30318,40383
+13746,26689,30317,40173
+14966,26690,30316,40428
+16078,26691,30315,40892
+15067,26692,30314,40068
+12069,26693,30313,40762
+15734,26694,30312,40106
+16682,26695,30311,40878
+12788,26696,30310,40354
+16730,26697,30309,40694
+17325,26698,30308,40387
+16485,26699,30307,40815
+10834,26700,30306,40992
+13612,26701,30305,40032
+10668,26702,30304,40150
+16325,26703,30303,40756
+17017,26704,30302,40895
+17515,26705,30301,40013
+17076,26706,30300,40452
+11647,26707,30299,40558
+18136,26708,30298,40354
+17302,26709,30297,40489
+14181,26710,30296,40108
+17919,26711,30295,40184
+17388,26712,30294,40174
+15368,26713,30293,40543
+12641,26714,30292,40321
+12446,26715,30291,40925
+13368,26716,30290,40378
+14066,26717,30289,40628
+17355,26718,30288,40942
+14707,26719,30287,40526
+13467,26720,30286,40376
+11120,26721,30285,40343
+17135,26722,30284,40690
+18023,26723,30283,40706
+15772,26724,30282,40706
+12709,26725,30281,40540
+14685,26726,30280,40371
+13147,26727,30279,40988
+11432,26728,30278,40173
+14438,26729,30277,40708
+15372,26730,30276,40831
+16960,26731,30275,40881
+11907,26732,30274,40265
+12290,26733,30273,40245
+16117,26734,30272,40526
+10025,26735,30271,40526
+13722,26736,30270,40814
+15126,26737,30269,40040
+10099,26738,30268,40809
+13009,26739,30267,40277
+11211,26740,30266,40623
+16254,26741,30265,40869
+12833,26742,30264,40493
+11419,26743,30263,40032
+16399,26744,30262,40923
+13239,26745,30261,40952
+15776,26746,30260,40784
+14769,26747,30259,40036
+10523,26748,30258,40390
+15513,26749,30257,40479
+12111,26750,30256,40723
+12606,26751,30255,40888
+11354,26752,30254,40940
+10035,26753,30253,40834
+17844,26754,30252,40534
+15823,26755,30251,40987
+17921,26756,30250,40367
+13442,26757,30249,40121
+12910,26758,30248,40886
+14536,26759,30247,40328
+17868,26760,30246,40515
+10122,26761,30245,40225
+17932,26762,30244,40401
+14644,26763,30243,40637
+13163,26764,30242,40784
+12243,26765,30241,40815
+14714,26766,30240,40847
+17146,26767,30239,40271
+14483,26768,30238,40191
+15670,26769,30237,40194
+10730,26770,30236,40059
+15608,26771,30235,40063
+13583,26772,30234,40812
+13065,26773,30233,40500
+15894,26774,30232,40064
+14313,26775,30231,40015
+13021,26776,30230,40770
+12094,26777,30229,40706
+14855,26778,30228,40101
+10746,26779,30227,40349
+16689,26780,30226,40945
+14949,26781,30225,40863
+16797,26782,30224,40345
+11962,26783,30223,40959
+14379,26784,30222,40323
+10911,26785,30221,40404
+15183,26786,30220,40086
+17650,26787,30219,40205
+11326,26788,30218,40066
+14396,26789,30217,40891
+13646,26790,30216,40305
+14256,26791,30215,40266
+11136,26792,30214,40012
+14346,26793,30213,40094
+11300,26794,30212,40120
+11416,26795,30211,40235
+10170,26796,30210,40943
+17454,26797,30209,40004
+13945,26798,30208,40626
+13314,26799,30207,40029
+10214,26800,30206,40156
+17133,26801,30205,40108
+17847,26802,30204,40234
+16519,26803,30203,40962
+16283,26804,30202,40207
+12574,26805,30201,40824
+14513,26806,30200,40844
+17188,26807,30199,40820
+15386,26808,30198,40236
+11947,26809,30197,40907
+10322,26810,30196,40886
+17896,26811,30195,40152
+16182,26812,30194,40116
+10054,26813,30193,40843
+17839,26814,30192,40736
+11212,26815,30191,40229
+15320,26816,30190,40918
+16627,26817,30189,40927
+10831,26818,30188,40076
+12189,26819,30187,40570
+12580,26820,30186,40405
+17807,26821,30185,40585
+12242,26822,30184,40094
+16442,26823,30183,40850
+14130,26824,30182,40200
+16287,26825,30181,40476
+17464,26826,30180,40460
+12067,26827,30179,40731
+15343,26828,30178,40860
+11292,26829,30177,40766
+10553,26830,30176,40166
+11945,26831,30175,40762
+11193,26832,30174,40532
+11859,26833,30173,40535
+11023,26834,30172,40494
+12966,26835,30171,40264
+15534,26836,30170,40066
+11365,26837,30169,40624
+13016,26838,30168,40345
+13797,26839,30167,40640
+12930,26840,30166,40954
+15886,26841,30165,40549
+17819,26842,30164,40564
+15967,26843,30163,40830
+15357,26844,30162,40797
+14647,26845,30161,40915
+12875,26846,30160,40664
+11732,26847,30159,40080
+18037,26848,30158,40211
+11903,26849,30157,40593
+16403,26850,30156,40550
+15944,26851,30155,40353
+15050,26852,30154,40351
+15869,26853,30153,40651
+16123,26854,30152,40435
+12593,26855,30151,40241
+13895,26856,30150,40191
+15874,26857,30149,40834
+15720,26858,30148,40205
+15977,26859,30147,40029
+15098,26860,30146,40481
+16196,26861,30145,40660
+15380,26862,30144,40619
+11272,26863,30143,40157
+14904,26864,30142,40767
+13199,26865,30141,40770
+13126,26866,30140,40544
+11873,26867,30139,40816
+15397,26868,30138,40371
+17864,26869,30137,40084
+16921,26870,30136,40767
+14927,26871,30135,40761
+16211,26872,30134,40795
+16159,26873,30133,40663
+12687,26874,30132,40520
+10664,26875,30131,40149
+11538,26876,30130,40205
+12340,26877,30129,40791
+13456,26878,30128,40954
+16737,26879,30127,40946
+18155,26880,30126,40988
+14145,26881,30125,40684
+10570,26882,30124,40526
+17029,26883,30123,40665
+14077,26884,30122,40753
+12774,26885,30121,40649
+16024,26886,30120,40834
+14507,26887,30119,40109
+14864,26888,30118,40554
+12135,26889,30117,40432
+13289,26890,30116,40763
+14977,26891,30115,40850
+10821,26892,30114,40877
+11670,26893,30113,40798
+11924,26894,30112,40249
+14501,26895,30111,40518
+16986,26896,30110,40309
+15645,26897,30109,40173
+12313,26898,30108,40198
+10980,26899,30107,40074
+14277,26900,30106,40340
+14086,26901,30105,40700
+15901,26902,30104,40983
+13589,26903,30103,40481
+17910,26904,30102,40263
+13134,26905,30101,40921
+10433,26906,30100,40625
+11808,26907,30099,40503
+13724,26908,30098,40453
+12477,26909,30097,40039
+14974,26910,30096,40908
+15235,26911,30095,40002
+14057,26912,30094,40159
+17646,26913,30093,40303
+10892,26914,30092,40419
+14430,26915,30091,40317
+13430,26916,30090,40057
+13005,26917,30089,40027
+12492,26918,30088,40906
+17048,26919,30087,40600
+16197,26920,30086,40511
+14454,26921,30085,40742
+10458,26922,30084,40038
+11551,26923,30083,40552
+13160,26924,30082,40632
+11885,26925,30081,40885
+12652,26926,30080,40368
+11002,26927,30079,40693
+12167,26928,30078,40118
+15326,26929,30077,40064
+16749,26930,30076,40810
+13261,26931,30075,40894
+14033,26932,30074,40356
+14155,26933,30073,40809
+12267,26934,30072,40559
+14588,26935,30071,40406
+17398,26936,30070,40094
+13858,26937,30069,40203
+15794,26938,30068,40918
+18116,26939,30067,40962
+16154,26940,30066,40276
+14015,26941,30065,40395
+12685,26942,30064,40296
+10623,26943,30063,40608
+13135,26944,30062,40270
+15310,26945,30061,40266
+14719,26946,30060,40037
+15925,26947,30059,40706
+17761,26948,30058,40241
+16049,26949,30057,40105
+14204,26950,30056,40507
+14051,26951,30055,40496
+17651,26952,30054,40056
+17988,26953,30053,40172
+12452,26954,30052,40382
+12988,26955,30051,40521
+13598,26956,30050,40619
+17660,26957,30049,40687
+11790,26958,30048,40424
+18090,26959,30047,40378
+13471,26960,30046,40275
+11399,26961,30045,40715
+15445,26962,30044,40122
+14784,26963,30043,40404
+15741,26964,30042,40707
+15885,26965,30041,40321
+18182,26966,30040,40105
+12310,26967,30039,40546
+10962,26968,30038,40824
+16472,26969,30037,40781
+16435,26970,30036,40961
+17117,26971,30035,40696
+10381,26972,30034,40970
+10103,26973,30033,40228
+17292,26974,30032,40893
+13328,26975,30031,40365
+15662,26976,30030,40672
+13863,26977,30029,40774
+10382,26978,30028,40372
+15125,26979,30027,40857
+16523,26980,30026,40111
+16499,26981,30025,40324
+10974,26982,30024,40293
+11688,26983,30023,40063
+11108,26984,30022,40211
+17541,26985,30021,40298
+16240,26986,30020,40940
+13849,26987,30019,40460
+16288,26988,30018,40148
+10349,26989,30017,40661
+11308,26990,30016,40410
+14797,26991,30015,40629
+18185,26992,30014,40146
+17444,26993,30013,40774
+11525,26994,30012,40520
+15860,26995,30011,40732
+10866,26996,30010,40529
+14696,26997,30009,40894
+15299,26998,30008,40635
+11978,26999,30007,40548
+15788,27000,30006,40565
+13426,27001,30005,40599
+14310,27002,30004,40337
+16765,27003,30003,40904
+11579,27004,30002,40702
+17078,27005,30001,40672
+14006,27006,30000,40811
+11077,27007,31000,40079
+10305,27008,30999,40853
+15259,27009,30998,40417
+10175,27010,30997,40065
+17178,27011,30996,40898
+16199,27012,30995,40357
+15266,27013,30994,40698
+16231,27014,30993,40728
+10351,27015,30992,40444
+10858,27016,30991,40674
+12251,27017,30990,40768
+17153,27018,30989,40718
+13625,27019,30988,40064
+12845,27020,30987,40685
+15875,27021,30986,40432
+16234,27022,30985,40772
+10327,27023,30984,40047
+15472,27024,30983,40285
+12099,27025,30982,40138
+16075,27026,30981,40299
+10691,27027,30980,40943
+11493,27028,30979,40478
+12005,27029,30978,40726
+14064,27030,30977,40687
+10772,27031,30976,40500
+14484,27032,30975,40694
+13666,27033,30974,40393
+11329,27034,30973,40535
+13856,27035,30972,40354
+11926,27036,30971,40627
+16891,27037,30970,40921
+11660,27038,30969,40050
+11148,27039,30968,40408
+17671,27040,30967,40029
+12474,27041,30966,40212
+11917,27042,30965,40186
+10557,27043,30964,40806
+17913,27044,30963,40970
+11447,27045,30962,40597
+14515,27046,30961,40764
+11060,27047,30960,40215
+17397,27048,30959,40308
+14022,27049,30958,40359
+12106,27050,30957,40166
+10396,27051,30956,40440
+18103,27052,30955,40060
+15669,27053,30954,40999
+15356,27054,30953,40408
+14485,27055,30952,40646
+15291,27056,30951,40123
+10431,27057,30950,40437
+14562,27058,30949,40147
+16219,27059,30948,40587
+11624,27060,30947,40259
+12995,27061,30946,40147
+16104,27062,30945,40100
+10133,27063,30944,40402
+14139,27064,30943,40561
+16632,27065,30942,40148
+16895,27066,30941,40625
+12392,27067,30940,40494
+15606,27068,30939,40958
+12472,27069,30938,40534
+13591,27070,30937,40002
+11323,27071,30936,40238
+12479,27072,30935,40649
+17300,27073,30934,40008
+16941,27074,30933,40121
+14510,27075,30932,40763
+14546,27076,30931,40582
+13647,27077,30930,40372
+14362,27078,30929,40786
+14479,27079,30928,40016
+10020,27080,30927,40849
+11662,27081,30926,40997
+17229,27082,30925,40715
+15803,27083,30924,40833
+16187,27084,30923,40475
+12716,27085,30922,40746
+10629,27086,30921,40988
+14731,27087,30920,40980
+11682,27088,30919,40284
+18189,27089,30918,40793
+16320,27090,30917,40592
+10271,27091,30916,40578
+15433,27092,30915,40598
+17126,27093,30914,40756
+14909,27094,30913,40122
+11325,27095,30912,40438
+11828,27096,30911,40952
+11641,27097,30910,40983
+10711,27098,30909,40560
+12979,27099,30908,40460
+13580,27100,30907,40320
+12860,27101,30906,40903
+15883,27102,30905,40939
+12408,27103,30904,40507
+13066,27104,30903,40919
+13049,27105,30902,40580
+12183,27106,30901,40765
+14870,27107,30900,40139
+16210,27108,30899,40601
+10769,27109,30898,40607
+14831,27110,30897,40001
+17331,27111,30896,40631
+11022,27112,30895,40986
+14612,27113,30894,40700
+11345,27114,30893,40829
+15785,27115,30892,40340
+17544,27116,30891,40777
+13164,27117,30890,40473
+11935,27118,30889,40130
+12635,27119,30888,40536
+13252,27120,30887,40808
+17089,27121,30886,40828
+16224,27122,30885,40658
+16833,27123,30884,40803
+15611,27124,30883,40065
+10107,27125,30882,40425
+10110,27126,30881,40257
+10627,27127,30880,40976
+14574,27128,30879,40858
+10999,27129,30878,40489
+14965,27130,30877,40085
+10743,27131,30876,40828
+15215,27132,30875,40946
+12858,27133,30874,40388
+16745,27134,30873,40934
+15592,27135,30872,40110
+12616,27136,30871,40377
+12968,27137,30870,40103
+10259,27138,30869,40693
+13564,27139,30868,40557
+15283,27140,30867,40427
+17119,27141,30866,40574
+12640,27142,30865,40912
+11146,27143,30864,40685
+15632,27144,30863,40181
+17892,27145,30862,40123
+17112,27146,30861,40076
+13142,27147,30860,40465
+14194,27148,30859,40638
+12660,27149,30858,40820
+15421,27150,30857,40249
+11427,27151,30856,40514
+13362,27152,30855,40343
+10649,27153,30854,40248
+15653,27154,30853,40730
+10248,27155,30852,40878
+15637,27156,30851,40923
+13372,27157,30850,40858
+13609,27158,30849,40490
+14297,27159,30848,40017
+10408,27160,30847,40631
+16865,27161,30846,40036
+15452,27162,30845,40559
+11355,27163,30844,40575
+11727,27164,30843,40898
+10013,27165,30842,40605
+15270,27166,30841,40873
+16207,27167,30840,40191
+14672,27168,30839,40886
+16868,27169,30838,40947
+16013,27170,30837,40835
+12458,27171,30836,40105
+15358,27172,30835,40468
+16020,27173,30834,40713
+17364,27174,30833,40297
+10578,27175,30832,40952
+14107,27176,30831,40283
+17780,27177,30830,40970
+15955,27178,30829,40352
+14136,27179,30828,40649
+16909,27180,30827,40418
+12688,27181,30826,40883
+12726,27182,30825,40201
+16867,27183,30824,40274
+14242,27184,30823,40137
+13998,27185,30822,40028
+16143,27186,30821,40086
+16396,27187,30820,40939
+16566,27188,30819,40308
+15811,27189,30818,40583
+17680,27190,30817,40419
+13684,27191,30816,40607
+17723,27192,30815,40527
+12148,27193,30814,40743
+13964,27194,30813,40989
+14898,27195,30812,40468
+12943,27196,30811,40394
+13718,27197,30810,40090
+11128,27198,30809,40150
+13720,27199,30808,40934
+10761,27200,30807,40845
+15060,27201,30806,40503
+16375,27202,30805,40462
+12829,27203,30804,40991
+11693,27204,30803,40477
+17881,27205,30802,40856
+11031,27206,30801,40443
+16208,27207,30800,40642
+10826,27208,30799,40821
+17210,27209,30798,40665
+16526,27210,30797,40778
+15396,27211,30796,40243
+10734,27212,30795,40053
+16333,27213,30794,40814
+11485,27214,30793,40723
+10700,27215,30792,40528
+13620,27216,30791,40406
+17288,27217,30790,40057
+10090,27218,30789,40648
+17639,27219,30788,40866
+14583,27220,30787,40668
+17492,27221,30786,40929
+10469,27222,30785,40606
+12520,27223,30784,40228
+12533,27224,30783,40570
+16957,27225,30782,40555
+14399,27226,30781,40681
+14167,27227,30780,40498
+13204,27228,30779,40201
+12586,27229,30778,40708
+10820,27230,30777,40316
+13921,27231,30776,40971
+13136,27232,30775,40608
+15486,27233,30774,40156
+13805,27234,30773,40631
+14900,27235,30772,40056
+10496,27236,30771,40036
+14344,27237,30770,40884
+13782,27238,30769,40894
+16204,27239,30768,40714
+14626,27240,30767,40786
+13115,27241,30766,40344
+15220,27242,30765,40744
+16273,27243,30764,40270
+10102,27244,30763,40208
+13041,27245,30762,40854
+16125,27246,30761,40429
+13024,27247,30760,40908
+10686,27248,30759,40207
+12791,27249,30758,40965
+14587,27250,30757,40983
+18110,27251,30756,40252
+15889,27252,30755,40467
+12105,27253,30754,40190
+11805,27254,30753,40529
+12151,27255,30752,40327
+10878,27256,30751,40384
+16437,27257,30750,40926
+14373,27258,30749,40442
+11816,27259,30748,40099
+16532,27260,30747,40839
+12817,27261,30746,40545
+12949,27262,30745,40835
+11744,27263,30744,40703
+10002,27264,30743,40806
+13840,27265,30742,40344
+15880,27266,30741,40108
+15243,27267,30740,40418
+16700,27268,30739,40427
+17609,27269,30738,40828
+15213,27270,30737,40330
+16365,27271,30736,40392
+17265,27272,30735,40810
+16473,27273,30734,40296
+14306,27274,30733,40284
+16188,27275,30732,40636
+13449,27276,30731,40225
+14335,27277,30730,40866
+18100,27278,30729,40137
+13151,27279,30728,40687
+15996,27280,30727,40923
+16060,27281,30726,40889
+10226,27282,30725,40523
+10853,27283,30724,40447
+11725,27284,30723,40393
+14729,27285,30722,40728
+11107,27286,30721,40406
+10658,27287,30720,40880
+14676,27288,30719,40938
+12975,27289,30718,40945
+11619,27290,30717,40465
+15960,27291,30716,40102
+15602,27292,30715,40100
+14834,27293,30714,40443
+13133,27294,30713,40563
+12919,27295,30712,40824
+18047,27296,30711,40000
+10568,27297,30710,40652
+10937,27298,30709,40639
+17616,27299,30708,40814
+13104,27300,30707,40345
+14366,27301,30706,40943
+10213,27302,30705,40728
+15753,27303,30704,40068
+12972,27304,30703,40365
+15730,27305,30702,40127
+14178,27306,30701,40349
+15949,27307,30700,40948
+14004,27308,30699,40810
+13112,27309,30698,40785
+12783,27310,30697,40149
+13454,27311,30696,40223
+10106,27312,30695,40313
+16704,27313,30694,40070
+15247,27314,30693,40044
+15132,27315,30692,40765
+16791,27316,30691,40591
+16028,27317,30690,40366
+18032,27318,30689,40676
+10679,27319,30688,40047
+17842,27320,30687,40382
+16615,27321,30686,40437
+10628,27322,30685,40948
+11184,27323,30684,40075
+16120,27324,30683,40424
+11218,27325,30682,40009
+12150,27326,30681,40380
+16823,27327,30680,40880
+17105,27328,30679,40180
+14354,27329,30678,40753
+18183,27330,30677,40582
+17198,27331,30676,40957
+13533,27332,30675,40774
+15971,27333,30674,40399
+13552,27334,30673,40062
+10856,27335,30672,40174
+13403,27336,30671,40819
+12217,27337,30670,40735
+17054,27338,30669,40355
+15538,27339,30668,40607
+11745,27340,30667,40914
+13909,27341,30666,40037
+18154,27342,30665,40291
+11293,27343,30664,40158
+13495,27344,30663,40883
+14950,27345,30662,40898
+11113,27346,30661,40170
+15560,27347,30660,40884
+12360,27348,30659,40043
+11559,27349,30658,40368
+12839,27350,30657,40995
+11242,27351,30656,40316
+11922,27352,30655,40030
+10490,27353,30654,40793
+15681,27354,30653,40804
+13834,27355,30652,40163
+11013,27356,30651,40891
+13697,27357,30650,40139
+15664,27358,30649,40142
+14113,27359,30648,40910
+10163,27360,30647,40187
+13319,27361,30646,40558
+16718,27362,30645,40523
+13795,27363,30644,40806
+11194,27364,30643,40050
+12325,27365,30642,40171
+11496,27366,30641,40350
+16429,27367,30640,40966
+14475,27368,30639,40460
+14514,27369,30638,40998
+14611,27370,30637,40526
+15279,27371,30636,40747
+17147,27372,30635,40072
+15558,27373,30634,40805
+10610,27374,30633,40314
+14661,27375,30632,40627
+11656,27376,30631,40048
+12878,27377,30630,40879
+17293,27378,30629,40917
+13249,27379,30628,40046
+10872,27380,30627,40279
+10677,27381,30626,40494
+12750,27382,30625,40547
+12658,27383,30624,40280
+17201,27384,30623,40185
+12355,27385,30622,40420
+12804,27386,30621,40218
+12274,27387,30620,40836
+14070,27388,30619,40830
+12807,27389,30618,40393
+17001,27390,30617,40546
+11437,27391,30616,40177
+13571,27392,30615,40150
+17360,27393,30614,40451
+13162,27394,30613,40917
+17795,27395,30612,40809
+17299,27396,30611,40501
+15603,27397,30610,40499
+17917,27398,30609,40471
+17808,27399,30608,40981
+14295,27400,30607,40784
+10227,27401,30606,40814
+11797,27402,30605,40022
+17812,27403,30604,40265
+17606,27404,30603,40230
+15921,27405,30602,40921
+12884,27406,30601,40570
+15338,27407,30600,40390
+16087,27408,30599,40273
+16732,27409,30598,40964
+11286,27410,30597,40022
+16569,27411,30596,40226
+17196,27412,30595,40986
+11939,27413,30594,40672
+12519,27414,30593,40212
+10363,27415,30592,40249
+10601,27416,30591,40663
+15804,27417,30590,40640
+11332,27418,30589,40413
+11877,27419,30588,40375
+15652,27420,30587,40927
+11814,27421,30586,40635
+13376,27422,30585,40438
+18058,27423,30584,40699
+12196,27424,30583,40612
+16997,27425,30582,40855
+15849,27426,30581,40636
+14941,27427,30580,40238
+16149,27428,30579,40224
+10355,27429,30578,40311
+11605,27430,30577,40440
+14059,27431,30576,40881
+15740,27432,30575,40880
+12534,27433,30574,40670
+16025,27434,30573,40403
+15733,27435,30572,40474
+10250,27436,30571,40308
+14080,27437,30570,40666
+17992,27438,30569,40835
+17936,27439,30568,40815
+14551,27440,30567,40876
+16685,27441,30566,40985
+11564,27442,30565,40067
+17684,27443,30564,40392
+10895,27444,30563,40132
+15282,27445,30562,40766
+16760,27446,30561,40703
+14809,27447,30560,40617
+12572,27448,30559,40672
+16491,27449,30558,40660
+10378,27450,30557,40740
+11294,27451,30556,40685
+12407,27452,30555,40664
+15242,27453,30554,40105
+17142,27454,30553,40324
+15764,27455,30552,40036
+12738,27456,30551,40853
+10465,27457,30550,40349
+10709,27458,30549,40088
+15575,27459,30548,40253
+15351,27460,30547,40769
+17385,27461,30546,40560
+11311,27462,30545,40651
+16496,27463,30544,40444
+13536,27464,30543,40651
+16352,27465,30542,40614
+17456,27466,30541,40637
+10694,27467,30540,40709
+14120,27468,30539,40680
+11743,27469,30538,40886
+15278,27470,30537,40157
+14952,27471,30536,40886
+12668,27472,30535,40642
+11253,27473,30534,40054
+18081,27474,30533,40230
+10758,27475,30532,40728
+12132,27476,30531,40405
+15155,27477,30530,40424
+15474,27478,30529,40202
+12666,27479,30528,40534
+10811,27480,30527,40452
+11843,27481,30526,40352
+13633,27482,30525,40033
+10927,27483,30524,40245
+13747,27484,30523,40008
+16427,27485,30522,40772
+17239,27486,30521,40147
+14666,27487,30520,40106
+16432,27488,30519,40953
+15812,27489,30518,40050
+15655,27490,30517,40233
+10141,27491,30516,40822
+17087,27492,30515,40930
+11469,27493,30514,40764
+14243,27494,30513,40024
+15363,27495,30512,40285
+16493,27496,30511,40080
+12885,27497,30510,40487
+16594,27498,30509,40093
+11519,27499,30508,40809
+15466,27500,30507,40387
+10740,27501,30506,40806
+16270,27502,30505,40857
+14393,27503,30504,40011
+12491,27504,30503,40988
+15203,27505,30502,40283
+11092,27506,30501,40638
+17403,27507,30500,40547
+17189,27508,30499,40621
+11362,27509,30498,40431
+13337,27510,30497,40087
+15325,27511,30496,40506
+11996,27512,30495,40571
+13588,27513,30494,40293
+13642,27514,30493,40347
+15914,27515,30492,40863
+15959,27516,30491,40559
+15482,27517,30490,40170
+10359,27518,30489,40211
+13924,27519,30488,40750
+14939,27520,30487,40120
+13428,27521,30486,40855
+17061,27522,30485,40583
+12287,27523,30484,40906
+10849,27524,30483,40305
+17326,27525,30482,40229
+10180,27526,30481,40773
+14994,27527,30480,40259
+15982,27528,30479,40761
+15239,27529,30478,40527
+11423,27530,30477,40905
+12034,27531,30476,40144
+12891,27532,30475,40790
+15036,27533,30474,40083
+16536,27534,30473,40901
+17452,27535,30472,40743
+17596,27536,30471,40208
+14675,27537,30470,40009
+11930,27538,30469,40938
+18095,27539,30468,40039
+18035,27540,30467,40580
+16726,27541,30466,40674
+14404,27542,30465,40629
+14517,27543,30464,40892
+16459,27544,30463,40206
+12585,27545,30462,40417
+12266,27546,30461,40307
+15103,27547,30460,40747
+16307,27548,30459,40773
+17668,27549,30458,40045
+16559,27550,30457,40548
+10053,27551,30456,40313
+11516,27552,30455,40282
+13068,27553,30454,40765
+13853,27554,30453,40339
+10109,27555,30452,40551
+15517,27556,30451,40968
+12644,27557,30450,40649
+11414,27558,30449,40331
+17521,27559,30448,40425
+15779,27560,30447,40205
+16814,27561,30446,40375
+11581,27562,30445,40259
+12081,27563,30444,40377
+11415,27564,30443,40135
+14602,27565,30442,40844
+10263,27566,30441,40027
+14642,27567,30440,40632
+11807,27568,30439,40176
+10930,27569,30438,40630
+16421,27570,30437,40724
+12149,27571,30436,40478
+14112,27572,30435,40946
+11180,27573,30434,40560
+11305,27574,30433,40892
+12680,27575,30432,40028
+13894,27576,30431,40147
+10904,27577,30430,40590
+14117,27578,30429,40319
+14478,27579,30428,40433
+14038,27580,30427,40746
+10914,27581,30426,40660
+11158,27582,30425,40310
+11374,27583,30424,40489
+12881,27584,30423,40703
+11050,27585,30422,40826
+11149,27586,30421,40915
+17470,27587,30420,40255
+17725,27588,30419,40446
+11379,27589,30418,40266
+11595,27590,30417,40291
+14273,27591,30416,40260
+11348,27592,30415,40140
+12358,27593,30414,40152
+14492,27594,30413,40930
+15518,27595,30412,40613
+10609,27596,30411,40856
+11298,27597,30410,40401
+11392,27598,30409,40234
+17281,27599,30408,40312
+13610,27600,30407,40595
+17399,27601,30406,40012
+13400,27602,30405,40388
+16670,27603,30404,40622
+14619,27604,30403,40336
+15932,27605,30402,40979
+14674,27606,30401,40428
+15348,27607,30400,40147
+17833,27608,30399,40316
+17080,27609,30398,40138
+18053,27610,30397,40984
+12830,27611,30396,40921
+10939,27612,30395,40742
+10531,27613,30394,40911
+10424,27614,30393,40314
+16750,27615,30392,40088
+14973,27616,30391,40992
+11700,27617,30390,40696
+10661,27618,30389,40905
+17450,27619,30388,40177
+17180,27620,30387,40619
+10331,27621,30386,40295
+12524,27622,30385,40259
+16753,27623,30384,40453
+14710,27624,30383,40535
+10543,27625,30382,40745
+15665,27626,30381,40982
+12347,27627,30380,40282
+12421,27628,30379,40608
+13431,27629,30378,40578
+11460,27630,30377,40194
+18044,27631,30376,40822
+16441,27632,30375,40633
+13587,27633,30374,40794
+13754,27634,30373,40424
+17221,27635,30372,40857
+13940,27636,30371,40317
+10919,27637,30370,40482
+16794,27638,30369,40773
+17389,27639,30368,40023
+17633,27640,30367,40592
+14627,27641,30366,40783
+10493,27642,30365,40303
+18046,27643,30364,40595
+15796,27644,30363,40210
+10174,27645,30362,40110
+15938,27646,30361,40902
+15484,27647,30360,40237
+17479,27648,30359,40835
+13003,27649,30358,40462
+11450,27650,30357,40780
+13282,27651,30356,40084
+10715,27652,30355,40824
+12254,27653,30354,40848
+11409,27654,30353,40517
+16515,27655,30352,40596
+16821,27656,30351,40236
+11954,27657,30350,40954
+12040,27658,30349,40839
+10203,27659,30348,40817
+11280,27660,30347,40983
+11043,27661,30346,40303
+11558,27662,30345,40219
+12384,27663,30344,40034
+15522,27664,30343,40410
+16233,27665,30342,40728
+15153,27666,30341,40593
+11537,27667,30340,40478
+12722,27668,30339,40024
+10341,27669,30338,40069
+14589,27670,30337,40783
+10916,27671,30336,40185
+16012,27672,30335,40799
+17703,27673,30334,40700
+17802,27674,30333,40443
+10787,27675,30332,40480
+13778,27676,30331,40244
+15844,27677,30330,40033
+15461,27678,30329,40372
+14825,27679,30328,40376
+12990,27680,30327,40242
+10788,27681,30326,40916
+15027,27682,30325,40380
+13785,27683,30324,40986
+10741,27684,30323,40923
+11130,27685,30322,40950
+10755,27686,30321,40048
+17437,27687,30320,40850
+15890,27688,30319,40412
+17199,27689,30318,40547
+16798,27690,30317,40565
+12077,27691,30316,40123
+13638,27692,30315,40837
+14119,27693,30314,40296
+17659,27694,30313,40437
+10569,27695,30312,40228
+11644,27696,30311,40638
+12206,27697,30310,40054
+16249,27698,30309,40512
+12547,27699,30308,40189
+16484,27700,30307,40339
+10935,27701,30306,40197
+16054,27702,30305,40659
+10532,27703,30304,40218
+12695,27704,30303,40361
+12228,27705,30302,40441
+11714,27706,30301,40180
+10333,27707,30300,40926
+11752,27708,30299,40966
+17283,27709,30298,40768
+12402,27710,30297,40658
+14538,27711,30296,40049
+16863,27712,30295,40748
+18150,27713,30294,40314
+13474,27714,30293,40033
+18142,27715,30292,40603
+10379,27716,30291,40645
+14340,27717,30290,40519
+17494,27718,30289,40780
+10436,27719,30288,40110
+14807,27720,30287,40664
+17948,27721,30286,40100
+16192,27722,30285,40297
+14290,27723,30284,40606
+14816,27724,30283,40770
+11620,27725,30282,40245
+13155,27726,30281,40936
+16761,27727,30280,40212
+11008,27728,30279,40046
+10767,27729,30278,40660
+15816,27730,30277,40043
+17859,27731,30276,40326
+11584,27732,30275,40621
+17072,27733,30274,40088
+15633,27734,30273,40639
+14889,27735,30272,40201
+10211,27736,30271,40574
+15892,27737,30270,40358
+12960,27738,30269,40193
+13460,27739,30268,40103
+17062,27740,30267,40567
+14662,27741,30266,40940
+13915,27742,30265,40178
+15339,27743,30264,40312
+17595,27744,30263,40889
+14706,27745,30262,40120
+16377,27746,30261,40571
+10326,27747,30260,40229
+15997,27748,30259,40775
+13195,27749,30258,40676
+15404,27750,30257,40033
+10877,27751,30256,40847
+14312,27752,30255,40343
+16971,27753,30254,40117
+12102,27754,30253,40457
+16364,27755,30252,40532
+15818,27756,30251,40223
+14933,27757,30250,40023
+16946,27758,30249,40820
+16433,27759,30248,40018
+15186,27760,30247,40544
+12258,27761,30246,40098
+14972,27762,30245,40252
+11779,27763,30244,40530
+15293,27764,30243,40063
+10461,27765,30242,40920
+13488,27766,30241,40669
+10402,27767,30240,40598
+14258,27768,30239,40270
+15958,27769,30238,40996
+17345,27770,30237,40629
+14339,27771,30236,40098
+17953,27772,30235,40159
+14821,27773,30234,40139
+16524,27774,30233,40803
+13168,27775,30232,40350
+13630,27776,30231,40825
+10442,27777,30230,40886
+16418,27778,30229,40739
+16178,27779,30228,40921
+10616,27780,30227,40256
+17227,27781,30226,40790
+15496,27782,30225,40287
+16423,27783,30224,40159
+13685,27784,30223,40137
+10672,27785,30222,40582
+13422,27786,30221,40015
+15810,27787,30220,40988
+17394,27788,30219,40946
+17121,27789,30218,40573
+18179,27790,30217,40178
+10803,27791,30216,40143
+17592,27792,30215,40366
+14725,27793,30214,40752
+12825,27794,30213,40026
+17709,27795,30212,40031
+14595,27796,30211,40655
+10017,27797,30210,40547
+13868,27798,30209,40508
+16641,27799,30208,40761
+11054,27800,30207,40618
+16801,27801,30206,40097
+15679,27802,30205,40157
+16514,27803,30204,40047
+17507,27804,30203,40294
+10794,27805,30202,40567
+11429,27806,30201,40821
+10247,27807,30200,40409
+12766,27808,30199,40988
+12552,27809,30198,40581
+17165,27810,30197,40511
+16811,27811,30196,40592
+12429,27812,30195,40264
+17834,27813,30194,40308
+15255,27814,30193,40159
+12140,27815,30192,40317
+12560,27816,30191,40622
+13184,27817,30190,40480
+11845,27818,30189,40114
+12066,27819,30188,40246
+12192,27820,30187,40517
+11809,27821,30186,40766
+13710,27822,30185,40313
+17313,27823,30184,40473
+16477,27824,30183,40167
+10652,27825,30182,40735
+11609,27826,30181,40557
+11213,27827,30180,40319
+12521,27828,30179,40958
+11201,27829,30178,40683
+16667,27830,30177,40276
+14872,27831,30176,40642
+10196,27832,30175,40424
+16695,27833,30174,40471
+14271,27834,30173,40882
+11405,27835,30172,40302
+11618,27836,30171,40044
+11368,27837,30170,40695
+16940,27838,30169,40680
+11588,27839,30168,40947
+11267,27840,30167,40586
+13958,27841,30166,40883
+13057,27842,30165,40145
+14885,27843,30164,40720
+14166,27844,30163,40599
+14203,27845,30162,40390
+15341,27846,30161,40991
+18145,27847,30160,40401
+11573,27848,30159,40614
+15643,27849,30158,40353
+10049,27850,30157,40266
+11707,27851,30156,40962
+10960,27852,30155,40871
+13543,27853,30154,40412
+18004,27854,30153,40128
+13182,27855,30152,40242
+15641,27856,30151,40987
+17589,27857,30150,40638
+15924,27858,30149,40085
+17732,27859,30148,40081
+13473,27860,30147,40301
+10839,27861,30146,40983
+16687,27862,30145,40741
+16626,27863,30144,40881
+10728,27864,30143,40242
+13081,27865,30142,40524
+15926,27866,30141,40116
+17764,27867,30140,40383
+13726,27868,30139,40289
+13699,27869,30138,40971
+16777,27870,30137,40589
+16776,27871,30136,40627
+15596,27872,30135,40372
+14812,27873,30134,40098
+13949,27874,30133,40615
+11851,27875,30132,40627
+16590,27876,30131,40061
+12822,27877,30130,40452
+16535,27878,30129,40953
+18149,27879,30128,40084
+10845,27880,30127,40489
+11974,27881,30126,40003
+15537,27882,30125,40455
+10647,27883,30124,40825
+13448,27884,30123,40512
+16479,27885,30122,40230
+12406,27886,30121,40945
+12719,27887,30120,40297
+13544,27888,30119,40826
+15973,27889,30118,40788
+16431,27890,30117,40907
+10988,27891,30116,40804
+15026,27892,30115,40666
+16625,27893,30114,40047
+17246,27894,30113,40575
+13912,27895,30112,40990
+16978,27896,30111,40311
+10592,27897,30110,40250
+14106,27898,30109,40052
+10286,27899,30108,40083
+10018,27900,30107,40030
+14357,27901,30106,40771
+15783,27902,30105,40618
+17963,27903,30104,40456
+16607,27904,30103,40024
+11858,27905,30102,40217
+15806,27906,30101,40843
+11465,27907,30100,40544
+14299,27908,30099,40426
+10516,27909,30098,40974
+13914,27910,30097,40507
+10321,27911,30096,40916
+14782,27912,30095,40527
+10861,27913,30094,40981
+14560,27914,30093,40153
+16852,27915,30092,40705
+15577,27916,30091,40112
+12114,27917,30090,40388
+17584,27918,30089,40411
+10517,27919,30088,40766
+10971,27920,30087,40780
+16752,27921,30086,40291
+11691,27922,30085,40589
+10126,27923,30084,40954
+10155,27924,30083,40423
+14039,27925,30082,40147
+17436,27926,30081,40986
+17502,27927,30080,40210
+15782,27928,30079,40487
+17622,27929,30078,40584
+18059,27930,30077,40233
+12815,27931,30076,40776
+11302,27932,30075,40441
+13158,27933,30074,40871
+12819,27934,30073,40537
+11993,27935,30072,40148
+13803,27936,30071,40392
+14063,27937,30070,40656
+16014,27938,30069,40604
+12937,27939,30068,40527
+12964,27940,30067,40054
+16275,27941,30066,40332
+14969,27942,30065,40983
+11099,27943,30064,40107
+14926,27944,30063,40353
+17532,27945,30062,40647
+17232,27946,30061,40544
+12894,27947,30060,40892
+12060,27948,30059,40487
+10417,27949,30058,40655
+13169,27950,30057,40734
+12517,27951,30056,40505
+14200,27952,30055,40737
+10806,27953,30054,40698
+13001,27954,30053,40509
+10889,27955,30052,40874
+15096,27956,30051,40035
+16724,27957,30050,40291
+18079,27958,30049,40039
+13076,27959,30048,40627
+12044,27960,30047,40194
+15429,27961,30046,40622
+15174,27962,30045,40235
+13615,27963,30044,40112
+15864,27964,30043,40364
+13836,27965,30042,40519
+11774,27966,30041,40088
+14447,27967,30040,40487
+16072,27968,30039,40699
+16042,27969,30038,40903
+12245,27970,30037,40634
+16202,27971,30036,40056
+14832,27972,30035,40930
+13304,27973,30034,40318
+13415,27974,30033,40220
+14852,27975,30032,40927
+12049,27976,30031,40637
+12349,27977,30030,40953
+17247,27978,30029,40772
+10258,27979,30028,40528
+18123,27980,30027,40485
+13475,27981,30026,40461
+14883,27982,30025,40123
+16480,27983,30024,40994
+14261,27984,30023,40098
+14556,27985,30022,40929
+11524,27986,30021,40241
+14476,27987,30020,40309
+17373,27988,30019,40262
+17183,27989,30018,40072
+13871,27990,30017,40108
+17286,27991,30016,40099
+11014,27992,30015,40104
+16888,27993,30014,40100
+17136,27994,30013,40017
+13060,27995,30012,40498
+17194,27996,30011,40294
+14503,27997,30010,40900
+16980,27998,30009,40297
+18094,27999,30008,40215
+10533,28000,30007,40961
+14561,28001,30006,40318
+15805,28002,30005,40013
+11603,28003,30004,40956
+16970,28004,30003,40148
+12301,28005,30002,40149
+14567,28006,30001,40018
+16222,28007,30000,40962
+18010,28008,31000,40885
+16922,28009,30999,40915
+14712,28010,30998,40286
+10525,28011,30997,40855
+15502,28012,30996,40786
+14705,28013,30995,40175
+15360,28014,30994,40840
+13477,28015,30993,40497
+14983,28016,30992,40293
+13332,28017,30991,40825
+15476,28018,30990,40867
+16734,28019,30989,40073
+16379,28020,30988,40104
+12596,28021,30987,40016
+13990,28022,30986,40787
+16065,28023,30985,40305
+14282,28024,30984,40097
+16447,28025,30983,40082
+10071,28026,30982,40782
+13412,28027,30981,40909
+11717,28028,30980,40231
+16981,28029,30979,40564
+17763,28030,30978,40275
+17488,28031,30977,40413
+11686,28032,30976,40722
+17933,28033,30975,40661
+13547,28034,30974,40742
+13698,28035,30973,40633
+14060,28036,30972,40430
+15492,28037,30971,40271
+16799,28038,30970,40554
+13023,28039,30969,40339
+10702,28040,30968,40256
+11459,28041,30967,40278
+12899,28042,30966,40668
+11103,28043,30965,40176
+15312,28044,30964,40207
+10511,28045,30963,40990
+15933,28046,30962,40879
+12337,28047,30961,40599
+13744,28048,30960,40572
+12145,28049,30959,40072
+17579,28050,30958,40946
+14452,28051,30957,40399
+16669,28052,30956,40770
+13563,28053,30955,40909
+10220,28054,30954,40403
+10607,28055,30953,40085
+17410,28056,30952,40710
+14127,28057,30951,40025
+17114,28058,30950,40791
+14778,28059,30949,40963
+17230,28060,30948,40551
+16567,28061,30947,40369
+10139,28062,30946,40646
+11024,28063,30945,40138
+12743,28064,30944,40590
+12780,28065,30943,40477
+13878,28066,30942,40355
+10130,28067,30941,40768
+17719,28068,30940,40839
+12951,28069,30939,40030
+11919,28070,30938,40002
+16457,28071,30937,40210
+13841,28072,30936,40118
+18137,28073,30935,40160
+11645,28074,30934,40400
+13648,28075,30933,40979
+10344,28076,30932,40323
+10818,28077,30931,40073
+16557,28078,30930,40430
+10199,28079,30929,40482
+10459,28080,30928,40806
+10501,28081,30927,40047
+12388,28082,30926,40294
+10565,28083,30925,40136
+14865,28084,30924,40146
+12239,28085,30923,40206
+13590,28086,30922,40804
+15034,28087,30921,40671
+12326,28088,30920,40141
+11318,28089,30919,40583
+15365,28090,30918,40405
+16164,28091,30917,40732
+14918,28092,30916,40030
+17421,28093,30915,40842
+10144,28094,30914,40234
+11639,28095,30913,40452
+15635,28096,30912,40150
+12723,28097,30911,40628
+10703,28098,30910,40622
+10756,28099,30909,40730
+15079,28100,30908,40415
+11614,28101,30907,40761
+12525,28102,30906,40131
+15640,28103,30905,40103
+14201,28104,30904,40047
+10836,28105,30903,40787
+12969,28106,30902,40513
+14999,28107,30901,40432
+12273,28108,30900,40413
+10050,28109,30899,40576
+12294,28110,30898,40735
+12505,28111,30897,40532
+10947,28112,30896,40746
+11266,28113,30895,40975
+14259,28114,30894,40375
+17339,28115,30893,40211
+13641,28116,30892,40650
+10499,28117,30891,40939
+17314,28118,30890,40702
+11056,28119,30889,40607
+15244,28120,30888,40360
+13241,28121,30887,40150
+18084,28122,30886,40488
+17041,28123,30885,40806
+12568,28124,30884,40596
+11795,28125,30883,40999
+12619,28126,30882,40312
+16629,28127,30881,40893
+12469,28128,30880,40376
+16118,28129,30879,40916
+13398,28130,30878,40038
+12955,28131,30877,40790
+12848,28132,30876,40718
+13389,28133,30875,40280
+18019,28134,30874,40806
+17832,28135,30873,40227
+16331,28136,30872,40315
+16586,28137,30871,40125
+17735,28138,30870,40583
+12712,28139,30869,40920
+12717,28140,30868,40151
+16780,28141,30867,40019
+10830,28142,30866,40841
+12122,28143,30865,40604
+14114,28144,30864,40602
+14079,28145,30863,40775
+14239,28146,30862,40069
+11541,28147,30861,40064
+14955,28148,30860,40112
+17473,28149,30859,40364
+13366,28150,30858,40041
+13534,28151,30857,40693
+17636,28152,30856,40638
+16436,28153,30855,40541
+16443,28154,30854,40153
+16097,28155,30853,40586
+17721,28156,30852,40122
+11321,28157,30851,40368
+11821,28158,30850,40462
+17738,28159,30849,40436
+13465,28160,30848,40318
+16767,28161,30847,40196
+11104,28162,30846,40586
+10903,28163,30845,40364
+14202,28164,30844,40932
+16067,28165,30843,40043
+11091,28166,30842,40346
+13111,28167,30841,40052
+14508,28168,30840,40659
+11078,28169,30839,40032
+11322,28170,30838,40506
+15851,28171,30837,40361
+11612,28172,30836,40664
+17801,28173,30835,40418
+12006,28174,30834,40460
+10777,28175,30833,40042
+18024,28176,30832,40822
+13693,28177,30831,40751
+17931,28178,30830,40129
+16132,28179,30829,40231
+14465,28180,30828,40454
+16158,28181,30827,40873
+12379,28182,30826,40349
+16009,28183,30825,40842
+15081,28184,30824,40943
+10537,28185,30823,40004
+10068,28186,30822,40343
+10016,28187,30821,40057
+11782,28188,30820,40998
+14792,28189,30819,40741
+17050,28190,30818,40950
+10450,28191,30817,40433
diff --git a/regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column.out b/regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column.out
new file mode 100644
index 00000000000000..34b36e765f4728
--- /dev/null
+++ b/regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select1 --
+12345	23083	\N	30920	40410
+
+-- !select2 --
+17320	24209	\N	30795	40000
+
+-- !select3 --
+59832	36673	\N	30343	40299
+
+-- !select1 --
+12345	23083	\N	30920	40410
+
+-- !select2 --
+17320	24209	\N	30795	40782
+
+-- !select3 --
+59832	36673	\N	30586	40739
+
diff --git a/regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column1.csv b/regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column1.csv
new file mode 100644
index 00000000000000..6cd6e2aec3c033
--- /dev/null
+++ b/regression-test/data/unique_with_mow_c_p0/test_schema_change_add_key_column1.csv
@@ -0,0 +1,20480 @@
+50019,20000,31000,40444
+57401,20001,30999,40084
+58347,20002,30998,40429
+52118,20003,30997,40438
+57056,20004,30996,40132
+54769,20005,30995,40174
+58735,20006,30994,40013
+56920,20007,30993,40432
+51358,20008,30992,40513
+59832,20009,30991,40589
+51219,20010,30990,40032
+52735,20011,30989,40174
+50975,20012,30988,40889
+57635,20013,30987,40742
+52952,20014,30986,40473
+57067,20015,30985,40642
+55892,20016,30984,40445
+57819,20017,30983,40389
+57041,20018,30982,40494
+52093,20019,30981,40636
+55585,20020,30980,40569
+51707,20021,30979,40994
+58147,20022,30978,40287
+58985,20023,30977,40001
+57405,20024,30976,40896
+57166,20025,30975,40774
+51045,20026,30974,40831
+56532,20027,30973,40335
+51375,20028,30972,40689
+58213,20029,30971,40372
+50128,20030,30970,40744
+51829,20031,30969,40736
+56546,20032,30968,40655
+57023,20033,30967,40218
+53808,20034,30966,40393
+53959,20035,30965,40260
+50497,20036,30964,40234
+56047,20037,30963,40823
+59841,20038,30962,40809
+56805,20039,30961,40177
+50235,20040,30960,40016
+59546,20041,30959,40940
+57795,20042,30958,40498
+57165,20043,30957,40516
+56783,20044,30956,40328
+51669,20045,30955,40981
+52646,20046,30954,40488
+53678,20047,30953,40743
+58209,20048,30952,40699
+51546,20049,30951,40775
+51550,20050,30950,40434
+57753,20051,30949,40843
+56019,20052,30948,40196
+51859,20053,30947,40513
+55791,20054,30946,40741
+59682,20055,30945,40364
+54508,20056,30944,40782
+55005,20057,30943,40661
+51233,20058,30942,40366
+59967,20059,30941,40690
+59428,20060,30940,40154
+57447,20061,30939,40165
+55453,20062,30938,40462
+56376,20063,30937,40940
+52197,20064,30936,40808
+54496,20065,30935,40072
+58909,20066,30934,40165
+50245,20067,30933,40484
+54427,20068,30932,40602
+57483,20069,30931,40588
+58207,20070,30930,40585
+52883,20071,30929,40199
+56740,20072,30928,40046
+58334,20073,30927,40595
+59923,20074,30926,40637
+56031,20075,30925,40564
+54033,20076,30924,40514
+56526,20077,30923,40712
+58504,20078,30922,40597
+58440,20079,30921,40362
+57203,20080,30920,40656
+55348,20081,30919,40673
+55186,20082,30918,40606
+56112,20083,30917,40508
+52686,20084,30916,40505
+52922,20085,30915,40148
+55862,20086,30914,40752
+55340,20087,30913,40022
+57672,20088,30912,40516
+51182,20089,30911,40947
+56060,20090,30910,40133
+54447,20091,30909,40054
+53210,20092,30908,40997
+55982,20093,30907,40042
+55324,20094,30906,40017
+53346,20095,30905,40381
+50157,20096,30904,40384
+57850,20097,30903,40365
+50545,20098,30902,40305
+58463,20099,30901,40510
+59342,20100,30900,40781
+51749,20101,30899,40925
+55752,20102,30898,40246
+52288,20103,30897,40300
+59531,20104,30896,40875
+50516,20105,30895,40393
+51921,20106,30894,40937
+58596,20107,30893,40837
+50243,20108,30892,40439
+55452,20109,30891,40623
+54044,20110,30890,40830
+57354,20111,30889,40218
+53466,20112,30888,40507
+52455,20113,30887,40754
+56861,20114,30886,40859
+59044,20115,30885,40566
+50546,20116,30884,40634
+57455,20117,30883,40242
+58969,20118,30882,40143
+52917,20119,30881,40621
+51649,20120,30880,40288
+54129,20121,30879,40561
+59622,20122,30878,40421
+55603,20123,30877,40308
+53105,20124,30876,40019
+59900,20125,30875,40290
+57214,20126,30874,40224
+59605,20127,30873,40437
+59867,20128,30872,40294
+52399,20129,30871,40666
+57388,20130,30870,40663
+52405,20131,30869,40705
+50091,20132,30868,40731
+55389,20133,30867,40017
+50844,20134,30866,40288
+52014,20135,30865,40101
+51663,20136,30864,40814
+57212,20137,30863,40111
+50735,20138,30862,40413
+51598,20139,30861,40792
+51271,20140,30860,40662
+53217,20141,30859,40025
+52324,20142,30858,40636
+58197,20143,30857,40561
+58262,20144,30856,40389
+51477,20145,30855,40539
+54352,20146,30854,40072
+55956,20147,30853,40946
+55887,20148,30852,40389
+58067,20149,30851,40068
+56375,20150,30850,40619
+51215,20151,30849,40417
+58840,20152,30848,40394
+54215,20153,30847,40887
+57327,20154,30846,40926
+59339,20155,30845,40636
+51882,20156,30844,40284
+58255,20157,30843,40022
+55700,20158,30842,40924
+55583,20159,30841,40730
+59870,20160,30840,40899
+57358,20161,30839,40615
+52670,20162,30838,40148
+55063,20163,30837,40734
+56673,20164,30836,40571
+51604,20165,30835,40881
+55324,20166,30834,40774
+54252,20167,30833,40493
+55696,20168,30832,40448
+55286,20169,30831,40909
+51916,20170,30830,40898
+57980,20171,30829,40372
+53420,20172,30828,40992
+58508,20173,30827,40127
+55215,20174,30826,40678
+52776,20175,30825,40829
+59943,20176,30824,40085
+57533,20177,30823,40067
+57002,20178,30822,40159
+51200,20179,30821,40400
+58933,20180,30820,40162
+51060,20181,30819,40210
+53423,20182,30818,40020
+59913,20183,30817,40164
+53499,20184,30816,40747
+51790,20185,30815,40789
+53560,20186,30814,40743
+53319,20187,30813,40357
+58611,20188,30812,40135
+54457,20189,30811,40190
+56380,20190,30810,40765
+52050,20191,30809,40541
+54278,20192,30808,40103
+57030,20193,30807,40931
+57751,20194,30806,40723
+52437,20195,30805,40965
+53119,20196,30804,40182
+57592,20197,30803,40844
+57030,20198,30802,40113
+58185,20199,30801,40794
+54302,20200,30800,40552
+59906,20201,30799,40986
+58152,20202,30798,40608
+52078,20203,30797,40178
+56594,20204,30796,40541
+56173,20205,30795,40328
+51838,20206,30794,40717
+55507,20207,30793,40990
+53165,20208,30792,40053
+59839,20209,30791,40371
+52986,20210,30790,40493
+50632,20211,30789,40376
+52772,20212,30788,40330
+56531,20213,30787,40323
+58339,20214,30786,40489
+55874,20215,30785,40035
+53212,20216,30784,40019
+54263,20217,30783,40546
+56735,20218,30782,40033
+56663,20219,30781,40136
+52686,20220,30780,40773
+53145,20221,30779,40105
+50584,20222,30778,40066
+55085,20223,30777,40253
+50147,20224,30776,40138
+59257,20225,30775,40407
+59431,20226,30774,40754
+50959,20227,30773,40381
+59303,20228,30772,40531
+56331,20229,30771,40588
+51355,20230,30770,40968
+51751,20231,30769,40020
+59897,20232,30768,40927
+52481,20233,30767,40855
+54371,20234,30766,40086
+52069,20235,30765,40250
+53641,20236,30764,40820
+54148,20237,30763,40373
+59663,20238,30762,40049
+55606,20239,30761,40565
+59017,20240,30760,40834
+59259,20241,30759,40507
+51138,20242,30758,40389
+50845,20243,30757,40377
+59677,20244,30756,40397
+53841,20245,30755,40808
+58418,20246,30754,40203
+53069,20247,30753,40049
+53978,20248,30752,40569
+51379,20249,30751,40758
+59734,20250,30750,40540
+55147,20251,30749,40932
+54153,20252,30748,40093
+57039,20253,30747,40872
+56519,20254,30746,40484
+57376,20255,30745,40531
+55765,20256,30744,40891
+55719,20257,30743,40923
+59935,20258,30742,40474
+59477,20259,30741,40111
+51640,20260,30740,40451
+52219,20261,30739,40038
+55596,20262,30738,40748
+53366,20263,30737,40173
+57890,20264,30736,40661
+50578,20265,30735,40171
+58093,20266,30734,40765
+56031,20267,30733,40299
+50738,20268,30732,40974
+59409,20269,30731,40710
+53827,20270,30730,40153
+56545,20271,30729,40138
+54170,20272,30728,40671
+56769,20273,30727,40907
+52737,20274,30726,40645
+53901,20275,30725,40955
+53614,20276,30724,40107
+58941,20277,30723,40985
+57005,20278,30722,40009
+51639,20279,30721,40868
+57157,20280,30720,40174
+57761,20281,30719,40811
+53948,20282,30718,40135
+52724,20283,30717,40034
+57670,20284,30716,40849
+55644,20285,30715,40082
+54041,20286,30714,40699
+58539,20287,30713,40590
+57148,20288,30712,40162
+58558,20289,30711,40486
+50510,20290,30710,40866
+58504,20291,30709,40218
+58482,20292,30708,40326
+56442,20293,30707,40181
+58680,20294,30706,40453
+52804,20295,30705,40139
+50023,20296,30704,40729
+50389,20297,30703,40480
+50823,20298,30702,40677
+55277,20299,30701,40134
+59499,20300,30700,40810
+53892,20301,30699,40578
+51003,20302,30698,40902
+58308,20303,30697,40964
+56479,20304,30696,40386
+58241,20305,30695,40882
+51512,20306,30694,40254
+51097,20307,30693,40896
+52041,20308,30692,40467
+53559,20309,30691,40941
+51506,20310,30690,40628
+57761,20311,30689,40285
+52733,20312,30688,40370
+56369,20313,30687,40598
+52481,20314,30686,40107
+56656,20315,30685,40186
+57364,20316,30684,40198
+51214,20317,30683,40129
+50334,20318,30682,40902
+53775,20319,30681,40255
+57208,20320,30680,40761
+54271,20321,30679,40313
+51475,20322,30678,40539
+54966,20323,30677,40703
+53589,20324,30676,40508
+56199,20325,30675,40591
+56399,20326,30674,40282
+50162,20327,30673,40990
+53098,20328,30672,40732
+50233,20329,30671,40571
+50351,20330,30670,40837
+56832,20331,30669,40954
+51893,20332,30668,40041
+56862,20333,30667,40842
+56885,20334,30666,40847
+55968,20335,30665,40196
+59610,20336,30664,40819
+55676,20337,30663,40208
+58197,20338,30662,40775
+53170,20339,30661,40142
+50237,20340,30660,40333
+51519,20341,30659,40533
+54898,20342,30658,40876
+51835,20343,30657,40315
+58082,20344,30656,40412
+50163,20345,30655,40466
+50613,20346,30654,40879
+58161,20347,30653,40795
+57203,20348,30652,40399
+51047,20349,30651,40074
+59206,20350,30650,40606
+56954,20351,30649,40758
+59482,20352,30648,40693
+51339,20353,30647,40395
+59408,20354,30646,40845
+51393,20355,30645,40710
+58754,20356,30644,40476
+52018,20357,30643,40068
+50257,20358,30642,40956
+53153,20359,30641,40471
+55898,20360,30640,40726
+54032,20361,30639,40703
+57495,20362,30638,40782
+50343,20363,30637,40249
+57825,20364,30636,40899
+56301,20365,30635,40846
+58243,20366,30634,40845
+53635,20367,30633,40426
+50458,20368,30632,40891
+57456,20369,30631,40454
+54121,20370,30630,40865
+52231,20371,30629,40341
+53977,20372,30628,40370
+50693,20373,30627,40112
+59214,20374,30626,40984
+54758,20375,30625,40799
+56708,20376,30624,40149
+50087,20377,30623,40053
+52584,20378,30622,40884
+59366,20379,30621,40942
+54035,20380,30620,40585
+56213,20381,30619,40088
+58593,20382,30618,40066
+58270,20383,30617,40211
+58304,20384,30616,40737
+56752,20385,30615,40053
+53604,20386,30614,40983
+52488,20387,30613,40222
+50983,20388,30612,40726
+58683,20389,30611,40830
+50878,20390,30610,40611
+57887,20391,30609,40822
+59735,20392,30608,40726
+53802,20393,30607,40834
+51165,20394,30606,40479
+59682,20395,30605,40080
+51870,20396,30604,40551
+58515,20397,30603,40181
+51284,20398,30602,40464
+50399,20399,30601,40381
+51008,20400,30600,40858
+59519,20401,30599,40980
+58533,20402,30598,40123
+50152,20403,30597,40062
+53936,20404,30596,40050
+53033,20405,30595,40648
+58869,20406,30594,40649
+58753,20407,30593,40265
+51540,20408,30592,40396
+57613,20409,30591,40520
+56587,20410,30590,40642
+53071,20411,30589,40651
+52485,20412,30588,40231
+54730,20413,30587,40137
+58976,20414,30586,40451
+50090,20415,30585,40285
+52476,20416,30584,40747
+55472,20417,30583,40975
+50495,20418,30582,40130
+50731,20419,30581,40632
+57766,20420,30580,40403
+58018,20421,30579,40834
+53270,20422,30578,40422
+56110,20423,30577,40035
+54497,20424,30576,40477
+55893,20425,30575,40964
+56931,20426,30574,40745
+58977,20427,30573,40346
+58092,20428,30572,40190
+54991,20429,30571,40607
+59176,20430,30570,40543
+53394,20431,30569,40238
+57578,20432,30568,40904
+55672,20433,30567,40850
+56711,20434,30566,40503
+59621,20435,30565,40753
+58287,20436,30564,40085
+51867,20437,30563,40626
+55030,20438,30562,40878
+57355,20439,30561,40816
+54544,20440,30560,40987
+55493,20441,30559,40514
+57964,20442,30558,40568
+50588,20443,30557,40777
+59815,20444,30556,40511
+59413,20445,30555,40301
+57169,20446,30554,40905
+53725,20447,30553,40156
+56317,20448,30552,40404
+50666,20449,30551,40056
+51937,20450,30550,40477
+53834,20451,30549,40556
+53374,20452,30548,40715
+56232,20453,30547,40099
+59225,20454,30546,40846
+56045,20455,30545,40501
+50072,20456,30544,40314
+55314,20457,30543,40198
+59320,20458,30542,40572
+54148,20459,30541,40237
+52251,20460,30540,40327
+50177,20461,30539,40406
+53183,20462,30538,40371
+56382,20463,30537,40252
+50041,20464,30536,40164
+53291,20465,30535,40951
+58177,20466,30534,40922
+59758,20467,30533,40485
+52583,20468,30532,40306
+51052,20469,30531,40353
+54419,20470,30530,40808
+52534,20471,30529,40709
+59146,20472,30528,40727
+52128,20473,30527,40164
+55085,20474,30526,40527
+51437,20475,30525,40809
+51707,20476,30524,40122
+51926,20477,30523,40293
+52102,20478,30522,40483
+56169,20479,30521,40417
+53395,20480,30520,40459
+59454,20481,30519,40801
+56043,20482,30518,40889
+51221,20483,30517,40041
+50543,20484,30516,40502
+51227,20485,30515,40087
+55352,20486,30514,40658
+53212,20487,30513,40148
+51516,20488,30512,40518
+50146,20489,30511,40016
+56796,20490,30510,40367
+51707,20491,30509,40377
+55739,20492,30508,40651
+50169,20493,30507,40120
+57418,20494,30506,40476
+51938,20495,30505,40840
+59118,20496,30504,40254
+58168,20497,30503,40140
+54866,20498,30502,40263
+58995,20499,30501,40795
+57832,20500,30500,40978
+58682,20501,30499,40027
+51851,20502,30498,40135
+57441,20503,30497,40471
+52609,20504,30496,40515
+55901,20505,30495,40630
+58012,20506,30494,40815
+53138,20507,30493,40578
+57183,20508,30492,40936
+52461,20509,30491,40297
+53134,20510,30490,40042
+55037,20511,30489,40788
+55758,20512,30488,40489
+52268,20513,30487,40450
+54459,20514,30486,40358
+57900,20515,30485,40504
+55769,20516,30484,40659
+57181,20517,30483,40311
+51890,20518,30482,40723
+54043,20519,30481,40618
+58653,20520,30480,40705
+51332,20521,30479,40698
+55888,20522,30478,40867
+54822,20523,30477,40112
+55020,20524,30476,40498
+52311,20525,30475,40678
+50855,20526,30474,40513
+50094,20527,30473,40034
+55655,20528,30472,40712
+58383,20529,30471,40776
+52922,20530,30470,40804
+57109,20531,30469,40195
+58547,20532,30468,40740
+56183,20533,30467,40864
+58159,20534,30466,40755
+53597,20535,30465,40359
+54741,20536,30464,40525
+58667,20537,30463,40511
+55259,20538,30462,40535
+57196,20539,30461,40533
+57973,20540,30460,40640
+50202,20541,30459,40201
+58473,20542,30458,40350
+57130,20543,30457,40853
+54894,20544,30456,40078
+55829,20545,30455,40442
+50563,20546,30454,40540
+51824,20547,30453,40705
+51996,20548,30452,40882
+58269,20549,30451,40842
+53382,20550,30450,40857
+54874,20551,30449,40165
+52124,20552,30448,40890
+54327,20553,30447,40475
+56098,20554,30446,40496
+57590,20555,30445,40111
+55631,20556,30444,40236
+52059,20557,30443,40311
+55799,20558,30442,40995
+56819,20559,30441,40145
+54342,20560,30440,40344
+58272,20561,30439,40044
+56345,20562,30438,40634
+54386,20563,30437,40484
+58055,20564,30436,40801
+51138,20565,30435,40649
+59965,20566,30434,40963
+51688,20567,30433,40837
+53046,20568,30432,40853
+59212,20569,30431,40688
+57929,20570,30430,40704
+52473,20571,30429,40922
+55260,20572,30428,40458
+55646,20573,30427,40372
+59095,20574,30426,40150
+58846,20575,30425,40003
+52787,20576,30424,40041
+59863,20577,30423,40722
+57618,20578,30422,40682
+58639,20579,30421,40211
+57496,20580,30420,40085
+50475,20581,30419,40737
+57558,20582,30418,40715
+53738,20583,30417,40661
+58455,20584,30416,40874
+58013,20585,30415,40187
+53156,20586,30414,40453
+55626,20587,30413,40897
+50610,20588,30412,40616
+50758,20589,30411,40964
+50779,20590,30410,40706
+50831,20591,30409,40638
+55626,20592,30408,40958
+54124,20593,30407,40776
+54256,20594,30406,40616
+56428,20595,30405,40551
+56611,20596,30404,40113
+51851,20597,30403,40382
+56726,20598,30402,40327
+59763,20599,30401,40673
+55956,20600,30400,40420
+57615,20601,30399,40793
+54646,20602,30398,40210
+52187,20603,30397,40704
+58164,20604,30396,40080
+50183,20605,30395,40123
+52504,20606,30394,40451
+52922,20607,30393,40999
+55646,20608,30392,40787
+53667,20609,30391,40137
+55143,20610,30390,40040
+50059,20611,30389,40606
+56733,20612,30388,40498
+54439,20613,30387,40076
+54177,20614,30386,40242
+50233,20615,30385,40731
+54288,20616,30384,40721
+58165,20617,30383,40556
+56692,20618,30382,40546
+59457,20619,30381,40508
+50919,20620,30380,40222
+54326,20621,30379,40572
+55298,20622,30378,40437
+55796,20623,30377,40617
+58547,20624,30376,40724
+51729,20625,30375,40427
+55421,20626,30374,40031
+58786,20627,30373,40100
+54889,20628,30372,40923
+51058,20629,30371,40359
+51387,20630,30370,40656
+58559,20631,30369,40186
+57103,20632,30368,40546
+53236,20633,30367,40225
+57522,20634,30366,40388
+51809,20635,30365,40744
+51373,20636,30364,40144
+54839,20637,30363,40789
+54828,20638,30362,40708
+53375,20639,30361,40328
+59486,20640,30360,40716
+56410,20641,30359,40808
+57956,20642,30358,40279
+50307,20643,30357,40076
+54716,20644,30356,40913
+59089,20645,30355,40210
+51143,20646,30354,40030
+56441,20647,30353,40575
+57862,20648,30352,40910
+53600,20649,30351,40308
+53428,20650,30350,40809
+55997,20651,30349,40609
+50285,20652,30348,40351
+52861,20653,30347,40001
+57388,20654,30346,40100
+57072,20655,30345,40929
+58129,20656,30344,40775
+58497,20657,30343,40838
+53698,20658,30342,40718
+55348,20659,30341,40570
+52024,20660,30340,40592
+58663,20661,30339,40291
+56538,20662,30338,40454
+53274,20663,30337,40520
+56728,20664,30336,40679
+58203,20665,30335,40505
+56206,20666,30334,40454
+55995,20667,30333,40898
+52744,20668,30332,40346
+50732,20669,30331,40539
+55588,20670,30330,40698
+56764,20671,30329,40351
+51090,20672,30328,40347
+58347,20673,30327,40407
+56249,20674,30326,40359
+54551,20675,30325,40731
+59683,20676,30324,40076
+54425,20677,30323,40276
+52473,20678,30322,40534
+59560,20679,30321,40521
+55881,20680,30320,40458
+59708,20681,30319,40627
+54915,20682,30318,40504
+52813,20683,30317,40335
+54611,20684,30316,40691
+50202,20685,30315,40256
+52889,20686,30314,40473
+56677,20687,30313,40479
+50037,20688,30312,40218
+51883,20689,30311,40384
+58973,20690,30310,40079
+53042,20691,30309,40506
+51873,20692,30308,40560
+50923,20693,30307,40802
+55957,20694,30306,40657
+56262,20695,30305,40068
+55885,20696,30304,40090
+57221,20697,30303,40056
+53784,20698,30302,40906
+55609,20699,30301,40549
+50867,20700,30300,40455
+57752,20701,30299,40517
+56078,20702,30298,40413
+50848,20703,30297,40484
+50012,20704,30296,40657
+57414,20705,30295,40997
+58212,20706,30294,40872
+54471,20707,30293,40779
+59284,20708,30292,40280
+52512,20709,30291,40266
+52581,20710,30290,40371
+55464,20711,30289,40784
+51417,20712,30288,40132
+54538,20713,30287,40867
+57953,20714,30286,40864
+53313,20715,30285,40463
+50539,20716,30284,40322
+59949,20717,30283,40436
+59289,20718,30282,40890
+59970,20719,30281,40098
+55197,20720,30280,40387
+50623,20721,30279,40858
+56192,20722,30278,40799
+52860,20723,30277,40187
+56530,20724,30276,40634
+52300,20725,30275,40246
+53531,20726,30274,40264
+58738,20727,30273,40510
+52512,20728,30272,40836
+54640,20729,30271,40847
+53602,20730,30270,40534
+54715,20731,30269,40354
+50032,20732,30268,40887
+52378,20733,30267,40234
+55964,20734,30266,40572
+59437,20735,30265,40110
+50411,20736,30264,40973
+52267,20737,30263,40491
+54646,20738,30262,40937
+55683,20739,30261,40325
+56846,20740,30260,40704
+57492,20741,30259,40771
+53372,20742,30258,40652
+54104,20743,30257,40788
+52598,20744,30256,40453
+55099,20745,30255,40353
+53214,20746,30254,40680
+51898,20747,30253,40841
+55733,20748,30252,40219
+58614,20749,30251,40523
+51420,20750,30250,40519
+50679,20751,30249,40443
+57227,20752,30248,40675
+50397,20753,30247,40148
+58122,20754,30246,40793
+59218,20755,30245,40230
+50764,20756,30244,40951
+54588,20757,30243,40853
+59324,20758,30242,40308
+51646,20759,30241,40878
+51920,20760,30240,40868
+51630,20761,30239,40534
+53338,20762,30238,40242
+50241,20763,30237,40600
+55022,20764,30236,40546
+50882,20765,30235,40483
+59837,20766,30234,40584
+58274,20767,30233,40054
+52175,20768,30232,40784
+57994,20769,30231,40369
+58000,20770,30230,40123
+57964,20771,30229,40305
+57617,20772,30228,40474
+53815,20773,30227,40231
+55759,20774,30226,40754
+55832,20775,30225,40159
+57745,20776,30224,40733
+58910,20777,30223,40392
+55722,20778,30222,40457
+56374,20779,30221,40084
+51967,20780,30220,40727
+54405,20781,30219,40213
+50771,20782,30218,40122
+52116,20783,30217,40634
+54285,20784,30216,40137
+54181,20785,30215,40589
+53499,20786,30214,40283
+57268,20787,30213,40635
+52768,20788,30212,40918
+51529,20789,30211,40079
+55736,20790,30210,40196
+57434,20791,30209,40955
+53741,20792,30208,40579
+59534,20793,30207,40332
+50002,20794,30206,40239
+58092,20795,30205,40240
+59660,20796,30204,40279
+54385,20797,30203,40847
+52929,20798,30202,40880
+53600,20799,30201,40555
+59992,20800,30200,40570
+50035,20801,30199,40198
+51504,20802,30198,40925
+59904,20803,30197,40223
+52269,20804,30196,40415
+57256,20805,30195,40744
+54925,20806,30194,40829
+56745,20807,30193,40807
+50472,20808,30192,40547
+59298,20809,30191,40838
+54358,20810,30190,40826
+55878,20811,30189,40777
+50890,20812,30188,40997
+57610,20813,30187,40132
+58412,20814,30186,40552
+54025,20815,30185,40678
+55621,20816,30184,40734
+56088,20817,30183,40526
+57971,20818,30182,40214
+54083,20819,30181,40673
+57708,20820,30180,40408
+56117,20821,30179,40397
+56336,20822,30178,40617
+54503,20823,30177,40222
+53431,20824,30176,40385
+55216,20825,30175,40986
+53129,20826,30174,40029
+51930,20827,30173,40370
+56268,20828,30172,40486
+51173,20829,30171,40959
+56792,20830,30170,40518
+52565,20831,30169,40215
+59943,20832,30168,40322
+55367,20833,30167,40291
+59096,20834,30166,40400
+52721,20835,30165,40854
+58610,20836,30164,40462
+59430,20837,30163,40188
+54614,20838,30162,40452
+54121,20839,30161,40396
+52061,20840,30160,40656
+58905,20841,30159,40276
+51456,20842,30158,40636
+58763,20843,30157,40287
+56231,20844,30156,40526
+58502,20845,30155,40182
+53433,20846,30154,40996
+54679,20847,30153,40640
+52546,20848,30152,40945
+59892,20849,30151,40783
+59474,20850,30150,40329
+51438,20851,30149,40353
+54898,20852,30148,40877
+57039,20853,30147,40591
+51138,20854,30146,40651
+59859,20855,30145,40838
+59150,20856,30144,40037
+55428,20857,30143,40815
+53359,20858,30142,40836
+52877,20859,30141,40733
+50493,20860,30140,40753
+59936,20861,30139,40667
+50354,20862,30138,40519
+56322,20863,30137,40959
+54986,20864,30136,40609
+58244,20865,30135,40695
+56375,20866,30134,40468
+58621,20867,30133,40511
+54209,20868,30132,40416
+59699,20869,30131,40563
+56922,20870,30130,40608
+59945,20871,30129,40719
+52031,20872,30128,40948
+57618,20873,30127,40121
+52250,20874,30126,40487
+54100,20875,30125,40960
+50436,20876,30124,40122
+56839,20877,30123,40939
+50393,20878,30122,40052
+53196,20879,30121,40480
+58564,20880,30120,40412
+51995,20881,30119,40751
+51945,20882,30118,40645
+51524,20883,30117,40835
+53041,20884,30116,40732
+53207,20885,30115,40541
+55376,20886,30114,40905
+54576,20887,30113,40596
+51442,20888,30112,40144
+51146,20889,30111,40861
+58404,20890,30110,40011
+57412,20891,30109,40674
+58944,20892,30108,40390
+51200,20893,30107,40818
+52960,20894,30106,40006
+51320,20895,30105,40646
+59603,20896,30104,40678
+55916,20897,30103,40619
+50589,20898,30102,40128
+58198,20899,30101,40217
+58040,20900,30100,40533
+50687,20901,30099,40295
+57465,20902,30098,40161
+58170,20903,30097,40928
+55038,20904,30096,40534
+57901,20905,30095,40918
+59939,20906,30094,40883
+51393,20907,30093,40641
+54115,20908,30092,40024
+52415,20909,30091,40180
+59579,20910,30090,40033
+57114,20911,30089,40157
+58523,20912,30088,40669
+56235,20913,30087,40335
+58870,20914,30086,40001
+59799,20915,30085,40070
+58409,20916,30084,40630
+53918,20917,30083,40990
+50248,20918,30082,40100
+56380,20919,30081,40192
+56532,20920,30080,40601
+57410,20921,30079,40401
+56153,20922,30078,40931
+53487,20923,30077,40577
+53139,20924,30076,40133
+50503,20925,30075,40069
+54544,20926,30074,40121
+52454,20927,30073,40962
+54529,20928,30072,40826
+56190,20929,30071,40007
+50586,20930,30070,40237
+51563,20931,30069,40853
+54937,20932,30068,40268
+53768,20933,30067,40686
+57939,20934,30066,40366
+50059,20935,30065,40030
+56922,20936,30064,40167
+52795,20937,30063,40675
+52490,20938,30062,40407
+58498,20939,30061,40053
+56387,20940,30060,40881
+54621,20941,30059,40745
+59937,20942,30058,40226
+55584,20943,30057,40240
+51511,20944,30056,40796
+54214,20945,30055,40271
+55179,20946,30054,40119
+56038,20947,30053,40987
+58266,20948,30052,40907
+53314,20949,30051,40746
+58108,20950,30050,40675
+58697,20951,30049,40649
+58642,20952,30048,40937
+59484,20953,30047,40926
+52460,20954,30046,40043
+57422,20955,30045,40585
+58967,20956,30044,40511
+52596,20957,30043,40324
+54020,20958,30042,40633
+55244,20959,30041,40203
+57683,20960,30040,40437
+57673,20961,30039,40434
+50510,20962,30038,40898
+53916,20963,30037,40451
+54594,20964,30036,40174
+55471,20965,30035,40400
+56760,20966,30034,40408
+59628,20967,30033,40002
+50723,20968,30032,40214
+50234,20969,30031,40223
+55761,20970,30030,40347
+58567,20971,30029,40842
+55204,20972,30028,40156
+54817,20973,30027,40242
+50203,20974,30026,40072
+56062,20975,30025,40832
+58185,20976,30024,40146
+50987,20977,30023,40486
+52615,20978,30022,40268
+57249,20979,30021,40065
+55333,20980,30020,40866
+56020,20981,30019,40091
+53147,20982,30018,40904
+54246,20983,30017,40064
+57006,20984,30016,40920
+59113,20985,30015,40882
+56079,20986,30014,40476
+55489,20987,30013,40163
+55165,20988,30012,40068
+50087,20989,30011,40774
+53162,20990,30010,40925
+53514,20991,30009,40891
+58843,20992,30008,40583
+57407,20993,30007,40861
+54281,20994,30006,40008
+50266,20995,30005,40162
+58661,20996,30004,40226
+59381,20997,30003,40798
+57913,20998,30002,40254
+57165,20999,30001,40460
+52665,21000,30000,40548
+51116,21001,31000,40195
+54952,21002,30999,40641
+53440,21003,30998,40424
+55878,21004,30997,40579
+56779,21005,30996,40086
+53655,21006,30995,40603
+54740,21007,30994,40766
+58191,21008,30993,40463
+52618,21009,30992,40597
+52506,21010,30991,40155
+55145,21011,30990,40585
+55864,21012,30989,40275
+57077,21013,30988,40039
+54757,21014,30987,40616
+52259,21015,30986,40908
+53655,21016,30985,40725
+54263,21017,30984,40413
+57328,21018,30983,40261
+51816,21019,30982,40102
+54596,21020,30981,40964
+52176,21021,30980,40572
+56820,21022,30979,40886
+50688,21023,30978,40240
+52520,21024,30977,40183
+53744,21025,30976,40296
+50172,21026,30975,40231
+57411,21027,30974,40787
+57921,21028,30973,40601
+53417,21029,30972,40184
+58004,21030,30971,40554
+51902,21031,30970,40243
+57292,21032,30969,40697
+52735,21033,30968,40094
+57661,21034,30967,40079
+55117,21035,30966,40977
+55929,21036,30965,40667
+54539,21037,30964,40381
+51078,21038,30963,40738
+56002,21039,30962,40242
+55675,21040,30961,40378
+51939,21041,30960,40907
+53908,21042,30959,40762
+54396,21043,30958,40139
+56757,21044,30957,40128
+54804,21045,30956,40870
+58282,21046,30955,40902
+54720,21047,30954,40190
+52125,21048,30953,40073
+52616,21049,30952,40289
+59441,21050,30951,40176
+55316,21051,30950,40147
+50908,21052,30949,40614
+57823,21053,30948,40933
+51329,21054,30947,40723
+51998,21055,30946,40434
+52571,21056,30945,40669
+50394,21057,30944,40482
+50757,21058,30943,40311
+50488,21059,30942,40861
+56742,21060,30941,40771
+53562,21061,30940,40317
+53291,21062,30939,40369
+56543,21063,30938,40616
+55277,21064,30937,40556
+59493,21065,30936,40801
+57169,21066,30935,40965
+53679,21067,30934,40930
+59086,21068,30933,40119
+53713,21069,30932,40081
+50985,21070,30931,40902
+57384,21071,30930,40956
+52172,21072,30929,40468
+59687,21073,30928,40505
+50901,21074,30927,40466
+59228,21075,30926,40998
+59098,21076,30925,40423
+50800,21077,30924,40385
+59259,21078,30923,40656
+59998,21079,30922,40694
+57200,21080,30921,40868
+52315,21081,30920,40812
+54465,21082,30919,40731
+56707,21083,30918,40178
+57988,21084,30917,40114
+52661,21085,30916,40043
+59249,21086,30915,40270
+57604,21087,30914,40626
+52306,21088,30913,40435
+54782,21089,30912,40332
+52966,21090,30911,40927
+52186,21091,30910,40361
+59253,21092,30909,40103
+59322,21093,30908,40894
+54909,21094,30907,40254
+53520,21095,30906,40027
+50378,21096,30905,40366
+52749,21097,30904,40228
+58261,21098,30903,40964
+58820,21099,30902,40606
+59251,21100,30901,40749
+55758,21101,30900,40743
+50750,21102,30899,40494
+58150,21103,30898,40210
+57586,21104,30897,40461
+53028,21105,30896,40255
+56205,21106,30895,40242
+56327,21107,30894,40043
+50745,21108,30893,40697
+52747,21109,30892,40496
+55046,21110,30891,40638
+58203,21111,30890,40259
+57200,21112,30889,40861
+55392,21113,30888,40278
+52784,21114,30887,40894
+51863,21115,30886,40066
+55743,21116,30885,40114
+54928,21117,30884,40627
+53252,21118,30883,40380
+52979,21119,30882,40200
+55518,21120,30881,40022
+58381,21121,30880,40789
+56206,21122,30879,40831
+54760,21123,30878,40967
+53852,21124,30877,40486
+59272,21125,30876,40531
+57722,21126,30875,40806
+56754,21127,30874,40427
+55429,21128,30873,40654
+50647,21129,30872,40534
+51383,21130,30871,40228
+55006,21131,30870,40827
+58090,21132,30869,40349
+50176,21133,30868,40874
+54071,21134,30867,40331
+56893,21135,30866,40463
+56669,21136,30865,40228
+57580,21137,30864,40420
+52061,21138,30863,40715
+55839,21139,30862,40769
+55345,21140,30861,40869
+53525,21141,30860,40603
+51837,21142,30859,40370
+55704,21143,30858,40846
+58253,21144,30857,40061
+53991,21145,30856,40712
+51790,21146,30855,40554
+55119,21147,30854,40980
+50426,21148,30853,40708
+56914,21149,30852,40392
+52683,21150,30851,40656
+53849,21151,30850,40075
+54768,21152,30849,40289
+52421,21153,30848,40086
+53293,21154,30847,40176
+58369,21155,30846,40534
+53916,21156,30845,40456
+55636,21157,30844,40307
+57378,21158,30843,40220
+57971,21159,30842,40857
+50173,21160,30841,40278
+54259,21161,30840,40664
+59353,21162,30839,40956
+53884,21163,30838,40674
+55410,21164,30837,40217
+52361,21165,30836,40800
+57571,21166,30835,40997
+57467,21167,30834,40425
+59117,21168,30833,40466
+50462,21169,30832,40665
+56611,21170,30831,40892
+50691,21171,30830,40338
+52146,21172,30829,40116
+52188,21173,30828,40529
+53518,21174,30827,40658
+55014,21175,30826,40639
+52056,21176,30825,40980
+58320,21177,30824,40738
+52636,21178,30823,40415
+51739,21179,30822,40416
+52246,21180,30821,40136
+52714,21181,30820,40712
+51972,21182,30819,40128
+58408,21183,30818,40204
+56935,21184,30817,40770
+51816,21185,30816,40135
+51347,21186,30815,40530
+53975,21187,30814,40647
+56838,21188,30813,40644
+59775,21189,30812,40391
+57333,21190,30811,40014
+55701,21191,30810,40644
+57314,21192,30809,40891
+53826,21193,30808,40442
+57930,21194,30807,40461
+52206,21195,30806,40738
+56683,21196,30805,40478
+56808,21197,30804,40538
+58322,21198,30803,40404
+52066,21199,30802,40969
+55170,21200,30801,40806
+53321,21201,30800,40965
+53905,21202,30799,40927
+50355,21203,30798,40347
+50492,21204,30797,40485
+58561,21205,30796,40202
+51451,21206,30795,40150
+54772,21207,30794,40989
+51967,21208,30793,40865
+51996,21209,30792,40955
+54376,21210,30791,40636
+50347,21211,30790,40621
+51172,21212,30789,40855
+56903,21213,30788,40947
+59025,21214,30787,40044
+56783,21215,30786,40403
+56979,21216,30785,40482
+59045,21217,30784,40311
+59952,21218,30783,40325
+52554,21219,30782,40858
+53849,21220,30781,40675
+59527,21221,30780,40055
+53140,21222,30779,40517
+50711,21223,30778,40869
+50292,21224,30777,40928
+53074,21225,30776,40875
+54346,21226,30775,40607
+53364,21227,30774,40236
+59845,21228,30773,40095
+52189,21229,30772,40143
+56740,21230,30771,40967
+58849,21231,30770,40167
+56588,21232,30769,40377
+53439,21233,30768,40137
+56566,21234,30767,40137
+50111,21235,30766,40823
+57808,21236,30765,40186
+56037,21237,30764,40824
+54211,21238,30763,40615
+51458,21239,30762,40993
+58948,21240,30761,40013
+54521,21241,30760,40228
+56196,21242,30759,40850
+56271,21243,30758,40212
+52706,21244,30757,40705
+55067,21245,30756,40610
+52753,21246,30755,40734
+52366,21247,30754,40671
+51668,21248,30753,40704
+50844,21249,30752,40911
+51173,21250,30751,40696
+54408,21251,30750,40654
+53589,21252,30749,40493
+56526,21253,30748,40066
+53181,21254,30747,40961
+55881,21255,30746,40628
+54803,21256,30745,40131
+57285,21257,30744,40780
+50521,21258,30743,40289
+51598,21259,30742,40088
+54640,21260,30741,40418
+59096,21261,30740,40160
+55930,21262,30739,40990
+59672,21263,30738,40258
+59484,21264,30737,40739
+56815,21265,30736,40165
+59506,21266,30735,40541
+51223,21267,30734,40700
+59564,21268,30733,40488
+52454,21269,30732,40129
+56486,21270,30731,40149
+53904,21271,30730,40259
+52182,21272,30729,40639
+58457,21273,30728,40829
+55134,21274,30727,40385
+59447,21275,30726,40068
+50930,21276,30725,40862
+54294,21277,30724,40853
+52339,21278,30723,40972
+54869,21279,30722,40905
+54218,21280,30721,40197
+50269,21281,30720,40871
+59536,21282,30719,40516
+56562,21283,30718,40692
+56458,21284,30717,40785
+55362,21285,30716,40937
+56546,21286,30715,40137
+53467,21287,30714,40672
+52048,21288,30713,40005
+54841,21289,30712,40612
+52599,21290,30711,40348
+59005,21291,30710,40669
+59356,21292,30709,40608
+55993,21293,30708,40333
+57346,21294,30707,40074
+59543,21295,30706,40922
+55361,21296,30705,40971
+52042,21297,30704,40848
+56181,21298,30703,40049
+57436,21299,30702,40746
+51794,21300,30701,40138
+50262,21301,30700,40917
+59700,21302,30699,40687
+51422,21303,30698,40306
+57781,21304,30697,40256
+56426,21305,30696,40069
+54728,21306,30695,40696
+51348,21307,30694,40718
+57043,21308,30693,40181
+58230,21309,30692,40164
+52009,21310,30691,40206
+51716,21311,30690,40225
+51733,21312,30689,40921
+56968,21313,30688,40638
+55635,21314,30687,40000
+56601,21315,30686,40531
+50952,21316,30685,40431
+54485,21317,30684,40083
+59234,21318,30683,40077
+51389,21319,30682,40906
+58425,21320,30681,40285
+54416,21321,30680,40860
+51397,21322,30679,40984
+57103,21323,30678,40956
+50233,21324,30677,40016
+57968,21325,30676,40341
+51022,21326,30675,40323
+53343,21327,30674,40777
+52122,21328,30673,40674
+56030,21329,30672,40475
+50479,21330,30671,40525
+54268,21331,30670,40579
+56528,21332,30669,40726
+51924,21333,30668,40606
+52734,21334,30667,40094
+54604,21335,30666,40510
+50289,21336,30665,40776
+53758,21337,30664,40157
+50913,21338,30663,40472
+59300,21339,30662,40522
+51525,21340,30661,40214
+52108,21341,30660,40270
+51298,21342,30659,40565
+58331,21343,30658,40326
+52485,21344,30657,40075
+52014,21345,30656,40773
+59821,21346,30655,40861
+58235,21347,30654,40636
+56191,21348,30653,40457
+58134,21349,30652,40305
+56556,21350,30651,40887
+56784,21351,30650,40498
+50608,21352,30649,40156
+59289,21353,30648,40108
+50113,21354,30647,40084
+55542,21355,30646,40452
+56376,21356,30645,40243
+51678,21357,30644,40337
+56700,21358,30643,40521
+52638,21359,30642,40709
+55993,21360,30641,40696
+57948,21361,30640,40182
+53040,21362,30639,40403
+53689,21363,30638,40941
+55192,21364,30637,40572
+53664,21365,30636,40225
+51666,21366,30635,40682
+52571,21367,30634,40441
+59562,21368,30633,40358
+54612,21369,30632,40949
+51545,21370,30631,40240
+52537,21371,30630,40415
+52616,21372,30629,40195
+56921,21373,30628,40420
+56111,21374,30627,40497
+51355,21375,30626,40337
+52475,21376,30625,40996
+53645,21377,30624,40403
+56685,21378,30623,40907
+53322,21379,30622,40343
+58191,21380,30621,40949
+56357,21381,30620,40771
+58466,21382,30619,40133
+51410,21383,30618,40551
+59764,21384,30617,40897
+54468,21385,30616,40843
+50253,21386,30615,40873
+53729,21387,30614,40511
+50040,21388,30613,40585
+56865,21389,30612,40097
+51294,21390,30611,40867
+56933,21391,30610,40014
+54349,21392,30609,40900
+56773,21393,30608,40096
+50376,21394,30607,40025
+51586,21395,30606,40304
+55865,21396,30605,40094
+58234,21397,30604,40041
+50434,21398,30603,40642
+59241,21399,30602,40809
+57744,21400,30601,40635
+57260,21401,30600,40148
+57776,21402,30599,40241
+56450,21403,30598,40382
+57596,21404,30597,40114
+52619,21405,30596,40813
+50814,21406,30595,40306
+54402,21407,30594,40233
+52773,21408,30593,40964
+52983,21409,30592,40129
+52341,21410,30591,40849
+53845,21411,30590,40740
+57049,21412,30589,40891
+56882,21413,30588,40616
+53780,21414,30587,40615
+57029,21415,30586,40338
+54805,21416,30585,40982
+52833,21417,30584,40810
+53864,21418,30583,40006
+51360,21419,30582,40730
+53368,21420,30581,40209
+57723,21421,30580,40084
+50083,21422,30579,40682
+57717,21423,30578,40093
+53111,21424,30577,40015
+51512,21425,30576,40930
+52072,21426,30575,40383
+56967,21427,30574,40029
+53091,21428,30573,40690
+54946,21429,30572,40188
+50636,21430,30571,40687
+58760,21431,30570,40982
+54324,21432,30569,40494
+51223,21433,30568,40363
+55399,21434,30567,40619
+56815,21435,30566,40319
+59441,21436,30565,40235
+54116,21437,30564,40538
+52064,21438,30563,40319
+58514,21439,30562,40167
+55077,21440,30561,40978
+57947,21441,30560,40119
+52612,21442,30559,40136
+53868,21443,30558,40993
+55851,21444,30557,40205
+56957,21445,30556,40457
+55440,21446,30555,40230
+56279,21447,30554,40498
+52083,21448,30553,40080
+57729,21449,30552,40233
+57634,21450,30551,40100
+50931,21451,30550,40568
+56858,21452,30549,40188
+59755,21453,30548,40187
+51403,21454,30547,40613
+56125,21455,30546,40818
+50453,21456,30545,40632
+52469,21457,30544,40300
+52671,21458,30543,40878
+54904,21459,30542,40676
+55078,21460,30541,40936
+50616,21461,30540,40059
+56722,21462,30539,40545
+53591,21463,30538,40874
+57636,21464,30537,40072
+52483,21465,30536,40241
+50748,21466,30535,40413
+56816,21467,30534,40273
+56280,21468,30533,40351
+59011,21469,30532,40051
+55430,21470,30531,40136
+51626,21471,30530,40875
+51818,21472,30529,40743
+54441,21473,30528,40540
+53899,21474,30527,40639
+56991,21475,30526,40521
+52391,21476,30525,40752
+58935,21477,30524,40517
+54807,21478,30523,40562
+51642,21479,30522,40152
+54110,21480,30521,40858
+58226,21481,30520,40287
+54251,21482,30519,40003
+53738,21483,30518,40391
+55213,21484,30517,40146
+57553,21485,30516,40901
+58726,21486,30515,40517
+57869,21487,30514,40109
+50616,21488,30513,40528
+53291,21489,30512,40141
+57052,21490,30511,40694
+52149,21491,30510,40748
+52626,21492,30509,40817
+56957,21493,30508,40117
+50323,21494,30507,40887
+53526,21495,30506,40330
+56338,21496,30505,40657
+59746,21497,30504,40407
+51169,21498,30503,40928
+50389,21499,30502,40949
+58111,21500,30501,40822
+58120,21501,30500,40175
+52799,21502,30499,40439
+51658,21503,30498,40420
+50836,21504,30497,40562
+52097,21505,30496,40313
+57784,21506,30495,40959
+54718,21507,30494,40842
+52732,21508,30493,40559
+55694,21509,30492,40613
+51450,21510,30491,40219
+56381,21511,30490,40220
+50134,21512,30489,40690
+58950,21513,30488,40669
+54675,21514,30487,40034
+50565,21515,30486,40023
+51281,21516,30485,40818
+54429,21517,30484,40483
+56356,21518,30483,40545
+50021,21519,30482,40069
+53905,21520,30481,40191
+51229,21521,30480,40093
+56849,21522,30479,40975
+51774,21523,30478,40239
+57208,21524,30477,40221
+51455,21525,30476,40107
+54559,21526,30475,40946
+57493,21527,30474,40977
+50845,21528,30473,40320
+51474,21529,30472,40605
+58148,21530,30471,40548
+51886,21531,30470,40306
+55256,21532,30469,40952
+59619,21533,30468,40351
+52715,21534,30467,40410
+58754,21535,30466,40019
+56371,21536,30465,40204
+53635,21537,30464,40942
+51837,21538,30463,40936
+53849,21539,30462,40053
+55562,21540,30461,40696
+56164,21541,30460,40753
+51959,21542,30459,40360
+57088,21543,30458,40301
+59387,21544,30457,40929
+53515,21545,30456,40018
+50378,21546,30455,40306
+53156,21547,30454,40978
+58199,21548,30453,40820
+57423,21549,30452,40365
+50502,21550,30451,40387
+53173,21551,30450,40682
+50558,21552,30449,40612
+51128,21553,30448,40558
+58589,21554,30447,40672
+53444,21555,30446,40378
+56310,21556,30445,40756
+59296,21557,30444,40692
+55606,21558,30443,40265
+59796,21559,30442,40541
+53925,21560,30441,40886
+52085,21561,30440,40626
+56903,21562,30439,40928
+52188,21563,30438,40001
+50546,21564,30437,40500
+54100,21565,30436,40330
+55962,21566,30435,40905
+53842,21567,30434,40071
+54615,21568,30433,40325
+51487,21569,30432,40274
+50632,21570,30431,40074
+56673,21571,30430,40730
+53937,21572,30429,40564
+59048,21573,30428,40353
+55216,21574,30427,40801
+50100,21575,30426,40284
+58078,21576,30425,40262
+57123,21577,30424,40480
+54736,21578,30423,40971
+55784,21579,30422,40413
+56093,21580,30421,40386
+51463,21581,30420,40242
+50442,21582,30419,40993
+56397,21583,30418,40942
+58929,21584,30417,40640
+54685,21585,30416,40198
+53697,21586,30415,40305
+58933,21587,30414,40767
+58750,21588,30413,40421
+57007,21589,30412,40194
+52473,21590,30411,40197
+50099,21591,30410,40687
+53962,21592,30409,40035
+56346,21593,30408,40223
+51337,21594,30407,40397
+54781,21595,30406,40458
+58781,21596,30405,40994
+53244,21597,30404,40377
+52188,21598,30403,40109
+58106,21599,30402,40458
+54769,21600,30401,40044
+56278,21601,30400,40806
+57435,21602,30399,40636
+53828,21603,30398,40816
+56049,21604,30397,40306
+58282,21605,30396,40604
+53972,21606,30395,40256
+53159,21607,30394,40049
+53517,21608,30393,40986
+54525,21609,30392,40540
+59798,21610,30391,40364
+52191,21611,30390,40011
+59701,21612,30389,40306
+53494,21613,30388,40795
+56018,21614,30387,40995
+59470,21615,30386,40551
+59289,21616,30385,40651
+59917,21617,30384,40821
+55755,21618,30383,40284
+55987,21619,30382,40950
+55487,21620,30381,40248
+58908,21621,30380,40698
+51334,21622,30379,40718
+52435,21623,30378,40590
+57372,21624,30377,40502
+56449,21625,30376,40860
+54506,21626,30375,40506
+58163,21627,30374,40724
+54265,21628,30373,40012
+55753,21629,30372,40900
+55553,21630,30371,40286
+57963,21631,30370,40337
+55471,21632,30369,40658
+58918,21633,30368,40489
+57045,21634,30367,40542
+56324,21635,30366,40295
+57851,21636,30365,40731
+58586,21637,30364,40570
+59107,21638,30363,40187
+51770,21639,30362,40491
+59290,21640,30361,40948
+52553,21641,30360,40769
+59499,21642,30359,40541
+55339,21643,30358,40290
+55949,21644,30357,40548
+55754,21645,30356,40416
+52381,21646,30355,40146
+55062,21647,30354,40285
+53014,21648,30353,40668
+54118,21649,30352,40357
+51388,21650,30351,40079
+51196,21651,30350,40423
+59465,21652,30349,40798
+56231,21653,30348,40031
+55247,21654,30347,40211
+51474,21655,30346,40059
+52747,21656,30345,40008
+53642,21657,30344,40800
+52699,21658,30343,40369
+57389,21659,30342,40832
+55734,21660,30341,40960
+56093,21661,30340,40178
+53177,21662,30339,40499
+58115,21663,30338,40383
+54035,21664,30337,40038
+59969,21665,30336,40932
+52950,21666,30335,40937
+53872,21667,30334,40721
+55645,21668,30333,40031
+51800,21669,30332,40028
+53981,21670,30331,40058
+50926,21671,30330,40755
+58500,21672,30329,40645
+56391,21673,30328,40589
+51902,21674,30327,40488
+52523,21675,30326,40363
+54343,21676,30325,40970
+52549,21677,30324,40400
+52955,21678,30323,40262
+51847,21679,30322,40578
+55384,21680,30321,40916
+58682,21681,30320,40908
+57589,21682,30319,40538
+56498,21683,30318,40625
+58252,21684,30317,40271
+54448,21685,30316,40631
+57602,21686,30315,40053
+55250,21687,30314,40638
+58744,21688,30313,40633
+58069,21689,30312,40683
+56778,21690,30311,40679
+55595,21691,30310,40126
+53956,21692,30309,40238
+52247,21693,30308,40116
+51669,21694,30307,40894
+52004,21695,30306,40698
+51051,21696,30305,40705
+56261,21697,30304,40551
+51054,21698,30303,40965
+50488,21699,30302,40948
+59593,21700,30301,40933
+55204,21701,30300,40442
+56050,21702,30299,40310
+55986,21703,30298,40080
+55227,21704,30297,40693
+58392,21705,30296,40320
+50593,21706,30295,40827
+54239,21707,30294,40837
+58130,21708,30293,40698
+57238,21709,30292,40092
+59660,21710,30291,40950
+56751,21711,30290,40939
+52128,21712,30289,40115
+54824,21713,30288,40018
+52533,21714,30287,40422
+58468,21715,30286,40181
+56648,21716,30285,40939
+57864,21717,30284,40995
+57313,21718,30283,40571
+55954,21719,30282,40494
+53697,21720,30281,40216
+59522,21721,30280,40495
+55384,21722,30279,40527
+52781,21723,30278,40389
+56596,21724,30277,40199
+54390,21725,30276,40442
+57395,21726,30275,40236
+50333,21727,30274,40762
+51449,21728,30273,40923
+56676,21729,30272,40593
+55949,21730,30271,40458
+54652,21731,30270,40192
+52718,21732,30269,40006
+50623,21733,30268,40220
+59267,21734,30267,40247
+55286,21735,30266,40323
+58783,21736,30265,40692
+55226,21737,30264,40299
+50499,21738,30263,40615
+56535,21739,30262,40281
+50158,21740,30261,40892
+59885,21741,30260,40715
+50400,21742,30259,40971
+55296,21743,30258,40131
+55804,21744,30257,40152
+51168,21745,30256,40384
+55675,21746,30255,40947
+53939,21747,30254,40362
+52909,21748,30253,40617
+55117,21749,30252,40138
+59499,21750,30251,40806
+59890,21751,30250,40756
+53448,21752,30249,40909
+59922,21753,30248,40258
+54454,21754,30247,40999
+57395,21755,30246,40406
+54932,21756,30245,40075
+58125,21757,30244,40043
+52133,21758,30243,40727
+59694,21759,30242,40105
+58220,21760,30241,40540
+55997,21761,30240,40651
+55523,21762,30239,40183
+54972,21763,30238,40239
+58939,21764,30237,40521
+54439,21765,30236,40921
+59406,21766,30235,40429
+56490,21767,30234,40460
+52400,21768,30233,40873
+57710,21769,30232,40686
+55700,21770,30231,40763
+52643,21771,30230,40951
+53163,21772,30229,40785
+56074,21773,30228,40286
+57463,21774,30227,40381
+58377,21775,30226,40352
+50526,21776,30225,40611
+54002,21777,30224,40646
+59449,21778,30223,40497
+51933,21779,30222,40449
+59532,21780,30221,40682
+52897,21781,30220,40453
+50736,21782,30219,40942
+54690,21783,30218,40815
+51964,21784,30217,40698
+50484,21785,30216,40533
+51299,21786,30215,40055
+55053,21787,30214,40912
+56902,21788,30213,40226
+58617,21789,30212,40584
+55978,21790,30211,40719
+51975,21791,30210,40901
+53412,21792,30209,40091
+59809,21793,30208,40029
+56353,21794,30207,40173
+53929,21795,30206,40374
+55666,21796,30205,40695
+54062,21797,30204,40690
+58293,21798,30203,40735
+58318,21799,30202,40338
+55663,21800,30201,40900
+56949,21801,30200,40705
+51669,21802,30199,40064
+52564,21803,30198,40542
+56505,21804,30197,40265
+53677,21805,30196,40565
+59537,21806,30195,40531
+55370,21807,30194,40347
+51108,21808,30193,40862
+58446,21809,30192,40739
+53886,21810,30191,40231
+59347,21811,30190,40482
+55891,21812,30189,40069
+55026,21813,30188,40107
+58736,21814,30187,40947
+50146,21815,30186,40248
+59101,21816,30185,40885
+56967,21817,30184,40879
+50542,21818,30183,40368
+51948,21819,30182,40572
+51993,21820,30181,40075
+59307,21821,30180,40720
+51298,21822,30179,40644
+58456,21823,30178,40470
+58273,21824,30177,40392
+55582,21825,30176,40318
+55541,21826,30175,40683
+54417,21827,30174,40776
+59187,21828,30173,40968
+53411,21829,30172,40878
+52710,21830,30171,40286
+56834,21831,30170,40118
+57722,21832,30169,40527
+56926,21833,30168,40035
+57476,21834,30167,40493
+53816,21835,30166,40391
+51929,21836,30165,40907
+50662,21837,30164,40495
+52879,21838,30163,40686
+52740,21839,30162,40307
+58672,21840,30161,40815
+55124,21841,30160,40752
+55674,21842,30159,40493
+51062,21843,30158,40388
+52612,21844,30157,40707
+51650,21845,30156,40670
+54956,21846,30155,40124
+59954,21847,30154,40535
+52107,21848,30153,40734
+59485,21849,30152,40233
+58137,21850,30151,40666
+52686,21851,30150,40922
+52461,21852,30149,40943
+50579,21853,30148,40893
+55581,21854,30147,40909
+55633,21855,30146,40165
+59716,21856,30145,40357
+56693,21857,30144,40943
+52529,21858,30143,40994
+50854,21859,30142,40859
+50844,21860,30141,40123
+52966,21861,30140,40118
+58691,21862,30139,40273
+50886,21863,30138,40680
+55203,21864,30137,40976
+56121,21865,30136,40821
+59042,21866,30135,40595
+59201,21867,30134,40641
+57529,21868,30133,40568
+50166,21869,30132,40389
+58800,21870,30131,40437
+50725,21871,30130,40505
+57969,21872,30129,40486
+53157,21873,30128,40045
+59070,21874,30127,40077
+52852,21875,30126,40152
+53326,21876,30125,40396
+51088,21877,30124,40628
+52450,21878,30123,40281
+56258,21879,30122,40916
+50742,21880,30121,40727
+50517,21881,30120,40306
+53715,21882,30119,40397
+51877,21883,30118,40409
+57346,21884,30117,40565
+55698,21885,30116,40012
+56267,21886,30115,40707
+51955,21887,30114,40226
+50395,21888,30113,40999
+52631,21889,30112,40615
+55460,21890,30111,40306
+55654,21891,30110,40600
+50606,21892,30109,40393
+57079,21893,30108,40954
+50363,21894,30107,40113
+51979,21895,30106,40703
+54537,21896,30105,40831
+54630,21897,30104,40942
+57760,21898,30103,40130
+54782,21899,30102,40326
+56283,21900,30101,40944
+58235,21901,30100,40135
+58419,21902,30099,40554
+54719,21903,30098,40484
+58455,21904,30097,40580
+57711,21905,30096,40745
+59480,21906,30095,40509
+51567,21907,30094,40230
+58161,21908,30093,40063
+52604,21909,30092,40301
+58420,21910,30091,40108
+50387,21911,30090,40526
+58718,21912,30089,40133
+51135,21913,30088,40267
+52781,21914,30087,40952
+53792,21915,30086,40458
+50920,21916,30085,40582
+53190,21917,30084,40432
+56364,21918,30083,40029
+52764,21919,30082,40252
+57944,21920,30081,40249
+55650,21921,30080,40999
+57518,21922,30079,40362
+52163,21923,30078,40302
+58714,21924,30077,40402
+55846,21925,30076,40849
+53358,21926,30075,40805
+57887,21927,30074,40492
+53452,21928,30073,40133
+57605,21929,30072,40847
+51789,21930,30071,40170
+55951,21931,30070,40355
+59134,21932,30069,40098
+58023,21933,30068,40852
+51034,21934,30067,40922
+50128,21935,30066,40350
+50738,21936,30065,40281
+57524,21937,30064,40989
+54442,21938,30063,40493
+57974,21939,30062,40893
+55437,21940,30061,40173
+57383,21941,30060,40370
+58480,21942,30059,40514
+53812,21943,30058,40781
+54292,21944,30057,40747
+51660,21945,30056,40076
+52645,21946,30055,40804
+57534,21947,30054,40149
+59471,21948,30053,40345
+52721,21949,30052,40751
+54753,21950,30051,40300
+56158,21951,30050,40940
+53226,21952,30049,40721
+54696,21953,30048,40343
+54175,21954,30047,40411
+50252,21955,30046,40150
+57422,21956,30045,40501
+57891,21957,30044,40996
+50077,21958,30043,40799
+58948,21959,30042,40934
+59724,21960,30041,40146
+59606,21961,30040,40943
+54745,21962,30039,40218
+52516,21963,30038,40779
+59810,21964,30037,40735
+57465,21965,30036,40684
+59876,21966,30035,40816
+58282,21967,30034,40189
+50290,21968,30033,40605
+58867,21969,30032,40480
+51730,21970,30031,40782
+53353,21971,30030,40006
+56978,21972,30029,40454
+55823,21973,30028,40483
+57384,21974,30027,40838
+58004,21975,30026,40690
+56230,21976,30025,40519
+53354,21977,30024,40397
+58332,21978,30023,40848
+54066,21979,30022,40993
+57928,21980,30021,40880
+56594,21981,30020,40425
+59032,21982,30019,40624
+56104,21983,30018,40266
+57744,21984,30017,40296
+57528,21985,30016,40318
+56887,21986,30015,40412
+56481,21987,30014,40448
+51772,21988,30013,40723
+57710,21989,30012,40738
+59382,21990,30011,40932
+58905,21991,30010,40502
+52601,21992,30009,40982
+54458,21993,30008,40664
+59764,21994,30007,40946
+55773,21995,30006,40004
+59735,21996,30005,40959
+53537,21997,30004,40929
+57713,21998,30003,40418
+57708,21999,30002,40649
+57608,22000,30001,40367
+51244,22001,30000,40724
+57570,22002,31000,40624
+50520,22003,30999,40304
+52576,22004,30998,40777
+53989,22005,30997,40791
+51467,22006,30996,40358
+58037,22007,30995,40709
+57174,22008,30994,40930
+59146,22009,30993,40941
+55747,22010,30992,40477
+57657,22011,30991,40143
+50225,22012,30990,40512
+54745,22013,30989,40231
+57651,22014,30988,40832
+59178,22015,30987,40749
+50076,22016,30986,40181
+53634,22017,30985,40125
+50279,22018,30984,40174
+58818,22019,30983,40939
+54915,22020,30982,40459
+58392,22021,30981,40261
+52747,22022,30980,40831
+56670,22023,30979,40332
+51619,22024,30978,40840
+57775,22025,30977,40327
+58126,22026,30976,40380
+52714,22027,30975,40766
+57501,22028,30974,40017
+59055,22029,30973,40593
+52546,22030,30972,40856
+59030,22031,30971,40019
+50098,22032,30970,40655
+54943,22033,30969,40350
+57513,22034,30968,40571
+56708,22035,30967,40052
+50776,22036,30966,40767
+52730,22037,30965,40735
+51193,22038,30964,40323
+59798,22039,30963,40934
+50483,22040,30962,40451
+55261,22041,30961,40372
+55753,22042,30960,40690
+52036,22043,30959,40750
+55996,22044,30958,40364
+55238,22045,30957,40161
+51873,22046,30956,40965
+59508,22047,30955,40926
+56045,22048,30954,40553
+53718,22049,30953,40916
+57661,22050,30952,40329
+51328,22051,30951,40392
+58003,22052,30950,40831
+51728,22053,30949,40890
+55898,22054,30948,40966
+50755,22055,30947,40399
+52554,22056,30946,40808
+54999,22057,30945,40327
+52439,22058,30944,40805
+59387,22059,30943,40587
+58180,22060,30942,40238
+54979,22061,30941,40109
+54213,22062,30940,40611
+57256,22063,30939,40683
+59474,22064,30938,40439
+55852,22065,30937,40769
+56471,22066,30936,40750
+55595,22067,30935,40386
+53272,22068,30934,40207
+51357,22069,30933,40571
+58130,22070,30932,40940
+50824,22071,30931,40724
+58125,22072,30930,40627
+57410,22073,30929,40011
+52813,22074,30928,40937
+50019,22075,30927,40183
+51816,22076,30926,40746
+57561,22077,30925,40711
+55138,22078,30924,40495
+54630,22079,30923,40621
+55092,22080,30922,40148
+57140,22081,30921,40962
+50987,22082,30920,40405
+58009,22083,30919,40222
+57340,22084,30918,40836
+59109,22085,30917,40908
+53957,22086,30916,40522
+54744,22087,30915,40128
+53471,22088,30914,40576
+54801,22089,30913,40574
+59792,22090,30912,40488
+54225,22091,30911,40561
+50156,22092,30910,40301
+52813,22093,30909,40012
+53676,22094,30908,40371
+57554,22095,30907,40432
+52759,22096,30906,40453
+55779,22097,30905,40779
+50476,22098,30904,40145
+54719,22099,30903,40497
+51170,22100,30902,40714
+55886,22101,30901,40106
+55670,22102,30900,40282
+52985,22103,30899,40480
+53940,22104,30898,40122
+51545,22105,30897,40674
+59953,22106,30896,40567
+58775,22107,30895,40378
+56069,22108,30894,40235
+53059,22109,30893,40488
+57798,22110,30892,40517
+51312,22111,30891,40001
+56863,22112,30890,40485
+54989,22113,30889,40256
+59007,22114,30888,40809
+58367,22115,30887,40662
+58798,22116,30886,40309
+53465,22117,30885,40617
+53963,22118,30884,40338
+53550,22119,30883,40743
+57088,22120,30882,40406
+52589,22121,30881,40509
+51351,22122,30880,40347
+53226,22123,30879,40640
+52614,22124,30878,40591
+50494,22125,30877,40831
+50501,22126,30876,40560
+51125,22127,30875,40395
+57857,22128,30874,40521
+54501,22129,30873,40044
+54889,22130,30872,40102
+53790,22131,30871,40506
+54853,22132,30870,40376
+54541,22133,30869,40517
+56070,22134,30868,40805
+54202,22135,30867,40681
+52408,22136,30866,40975
+56338,22137,30865,40553
+53812,22138,30864,40687
+53983,22139,30863,40470
+59315,22140,30862,40542
+59106,22141,30861,40493
+51005,22142,30860,40968
+50531,22143,30859,40675
+58463,22144,30858,40469
+58527,22145,30857,40317
+54587,22146,30856,40893
+56721,22147,30855,40264
+54090,22148,30854,40832
+58331,22149,30853,40441
+58316,22150,30852,40303
+57875,22151,30851,40643
+54063,22152,30850,40786
+56683,22153,30849,40963
+51806,22154,30848,40607
+51464,22155,30847,40305
+52305,22156,30846,40460
+58372,22157,30845,40645
+55858,22158,30844,40425
+58335,22159,30843,40172
+53680,22160,30842,40549
+53664,22161,30841,40774
+58924,22162,30840,40611
+55818,22163,30839,40468
+52065,22164,30838,40952
+57818,22165,30837,40259
+57456,22166,30836,40316
+56604,22167,30835,40896
+56441,22168,30834,40827
+57645,22169,30833,40119
+54135,22170,30832,40487
+55035,22171,30831,40620
+53682,22172,30830,40652
+56904,22173,30829,40197
+59624,22174,30828,40935
+50526,22175,30827,40866
+59221,22176,30826,40900
+51260,22177,30825,40394
+57390,22178,30824,40058
+52302,22179,30823,40483
+58766,22180,30822,40869
+55260,22181,30821,40118
+52648,22182,30820,40009
+50537,22183,30819,40340
+51771,22184,30818,40413
+51755,22185,30817,40049
+51731,22186,30816,40947
+50090,22187,30815,40964
+55692,22188,30814,40883
+52465,22189,30813,40392
+57234,22190,30812,40335
+57234,22191,30811,40882
+52004,22192,30810,40438
+59846,22193,30809,40126
+58390,22194,30808,40512
+57250,22195,30807,40073
+55958,22196,30806,40987
+58517,22197,30805,40005
+52694,22198,30804,40728
+56414,22199,30803,40589
+54328,22200,30802,40941
+59162,22201,30801,40237
+58760,22202,30800,40976
+53850,22203,30799,40932
+57763,22204,30798,40388
+51940,22205,30797,40889
+56997,22206,30796,40587
+57248,22207,30795,40799
+51648,22208,30794,40249
+57443,22209,30793,40319
+52072,22210,30792,40232
+53096,22211,30791,40712
+51260,22212,30790,40861
+50122,22213,30789,40910
+54735,22214,30788,40783
+57342,22215,30787,40956
+51172,22216,30786,40719
+57773,22217,30785,40255
+59800,22218,30784,40920
+59664,22219,30783,40448
+54713,22220,30782,40661
+50372,22221,30781,40949
+55957,22222,30780,40153
+56749,22223,30779,40182
+51309,22224,30778,40319
+56630,22225,30777,40015
+57448,22226,30776,40198
+52678,22227,30775,40470
+57700,22228,30774,40481
+59038,22229,30773,40731
+52807,22230,30772,40553
+51231,22231,30771,40711
+51235,22232,30770,40959
+54121,22233,30769,40242
+56824,22234,30768,40729
+50015,22235,30767,40637
+57828,22236,30766,40912
+57254,22237,30765,40985
+57926,22238,30764,40476
+56539,22239,30763,40586
+50608,22240,30762,40589
+55028,22241,30761,40771
+56442,22242,30760,40263
+56942,22243,30759,40391
+52706,22244,30758,40176
+52342,22245,30757,40033
+58786,22246,30756,40818
+58756,22247,30755,40921
+57418,22248,30754,40440
+57970,22249,30753,40396
+56844,22250,30752,40926
+51690,22251,30751,40942
+54391,22252,30750,40953
+53645,22253,30749,40327
+51035,22254,30748,40544
+57024,22255,30747,40191
+59515,22256,30746,40046
+56656,22257,30745,40335
+57761,22258,30744,40581
+53158,22259,30743,40854
+52699,22260,30742,40825
+58774,22261,30741,40124
+50371,22262,30740,40774
+50801,22263,30739,40279
+52444,22264,30738,40446
+57771,22265,30737,40175
+51928,22266,30736,40851
+55063,22267,30735,40248
+55495,22268,30734,40109
+55726,22269,30733,40252
+57961,22270,30732,40289
+58158,22271,30731,40434
+56229,22272,30730,40543
+52351,22273,30729,40255
+50530,22274,30728,40462
+53425,22275,30727,40567
+54162,22276,30726,40382
+54913,22277,30725,40048
+51583,22278,30724,40617
+54490,22279,30723,40468
+59485,22280,30722,40559
+53256,22281,30721,40945
+52719,22282,30720,40916
+53768,22283,30719,40302
+52310,22284,30718,40544
+58236,22285,30717,40868
+51204,22286,30716,40580
+58907,22287,30715,40630
+55999,22288,30714,40140
+57412,22289,30713,40999
+59377,22290,30712,40733
+54626,22291,30711,40703
+55980,22292,30710,40059
+52283,22293,30709,40961
+57466,22294,30708,40167
+58819,22295,30707,40788
+53026,22296,30706,40477
+54542,22297,30705,40796
+52567,22298,30704,40902
+58499,22299,30703,40620
+56170,22300,30702,40382
+52460,22301,30701,40187
+57264,22302,30700,40163
+58025,22303,30699,40824
+53951,22304,30698,40096
+50196,22305,30697,40355
+58465,22306,30696,40934
+58687,22307,30695,40017
+53659,22308,30694,40430
+58969,22309,30693,40995
+52248,22310,30692,40005
+53344,22311,30691,40748
+58014,22312,30690,40404
+51073,22313,30689,40805
+55842,22314,30688,40139
+59307,22315,30687,40159
+55223,22316,30686,40314
+55985,22317,30685,40316
+50593,22318,30684,40628
+57998,22319,30683,40867
+56152,22320,30682,40872
+58754,22321,30681,40934
+55224,22322,30680,40677
+56155,22323,30679,40945
+52933,22324,30678,40929
+50271,22325,30677,40226
+56300,22326,30676,40350
+50117,22327,30675,40023
+51141,22328,30674,40706
+53300,22329,30673,40528
+50773,22330,30672,40977
+57555,22331,30671,40170
+51247,22332,30670,40041
+57952,22333,30669,40469
+51722,22334,30668,40426
+58540,22335,30667,40779
+53726,22336,30666,40072
+59363,22337,30665,40366
+53832,22338,30664,40534
+59145,22339,30663,40500
+56189,22340,30662,40289
+57520,22341,30661,40425
+55611,22342,30660,40454
+55031,22343,30659,40386
+56763,22344,30658,40417
+55276,22345,30657,40761
+59043,22346,30656,40315
+51793,22347,30655,40066
+57640,22348,30654,40566
+53244,22349,30653,40441
+58944,22350,30652,40321
+51369,22351,30651,40933
+55221,22352,30650,40971
+58635,22353,30649,40941
+52054,22354,30648,40093
+57487,22355,30647,40030
+55576,22356,30646,40905
+54177,22357,30645,40606
+54521,22358,30644,40329
+59704,22359,30643,40240
+54949,22360,30642,40309
+50796,22361,30641,40300
+55313,22362,30640,40757
+59624,22363,30639,40818
+56613,22364,30638,40735
+57706,22365,30637,40923
+55927,22366,30636,40733
+50888,22367,30635,40869
+54722,22368,30634,40062
+59149,22369,30633,40193
+53366,22370,30632,40727
+59888,22371,30631,40946
+56233,22372,30630,40006
+54549,22373,30629,40011
+58394,22374,30628,40859
+56429,22375,30627,40589
+59574,22376,30626,40420
+54105,22377,30625,40781
+54580,22378,30624,40555
+55388,22379,30623,40071
+58507,22380,30622,40381
+52302,22381,30621,40977
+53699,22382,30620,40026
+50732,22383,30619,40397
+51048,22384,30618,40293
+53389,22385,30617,40276
+55003,22386,30616,40695
+55472,22387,30615,40500
+59716,22388,30614,40532
+51681,22389,30613,40958
+50370,22390,30612,40969
+55418,22391,30611,40206
+54082,22392,30610,40703
+58742,22393,30609,40536
+56622,22394,30608,40225
+58305,22395,30607,40435
+52076,22396,30606,40263
+59115,22397,30605,40925
+51527,22398,30604,40877
+56235,22399,30603,40115
+57164,22400,30602,40027
+53904,22401,30601,40043
+52379,22402,30600,40721
+52099,22403,30599,40528
+57677,22404,30598,40630
+53272,22405,30597,40641
+50756,22406,30596,40045
+56995,22407,30595,40175
+52702,22408,30594,40861
+56360,22409,30593,40048
+57347,22410,30592,40364
+57371,22411,30591,40797
+52430,22412,30590,40767
+53717,22413,30589,40484
+52857,22414,30588,40288
+56392,22415,30587,40963
+52432,22416,30586,40941
+55290,22417,30585,40570
+53880,22418,30584,40161
+55181,22419,30583,40651
+54792,22420,30582,40865
+56767,22421,30581,40865
+50629,22422,30580,40717
+51501,22423,30579,40802
+52028,22424,30578,40712
+53682,22425,30577,40539
+52761,22426,30576,40134
+57662,22427,30575,40206
+58052,22428,30574,40530
+55601,22429,30573,40002
+55675,22430,30572,40801
+56215,22431,30571,40656
+52506,22432,30570,40102
+55230,22433,30569,40205
+54743,22434,30568,40443
+54098,22435,30567,40441
+56410,22436,30566,40281
+50713,22437,30565,40314
+53353,22438,30564,40772
+57786,22439,30563,40445
+59955,22440,30562,40317
+53081,22441,30561,40627
+59826,22442,30560,40493
+57853,22443,30559,40732
+59123,22444,30558,40595
+53219,22445,30557,40707
+52530,22446,30556,40179
+56827,22447,30555,40350
+51934,22448,30554,40468
+57360,22449,30553,40944
+56843,22450,30552,40523
+53702,22451,30551,40284
+52871,22452,30550,40641
+59755,22453,30549,40928
+50436,22454,30548,40950
+53071,22455,30547,40996
+55780,22456,30546,40888
+52240,22457,30545,40126
+50771,22458,30544,40548
+54686,22459,30543,40899
+53166,22460,30542,40887
+50734,22461,30541,40123
+55793,22462,30540,40994
+56890,22463,30539,40882
+55342,22464,30538,40080
+51301,22465,30537,40271
+54730,22466,30536,40614
+52477,22467,30535,40331
+54769,22468,30534,40006
+54137,22469,30533,40063
+50714,22470,30532,40007
+51393,22471,30531,40394
+54149,22472,30530,40148
+51900,22473,30529,40374
+59824,22474,30528,40655
+50404,22475,30527,40134
+56457,22476,30526,40466
+53031,22477,30525,40902
+57166,22478,30524,40154
+53527,22479,30523,40831
+56880,22480,30522,40225
+59105,22481,30521,40861
+56758,22482,30520,40038
+52139,22483,30519,40775
+58542,22484,30518,40651
+51894,22485,30517,40795
+51166,22486,30516,40171
+58818,22487,30515,40507
+56392,22488,30514,40022
+54577,22489,30513,40592
+50684,22490,30512,40186
+50787,22491,30511,40105
+57352,22492,30510,40800
+57841,22493,30509,40008
+57019,22494,30508,40568
+51878,22495,30507,40329
+53511,22496,30506,40689
+57111,22497,30505,40328
+57876,22498,30504,40137
+50372,22499,30503,40062
+57713,22500,30502,40622
+53798,22501,30501,40264
+50117,22502,30500,40759
+52792,22503,30499,40812
+53124,22504,30498,40585
+55010,22505,30497,40184
+58189,22506,30496,40202
+58992,22507,30495,40305
+54432,22508,30494,40820
+52481,22509,30493,40781
+56093,22510,30492,40638
+57803,22511,30491,40285
+54935,22512,30490,40277
+50444,22513,30489,40955
+54808,22514,30488,40690
+53932,22515,30487,40060
+51502,22516,30486,40273
+58158,22517,30485,40681
+55715,22518,30484,40181
+56357,22519,30483,40664
+50458,22520,30482,40291
+52403,22521,30481,40002
+58821,22522,30480,40033
+53373,22523,30479,40689
+55748,22524,30478,40882
+53242,22525,30477,40000
+57032,22526,30476,40474
+50970,22527,30475,40591
+54533,22528,30474,40489
+58451,22529,30473,40473
+54645,22530,30472,40349
+56775,22531,30471,40571
+50986,22532,30470,40744
+51202,22533,30469,40084
+54091,22534,30468,40875
+57455,22535,30467,40612
+56672,22536,30466,40158
+53577,22537,30465,40618
+57433,22538,30464,40421
+51375,22539,30463,40861
+56408,22540,30462,40736
+51153,22541,30461,40015
+52863,22542,30460,40055
+56631,22543,30459,40505
+50580,22544,30458,40336
+51025,22545,30457,40351
+55025,22546,30456,40295
+57737,22547,30455,40882
+50039,22548,30454,40776
+56373,22549,30453,40635
+55507,22550,30452,40156
+53096,22551,30451,40946
+52489,22552,30450,40830
+50616,22553,30449,40796
+59037,22554,30448,40969
+54570,22555,30447,40850
+51856,22556,30446,40928
+56745,22557,30445,40148
+55319,22558,30444,40873
+53190,22559,30443,40025
+52136,22560,30442,40087
+56682,22561,30441,40361
+59470,22562,30440,40530
+51222,22563,30439,40746
+55468,22564,30438,40861
+50971,22565,30437,40732
+56525,22566,30436,40756
+57670,22567,30435,40057
+51673,22568,30434,40952
+54835,22569,30433,40731
+52212,22570,30432,40745
+55170,22571,30431,40188
+55841,22572,30430,40815
+55783,22573,30429,40948
+54564,22574,30428,40828
+52107,22575,30427,40779
+55205,22576,30426,40497
+54224,22577,30425,40140
+54321,22578,30424,40476
+55810,22579,30423,40897
+51914,22580,30422,40331
+50889,22581,30421,40361
+58414,22582,30420,40583
+57067,22583,30419,40907
+57269,22584,30418,40816
+53487,22585,30417,40845
+58660,22586,30416,40553
+51931,22587,30415,40341
+55966,22588,30414,40768
+56702,22589,30413,40659
+51594,22590,30412,40357
+53594,22591,30411,40850
+54512,22592,30410,40859
+52650,22593,30409,40306
+57181,22594,30408,40144
+53042,22595,30407,40867
+54327,22596,30406,40781
+55395,22597,30405,40559
+53338,22598,30404,40866
+57885,22599,30403,40020
+55312,22600,30402,40922
+59135,22601,30401,40172
+51535,22602,30400,40405
+57530,22603,30399,40284
+58396,22604,30398,40096
+54419,22605,30397,40222
+57453,22606,30396,40350
+50056,22607,30395,40074
+59068,22608,30394,40649
+53690,22609,30393,40816
+57896,22610,30392,40185
+51416,22611,30391,40451
+54134,22612,30390,40784
+51772,22613,30389,40770
+53124,22614,30388,40167
+59615,22615,30387,40296
+58654,22616,30386,40513
+58119,22617,30385,40182
+58986,22618,30384,40666
+55329,22619,30383,40114
+59197,22620,30382,40235
+58682,22621,30381,40751
+51472,22622,30380,40605
+51799,22623,30379,40728
+57134,22624,30378,40935
+56116,22625,30377,40409
+51988,22626,30376,40062
+51513,22627,30375,40309
+59323,22628,30374,40287
+53657,22629,30373,40302
+57714,22630,30372,40380
+53796,22631,30371,40332
+52979,22632,30370,40617
+50749,22633,30369,40133
+58360,22634,30368,40579
+50325,22635,30367,40285
+55911,22636,30366,40820
+56902,22637,30365,40770
+56329,22638,30364,40397
+54012,22639,30363,40391
+51320,22640,30362,40736
+59165,22641,30361,40459
+57446,22642,30360,40672
+58090,22643,30359,40501
+54714,22644,30358,40895
+52061,22645,30357,40687
+55238,22646,30356,40052
+57106,22647,30355,40812
+56452,22648,30354,40176
+58818,22649,30353,40046
+52911,22650,30352,40931
+55835,22651,30351,40712
+54685,22652,30350,40088
+53291,22653,30349,40925
+59418,22654,30348,40785
+50603,22655,30347,40332
+52016,22656,30346,40792
+55017,22657,30345,40265
+50322,22658,30344,40572
+57885,22659,30343,40457
+54248,22660,30342,40300
+53613,22661,30341,40953
+53541,22662,30340,40227
+55868,22663,30339,40514
+57908,22664,30338,40968
+58509,22665,30337,40029
+53633,22666,30336,40512
+58949,22667,30335,40896
+55876,22668,30334,40931
+59902,22669,30333,40627
+58343,22670,30332,40976
+55656,22671,30331,40474
+55673,22672,30330,40989
+54859,22673,30329,40174
+54748,22674,30328,40655
+56167,22675,30327,40578
+59816,22676,30326,40186
+55132,22677,30325,40167
+55218,22678,30324,40437
+52742,22679,30323,40341
+51511,22680,30322,40222
+52367,22681,30321,40622
+58195,22682,30320,40182
+51736,22683,30319,40632
+58863,22684,30318,40856
+54787,22685,30317,40056
+56246,22686,30316,40867
+56344,22687,30315,40053
+50472,22688,30314,40425
+57655,22689,30313,40080
+54343,22690,30312,40947
+55469,22691,30311,40582
+59407,22692,30310,40820
+54607,22693,30309,40119
+52339,22694,30308,40095
+52947,22695,30307,40302
+54418,22696,30306,40485
+59375,22697,30305,40277
+54057,22698,30304,40847
+52025,22699,30303,40325
+58978,22700,30302,40492
+57469,22701,30301,40279
+55961,22702,30300,40623
+57917,22703,30299,40690
+54959,22704,30298,40017
+53442,22705,30297,40033
+55336,22706,30296,40934
+56577,22707,30295,40528
+55439,22708,30294,40751
+58377,22709,30293,40971
+53610,22710,30292,40331
+59145,22711,30291,40883
+57256,22712,30290,40066
+50575,22713,30289,40966
+56918,22714,30288,40000
+57354,22715,30287,40298
+55409,22716,30286,40643
+58017,22717,30285,40782
+56597,22718,30284,40713
+54748,22719,30283,40444
+53252,22720,30282,40112
+55680,22721,30281,40959
+52019,22722,30280,40163
+58073,22723,30279,40229
+55118,22724,30278,40558
+53903,22725,30277,40833
+57708,22726,30276,40055
+53963,22727,30275,40608
+58557,22728,30274,40385
+57251,22729,30273,40037
+57007,22730,30272,40566
+58385,22731,30271,40299
+53104,22732,30270,40184
+50594,22733,30269,40055
+57182,22734,30268,40964
+58042,22735,30267,40808
+57675,22736,30266,40859
+57188,22737,30265,40636
+53916,22738,30264,40390
+55543,22739,30263,40648
+53680,22740,30262,40934
+57797,22741,30261,40819
+54461,22742,30260,40694
+52941,22743,30259,40032
+55653,22744,30258,40061
+53728,22745,30257,40326
+56921,22746,30256,40012
+56416,22747,30255,40357
+51060,22748,30254,40384
+52501,22749,30253,40415
+57320,22750,30252,40220
+54057,22751,30251,40457
+51606,22752,30250,40646
+50675,22753,30249,40685
+51638,22754,30248,40292
+51921,22755,30247,40973
+55191,22756,30246,40003
+55790,22757,30245,40326
+54080,22758,30244,40452
+50659,22759,30243,40111
+58581,22760,30242,40099
+52842,22761,30241,40276
+51077,22762,30240,40493
+54957,22763,30239,40179
+57312,22764,30238,40486
+57693,22765,30237,40010
+53030,22766,30236,40237
+59007,22767,30235,40515
+51659,22768,30234,40308
+54608,22769,30233,40638
+52567,22770,30232,40303
+51145,22771,30231,40257
+58009,22772,30230,40968
+56206,22773,30229,40742
+51419,22774,30228,40309
+58225,22775,30227,40009
+55040,22776,30226,40807
+52260,22777,30225,40077
+53082,22778,30224,40952
+57163,22779,30223,40610
+50441,22780,30222,40832
+50255,22781,30221,40868
+59598,22782,30220,40067
+59054,22783,30219,40003
+59117,22784,30218,40394
+51792,22785,30217,40179
+53585,22786,30216,40509
+56725,22787,30215,40984
+54137,22788,30214,40874
+59834,22789,30213,40719
+50160,22790,30212,40941
+51115,22791,30211,40451
+50122,22792,30210,40924
+58066,22793,30209,40635
+52811,22794,30208,40422
+58269,22795,30207,40981
+53058,22796,30206,40943
+51534,22797,30205,40433
+56898,22798,30204,40386
+55228,22799,30203,40260
+55169,22800,30202,40732
+58006,22801,30201,40136
+51624,22802,30200,40289
+55256,22803,30199,40300
+56891,22804,30198,40117
+51542,22805,30197,40997
+59332,22806,30196,40217
+59317,22807,30195,40273
+57096,22808,30194,40671
+51081,22809,30193,40331
+55995,22810,30192,40188
+57545,22811,30191,40944
+58287,22812,30190,40690
+52639,22813,30189,40733
+52679,22814,30188,40551
+52363,22815,30187,40667
+51881,22816,30186,40701
+53223,22817,30185,40920
+59506,22818,30184,40495
+54965,22819,30183,40547
+56116,22820,30182,40124
+56405,22821,30181,40268
+59627,22822,30180,40963
+56067,22823,30179,40702
+55789,22824,30178,40468
+51261,22825,30177,40610
+53535,22826,30176,40023
+56050,22827,30175,40832
+53636,22828,30174,40283
+50597,22829,30173,40556
+58320,22830,30172,40061
+59441,22831,30171,40355
+52379,22832,30170,40866
+52887,22833,30169,40961
+58770,22834,30168,40443
+51922,22835,30167,40505
+58999,22836,30166,40845
+57708,22837,30165,40056
+56760,22838,30164,40889
+53353,22839,30163,40134
+51901,22840,30162,40870
+55777,22841,30161,40698
+53307,22842,30160,40153
+50395,22843,30159,40440
+58805,22844,30158,40775
+58117,22845,30157,40330
+55123,22846,30156,40399
+56656,22847,30155,40497
+53114,22848,30154,40311
+56360,22849,30153,40526
+54027,22850,30152,40025
+58434,22851,30151,40809
+54792,22852,30150,40809
+53918,22853,30149,40580
+57216,22854,30148,40846
+51547,22855,30147,40862
+54860,22856,30146,40418
+56122,22857,30145,40704
+57106,22858,30144,40968
+58666,22859,30143,40821
+53591,22860,30142,40858
+56882,22861,30141,40586
+58738,22862,30140,40171
+53886,22863,30139,40528
+52635,22864,30138,40223
+51793,22865,30137,40594
+58082,22866,30136,40101
+51129,22867,30135,40103
+52300,22868,30134,40135
+55647,22869,30133,40638
+57511,22870,30132,40598
+53917,22871,30131,40481
+52143,22872,30130,40164
+57003,22873,30129,40656
+55626,22874,30128,40799
+55858,22875,30127,40370
+58781,22876,30126,40919
+55521,22877,30125,40666
+54946,22878,30124,40429
+56830,22879,30123,40892
+57857,22880,30122,40703
+52740,22881,30121,40530
+52090,22882,30120,40604
+50154,22883,30119,40880
+52341,22884,30118,40561
+58069,22885,30117,40632
+53953,22886,30116,40805
+57972,22887,30115,40435
+55657,22888,30114,40759
+55209,22889,30113,40621
+53973,22890,30112,40202
+55255,22891,30111,40812
+56332,22892,30110,40531
+53686,22893,30109,40652
+51686,22894,30108,40479
+51715,22895,30107,40313
+55072,22896,30106,40355
+50133,22897,30105,40339
+56489,22898,30104,40513
+59003,22899,30103,40759
+54745,22900,30102,40131
+53313,22901,30101,40232
+52274,22902,30100,40880
+55458,22903,30099,40873
+55534,22904,30098,40464
+53911,22905,30097,40901
+50632,22906,30096,40502
+57437,22907,30095,40885
+59702,22908,30094,40305
+53636,22909,30093,40958
+59860,22910,30092,40294
+59571,22911,30091,40813
+58460,22912,30090,40398
+54377,22913,30089,40663
+53441,22914,30088,40064
+53695,22915,30087,40345
+57484,22916,30086,40747
+51850,22917,30085,40920
+53293,22918,30084,40985
+52716,22919,30083,40230
+51523,22920,30082,40771
+54234,22921,30081,40059
+59798,22922,30080,40589
+58872,22923,30079,40228
+59009,22924,30078,40238
+56991,22925,30077,40842
+50254,22926,30076,40863
+57297,22927,30075,40729
+55098,22928,30074,40381
+50900,22929,30073,40651
+51815,22930,30072,40661
+53402,22931,30071,40873
+59648,22932,30070,40283
+53026,22933,30069,40949
+59271,22934,30068,40854
+56038,22935,30067,40551
+52602,22936,30066,40003
+53077,22937,30065,40138
+55269,22938,30064,40989
+50843,22939,30063,40842
+51745,22940,30062,40958
+54397,22941,30061,40765
+54777,22942,30060,40861
+55473,22943,30059,40899
+58413,22944,30058,40004
+58537,22945,30057,40339
+50847,22946,30056,40132
+54208,22947,30055,40843
+52667,22948,30054,40655
+59181,22949,30053,40243
+58180,22950,30052,40874
+58325,22951,30051,40815
+56906,22952,30050,40421
+57389,22953,30049,40664
+54999,22954,30048,40121
+58908,22955,30047,40693
+54664,22956,30046,40868
+51423,22957,30045,40083
+58281,22958,30044,40140
+57559,22959,30043,40134
+58137,22960,30042,40134
+58669,22961,30041,40547
+51053,22962,30040,40873
+58645,22963,30039,40684
+54672,22964,30038,40860
+58800,22965,30037,40110
+53128,22966,30036,40273
+55339,22967,30035,40243
+59988,22968,30034,40765
+57397,22969,30033,40655
+59921,22970,30032,40828
+53630,22971,30031,40313
+54076,22972,30030,40494
+59835,22973,30029,40336
+52930,22974,30028,40184
+59479,22975,30027,40503
+56003,22976,30026,40400
+56243,22977,30025,40111
+53700,22978,30024,40857
+51325,22979,30023,40159
+50276,22980,30022,40182
+50526,22981,30021,40330
+59830,22982,30020,40111
+59613,22983,30019,40902
+50574,22984,30018,40838
+51698,22985,30017,40300
+56135,22986,30016,40573
+59013,22987,30015,40993
+52576,22988,30014,40181
+55349,22989,30013,40064
+51399,22990,30012,40565
+59652,22991,30011,40433
+56989,22992,30010,40213
+57087,22993,30009,40810
+55770,22994,30008,40470
+52924,22995,30007,40697
+59849,22996,30006,40633
+56348,22997,30005,40091
+50052,22998,30004,40720
+53824,22999,30003,40996
+54650,23000,30002,40885
+56840,23001,30001,40939
+55133,23002,30000,40421
+50453,23003,31000,40233
+57960,23004,30999,40136
+55216,23005,30998,40623
+55197,23006,30997,40351
+56708,23007,30996,40985
+52669,23008,30995,40019
+58871,23009,30994,40746
+53473,23010,30993,40702
+56614,23011,30992,40221
+50693,23012,30991,40896
+56462,23013,30990,40909
+52148,23014,30989,40193
+54646,23015,30988,40458
+52575,23016,30987,40525
+59577,23017,30986,40592
+52678,23018,30985,40649
+57683,23019,30984,40182
+54937,23020,30983,40217
+54065,23021,30982,40824
+57352,23022,30981,40067
+58586,23023,30980,40114
+54695,23024,30979,40195
+53756,23025,30978,40761
+53021,23026,30977,40788
+53919,23027,30976,40903
+55692,23028,30975,40744
+57767,23029,30974,40027
+53115,23030,30973,40133
+51621,23031,30972,40571
+55342,23032,30971,40852
+52812,23033,30970,40319
+50311,23034,30969,40014
+57233,23035,30968,40739
+56182,23036,30967,40589
+57803,23037,30966,40689
+52108,23038,30965,40644
+53786,23039,30964,40512
+59437,23040,30963,40796
+57541,23041,30962,40093
+52788,23042,30961,40388
+51003,23043,30960,40716
+52224,23044,30959,40197
+57819,23045,30958,40339
+51388,23046,30957,40751
+56367,23047,30956,40787
+54813,23048,30955,40659
+58659,23049,30954,40604
+59284,23050,30953,40815
+54356,23051,30952,40724
+59027,23052,30951,40799
+58483,23053,30950,40908
+52508,23054,30949,40222
+51304,23055,30948,40624
+50069,23056,30947,40272
+59540,23057,30946,40696
+59880,23058,30945,40467
+53566,23059,30944,40847
+59383,23060,30943,40322
+51471,23061,30942,40431
+51965,23062,30941,40469
+55866,23063,30940,40095
+51343,23064,30939,40729
+53443,23065,30938,40275
+50513,23066,30937,40279
+53648,23067,30936,40990
+51493,23068,30935,40988
+55468,23069,30934,40015
+52286,23070,30933,40957
+51896,23071,30932,40589
+52498,23072,30931,40276
+59773,23073,30930,40335
+56350,23074,30929,40219
+59469,23075,30928,40973
+59092,23076,30927,40556
+52023,23077,30926,40621
+57808,23078,30925,40674
+53512,23079,30924,40059
+51592,23080,30923,40888
+55231,23081,30922,40264
+56273,23082,30921,40500
+51833,23083,30920,40781
+52062,23084,30919,40008
+58379,23085,30918,40132
+51172,23086,30917,40365
+51550,23087,30916,40827
+52393,23088,30915,40340
+54325,23089,30914,40887
+56170,23090,30913,40890
+52666,23091,30912,40561
+50124,23092,30911,40238
+58710,23093,30910,40479
+58986,23094,30909,40115
+58703,23095,30908,40873
+58461,23096,30907,40589
+55179,23097,30906,40086
+57786,23098,30905,40042
+54659,23099,30904,40387
+59749,23100,30903,40190
+57284,23101,30902,40024
+52627,23102,30901,40618
+50879,23103,30900,40735
+58765,23104,30899,40739
+55167,23105,30898,40330
+54544,23106,30897,40394
+57578,23107,30896,40695
+50481,23108,30895,40502
+51409,23109,30894,40146
+59039,23110,30893,40026
+56898,23111,30892,40448
+51073,23112,30891,40043
+55885,23113,30890,40515
+59134,23114,30889,40579
+50547,23115,30888,40463
+58843,23116,30887,40563
+59257,23117,30886,40911
+50398,23118,30885,40801
+53069,23119,30884,40021
+50530,23120,30883,40293
+52752,23121,30882,40789
+59876,23122,30881,40889
+52065,23123,30880,40156
+59977,23124,30879,40429
+59915,23125,30878,40394
+55426,23126,30877,40021
+55508,23127,30876,40977
+50536,23128,30875,40504
+54486,23129,30874,40673
+50722,23130,30873,40732
+58347,23131,30872,40810
+58421,23132,30871,40603
+57601,23133,30870,40453
+51275,23134,30869,40396
+51405,23135,30868,40023
+55036,23136,30867,40536
+58476,23137,30866,40851
+56185,23138,30865,40866
+52596,23139,30864,40667
+53064,23140,30863,40626
+56792,23141,30862,40587
+58969,23142,30861,40174
+50267,23143,30860,40362
+58582,23144,30859,40117
+50076,23145,30858,40326
+53529,23146,30857,40282
+55504,23147,30856,40354
+52072,23148,30855,40769
+53774,23149,30854,40446
+50188,23150,30853,40977
+52643,23151,30852,40100
+51328,23152,30851,40462
+54502,23153,30850,40514
+59940,23154,30849,40178
+55445,23155,30848,40345
+50912,23156,30847,40239
+53854,23157,30846,40561
+51142,23158,30845,40227
+58362,23159,30844,40822
+57264,23160,30843,40505
+51267,23161,30842,40180
+53977,23162,30841,40352
+54791,23163,30840,40770
+57784,23164,30839,40594
+52932,23165,30838,40574
+54214,23166,30837,40955
+55363,23167,30836,40397
+50913,23168,30835,40932
+53458,23169,30834,40237
+53001,23170,30833,40475
+56313,23171,30832,40017
+51863,23172,30831,40857
+58559,23173,30830,40824
+59294,23174,30829,40703
+51108,23175,30828,40298
+56249,23176,30827,40502
+56302,23177,30826,40200
+57572,23178,30825,40554
+56978,23179,30824,40943
+52647,23180,30823,40074
+55832,23181,30822,40337
+50941,23182,30821,40222
+57826,23183,30820,40456
+59619,23184,30819,40669
+54549,23185,30818,40271
+55259,23186,30817,40171
+55029,23187,30816,40040
+59360,23188,30815,40211
+54991,23189,30814,40843
+59844,23190,30813,40239
+52447,23191,30812,40937
+55637,23192,30811,40809
+57794,23193,30810,40470
+56943,23194,30809,40988
+52086,23195,30808,40012
+59483,23196,30807,40500
+52446,23197,30806,40274
+51828,23198,30805,40815
+54450,23199,30804,40385
+54250,23200,30803,40374
+56835,23201,30802,40862
+52448,23202,30801,40924
+50683,23203,30800,40040
+51904,23204,30799,40938
+55699,23205,30798,40896
+50222,23206,30797,40253
+57918,23207,30796,40888
+58137,23208,30795,40085
+52835,23209,30794,40418
+54062,23210,30793,40202
+58328,23211,30792,40910
+50983,23212,30791,40949
+54777,23213,30790,40831
+54657,23214,30789,40744
+59096,23215,30788,40322
+57499,23216,30787,40139
+55279,23217,30786,40308
+52010,23218,30785,40188
+57137,23219,30784,40160
+57785,23220,30783,40610
+56222,23221,30782,40409
+53374,23222,30781,40422
+51349,23223,30780,40418
+50536,23224,30779,40783
+54900,23225,30778,40000
+50500,23226,30777,40009
+57866,23227,30776,40952
+59551,23228,30775,40617
+57614,23229,30774,40402
+53670,23230,30773,40039
+58729,23231,30772,40426
+59545,23232,30771,40864
+51261,23233,30770,40015
+53070,23234,30769,40798
+58953,23235,30768,40401
+59009,23236,30767,40471
+56837,23237,30766,40516
+56553,23238,30765,40959
+51896,23239,30764,40340
+58054,23240,30763,40075
+59369,23241,30762,40131
+51879,23242,30761,40963
+50274,23243,30760,40734
+54536,23244,30759,40755
+54960,23245,30758,40321
+50568,23246,30757,40598
+54823,23247,30756,40138
+53448,23248,30755,40088
+58563,23249,30754,40529
+59055,23250,30753,40158
+57833,23251,30752,40705
+52827,23252,30751,40036
+59819,23253,30750,40635
+59715,23254,30749,40099
+50683,23255,30748,40760
+55287,23256,30747,40812
+58135,23257,30746,40841
+54809,23258,30745,40702
+59970,23259,30744,40453
+55157,23260,30743,40864
+58996,23261,30742,40134
+57786,23262,30741,40779
+56103,23263,30740,40576
+59805,23264,30739,40527
+53810,23265,30738,40439
+53603,23266,30737,40065
+50291,23267,30736,40372
+51516,23268,30735,40214
+52037,23269,30734,40843
+54799,23270,30733,40878
+55090,23271,30732,40197
+51920,23272,30731,40511
+58641,23273,30730,40317
+59103,23274,30729,40885
+56080,23275,30728,40160
+50010,23276,30727,40611
+54461,23277,30726,40470
+58978,23278,30725,40161
+53486,23279,30724,40424
+53435,23280,30723,40803
+56049,23281,30722,40909
+53452,23282,30721,40641
+58541,23283,30720,40093
+51527,23284,30719,40768
+58471,23285,30718,40752
+58147,23286,30717,40820
+50753,23287,30716,40604
+59492,23288,30715,40088
+52140,23289,30714,40970
+53292,23290,30713,40700
+58847,23291,30712,40628
+56731,23292,30711,40826
+57712,23293,30710,40789
+56370,23294,30709,40677
+57294,23295,30708,40661
+57574,23296,30707,40390
+50769,23297,30706,40847
+56639,23298,30705,40860
+51802,23299,30704,40504
+54656,23300,30703,40063
+55652,23301,30702,40174
+50145,23302,30701,40433
+55804,23303,30700,40419
+59287,23304,30699,40970
+57196,23305,30698,40749
+50822,23306,30697,40919
+53058,23307,30696,40036
+58237,23308,30695,40031
+55536,23309,30694,40071
+57903,23310,30693,40809
+54925,23311,30692,40971
+51858,23312,30691,40412
+53065,23313,30690,40289
+53088,23314,30689,40061
+55277,23315,30688,40290
+58459,23316,30687,40911
+55216,23317,30686,40465
+57198,23318,30685,40042
+56471,23319,30684,40334
+57758,23320,30683,40186
+57812,23321,30682,40467
+56847,23322,30681,40387
+50863,23323,30680,40548
+51011,23324,30679,40774
+55938,23325,30678,40912
+50557,23326,30677,40691
+52541,23327,30676,40686
+56868,23328,30675,40659
+51595,23329,30674,40118
+51315,23330,30673,40520
+54156,23331,30672,40454
+55981,23332,30671,40827
+56305,23333,30670,40837
+50797,23334,30669,40834
+50326,23335,30668,40912
+52940,23336,30667,40252
+51529,23337,30666,40078
+59484,23338,30665,40426
+59577,23339,30664,40545
+55377,23340,30663,40260
+51513,23341,30662,40516
+51440,23342,30661,40085
+58980,23343,30660,40891
+58943,23344,30659,40733
+54379,23345,30658,40020
+54011,23346,30657,40553
+57034,23347,30656,40737
+58377,23348,30655,40560
+59960,23349,30654,40646
+58100,23350,30653,40412
+51837,23351,30652,40997
+56645,23352,30651,40499
+58366,23353,30650,40979
+56001,23354,30649,40498
+52659,23355,30648,40278
+53228,23356,30647,40533
+54666,23357,30646,40726
+54251,23358,30645,40621
+51109,23359,30644,40934
+55079,23360,30643,40318
+54590,23361,30642,40398
+51501,23362,30641,40540
+53546,23363,30640,40773
+58739,23364,30639,40618
+50619,23365,30638,40112
+52309,23366,30637,40286
+52864,23367,30636,40017
+54941,23368,30635,40148
+57031,23369,30634,40086
+53585,23370,30633,40238
+54739,23371,30632,40168
+56919,23372,30631,40205
+50953,23373,30630,40660
+55395,23374,30629,40446
+54915,23375,30628,40282
+52688,23376,30627,40268
+55335,23377,30626,40823
+56781,23378,30625,40669
+59538,23379,30624,40713
+56927,23380,30623,40772
+53118,23381,30622,40310
+59973,23382,30621,40731
+56275,23383,30620,40378
+55225,23384,30619,40227
+55626,23385,30618,40507
+51410,23386,30617,40114
+57100,23387,30616,40298
+54678,23388,30615,40439
+53320,23389,30614,40041
+57100,23390,30613,40300
+51700,23391,30612,40195
+50383,23392,30611,40353
+59035,23393,30610,40228
+50319,23394,30609,40785
+56437,23395,30608,40911
+53714,23396,30607,40859
+59620,23397,30606,40894
+54568,23398,30605,40536
+53690,23399,30604,40337
+51708,23400,30603,40475
+53256,23401,30602,40534
+55382,23402,30601,40746
+56274,23403,30600,40201
+58941,23404,30599,40776
+58126,23405,30598,40528
+57610,23406,30597,40548
+50939,23407,30596,40094
+54439,23408,30595,40529
+53086,23409,30594,40610
+55495,23410,30593,40347
+51815,23411,30592,40915
+59711,23412,30591,40555
+53826,23413,30590,40434
+53218,23414,30589,40063
+59050,23415,30588,40444
+55610,23416,30587,40393
+59800,23417,30586,40578
+50674,23418,30585,40970
+59449,23419,30584,40007
+53089,23420,30583,40267
+55693,23421,30582,40822
+53792,23422,30581,40216
+54603,23423,30580,40334
+51821,23424,30579,40346
+50931,23425,30578,40307
+53800,23426,30577,40567
+52396,23427,30576,40825
+59018,23428,30575,40503
+52395,23429,30574,40143
+55238,23430,30573,40310
+56309,23431,30572,40046
+55528,23432,30571,40849
+51035,23433,30570,40133
+59514,23434,30569,40735
+50276,23435,30568,40131
+54757,23436,30567,40101
+55354,23437,30566,40112
+50819,23438,30565,40926
+59098,23439,30564,40383
+59827,23440,30563,40565
+56846,23441,30562,40332
+58791,23442,30561,40249
+55416,23443,30560,40758
+52120,23444,30559,40012
+53652,23445,30558,40921
+51113,23446,30557,40463
+58368,23447,30556,40652
+56280,23448,30555,40902
+58408,23449,30554,40562
+51473,23450,30553,40683
+59737,23451,30552,40846
+52013,23452,30551,40130
+51384,23453,30550,40872
+54264,23454,30549,40657
+58523,23455,30548,40406
+54879,23456,30547,40345
+55060,23457,30546,40201
+53576,23458,30545,40029
+50670,23459,30544,40870
+59980,23460,30543,40862
+52560,23461,30542,40077
+53167,23462,30541,40328
+54719,23463,30540,40481
+57809,23464,30539,40949
+58625,23465,30538,40084
+55244,23466,30537,40700
+53627,23467,30536,40982
+51645,23468,30535,40646
+51659,23469,30534,40379
+55616,23470,30533,40052
+56626,23471,30532,40599
+58053,23472,30531,40104
+52713,23473,30530,40406
+58815,23474,30529,40503
+55508,23475,30528,40187
+50090,23476,30527,40216
+53924,23477,30526,40781
+51388,23478,30525,40676
+57623,23479,30524,40105
+58084,23480,30523,40434
+55178,23481,30522,40776
+51151,23482,30521,40533
+55230,23483,30520,40455
+59370,23484,30519,40116
+56264,23485,30518,40685
+52114,23486,30517,40566
+50899,23487,30516,40932
+59751,23488,30515,40879
+58912,23489,30514,40279
+52367,23490,30513,40164
+59437,23491,30512,40468
+57238,23492,30511,40055
+56452,23493,30510,40733
+58115,23494,30509,40279
+55114,23495,30508,40985
+51479,23496,30507,40176
+50648,23497,30506,40467
+51502,23498,30505,40098
+53670,23499,30504,40276
+51207,23500,30503,40944
+57875,23501,30502,40361
+58912,23502,30501,40896
+56306,23503,30500,40299
+51947,23504,30499,40850
+50334,23505,30498,40551
+54045,23506,30497,40814
+54637,23507,30496,40042
+50949,23508,30495,40767
+53971,23509,30494,40675
+56938,23510,30493,40926
+54430,23511,30492,40093
+54796,23512,30491,40676
+58094,23513,30490,40463
+55325,23514,30489,40215
+56503,23515,30488,40265
+52313,23516,30487,40416
+53039,23517,30486,40301
+53936,23518,30485,40567
+52915,23519,30484,40360
+52388,23520,30483,40052
+51158,23521,30482,40956
+57549,23522,30481,40851
+55017,23523,30480,40283
+51162,23524,30479,40555
+53173,23525,30478,40050
+53283,23526,30477,40299
+57669,23527,30476,40375
+59706,23528,30475,40273
+59713,23529,30474,40601
+57295,23530,30473,40841
+57989,23531,30472,40139
+58738,23532,30471,40550
+56177,23533,30470,40340
+55892,23534,30469,40744
+55049,23535,30468,40438
+50838,23536,30467,40210
+54957,23537,30466,40688
+54675,23538,30465,40726
+57523,23539,30464,40339
+53290,23540,30463,40192
+58383,23541,30462,40222
+54627,23542,30461,40196
+54228,23543,30460,40422
+54594,23544,30459,40692
+54581,23545,30458,40533
+51049,23546,30457,40506
+59845,23547,30456,40709
+50296,23548,30455,40847
+56236,23549,30454,40529
+51312,23550,30453,40238
+54346,23551,30452,40697
+54582,23552,30451,40327
+52806,23553,30450,40689
+55763,23554,30449,40515
+57536,23555,30448,40316
+52583,23556,30447,40865
+52853,23557,30446,40194
+54157,23558,30445,40420
+51008,23559,30444,40203
+50122,23560,30443,40329
+57181,23561,30442,40559
+53983,23562,30441,40480
+51321,23563,30440,40388
+59054,23564,30439,40590
+53365,23565,30438,40994
+56650,23566,30437,40996
+59738,23567,30436,40086
+55143,23568,30435,40769
+59574,23569,30434,40193
+54432,23570,30433,40671
+57460,23571,30432,40277
+53159,23572,30431,40092
+55322,23573,30430,40715
+56887,23574,30429,40199
+56556,23575,30428,40724
+58789,23576,30427,40388
+56909,23577,30426,40714
+51845,23578,30425,40726
+53398,23579,30424,40114
+54473,23580,30423,40040
+54591,23581,30422,40729
+59268,23582,30421,40008
+57691,23583,30420,40856
+55901,23584,30419,40505
+53856,23585,30418,40453
+51951,23586,30417,40284
+55619,23587,30416,40247
+54573,23588,30415,40946
+54866,23589,30414,40166
+50979,23590,30413,40633
+54875,23591,30412,40396
+52663,23592,30411,40112
+56603,23593,30410,40812
+58463,23594,30409,40768
+56208,23595,30408,40222
+54988,23596,30407,40241
+59519,23597,30406,40228
+58211,23598,30405,40110
+57618,23599,30404,40892
+58348,23600,30403,40726
+55145,23601,30402,40433
+56964,23602,30401,40944
+50241,23603,30400,40936
+50349,23604,30399,40250
+59250,23605,30398,40957
+57878,23606,30397,40422
+53171,23607,30396,40024
+58773,23608,30395,40642
+51912,23609,30394,40960
+50800,23610,30393,40282
+59499,23611,30392,40793
+57179,23612,30391,40900
+50569,23613,30390,40213
+52795,23614,30389,40616
+57539,23615,30388,40869
+59241,23616,30387,40682
+58164,23617,30386,40618
+53907,23618,30385,40173
+51987,23619,30384,40764
+57708,23620,30383,40310
+59678,23621,30382,40717
+51300,23622,30381,40458
+53718,23623,30380,40811
+56434,23624,30379,40893
+50778,23625,30378,40161
+59965,23626,30377,40368
+59980,23627,30376,40482
+57738,23628,30375,40758
+52160,23629,30374,40737
+51104,23630,30373,40915
+50322,23631,30372,40743
+51671,23632,30371,40010
+58214,23633,30370,40089
+54895,23634,30369,40633
+53323,23635,30368,40702
+54653,23636,30367,40955
+55529,23637,30366,40450
+58672,23638,30365,40922
+59802,23639,30364,40922
+59282,23640,30363,40598
+52677,23641,30362,40989
+59306,23642,30361,40367
+56293,23643,30360,40466
+54910,23644,30359,40563
+54568,23645,30358,40148
+57125,23646,30357,40180
+56531,23647,30356,40831
+57291,23648,30355,40641
+52739,23649,30354,40485
+54077,23650,30353,40663
+51854,23651,30352,40970
+53668,23652,30351,40028
+57128,23653,30350,40016
+52383,23654,30349,40777
+56594,23655,30348,40232
+57609,23656,30347,40721
+56204,23657,30346,40558
+54122,23658,30345,40594
+56283,23659,30344,40111
+57194,23660,30343,40478
+53078,23661,30342,40306
+51439,23662,30341,40038
+52442,23663,30340,40992
+54087,23664,30339,40836
+55662,23665,30338,40865
+54385,23666,30337,40772
+59681,23667,30336,40653
+57494,23668,30335,40399
+56862,23669,30334,40051
+54294,23670,30333,40307
+56097,23671,30332,40873
+57164,23672,30331,40034
+51972,23673,30330,40761
+51043,23674,30329,40620
+52835,23675,30328,40524
+50578,23676,30327,40972
+57214,23677,30326,40783
+53378,23678,30325,40717
+52908,23679,30324,40222
+57201,23680,30323,40751
+50779,23681,30322,40753
+59583,23682,30321,40114
+55766,23683,30320,40354
+54086,23684,30319,40189
+59152,23685,30318,40391
+59238,23686,30317,40332
+52573,23687,30316,40154
+52718,23688,30315,40012
+51008,23689,30314,40210
+57127,23690,30313,40939
+59295,23691,30312,40646
+56810,23692,30311,40969
+58915,23693,30310,40958
+58370,23694,30309,40093
+58315,23695,30308,40294
+57996,23696,30307,40238
+50495,23697,30306,40688
+58715,23698,30305,40383
+53394,23699,30304,40645
+54600,23700,30303,40061
+58769,23701,30302,40696
+56262,23702,30301,40296
+59193,23703,30300,40198
+52819,23704,30299,40763
+52117,23705,30298,40733
+50651,23706,30297,40650
+57940,23707,30296,40576
+57367,23708,30295,40480
+57145,23709,30294,40285
+58771,23710,30293,40562
+51932,23711,30292,40564
+52525,23712,30291,40124
+54686,23713,30290,40820
+54904,23714,30289,40221
+58874,23715,30288,40499
+50560,23716,30287,40249
+52598,23717,30286,40225
+59785,23718,30285,40387
+53417,23719,30284,40906
+58240,23720,30283,40645
+52982,23721,30282,40526
+52435,23722,30281,40646
+58967,23723,30280,40095
+59404,23724,30279,40910
+57925,23725,30278,40375
+53400,23726,30277,40577
+56883,23727,30276,40659
+53782,23728,30275,40613
+55723,23729,30274,40895
+52907,23730,30273,40866
+57088,23731,30272,40287
+54051,23732,30271,40582
+56915,23733,30270,40520
+58626,23734,30269,40826
+54545,23735,30268,40057
+57144,23736,30267,40132
+56584,23737,30266,40025
+56018,23738,30265,40169
+50762,23739,30264,40534
+59165,23740,30263,40487
+52427,23741,30262,40679
+59622,23742,30261,40244
+57973,23743,30260,40151
+50525,23744,30259,40039
+59979,23745,30258,40830
+51016,23746,30257,40497
+50186,23747,30256,40046
+55332,23748,30255,40551
+53771,23749,30254,40175
+50118,23750,30253,40647
+53508,23751,30252,40015
+53951,23752,30251,40029
+54879,23753,30250,40210
+56750,23754,30249,40113
+58819,23755,30248,40953
+53855,23756,30247,40070
+56030,23757,30246,40514
+50609,23758,30245,40537
+54900,23759,30244,40757
+59062,23760,30243,40620
+53729,23761,30242,40902
+57251,23762,30241,40166
+57578,23763,30240,40899
+54295,23764,30239,40433
+58994,23765,30238,40467
+53273,23766,30237,40192
+54968,23767,30236,40113
+59385,23768,30235,40411
+53454,23769,30234,40796
+59593,23770,30233,40745
+50516,23771,30232,40797
+54485,23772,30231,40568
+55571,23773,30230,40861
+54205,23774,30229,40731
+56806,23775,30228,40056
+51475,23776,30227,40929
+55538,23777,30226,40753
+51664,23778,30225,40606
+53670,23779,30224,40930
+51233,23780,30223,40800
+54013,23781,30222,40593
+51502,23782,30221,40144
+53408,23783,30220,40746
+54347,23784,30219,40131
+58729,23785,30218,40509
+50440,23786,30217,40876
+53011,23787,30216,40021
+51779,23788,30215,40656
+57293,23789,30214,40914
+55085,23790,30213,40694
+53613,23791,30212,40556
+50602,23792,30211,40953
+53857,23793,30210,40620
+57394,23794,30209,40035
+56896,23795,30208,40144
+55908,23796,30207,40208
+59295,23797,30206,40235
+59675,23798,30205,40275
+50592,23799,30204,40335
+56728,23800,30203,40297
+57077,23801,30202,40480
+52494,23802,30201,40408
+59827,23803,30200,40646
+52237,23804,30199,40292
+51361,23805,30198,40470
+52166,23806,30197,40942
+58703,23807,30196,40574
+50494,23808,30195,40329
+56643,23809,30194,40539
+51614,23810,30193,40019
+57438,23811,30192,40649
+59718,23812,30191,40924
+58781,23813,30190,40510
+52866,23814,30189,40686
+55260,23815,30188,40995
+50164,23816,30187,40312
+52966,23817,30186,40192
+56161,23818,30185,40961
+59607,23819,30184,40779
+50648,23820,30183,40174
+54370,23821,30182,40189
+59437,23822,30181,40176
+56494,23823,30180,40753
+57156,23824,30179,40765
+58075,23825,30178,40898
+50632,23826,30177,40617
+59763,23827,30176,40335
+55083,23828,30175,40408
+56059,23829,30174,40590
+51896,23830,30173,40211
+57832,23831,30172,40594
+58564,23832,30171,40189
+57007,23833,30170,40090
+54947,23834,30169,40233
+58981,23835,30168,40360
+53015,23836,30167,40114
+53701,23837,30166,40619
+55555,23838,30165,40358
+52432,23839,30164,40402
+56782,23840,30163,40984
+51630,23841,30162,40492
+59347,23842,30161,40529
+53970,23843,30160,40841
+50757,23844,30159,40202
+50460,23845,30158,40942
+58800,23846,30157,40512
+51816,23847,30156,40705
+51627,23848,30155,40828
+50452,23849,30154,40434
+59337,23850,30153,40270
+52176,23851,30152,40082
+50398,23852,30151,40245
+57293,23853,30150,40339
+52287,23854,30149,40926
+57751,23855,30148,40876
+59554,23856,30147,40511
+51327,23857,30146,40815
+58645,23858,30145,40295
+51025,23859,30144,40536
+58026,23860,30143,40028
+54213,23861,30142,40660
+53808,23862,30141,40736
+58465,23863,30140,40588
+58845,23864,30139,40568
+52299,23865,30138,40580
+58567,23866,30137,40818
+57451,23867,30136,40789
+56690,23868,30135,40483
+57023,23869,30134,40699
+59123,23870,30133,40059
+55705,23871,30132,40315
+52819,23872,30131,40974
+50275,23873,30130,40549
+54304,23874,30129,40375
+56659,23875,30128,40322
+56334,23876,30127,40010
+53849,23877,30126,40051
+55642,23878,30125,40566
+52725,23879,30124,40829
+54494,23880,30123,40894
+51893,23881,30122,40452
+54016,23882,30121,40819
+52172,23883,30120,40123
+55670,23884,30119,40592
+51151,23885,30118,40894
+56612,23886,30117,40913
+53640,23887,30116,40892
+51718,23888,30115,40304
+52670,23889,30114,40948
+51145,23890,30113,40749
+55772,23891,30112,40282
+54515,23892,30111,40106
+54557,23893,30110,40022
+53459,23894,30109,40089
+58665,23895,30108,40610
+55699,23896,30107,40338
+56856,23897,30106,40974
+54970,23898,30105,40284
+57270,23899,30104,40586
+57278,23900,30103,40447
+51312,23901,30102,40231
+54118,23902,30101,40268
+56410,23903,30100,40058
+54478,23904,30099,40346
+56866,23905,30098,40414
+59636,23906,30097,40286
+50783,23907,30096,40845
+59455,23908,30095,40182
+58916,23909,30094,40267
+56703,23910,30093,40386
+52673,23911,30092,40679
+57839,23912,30091,40311
+55688,23913,30090,40622
+57879,23914,30089,40601
+54125,23915,30088,40134
+57886,23916,30087,40789
+59398,23917,30086,40684
+57229,23918,30085,40600
+55634,23919,30084,40194
+53519,23920,30083,40528
+55558,23921,30082,40785
+53140,23922,30081,40629
+56834,23923,30080,40918
+57765,23924,30079,40426
+51971,23925,30078,40756
+59169,23926,30077,40126
+59777,23927,30076,40951
+51142,23928,30075,40490
+52856,23929,30074,40656
+55292,23930,30073,40287
+50837,23931,30072,40018
+52363,23932,30071,40989
+57222,23933,30070,40439
+53117,23934,30069,40754
+54884,23935,30068,40015
+54749,23936,30067,40717
+55948,23937,30066,40775
+51279,23938,30065,40022
+57416,23939,30064,40010
+57593,23940,30063,40812
+59997,23941,30062,40926
+51119,23942,30061,40898
+52206,23943,30060,40358
+50157,23944,30059,40351
+50461,23945,30058,40343
+55453,23946,30057,40992
+58081,23947,30056,40634
+58376,23948,30055,40012
+50660,23949,30054,40813
+51081,23950,30053,40092
+54371,23951,30052,40030
+59965,23952,30051,40718
+59049,23953,30050,40869
+58519,23954,30049,40429
+54021,23955,30048,40788
+51568,23956,30047,40014
+51831,23957,30046,40118
+52891,23958,30045,40104
+54714,23959,30044,40841
+52518,23960,30043,40273
+59675,23961,30042,40513
+59805,23962,30041,40242
+51655,23963,30040,40610
+57081,23964,30039,40518
+58339,23965,30038,40386
+51990,23966,30037,40604
+50613,23967,30036,40810
+52034,23968,30035,40782
+59792,23969,30034,40207
+54289,23970,30033,40350
+57165,23971,30032,40384
+58109,23972,30031,40556
+59428,23973,30030,40574
+58202,23974,30029,40966
+54042,23975,30028,40883
+50728,23976,30027,40613
+50388,23977,30026,40224
+55761,23978,30025,40495
+54730,23979,30024,40608
+53219,23980,30023,40271
+56567,23981,30022,40554
+51016,23982,30021,40781
+58301,23983,30020,40255
+57648,23984,30019,40411
+55317,23985,30018,40097
+57376,23986,30017,40693
+53316,23987,30016,40803
+52932,23988,30015,40948
+57337,23989,30014,40419
+50960,23990,30013,40615
+56898,23991,30012,40728
+52148,23992,30011,40862
+59135,23993,30010,40106
+56429,23994,30009,40757
+56670,23995,30008,40189
+58072,23996,30007,40210
+59877,23997,30006,40673
+55618,23998,30005,40803
+55391,23999,30004,40210
+59340,24000,30003,40687
+56909,24001,30002,40510
+54682,24002,30001,40471
+56718,24003,30000,40056
+53513,24004,31000,40560
+54921,24005,30999,40121
+59872,24006,30998,40888
+59367,24007,30997,40197
+53904,24008,30996,40386
+57662,24009,30995,40529
+51013,24010,30994,40864
+54263,24011,30993,40223
+56303,24012,30992,40436
+54033,24013,30991,40195
+56954,24014,30990,40555
+56751,24015,30989,40646
+50344,24016,30988,40734
+53628,24017,30987,40924
+55073,24018,30986,40718
+54415,24019,30985,40348
+56449,24020,30984,40332
+56809,24021,30983,40467
+51984,24022,30982,40491
+53899,24023,30981,40998
+54238,24024,30980,40171
+53557,24025,30979,40802
+56765,24026,30978,40442
+56937,24027,30977,40353
+59289,24028,30976,40136
+51950,24029,30975,40044
+50866,24030,30974,40479
+52785,24031,30973,40847
+54394,24032,30972,40709
+54181,24033,30971,40254
+51242,24034,30970,40944
+55355,24035,30969,40226
+56831,24036,30968,40869
+57725,24037,30967,40975
+53186,24038,30966,40701
+58676,24039,30965,40765
+56885,24040,30964,40917
+56937,24041,30963,40425
+54312,24042,30962,40480
+59672,24043,30961,40888
+56940,24044,30960,40483
+53969,24045,30959,40520
+57023,24046,30958,40029
+58505,24047,30957,40530
+57286,24048,30956,40080
+55085,24049,30955,40191
+55849,24050,30954,40576
+57158,24051,30953,40782
+52407,24052,30952,40583
+50136,24053,30951,40328
+58474,24054,30950,40675
+55492,24055,30949,40064
+54598,24056,30948,40101
+52830,24057,30947,40898
+52779,24058,30946,40751
+50025,24059,30945,40279
+51419,24060,30944,40137
+58205,24061,30943,40906
+53989,24062,30942,40007
+51227,24063,30941,40946
+50670,24064,30940,40219
+59767,24065,30939,40525
+54593,24066,30938,40458
+56964,24067,30937,40877
+54222,24068,30936,40302
+51822,24069,30935,40025
+50126,24070,30934,40456
+57185,24071,30933,40059
+57591,24072,30932,40711
+56919,24073,30931,40721
+53690,24074,30930,40897
+51902,24075,30929,40091
+55690,24076,30928,40757
+55383,24077,30927,40608
+52311,24078,30926,40987
+55883,24079,30925,40164
+55048,24080,30924,40041
+58424,24081,30923,40914
+54648,24082,30922,40347
+59945,24083,30921,40818
+57575,24084,30920,40037
+58836,24085,30919,40936
+50209,24086,30918,40192
+59117,24087,30917,40737
+57483,24088,30916,40607
+56040,24089,30915,40634
+53721,24090,30914,40324
+58860,24091,30913,40134
+58573,24092,30912,40956
+53618,24093,30911,40172
+57323,24094,30910,40084
+57197,24095,30909,40979
+58375,24096,30908,40071
+51651,24097,30907,40299
+50832,24098,30906,40285
+50138,24099,30905,40250
+53926,24100,30904,40273
+58510,24101,30903,40943
+59126,24102,30902,40103
+52910,24103,30901,40030
+57910,24104,30900,40976
+55686,24105,30899,40881
+57180,24106,30898,40747
+51447,24107,30897,40983
+50969,24108,30896,40794
+53219,24109,30895,40659
+53475,24110,30894,40918
+52309,24111,30893,40596
+59536,24112,30892,40736
+53885,24113,30891,40820
+59854,24114,30890,40558
+55289,24115,30889,40496
+55951,24116,30888,40302
+57855,24117,30887,40387
+59442,24118,30886,40545
+58822,24119,30885,40967
+59837,24120,30884,40287
+51317,24121,30883,40305
+55574,24122,30882,40363
+52307,24123,30881,40648
+58593,24124,30880,40050
+50494,24125,30879,40545
+54867,24126,30878,40654
+59191,24127,30877,40754
+50950,24128,30876,40145
+53322,24129,30875,40665
+58773,24130,30874,40704
+50251,24131,30873,40280
+57063,24132,30872,40839
+55532,24133,30871,40288
+56680,24134,30870,40235
+51195,24135,30869,40071
+57043,24136,30868,40983
+50456,24137,30867,40232
+55020,24138,30866,40454
+53445,24139,30865,40693
+54937,24140,30864,40001
+54278,24141,30863,40267
+59073,24142,30862,40742
+52807,24143,30861,40569
+53341,24144,30860,40334
+56237,24145,30859,40022
+58699,24146,30858,40705
+56404,24147,30857,40791
+58673,24148,30856,40633
+55688,24149,30855,40322
+55278,24150,30854,40567
+59840,24151,30853,40795
+55500,24152,30852,40973
+59284,24153,30851,40232
+52498,24154,30850,40433
+51247,24155,30849,40050
+50742,24156,30848,40899
+51599,24157,30847,40243
+52560,24158,30846,40100
+50841,24159,30845,40710
+54233,24160,30844,40571
+55898,24161,30843,40423
+50002,24162,30842,40100
+54089,24163,30841,40008
+58401,24164,30840,40055
+57128,24165,30839,40693
+56230,24166,30838,40451
+57320,24167,30837,40310
+59276,24168,30836,40453
+58380,24169,30835,40566
+51020,24170,30834,40265
+52854,24171,30833,40788
+55579,24172,30832,40603
+54990,24173,30831,40357
+52243,24174,30830,40579
+53393,24175,30829,40452
+50367,24176,30828,40203
+51414,24177,30827,40136
+52767,24178,30826,40814
+55532,24179,30825,40304
+58476,24180,30824,40630
+58943,24181,30823,40981
+56117,24182,30822,40234
+55705,24183,30821,40276
+55736,24184,30820,40690
+53781,24185,30819,40511
+54461,24186,30818,40025
+59215,24187,30817,40119
+58652,24188,30816,40990
+59879,24189,30815,40455
+54863,24190,30814,40815
+59540,24191,30813,40368
+54786,24192,30812,40027
+52933,24193,30811,40189
+59702,24194,30810,40561
+58210,24195,30809,40434
+52629,24196,30808,40546
+52290,24197,30807,40198
+56063,24198,30806,40328
+56502,24199,30805,40098
+51859,24200,30804,40386
+58130,24201,30803,40765
+59418,24202,30802,40493
+56908,24203,30801,40061
+52526,24204,30800,40552
+50109,24205,30799,40996
+57208,24206,30798,40188
+58804,24207,30797,40429
+59606,24208,30796,40198
+54074,24209,30795,40313
+54282,24210,30794,40371
+59520,24211,30793,40859
+51296,24212,30792,40623
+55108,24213,30791,40988
+50312,24214,30790,40449
+53715,24215,30789,40843
+55265,24216,30788,40526
+57144,24217,30787,40749
+56855,24218,30786,40177
+55714,24219,30785,40292
+59843,24220,30784,40548
+59764,24221,30783,40136
+54033,24222,30782,40102
+59494,24223,30781,40855
+57231,24224,30780,40257
+57065,24225,30779,40354
+58529,24226,30778,40686
+58232,24227,30777,40453
+54241,24228,30776,40280
+55690,24229,30775,40442
+53667,24230,30774,40905
+54675,24231,30773,40132
+58247,24232,30772,40509
+51578,24233,30771,40923
+52536,24234,30770,40425
+52118,24235,30769,40464
+56497,24236,30768,40999
+52305,24237,30767,40150
+53183,24238,30766,40618
+58303,24239,30765,40491
+58882,24240,30764,40104
+52407,24241,30763,40298
+53437,24242,30762,40824
+51856,24243,30761,40255
+57149,24244,30760,40889
+54471,24245,30759,40358
+59622,24246,30758,40629
+58460,24247,30757,40576
+51912,24248,30756,40713
+56397,24249,30755,40995
+57249,24250,30754,40914
+52330,24251,30753,40244
+52380,24252,30752,40397
+57768,24253,30751,40748
+58535,24254,30750,40570
+58529,24255,30749,40940
+54402,24256,30748,40734
+56648,24257,30747,40774
+52563,24258,30746,40124
+57130,24259,30745,40890
+56455,24260,30744,40368
+56816,24261,30743,40290
+55459,24262,30742,40776
+53025,24263,30741,40411
+58744,24264,30740,40455
+54501,24265,30739,40208
+55813,24266,30738,40982
+53012,24267,30737,40124
+50470,24268,30736,40385
+59980,24269,30735,40668
+53658,24270,30734,40820
+53345,24271,30733,40511
+51267,24272,30732,40632
+50223,24273,30731,40875
+53799,24274,30730,40694
+50076,24275,30729,40045
+58147,24276,30728,40039
+52825,24277,30727,40515
+54398,24278,30726,40309
+50870,24279,30725,40570
+54459,24280,30724,40266
+54005,24281,30723,40408
+52233,24282,30722,40939
+54214,24283,30721,40117
+53381,24284,30720,40920
+55618,24285,30719,40523
+56959,24286,30718,40267
+57556,24287,30717,40567
+50154,24288,30716,40106
+54225,24289,30715,40851
+52412,24290,30714,40907
+57288,24291,30713,40905
+51157,24292,30712,40667
+54613,24293,30711,40494
+52013,24294,30710,40858
+51535,24295,30709,40592
+53686,24296,30708,40623
+56433,24297,30707,40701
+51873,24298,30706,40508
+50966,24299,30705,40754
+52828,24300,30704,40918
+53149,24301,30703,40869
+54825,24302,30702,40123
+50748,24303,30701,40464
+56186,24304,30700,40447
+54529,24305,30699,40859
+53223,24306,30698,40837
+58676,24307,30697,40108
+52018,24308,30696,40183
+51560,24309,30695,40181
+50626,24310,30694,40907
+54512,24311,30693,40034
+55884,24312,30692,40706
+50350,24313,30691,40141
+50593,24314,30690,40005
+53430,24315,30689,40976
+52096,24316,30688,40342
+56322,24317,30687,40630
+58139,24318,30686,40248
+56367,24319,30685,40471
+53095,24320,30684,40332
+54036,24321,30683,40151
+52285,24322,30682,40698
+52386,24323,30681,40553
+55997,24324,30680,40696
+53847,24325,30679,40437
+55155,24326,30678,40202
+50749,24327,30677,40011
+59553,24328,30676,40774
+58608,24329,30675,40834
+56444,24330,30674,40954
+53004,24331,30673,40082
+57651,24332,30672,40653
+50372,24333,30671,40346
+53034,24334,30670,40318
+59343,24335,30669,40863
+56709,24336,30668,40637
+55789,24337,30667,40666
+59327,24338,30666,40677
+54696,24339,30665,40661
+53835,24340,30664,40842
+56790,24341,30663,40098
+55169,24342,30662,40204
+55514,24343,30661,40591
+57626,24344,30660,40112
+51850,24345,30659,40354
+53735,24346,30658,40432
+55099,24347,30657,40516
+50235,24348,30656,40510
+55480,24349,30655,40069
+54186,24350,30654,40834
+57762,24351,30653,40245
+59465,24352,30652,40253
+54209,24353,30651,40906
+59972,24354,30650,40362
+54921,24355,30649,40019
+55214,24356,30648,40120
+54280,24357,30647,40736
+54883,24358,30646,40046
+53499,24359,30645,40971
+57626,24360,30644,40884
+57520,24361,30643,40759
+50598,24362,30642,40441
+58807,24363,30641,40251
+50508,24364,30640,40502
+57409,24365,30639,40435
+55054,24366,30638,40367
+59562,24367,30637,40212
+55190,24368,30636,40561
+52740,24369,30635,40018
+50334,24370,30634,40813
+56104,24371,30633,40304
+56764,24372,30632,40463
+53969,24373,30631,40970
+53692,24374,30630,40587
+50692,24375,30629,40850
+58293,24376,30628,40317
+50663,24377,30627,40918
+54333,24378,30626,40539
+56083,24379,30625,40987
+59576,24380,30624,40057
+58228,24381,30623,40552
+50668,24382,30622,40649
+50573,24383,30621,40073
+56800,24384,30620,40661
+56904,24385,30619,40757
+55987,24386,30618,40090
+57211,24387,30617,40351
+53144,24388,30616,40547
+56151,24389,30615,40694
+50175,24390,30614,40983
+59433,24391,30613,40241
+58271,24392,30612,40800
+56451,24393,30611,40499
+54848,24394,30610,40969
+56741,24395,30609,40670
+57861,24396,30608,40971
+55029,24397,30607,40487
+57350,24398,30606,40252
+56510,24399,30605,40670
+52379,24400,30604,40852
+59060,24401,30603,40257
+53317,24402,30602,40965
+59205,24403,30601,40444
+57383,24404,30600,40687
+55195,24405,30599,40439
+59489,24406,30598,40294
+56591,24407,30597,40827
+53277,24408,30596,40545
+56391,24409,30595,40146
+55874,24410,30594,40103
+53904,24411,30593,40482
+54118,24412,30592,40711
+50615,24413,30591,40278
+58641,24414,30590,40460
+57376,24415,30589,40970
+53228,24416,30588,40579
+55978,24417,30587,40536
+59832,36673,30586,40739
+53450,24419,30585,40460
+58636,24420,30584,40720
+50227,24421,30583,40488
+57734,24422,30582,40956
+58568,24423,30581,40461
+51426,24424,30580,40923
+53560,24425,30579,40903
+51448,24426,30578,40153
+58104,24427,30577,40827
+59280,24428,30576,40154
+55838,24429,30575,40759
+50864,24430,30574,40917
+51241,24431,30573,40278
+58534,24432,30572,40678
+55422,24433,30571,40359
+53218,24434,30570,40999
+50606,24435,30569,40930
+54731,24436,30568,40634
+54707,24437,30567,40654
+56065,24438,30566,40941
+58489,24439,30565,40393
+58114,24440,30564,40019
+51385,24441,30563,40133
+54886,24442,30562,40667
+53600,24443,30561,40988
+55861,24444,30560,40223
+50127,24445,30559,40344
+58537,24446,30558,40988
+54876,24447,30557,40992
+59310,24448,30556,40920
+54485,24449,30555,40779
+55557,24450,30554,40781
+52533,24451,30553,40146
+54024,24452,30552,40904
+59710,24453,30551,40446
+50024,24454,30550,40246
+52102,24455,30549,40166
+58114,24456,30548,40092
+57119,24457,30547,40322
+53840,24458,30546,40466
+57806,24459,30545,40777
+56779,24460,30544,40891
+52974,24461,30543,40675
+53070,24462,30542,40891
+57549,24463,30541,40388
+50710,24464,30540,40678
+57987,24465,30539,40782
+51974,24466,30538,40913
+55634,24467,30537,40353
+59208,24468,30536,40511
+58953,24469,30535,40138
+55901,24470,30534,40194
+53971,24471,30533,40173
+52628,24472,30532,40132
+50493,24473,30531,40256
+53257,24474,30530,40304
+52117,24475,30529,40790
+58789,24476,30528,40858
+55246,24477,30527,40637
+50372,24478,30526,40976
+55199,24479,30525,40061
+57268,24480,30524,40236
+50395,24481,30523,40333
+55257,24482,30522,40223
+57339,24483,30521,40219
+51373,24484,30520,40251
+50216,24485,30519,40999
+50437,24486,30518,40885
+59685,24487,30517,40429
+57343,24488,30516,40093
+56248,24489,30515,40685
+52663,24490,30514,40035
+55849,24491,30513,40137
+59800,24492,30512,40000
+56995,24493,30511,40482
+56335,24494,30510,40398
+58712,24495,30509,40527
+51100,24496,30508,40405
+53218,24497,30507,40052
+53954,24498,30506,40227
+50243,24499,30505,40099
+57918,24500,30504,40967
+50269,24501,30503,40713
+53908,24502,30502,40107
+58350,24503,30501,40815
+58820,24504,30500,40283
+51844,24505,30499,40931
+52284,24506,30498,40270
+56990,24507,30497,40403
+59175,24508,30496,40848
+58739,24509,30495,40422
+57571,24510,30494,40769
+54459,24511,30493,40231
+51136,24512,30492,40550
+51451,24513,30491,40046
+53748,24514,30490,40978
+53982,24515,30489,40581
+53276,24516,30488,40248
+57151,24517,30487,40900
+50317,24518,30486,40280
+51108,24519,30485,40889
+58505,24520,30484,40955
+56212,24521,30483,40999
+54316,24522,30482,40821
+54296,24523,30481,40578
+55430,24524,30480,40543
+52751,24525,30479,40581
+59091,24526,30478,40861
+55164,24527,30477,40600
+55334,24528,30476,40970
+59962,24529,30475,40681
+52308,24530,30474,40094
+53008,24531,30473,40765
+54573,24532,30472,40522
+57824,24533,30471,40948
+56261,24534,30470,40032
+51817,24535,30469,40766
+55888,24536,30468,40465
+53004,24537,30467,40806
+58573,24538,30466,40236
+50667,24539,30465,40505
+52329,24540,30464,40056
+54445,24541,30463,40059
+50884,24542,30462,40503
+53167,24543,30461,40102
+54033,24544,30460,40935
+51765,24545,30459,40552
+50246,24546,30458,40605
+55613,24547,30457,40288
+57268,24548,30456,40728
+55764,24549,30455,40236
+58784,24550,30454,40344
+59042,24551,30453,40739
+57392,24552,30452,40401
+57329,24553,30451,40954
+52321,24554,30450,40592
+59755,24555,30449,40109
+55205,24556,30448,40888
+59857,24557,30447,40819
+59674,24558,30446,40841
+55768,24559,30445,40870
+52517,24560,30444,40423
+53389,24561,30443,40990
+53534,24562,30442,40970
+51690,24563,30441,40210
+55717,24564,30440,40447
+54064,24565,30439,40914
+57442,24566,30438,40082
+57636,24567,30437,40667
+57242,24568,30436,40630
+52646,24569,30435,40880
+54510,24570,30434,40659
+52908,24571,30433,40689
+57437,24572,30432,40203
+59694,24573,30431,40523
+56745,24574,30430,40025
+52971,24575,30429,40564
+52023,24576,30428,40166
+52153,24577,30427,40129
+51216,24578,30426,40617
+53497,24579,30425,40814
+53861,24580,30424,40082
+54808,24581,30423,40880
+57935,24582,30422,40996
+57969,24583,30421,40171
+55828,24584,30420,40613
+56561,24585,30419,40540
+50534,24586,30418,40163
+57587,24587,30417,40394
+54453,24588,30416,40860
+51385,24589,30415,40293
+53781,24590,30414,40658
+50956,24591,30413,40746
+57242,24592,30412,40865
+57946,24593,30411,40195
+57407,24594,30410,40551
+58815,24595,30409,40328
+52454,24596,30408,40671
+53839,24597,30407,40508
+55482,24598,30406,40209
+50175,24599,30405,40654
+50993,24600,30404,40745
+58531,24601,30403,40169
+58685,24602,30402,40059
+59634,24603,30401,40115
+57889,24604,30400,40408
+57558,24605,30399,40135
+57473,24606,30398,40800
+57951,24607,30397,40967
+56866,24608,30396,40180
+58551,24609,30395,40313
+57520,24610,30394,40286
+55344,24611,30393,40276
+58319,24612,30392,40392
+51114,24613,30391,40032
+58009,24614,30390,40469
+56032,24615,30389,40128
+59509,24616,30388,40448
+57426,24617,30387,40237
+54441,24618,30386,40311
+52558,24619,30385,40800
+56326,24620,30384,40915
+54407,24621,30383,40763
+52962,24622,30382,40151
+59480,24623,30381,40884
+58739,24624,30380,40659
+51362,24625,30379,40647
+54311,24626,30378,40860
+50064,24627,30377,40975
+50257,24628,30376,40241
+54428,24629,30375,40273
+52590,24630,30374,40196
+58570,24631,30373,40574
+57845,24632,30372,40043
+50994,24633,30371,40467
+50892,24634,30370,40691
+58907,24635,30369,40598
+57127,24636,30368,40648
+59203,24637,30367,40958
+57932,24638,30366,40243
+57771,24639,30365,40159
+53458,24640,30364,40145
+59583,24641,30363,40881
+54504,24642,30362,40531
+54563,24643,30361,40421
+52274,24644,30360,40575
+56969,24645,30359,40732
+53597,24646,30358,40635
+59440,24647,30357,40099
+54052,24648,30356,40793
+52470,24649,30355,40758
+56203,24650,30354,40075
+58525,24651,30353,40360
+53913,24652,30352,40820
+58620,24653,30351,40628
+58371,24654,30350,40593
+56076,24655,30349,40796
+50946,24656,30348,40715
+53891,24657,30347,40507
+54294,24658,30346,40009
+53438,24659,30345,40985
+57109,24660,30344,40646
+51078,24661,30343,40955
+59516,24662,30342,40508
+50229,24663,30341,40819
+54358,24664,30340,40956
+51811,24665,30339,40067
+55322,24666,30338,40708
+53497,24667,30337,40386
+51614,24668,30336,40011
+58918,24669,30335,40817
+59322,24670,30334,40324
+58911,24671,30333,40429
+58056,24672,30332,40148
+56801,24673,30331,40393
+51911,24674,30330,40793
+57715,24675,30329,40783
+59086,24676,30328,40586
+56519,24677,30327,40635
+56989,24678,30326,40781
+55199,24679,30325,40171
+52767,24680,30324,40601
+56764,24681,30323,40703
+56013,24682,30322,40609
+53617,24683,30321,40825
+51500,24684,30320,40656
+52358,24685,30319,40521
+53782,24686,30318,40455
+59097,24687,30317,40480
+50902,24688,30316,40873
+55086,24689,30315,40159
+57981,24690,30314,40928
+54999,24691,30313,40796
+56081,24692,30312,40102
+51567,24693,30311,40791
+58005,24694,30310,40161
+50785,24695,30309,40311
+57124,24696,30308,40019
+53158,24697,30307,40708
+50087,24698,30306,40413
+54090,24699,30305,40313
+52150,24700,30304,40680
+54640,24701,30303,40408
+52123,24702,30302,40608
+54236,24703,30301,40657
+50242,24704,30300,40798
+58670,24705,30299,40241
+52141,24706,30298,40800
+55583,24707,30297,40195
+56467,24708,30296,40087
+58483,24709,30295,40053
+57960,24710,30294,40468
+57343,24711,30293,40762
+58184,24712,30292,40015
+54084,24713,30291,40479
+53421,24714,30290,40950
+55385,24715,30289,40860
+55809,24716,30288,40023
+53991,24717,30287,40810
+50349,24718,30286,40963
+55533,24719,30285,40507
+55103,24720,30284,40670
+54868,24721,30283,40298
+51098,24722,30282,40535
+58925,24723,30281,40000
+51527,24724,30280,40048
+50855,24725,30279,40600
+52394,24726,30278,40098
+51308,24727,30277,40856
+56095,24728,30276,40153
+55595,24729,30275,40764
+50355,24730,30274,40191
+52736,24731,30273,40801
+50560,24732,30272,40424
+55743,24733,30271,40082
+54544,24734,30270,40042
+56048,24735,30269,40490
+51218,24736,30268,40974
+55333,24737,30267,40072
+54697,24738,30266,40375
+54215,24739,30265,40368
+52722,24740,30264,40582
+53565,24741,30263,40499
+57182,24742,30262,40536
+54190,24743,30261,40549
+58942,24744,30260,40637
+53998,24745,30259,40839
+53229,24746,30258,40997
+50517,24747,30257,40921
+54487,24748,30256,40456
+50573,24749,30255,40662
+58615,24750,30254,40570
+57642,24751,30253,40875
+55133,24752,30252,40890
+57524,24753,30251,40581
+56271,24754,30250,40886
+56831,24755,30249,40262
+53606,24756,30248,40725
+52876,24757,30247,40244
+54826,24758,30246,40480
+53736,24759,30245,40310
+53288,24760,30244,40686
+50610,24761,30243,40124
+52527,24762,30242,40586
+57066,24763,30241,40898
+59994,24764,30240,40699
+53509,24765,30239,40008
+53639,24766,30238,40520
+54653,24767,30237,40673
+59350,24768,30236,40662
+58885,24769,30235,40366
+56270,24770,30234,40863
+56059,24771,30233,40927
+56601,24772,30232,40104
+55350,24773,30231,40901
+55952,24774,30230,40968
+58286,24775,30229,40919
+52193,24776,30228,40120
+57162,24777,30227,40623
+50524,24778,30226,40804
+58165,24779,30225,40679
+51053,24780,30224,40220
+56602,24781,30223,40078
+56036,24782,30222,40751
+50735,24783,30221,40161
+58673,24784,30220,40213
+59198,24785,30219,40184
+53962,24786,30218,40286
+51533,24787,30217,40222
+58063,24788,30216,40677
+59727,24789,30215,40188
+52324,24790,30214,40643
+57076,24791,30213,40171
+57933,24792,30212,40753
+51371,24793,30211,40155
+57305,24794,30210,40405
+55432,24795,30209,40109
+57093,24796,30208,40960
+57089,24797,30207,40610
+56402,24798,30206,40139
+57010,24799,30205,40046
+56640,24800,30204,40579
+52800,24801,30203,40629
+57001,24802,30202,40872
+54587,24803,30201,40168
+59018,24804,30200,40629
+57644,24805,30199,40379
+56012,24806,30198,40189
+52654,24807,30197,40774
+52321,24808,30196,40915
+53289,24809,30195,40062
+58394,24810,30194,40083
+52425,24811,30193,40319
+56981,24812,30192,40251
+53549,24813,30191,40369
+53524,24814,30190,40000
+52571,24815,30189,40261
+50473,24816,30188,40534
+59122,24817,30187,40626
+59741,24818,30186,40311
+53607,24819,30185,40295
+57984,24820,30184,40503
+50817,24821,30183,40143
+51946,24822,30182,40858
+50192,24823,30181,40445
+57008,24824,30180,40147
+53447,24825,30179,40050
+55699,24826,30178,40329
+54842,24827,30177,40490
+52354,24828,30176,40648
+50387,24829,30175,40255
+59999,24830,30174,40298
+55231,24831,30173,40640
+56095,24832,30172,40728
+52133,24833,30171,40491
+58747,24834,30170,40873
+53607,24835,30169,40618
+55668,24836,30168,40829
+55441,24837,30167,40932
+53811,24838,30166,40102
+53502,24839,30165,40915
+51518,24840,30164,40796
+51913,24841,30163,40757
+51959,24842,30162,40332
+50193,24843,30161,40137
+57519,24844,30160,40660
+52198,24845,30159,40776
+58439,24846,30158,40037
+50144,24847,30157,40118
+53431,24848,30156,40789
+56224,24849,30155,40859
+51511,24850,30154,40281
+56974,24851,30153,40107
+50938,24852,30152,40356
+51984,24853,30151,40038
+56971,24854,30150,40414
+58892,24855,30149,40772
+57616,24856,30148,40299
+52788,24857,30147,40321
+53915,24858,30146,40838
+54273,24859,30145,40769
+55872,24860,30144,40618
+53499,24861,30143,40632
+50224,24862,30142,40718
+57876,24863,30141,40560
+57315,24864,30140,40659
+57476,24865,30139,40784
+50469,24866,30138,40715
+52895,24867,30137,40622
+58679,24868,30136,40071
+56787,24869,30135,40633
+57558,24870,30134,40804
+55598,24871,30133,40861
+51273,24872,30132,40668
+59484,24873,30131,40803
+53443,24874,30130,40389
+53581,24875,30129,40068
+53833,24876,30128,40560
+55893,24877,30127,40123
+50254,24878,30126,40751
+57374,24879,30125,40476
+57737,24880,30124,40156
+54888,24881,30123,40861
+54952,24882,30122,40219
+59984,24883,30121,40759
+57022,24884,30120,40699
+58238,24885,30119,40712
+58472,24886,30118,40254
+59592,24887,30117,40585
+52989,24888,30116,40954
+52868,24889,30115,40671
+57539,24890,30114,40939
+53483,24891,30113,40505
+52238,24892,30112,40951
+55881,24893,30111,40500
+50724,24894,30110,40182
+50401,24895,30109,40958
+52044,24896,30108,40380
+52526,24897,30107,40413
+56421,24898,30106,40453
+52852,24899,30105,40963
+52003,24900,30104,40996
+58276,24901,30103,40897
+52484,24902,30102,40503
+52206,24903,30101,40109
+59232,24904,30100,40626
+53713,24905,30099,40889
+50735,24906,30098,40910
+58782,24907,30097,40001
+53364,24908,30096,40502
+57403,24909,30095,40201
+53479,24910,30094,40758
+50883,24911,30093,40545
+56481,24912,30092,40993
+53386,24913,30091,40950
+58112,24914,30090,40797
+53774,24915,30089,40741
+56059,24916,30088,40981
+54826,24917,30087,40839
+50495,24918,30086,40587
+53764,24919,30085,40884
+51287,24920,30084,40863
+57190,24921,30083,40317
+58250,24922,30082,40329
+59098,24923,30081,40794
+59998,24924,30080,40911
+53262,24925,30079,40541
+56923,24926,30078,40267
+50983,24927,30077,40681
+56987,24928,30076,40213
+53377,24929,30075,40250
+57965,24930,30074,40688
+51956,24931,30073,40738
+58536,24932,30072,40987
+52387,24933,30071,40557
+53245,24934,30070,40054
+58638,24935,30069,40538
+57109,24936,30068,40231
+59784,24937,30067,40007
+59689,24938,30066,40344
+55715,24939,30065,40648
+56497,24940,30064,40367
+53046,24941,30063,40307
+58804,24942,30062,40338
+56816,24943,30061,40275
+59326,24944,30060,40572
+57815,24945,30059,40854
+57728,24946,30058,40417
+54233,24947,30057,40725
+51936,24948,30056,40978
+58964,24949,30055,40849
+55345,24950,30054,40997
+50593,24951,30053,40684
+51534,24952,30052,40100
+50954,24953,30051,40627
+56176,24954,30050,40240
+53356,24955,30049,40958
+55535,24956,30048,40633
+50872,24957,30047,40688
+51669,24958,30046,40909
+59956,24959,30045,40778
+54830,24960,30044,40572
+53455,24961,30043,40591
+59932,24962,30042,40164
+58785,24963,30041,40803
+59095,24964,30040,40573
+51152,24965,30039,40493
+51953,24966,30038,40114
+55903,24967,30037,40086
+56663,24968,30036,40154
+55585,24969,30035,40964
+55459,24970,30034,40819
+57771,24971,30033,40864
+57790,24972,30032,40571
+55032,24973,30031,40744
+58788,24974,30030,40058
+59382,24975,30029,40442
+50880,24976,30028,40495
+58701,24977,30027,40855
+53750,24978,30026,40733
+58528,24979,30025,40055
+52575,24980,30024,40937
+56380,24981,30023,40790
+59647,24982,30022,40467
+50541,24983,30021,40914
+51538,24984,30020,40167
+56336,24985,30019,40672
+55733,24986,30018,40127
+56611,24987,30017,40251
+59890,24988,30016,40854
+50297,24989,30015,40512
+55443,24990,30014,40364
+55706,24991,30013,40264
+56183,24992,30012,40833
+53686,24993,30011,40171
+56962,24994,30010,40984
+52106,24995,30009,40117
+55006,24996,30008,40074
+58027,24997,30007,40288
+54288,24998,30006,40387
+53248,24999,30005,40030
+53894,25000,30004,40925
+50684,25001,30003,40329
+56620,25002,30002,40054
+57880,25003,30001,40050
+56365,25004,30000,40945
+57407,25005,31000,40946
+56231,25006,30999,40591
+57603,25007,30998,40751
+59847,25008,30997,40295
+50414,25009,30996,40670
+56675,25010,30995,40209
+53491,25011,30994,40906
+57031,25012,30993,40043
+52162,25013,30992,40263
+51447,25014,30991,40782
+50778,25015,30990,40686
+52004,25016,30989,40192
+51384,25017,30988,40397
+55210,25018,30987,40342
+58846,25019,30986,40841
+56229,25020,30985,40718
+50880,25021,30984,40259
+55715,25022,30983,40247
+50519,25023,30982,40564
+52421,25024,30981,40757
+57368,25025,30980,40742
+55343,25026,30979,40797
+59311,25027,30978,40313
+56207,25028,30977,40881
+51620,25029,30976,40743
+55578,25030,30975,40008
+54576,25031,30974,40437
+54146,25032,30973,40285
+57033,25033,30972,40710
+54676,25034,30971,40130
+58185,25035,30970,40053
+54365,25036,30969,40418
+52925,25037,30968,40742
+58045,25038,30967,40104
+50281,25039,30966,40850
+56387,25040,30965,40983
+59593,25041,30964,40838
+51064,25042,30963,40414
+58434,25043,30962,40303
+58104,25044,30961,40439
+54534,25045,30960,40508
+58107,25046,30959,40725
+59623,25047,30958,40423
+58979,25048,30957,40864
+59827,25049,30956,40585
+50781,25050,30955,40295
+57517,25051,30954,40049
+52137,25052,30953,40792
+56636,25053,30952,40169
+50743,25054,30951,40374
+55469,25055,30950,40694
+51207,25056,30949,40176
+52292,25057,30948,40322
+57523,25058,30947,40666
+57392,25059,30946,40919
+58361,25060,30945,40227
+50204,25061,30944,40945
+54271,25062,30943,40674
+53761,25063,30942,40321
+55419,25064,30941,40637
+50585,25065,30940,40226
+51771,25066,30939,40759
+59822,25067,30938,40986
+55518,25068,30937,40253
+54465,25069,30936,40607
+57334,25070,30935,40662
+53623,25071,30934,40203
+59573,25072,30933,40117
+55438,25073,30932,40365
+55636,25074,30931,40156
+59005,25075,30930,40509
+56803,25076,30929,40519
+58473,25077,30928,40868
+51578,25078,30927,40841
+56500,25079,30926,40902
+57086,25080,30925,40743
+57327,25081,30924,40205
+52512,25082,30923,40352
+59136,25083,30922,40091
+52559,25084,30921,40997
+54895,25085,30920,40291
+53453,25086,30919,40571
+59035,25087,30918,40173
+57519,25088,30917,40833
+54602,25089,30916,40237
+58478,25090,30915,40544
+59549,25091,30914,40158
+51835,25092,30913,40935
+51857,25093,30912,40312
+56192,25094,30911,40088
+57346,25095,30910,40618
+58001,25096,30909,40755
+55706,25097,30908,40374
+57553,25098,30907,40284
+56184,25099,30906,40361
+59072,25100,30905,40371
+56427,25101,30904,40563
+52610,25102,30903,40441
+54381,25103,30902,40929
+57387,25104,30901,40986
+57261,25105,30900,40932
+50503,25106,30899,40955
+50068,25107,30898,40573
+58534,25108,30897,40416
+56713,25109,30896,40056
+58116,25110,30895,40922
+51857,25111,30894,40202
+50151,25112,30893,40780
+53934,25113,30892,40313
+52897,25114,30891,40121
+58936,25115,30890,40791
+58374,25116,30889,40740
+57737,25117,30888,40321
+52599,25118,30887,40332
+58401,25119,30886,40146
+52877,25120,30885,40305
+59098,25121,30884,40551
+52049,25122,30883,40319
+52582,25123,30882,40887
+55239,25124,30881,40422
+57157,25125,30880,40481
+55243,25126,30879,40027
+56275,25127,30878,40826
+55956,25128,30877,40407
+57678,25129,30876,40857
+59526,25130,30875,40482
+50869,25131,30874,40321
+55588,25132,30873,40708
+55275,25133,30872,40843
+56289,25134,30871,40840
+56603,25135,30870,40556
+59655,25136,30869,40693
+55654,25137,30868,40778
+51665,25138,30867,40538
+53144,25139,30866,40375
+58451,25140,30865,40181
+56136,25141,30864,40384
+50793,25142,30863,40944
+58942,25143,30862,40981
+52605,25144,30861,40400
+55807,25145,30860,40460
+56450,25146,30859,40180
+56979,25147,30858,40908
+51294,25148,30857,40374
+56104,25149,30856,40299
+56580,25150,30855,40161
+57071,25151,30854,40764
+59951,25152,30853,40664
+53156,25153,30852,40289
+50075,25154,30851,40054
+53436,25155,30850,40273
+51628,25156,30849,40272
+53847,25157,30848,40887
+50434,25158,30847,40203
+53124,25159,30846,40735
+51285,25160,30845,40363
+55284,25161,30844,40684
+53626,25162,30843,40033
+51014,25163,30842,40011
+53640,25164,30841,40008
+51810,25165,30840,40214
+56676,25166,30839,40044
+59297,25167,30838,40278
+58104,25168,30837,40863
+52809,25169,30836,40024
+52898,25170,30835,40657
+50167,25171,30834,40514
+58648,25172,30833,40740
+59653,25173,30832,40907
+55626,25174,30831,40266
+52843,25175,30830,40188
+54640,25176,30829,40843
+51377,25177,30828,40775
+51010,25178,30827,40526
+53299,25179,30826,40618
+57783,25180,30825,40859
+56717,25181,30824,40160
+55476,25182,30823,40819
+59878,25183,30822,40373
+50918,25184,30821,40869
+57913,25185,30820,40331
+55963,25186,30819,40214
+55709,25187,30818,40261
+51163,25188,30817,40652
+58647,25189,30816,40120
+54756,25190,30815,40855
+59302,25191,30814,40657
+54530,25192,30813,40365
+59724,25193,30812,40823
+55454,25194,30811,40496
+51532,25195,30810,40630
+51085,25196,30809,40470
+57182,25197,30808,40629
+52746,25198,30807,40196
+51360,25199,30806,40933
+54638,25200,30805,40015
+53623,25201,30804,40382
+54778,25202,30803,40002
+50995,25203,30802,40479
+59799,25204,30801,40893
+58064,25205,30800,40778
+59557,25206,30799,40504
+51256,25207,30798,40874
+59088,25208,30797,40124
+57478,25209,30796,40249
+50097,25210,30795,40546
+54886,25211,30794,40371
+52129,25212,30793,40690
+50713,25213,30792,40369
+54936,25214,30791,40018
+55124,25215,30790,40178
+55651,25216,30789,40665
+59415,25217,30788,40460
+52003,25218,30787,40431
+54379,25219,30786,40779
+51842,25220,30785,40883
+52805,25221,30784,40028
+56737,25222,30783,40144
+53159,25223,30782,40393
+54413,25224,30781,40317
+59846,25225,30780,40530
+54432,25226,30779,40608
+53910,25227,30778,40275
+58996,25228,30777,40494
+59697,25229,30776,40023
+52546,25230,30775,40137
+54710,25231,30774,40497
+55113,25232,30773,40335
+52430,25233,30772,40355
+52858,25234,30771,40483
+51691,25235,30770,40006
+52879,25236,30769,40458
+59177,25237,30768,40972
+56524,25238,30767,40556
+54534,25239,30766,40781
+58398,25240,30765,40908
+51102,25241,30764,40292
+50511,25242,30763,40174
+59950,25243,30762,40245
+51372,25244,30761,40740
+51434,25245,30760,40997
+56382,25246,30759,40902
+56827,25247,30758,40558
+55540,25248,30757,40388
+53520,25249,30756,40494
+52050,25250,30755,40960
+57781,25251,30754,40563
+53043,25252,30753,40861
+59377,25253,30752,40205
+50201,25254,30751,40214
+52665,25255,30750,40407
+54725,25256,30749,40196
+57474,25257,30748,40067
+59698,25258,30747,40728
+53220,25259,30746,40578
+57452,25260,30745,40025
+56297,25261,30744,40677
+56158,25262,30743,40403
+53749,25263,30742,40428
+53081,25264,30741,40165
+56880,25265,30740,40776
+54992,25266,30739,40028
+59263,25267,30738,40070
+54512,25268,30737,40137
+59728,25269,30736,40762
+53613,25270,30735,40939
+50980,25271,30734,40213
+50632,25272,30733,40531
+51815,25273,30732,40189
+57453,25274,30731,40443
+59434,25275,30730,40872
+51016,25276,30729,40009
+50434,25277,30728,40421
+54487,25278,30727,40214
+50569,25279,30726,40199
+55403,25280,30725,40741
+56979,25281,30724,40446
+59720,25282,30723,40106
+56859,25283,30722,40004
+59964,25284,30721,40505
+56955,25285,30720,40313
+50476,25286,30719,40760
+51691,25287,30718,40139
+51322,25288,30717,40354
+55408,25289,30716,40213
+53251,25290,30715,40369
+51446,25291,30714,40555
+56837,25292,30713,40063
+57544,25293,30712,40951
+54973,25294,30711,40193
+50316,25295,30710,40027
+59854,25296,30709,40065
+52716,25297,30708,40420
+52158,25298,30707,40694
+59071,25299,30706,40776
+52282,25300,30705,40730
+52678,25301,30704,40212
+58138,25302,30703,40704
+56833,25303,30702,40219
+55440,25304,30701,40104
+51698,25305,30700,40135
+53945,25306,30699,40120
+59450,25307,30698,40677
+59078,25308,30697,40640
+57904,25309,30696,40225
+53796,25310,30695,40010
+55680,25311,30694,40693
+56295,25312,30693,40429
+51456,25313,30692,40487
+50197,25314,30691,40317
+58973,25315,30690,40446
+53455,25316,30689,40064
+55124,25317,30688,40696
+55851,25318,30687,40271
+50860,25319,30686,40652
+52604,25320,30685,40199
+51745,25321,30684,40614
+58611,25322,30683,40467
+58076,25323,30682,40237
+52623,25324,30681,40424
+57277,25325,30680,40765
+56479,25326,30679,40788
+54971,25327,30678,40969
+58313,25328,30677,40724
+52306,25329,30676,40833
+53955,25330,30675,40985
+55979,25331,30674,40875
+55550,25332,30673,40590
+51769,25333,30672,40937
+59138,25334,30671,40185
+58927,25335,30670,40927
+51060,25336,30669,40715
+57824,25337,30668,40216
+53189,25338,30667,40134
+55566,25339,30666,40324
+53620,25340,30665,40498
+50500,25341,30664,40888
+50724,25342,30663,40300
+52911,25343,30662,40986
+56536,25344,30661,40459
+56439,25345,30660,40630
+56602,25346,30659,40432
+55323,25347,30658,40802
+54523,25348,30657,40737
+50010,25349,30656,40179
+50196,25350,30655,40066
+50254,25351,30654,40418
+58331,25352,30653,40378
+54291,25353,30652,40599
+52146,25354,30651,40325
+54438,25355,30650,40288
+52138,25356,30649,40947
+53064,25357,30648,40566
+52661,25358,30647,40044
+50130,25359,30646,40244
+57496,25360,30645,40542
+53954,25361,30644,40004
+56783,25362,30643,40029
+57107,25363,30642,40072
+51642,25364,30641,40005
+52539,25365,30640,40666
+54517,25366,30639,40791
+58253,25367,30638,40201
+57526,25368,30637,40006
+51183,25369,30636,40968
+54399,25370,30635,40981
+52471,25371,30634,40781
+57684,25372,30633,40062
+55486,25373,30632,40699
+51062,25374,30631,40521
+53030,25375,30630,40252
+57294,25376,30629,40600
+54136,25377,30628,40462
+59029,25378,30627,40900
+52934,25379,30626,40432
+54096,25380,30625,40749
+58863,25381,30624,40962
+59897,25382,30623,40714
+52897,25383,30622,40087
+52872,25384,30621,40691
+51304,25385,30620,40824
+57201,25386,30619,40472
+53284,25387,30618,40757
+54006,25388,30617,40857
+53097,25389,30616,40634
+50253,25390,30615,40040
+56817,25391,30614,40397
+53645,25392,30613,40422
+54956,25393,30612,40488
+58338,25394,30611,40185
+56695,25395,30610,40479
+53088,25396,30609,40423
+59642,25397,30608,40935
+57075,25398,30607,40641
+50796,25399,30606,40560
+51845,25400,30605,40543
+56529,25401,30604,40785
+59689,25402,30603,40972
+55491,25403,30602,40535
+56885,25404,30601,40718
+58494,25405,30600,40969
+52044,25406,30599,40420
+58085,25407,30598,40579
+57193,25408,30597,40748
+55155,25409,30596,40469
+53688,25410,30595,40552
+51196,25411,30594,40880
+53950,25412,30593,40444
+55538,25413,30592,40780
+50057,25414,30591,40599
+50227,25415,30590,40849
+56657,25416,30589,40340
+51247,25417,30588,40552
+56302,25418,30587,40076
+55895,25419,30586,40628
+59316,25420,30585,40961
+56705,25421,30584,40411
+55433,25422,30583,40289
+55032,25423,30582,40651
+54855,25424,30581,40717
+52736,25425,30580,40787
+54908,25426,30579,40341
+54328,25427,30578,40335
+51038,25428,30577,40363
+56856,25429,30576,40711
+59821,25430,30575,40768
+57514,25431,30574,40901
+57617,25432,30573,40870
+52181,25433,30572,40345
+57138,25434,30571,40932
+52212,25435,30570,40946
+56983,25436,30569,40287
+58549,25437,30568,40235
+53430,25438,30567,40699
+58173,25439,30566,40856
+53369,25440,30565,40022
+50632,25441,30564,40124
+54706,25442,30563,40589
+51087,25443,30562,40577
+57349,25444,30561,40646
+54195,25445,30560,40682
+51492,25446,30559,40456
+52005,25447,30558,40384
+50440,25448,30557,40715
+57240,25449,30556,40127
+50762,25450,30555,40328
+58078,25451,30554,40637
+53826,25452,30553,40188
+51182,25453,30552,40791
+59543,25454,30551,40967
+55975,25455,30550,40727
+52997,25456,30549,40011
+58783,25457,30548,40645
+55933,25458,30547,40347
+58789,25459,30546,40993
+50678,25460,30545,40817
+59876,25461,30544,40971
+51121,25462,30543,40636
+58760,25463,30542,40943
+55562,25464,30541,40821
+54395,25465,30540,40608
+54090,25466,30539,40100
+55688,25467,30538,40528
+57898,25468,30537,40796
+53185,25469,30536,40777
+57746,25470,30535,40367
+52764,25471,30534,40035
+57903,25472,30533,40959
+53384,25473,30532,40098
+58610,25474,30531,40347
+54477,25475,30530,40755
+58149,25476,30529,40522
+51202,25477,30528,40716
+50522,25478,30527,40822
+53480,25479,30526,40941
+50332,25480,30525,40176
+58643,25481,30524,40783
+54710,25482,30523,40981
+58968,25483,30522,40759
+59202,25484,30521,40794
+55074,25485,30520,40926
+54793,25486,30519,40649
+58831,25487,30518,40736
+50201,25488,30517,40183
+56200,25489,30516,40728
+50294,25490,30515,40777
+58237,25491,30514,40812
+57415,25492,30513,40987
+50030,25493,30512,40498
+59953,25494,30511,40516
+58791,25495,30510,40887
+51441,25496,30509,40273
+52643,25497,30508,40255
+52050,25498,30507,40821
+55185,25499,30506,40471
+54046,25500,30505,40714
+50708,25501,30504,40162
+51441,25502,30503,40863
+53764,25503,30502,40195
+57993,25504,30501,40318
+59154,25505,30500,40872
+59850,25506,30499,40289
+54232,25507,30498,40032
+55151,25508,30497,40064
+55119,25509,30496,40881
+58731,25510,30495,40041
+57242,25511,30494,40920
+53885,25512,30493,40260
+59088,25513,30492,40582
+51921,25514,30491,40258
+58983,25515,30490,40443
+55154,25516,30489,40890
+56828,25517,30488,40345
+58030,25518,30487,40607
+55631,25519,30486,40528
+55299,25520,30485,40906
+54522,25521,30484,40265
+50933,25522,30483,40599
+55762,25523,30482,40322
+53169,25524,30481,40854
+50663,25525,30480,40933
+57467,25526,30479,40833
+57542,25527,30478,40221
+50164,25528,30477,40645
+59453,25529,30476,40910
+56264,25530,30475,40863
+55806,25531,30474,40863
+57347,25532,30473,40364
+52541,25533,30472,40479
+58708,25534,30471,40636
+53251,25535,30470,40633
+58681,25536,30469,40917
+51349,25537,30468,40070
+50746,25538,30467,40952
+51695,25539,30466,40790
+51954,25540,30465,40185
+50426,25541,30464,40586
+56343,25542,30463,40888
+56685,25543,30462,40721
+59659,25544,30461,40687
+54479,25545,30460,40052
+56267,25546,30459,40216
+54906,25547,30458,40920
+59796,25548,30457,40125
+54363,25549,30456,40520
+56502,25550,30455,40779
+58102,25551,30454,40053
+52129,25552,30453,40403
+56391,25553,30452,40971
+52972,25554,30451,40414
+56915,25555,30450,40584
+54388,25556,30449,40453
+51725,25557,30448,40804
+54065,25558,30447,40011
+52372,25559,30446,40904
+57301,25560,30445,40231
+59274,25561,30444,40516
+57981,25562,30443,40577
+54575,25563,30442,40873
+50330,25564,30441,40280
+59196,25565,30440,40408
+59865,25566,30439,40390
+51067,25567,30438,40749
+53086,25568,30437,40497
+57763,25569,30436,40947
+56944,25570,30435,40927
+56850,25571,30434,40533
+54205,25572,30433,40373
+51219,25573,30432,40183
+54750,25574,30431,40756
+52384,25575,30430,40799
+52422,25576,30429,40393
+58257,25577,30428,40307
+56724,25578,30427,40479
+58721,25579,30426,40727
+52663,25580,30425,40330
+58385,25581,30424,40515
+54401,25582,30423,40587
+53414,25583,30422,40535
+51460,25584,30421,40011
+58654,25585,30420,40860
+54618,25586,30419,40942
+58848,25587,30418,40889
+56513,25588,30417,40502
+52577,25589,30416,40934
+51842,25590,30415,40993
+52483,25591,30414,40761
+57842,25592,30413,40520
+51080,25593,30412,40218
+54914,25594,30411,40508
+52020,25595,30410,40605
+59201,25596,30409,40439
+58913,25597,30408,40388
+56194,25598,30407,40836
+53435,25599,30406,40606
+53304,25600,30405,40093
+58075,25601,30404,40552
+56493,25602,30403,40111
+54802,25603,30402,40151
+55746,25604,30401,40908
+54875,25605,30400,40839
+55361,25606,30399,40396
+52915,25607,30398,40394
+58190,25608,30397,40417
+51220,25609,30396,40552
+58700,25610,30395,40478
+53708,25611,30394,40376
+50540,25612,30393,40674
+55533,25613,30392,40489
+57615,25614,30391,40975
+59198,25615,30390,40423
+57469,25616,30389,40706
+59103,25617,30388,40285
+58990,25618,30387,40087
+53322,25619,30386,40235
+57462,25620,30385,40962
+51108,25621,30384,40468
+53486,25622,30383,40619
+52610,25623,30382,40079
+50349,25624,30381,40599
+59065,25625,30380,40024
+54374,25626,30379,40616
+54542,25627,30378,40459
+59594,25628,30377,40505
+53692,25629,30376,40052
+59198,25630,30375,40471
+56963,25631,30374,40405
+57879,25632,30373,40479
+50509,25633,30372,40333
+50770,25634,30371,40976
+55997,25635,30370,40201
+53088,25636,30369,40341
+52977,25637,30368,40250
+55710,25638,30367,40338
+50230,25639,30366,40764
+56659,25640,30365,40999
+55914,25641,30364,40334
+57219,25642,30363,40606
+53212,25643,30362,40754
+53249,25644,30361,40245
+58928,25645,30360,40058
+53985,25646,30359,40521
+52062,25647,30358,40677
+57834,25648,30357,40303
+56698,25649,30356,40341
+50507,25650,30355,40600
+54959,25651,30354,40902
+52981,25652,30353,40792
+54709,25653,30352,40398
+57493,25654,30351,40323
+51025,25655,30350,40179
+54837,25656,30349,40701
+56123,25657,30348,40157
+59208,25658,30347,40356
+50523,25659,30346,40698
+58756,25660,30345,40765
+58813,25661,30344,40271
+58077,25662,30343,40799
+51796,25663,30342,40458
+51819,25664,30341,40895
+54810,25665,30340,40473
+59021,25666,30339,40937
+50884,25667,30338,40838
+51623,25668,30337,40945
+55988,25669,30336,40154
+51079,25670,30335,40099
+50537,25671,30334,40674
+50774,25672,30333,40645
+58274,25673,30332,40867
+57369,25674,30331,40742
+53967,25675,30330,40030
+59415,25676,30329,40971
+51384,25677,30328,40680
+56229,25678,30327,40582
+59847,25679,30326,40432
+54749,25680,30325,40403
+50944,25681,30324,40613
+59342,25682,30323,40377
+50545,25683,30322,40116
+52892,25684,30321,40921
+51220,25685,30320,40651
+56056,25686,30319,40919
+50465,25687,30318,40597
+52394,25688,30317,40842
+53664,25689,30316,40842
+56410,25690,30315,40193
+59117,25691,30314,40746
+50415,25692,30313,40315
+53396,25693,30312,40767
+57295,25694,30311,40579
+56492,25695,30310,40836
+59541,25696,30309,40087
+58960,25697,30308,40437
+51936,25698,30307,40803
+58312,25699,30306,40324
+51927,25700,30305,40311
+58710,25701,30304,40341
+57835,25702,30303,40730
+56908,25703,30302,40871
+59221,25704,30301,40857
+51948,25705,30300,40456
+59311,25706,30299,40767
+59527,25707,30298,40467
+56783,25708,30297,40379
+54134,25709,30296,40480
+55789,25710,30295,40481
+55607,25711,30294,40985
+54133,25712,30293,40118
+58527,25713,30292,40813
+50584,25714,30291,40484
+50990,25715,30290,40379
+56381,25716,30289,40349
+58081,25717,30288,40908
+51285,25718,30287,40591
+56192,25719,30286,40364
+50300,25720,30285,40725
+52327,25721,30284,40943
+51227,25722,30283,40446
+53586,25723,30282,40581
+56907,25724,30281,40877
+56338,25725,30280,40681
+51284,25726,30279,40932
+56652,25727,30278,40580
+53059,25728,30277,40493
+58449,25729,30276,40016
+52602,25730,30275,40614
+58360,25731,30274,40596
+59553,25732,30273,40624
+57590,25733,30272,40756
+51466,25734,30271,40957
+56576,25735,30270,40745
+50530,25736,30269,40211
+56360,25737,30268,40708
+53252,25738,30267,40673
+52579,25739,30266,40506
+52631,25740,30265,40045
+53627,25741,30264,40877
+55816,25742,30263,40344
+50752,25743,30262,40224
+54818,25744,30261,40839
+58169,25745,30260,40460
+51465,25746,30259,40087
+55160,25747,30258,40371
+55607,25748,30257,40827
+55100,25749,30256,40512
+51278,25750,30255,40749
+53216,25751,30254,40425
+50764,25752,30253,40319
+58208,25753,30252,40755
+53948,25754,30251,40924
+59212,25755,30250,40441
+57752,25756,30249,40689
+56805,25757,30248,40664
+54652,25758,30247,40409
+56213,25759,30246,40761
+51223,25760,30245,40521
+57515,25761,30244,40403
+50139,25762,30243,40673
+58915,25763,30242,40029
+56803,25764,30241,40787
+59893,25765,30240,40686
+50804,25766,30239,40140
+57124,25767,30238,40865
+58204,25768,30237,40324
+52854,25769,30236,40664
+58983,25770,30235,40956
+51139,25771,30234,40107
+53089,25772,30233,40418
+53014,25773,30232,40525
+52246,25774,30231,40348
+59088,25775,30230,40732
+59588,25776,30229,40383
+50844,25777,30228,40475
+57746,25778,30227,40898
+56366,25779,30226,40757
+54570,25780,30225,40436
+58133,25781,30224,40056
+52625,25782,30223,40183
+52655,25783,30222,40444
+50568,25784,30221,40583
+50812,25785,30220,40231
+58539,25786,30219,40651
+58458,25787,30218,40158
+59014,25788,30217,40855
+52943,25789,30216,40562
+55192,25790,30215,40979
+55062,25791,30214,40572
+52559,25792,30213,40156
+53072,25793,30212,40766
+59651,25794,30211,40509
+57627,25795,30210,40640
+51647,25796,30209,40522
+59185,25797,30208,40388
+54233,25798,30207,40600
+55607,25799,30206,40934
+54067,25800,30205,40452
+56342,25801,30204,40355
+53626,25802,30203,40735
+56343,25803,30202,40710
+59020,25804,30201,40906
+58070,25805,30200,40057
+58615,25806,30199,40276
+53545,25807,30198,40342
+55244,25808,30197,40453
+57402,25809,30196,40145
+55474,25810,30195,40286
+54368,25811,30194,40609
+59958,25812,30193,40200
+54620,25813,30192,40524
+52896,25814,30191,40565
+50655,25815,30190,40932
+52578,25816,30189,40870
+54745,25817,30188,40002
+54905,25818,30187,40026
+52460,25819,30186,40341
+52758,25820,30185,40120
+50543,25821,30184,40087
+54966,25822,30183,40458
+50398,25823,30182,40269
+51741,25824,30181,40702
+54943,25825,30180,40144
+54978,25826,30179,40970
+55712,25827,30178,40789
+54020,25828,30177,40667
+52683,25829,30176,40749
+59386,25830,30175,40449
+51163,25831,30174,40793
+58743,25832,30173,40227
+57865,25833,30172,40999
+52044,25834,30171,40138
+52223,25835,30170,40290
+51752,25836,30169,40469
+57677,25837,30168,40968
+59899,25838,30167,40513
+54746,25839,30166,40581
+55464,25840,30165,40794
+53314,25841,30164,40951
+56488,25842,30163,40947
+50046,25843,30162,40631
+51019,25844,30161,40726
+58146,25845,30160,40291
+58618,25846,30159,40208
+50223,25847,30158,40050
+58968,25848,30157,40772
+59497,25849,30156,40965
+55703,25850,30155,40730
+55314,25851,30154,40174
+50118,25852,30153,40092
+59925,25853,30152,40984
+56904,25854,30151,40781
+52527,25855,30150,40971
+53312,25856,30149,40030
+51608,25857,30148,40850
+57768,25858,30147,40879
+59024,25859,30146,40331
+59265,25860,30145,40750
+57620,25861,30144,40330
+52933,25862,30143,40039
+54063,25863,30142,40090
+55880,25864,30141,40784
+51709,25865,30140,40322
+52765,25866,30139,40839
+57211,25867,30138,40551
+55253,25868,30137,40530
+50746,25869,30136,40771
+53618,25870,30135,40611
+57795,25871,30134,40586
+53642,25872,30133,40182
+50820,25873,30132,40597
+59302,25874,30131,40081
+57464,25875,30130,40870
+50346,25876,30129,40111
+54853,25877,30128,40502
+54991,25878,30127,40256
+54799,25879,30126,40150
+51852,25880,30125,40401
+55658,25881,30124,40184
+51364,25882,30123,40595
+51981,25883,30122,40129
+54344,25884,30121,40323
+52785,25885,30120,40813
+52301,25886,30119,40613
+54392,25887,30118,40191
+56328,25888,30117,40093
+54155,25889,30116,40892
+51110,25890,30115,40492
+53692,25891,30114,40259
+52616,25892,30113,40820
+55002,25893,30112,40315
+58588,25894,30111,40969
+57443,25895,30110,40976
+50935,25896,30109,40027
+50612,25897,30108,40044
+53870,25898,30107,40964
+56555,25899,30106,40862
+54627,25900,30105,40042
+54878,25901,30104,40762
+57388,25902,30103,40753
+55380,25903,30102,40373
+59328,25904,30101,40456
+57392,25905,30100,40307
+54539,25906,30099,40295
+58479,25907,30098,40006
+51921,25908,30097,40290
+58071,25909,30096,40949
+55695,25910,30095,40860
+50847,25911,30094,40751
+54685,25912,30093,40412
+56526,25913,30092,40762
+53199,25914,30091,40092
+58859,25915,30090,40533
+52043,25916,30089,40580
+58195,25917,30088,40702
+54645,25918,30087,40682
+59194,25919,30086,40160
+55445,25920,30085,40883
+54664,25921,30084,40493
+56779,25922,30083,40979
+58069,25923,30082,40732
+56345,25924,30081,40189
+53215,25925,30080,40023
+54942,25926,30079,40711
+51831,25927,30078,40140
+51682,25928,30077,40230
+56589,25929,30076,40974
+54381,25930,30075,40809
+56592,25931,30074,40567
+56467,25932,30073,40684
+59575,25933,30072,40348
+53610,25934,30071,40260
+54493,25935,30070,40037
+57484,25936,30069,40633
+50519,25937,30068,40792
+51287,25938,30067,40957
+56625,25939,30066,40945
+51026,25940,30065,40824
+59255,25941,30064,40159
+55371,25942,30063,40845
+56080,25943,30062,40557
+57419,25944,30061,40405
+52500,25945,30060,40052
+58490,25946,30059,40790
+57166,25947,30058,40218
+51695,25948,30057,40695
+56684,25949,30056,40546
+56240,25950,30055,40323
+50803,25951,30054,40424
+54552,25952,30053,40639
+59280,25953,30052,40342
+58934,25954,30051,40514
+50185,25955,30050,40135
+56882,25956,30049,40032
+56306,25957,30048,40687
+59882,25958,30047,40147
+50460,25959,30046,40382
+57883,25960,30045,40470
+57015,25961,30044,40368
+57388,25962,30043,40223
+53576,25963,30042,40042
+56467,25964,30041,40937
+51318,25965,30040,40586
+58843,25966,30039,40171
+53247,25967,30038,40387
+58509,25968,30037,40761
+51639,25969,30036,40372
+58011,25970,30035,40221
+54496,25971,30034,40628
+53164,25972,30033,40129
+56197,25973,30032,40213
+56657,25974,30031,40819
+52928,25975,30030,40610
+54326,25976,30029,40181
+51804,25977,30028,40166
+50820,25978,30027,40201
+55332,25979,30026,40210
+56724,25980,30025,40550
+55354,25981,30024,40290
+53854,25982,30023,40881
+56852,25983,30022,40396
+56975,25984,30021,40522
+59361,25985,30020,40398
+50361,25986,30019,40121
+52474,25987,30018,40340
+57444,25988,30017,40366
+51299,25989,30016,40083
+57554,25990,30015,40955
+56560,25991,30014,40469
+58209,25992,30013,40414
+53637,25993,30012,40497
+50553,25994,30011,40494
+50597,25995,30010,40238
+57013,25996,30009,40217
+54956,25997,30008,40740
+51262,25998,30007,40223
+58472,25999,30006,40431
+52963,26000,30005,40615
+50050,26001,30004,40360
+58021,26002,30003,40549
+53915,26003,30002,40402
+50383,26004,30001,40972
+50385,26005,30000,40598
+50851,26006,31000,40901
+54001,26007,30999,40236
+59087,26008,30998,40260
+59414,26009,30997,40007
+50638,26010,30996,40764
+58799,26011,30995,40195
+55834,26012,30994,40615
+56495,26013,30993,40653
+56223,26014,30992,40243
+51918,26015,30991,40487
+55532,26016,30990,40348
+58141,26017,30989,40502
+58931,26018,30988,40298
+52807,26019,30987,40234
+57920,26020,30986,40823
+56418,26021,30985,40834
+50006,26022,30984,40679
+57797,26023,30983,40890
+51732,26024,30982,40516
+51412,26025,30981,40778
+53766,26026,30980,40557
+55526,26027,30979,40620
+52968,26028,30978,40475
+54248,26029,30977,40289
+57269,26030,30976,40312
+54362,26031,30975,40368
+54366,26032,30974,40972
+52311,26033,30973,40356
+53975,26034,30972,40325
+57026,26035,30971,40353
+50139,26036,30970,40050
+59739,26037,30969,40203
+57711,26038,30968,40364
+52913,26039,30967,40915
+58084,26040,30966,40610
+58658,26041,30965,40143
+51062,26042,30964,40631
+58702,26043,30963,40037
+50610,26044,30962,40946
+50874,26045,30961,40588
+52468,26046,30960,40250
+54192,26047,30959,40127
+59022,26048,30958,40788
+57347,26049,30957,40358
+51204,26050,30956,40731
+54688,26051,30955,40106
+58372,26052,30954,40746
+57520,26053,30953,40277
+53268,26054,30952,40352
+52554,26055,30951,40236
+52084,26056,30950,40033
+54757,26057,30949,40979
+51474,26058,30948,40413
+57051,26059,30947,40715
+58380,26060,30946,40344
+57785,26061,30945,40314
+54415,26062,30944,40645
+58244,26063,30943,40719
+57502,26064,30942,40128
+52437,26065,30941,40568
+51519,26066,30940,40094
+58293,26067,30939,40498
+54384,26068,30938,40195
+51444,26069,30937,40404
+56999,26070,30936,40592
+52386,26071,30935,40057
+55559,26072,30934,40750
+52912,26073,30933,40536
+52029,26074,30932,40308
+59841,26075,30931,40073
+53082,26076,30930,40312
+56411,26077,30929,40775
+53180,26078,30928,40406
+56842,26079,30927,40012
+51396,26080,30926,40523
+52035,26081,30925,40429
+55996,26082,30924,40396
+57413,26083,30923,40404
+59810,26084,30922,40792
+53547,26085,30921,40532
+56237,26086,30920,40918
+55374,26087,30919,40372
+54417,26088,30918,40108
+58216,26089,30917,40059
+55805,26090,30916,40510
+59860,26091,30915,40340
+50449,26092,30914,40079
+56769,26093,30913,40382
+52540,26094,30912,40271
+50101,26095,30911,40787
+59396,26096,30910,40106
+54078,26097,30909,40579
+52911,26098,30908,40696
+58228,26099,30907,40060
+50254,26100,30906,40366
+52858,26101,30905,40824
+56458,26102,30904,40784
+54195,26103,30903,40444
+52246,26104,30902,40249
+55377,26105,30901,40472
+56164,26106,30900,40777
+50376,26107,30899,40207
+54340,26108,30898,40009
+54114,26109,30897,40253
+50662,26110,30896,40982
+57451,26111,30895,40664
+53728,26112,30894,40080
+52685,26113,30893,40988
+59932,26114,30892,40102
+56198,26115,30891,40647
+51401,26116,30890,40256
+55733,26117,30889,40190
+59522,26118,30888,40145
+50483,26119,30887,40050
+51387,26120,30886,40847
+54526,26121,30885,40611
+52487,26122,30884,40575
+50737,26123,30883,40679
+54407,26124,30882,40770
+58287,26125,30881,40702
+55821,26126,30880,40222
+52063,26127,30879,40853
+57678,26128,30878,40890
+50769,26129,30877,40678
+56134,26130,30876,40091
+55455,26131,30875,40960
+55446,26132,30874,40260
+57935,26133,30873,40248
+58824,26134,30872,40797
+52740,26135,30871,40836
+50595,26136,30870,40002
+58004,26137,30869,40472
+57450,26138,30868,40910
+55140,26139,30867,40623
+55678,26140,30866,40813
+58575,26141,30865,40207
+57380,26142,30864,40275
+57558,26143,30863,40545
+51554,26144,30862,40348
+58209,26145,30861,40187
+51890,26146,30860,40113
+58755,26147,30859,40252
+54825,26148,30858,40443
+54876,26149,30857,40212
+55201,26150,30856,40872
+52697,26151,30855,40901
+55586,26152,30854,40575
+59814,26153,30853,40170
+51328,26154,30852,40937
+52739,26155,30851,40571
+59310,26156,30850,40158
+59380,26157,30849,40538
+54784,26158,30848,40049
+51506,26159,30847,40221
+51502,26160,30846,40866
+57103,26161,30845,40672
+53044,26162,30844,40558
+55911,26163,30843,40382
+53558,26164,30842,40308
+51841,26165,30841,40556
+54337,26166,30840,40726
+56821,26167,30839,40834
+59821,26168,30838,40610
+54984,26169,30837,40695
+57039,26170,30836,40364
+55395,26171,30835,40056
+50794,26172,30834,40619
+54914,26173,30833,40900
+58837,26174,30832,40405
+56616,26175,30831,40447
+54072,26176,30830,40647
+59871,26177,30829,40281
+52144,26178,30828,40812
+54541,26179,30827,40026
+52370,26180,30826,40873
+51426,26181,30825,40197
+58444,26182,30824,40691
+54526,26183,30823,40230
+59111,26184,30822,40854
+58079,26185,30821,40129
+58425,26186,30820,40180
+55000,26187,30819,40121
+51094,26188,30818,40678
+59438,26189,30817,40889
+50183,26190,30816,40609
+55068,26191,30815,40290
+59027,26192,30814,40224
+58102,26193,30813,40282
+55721,26194,30812,40105
+54842,26195,30811,40141
+56704,26196,30810,40463
+51906,26197,30809,40219
+56870,26198,30808,40164
+50438,26199,30807,40062
+59172,26200,30806,40073
+55263,26201,30805,40726
+59282,26202,30804,40219
+53031,26203,30803,40386
+51591,26204,30802,40515
+58267,26205,30801,40453
+59257,26206,30800,40747
+58770,26207,30799,40027
+51555,26208,30798,40416
+57879,26209,30797,40527
+53221,26210,30796,40208
+50322,26211,30795,40763
+54996,26212,30794,40473
+51878,26213,30793,40402
+58146,26214,30792,40372
+59859,26215,30791,40477
+58760,26216,30790,40452
+53627,26217,30789,40336
+52112,26218,30788,40823
+55090,26219,30787,40216
+56544,26220,30786,40392
+59742,26221,30785,40084
+57776,26222,30784,40662
+54526,26223,30783,40021
+56509,26224,30782,40524
+52215,26225,30781,40076
+54678,26226,30780,40734
+56074,26227,30779,40318
+50787,26228,30778,40725
+53121,26229,30777,40302
+50965,26230,30776,40663
+53688,26231,30775,40855
+57640,26232,30774,40398
+53256,26233,30773,40804
+58422,26234,30772,40051
+59438,26235,30771,40031
+59365,26236,30770,40063
+52555,26237,30769,40250
+52768,26238,30768,40742
+51477,26239,30767,40773
+59076,26240,30766,40760
+55370,26241,30765,40988
+50864,26242,30764,40773
+59492,26243,30763,40612
+51312,26244,30762,40204
+56088,26245,30761,40131
+59427,26246,30760,40121
+53833,26247,30759,40537
+53783,26248,30758,40585
+56425,26249,30757,40014
+56999,26250,30756,40323
+56048,26251,30755,40257
+58171,26252,30754,40612
+56741,26253,30753,40171
+52141,26254,30752,40183
+55337,26255,30751,40126
+53964,26256,30750,40335
+53459,26257,30749,40149
+58793,26258,30748,40131
+51238,26259,30747,40505
+51823,26260,30746,40749
+57262,26261,30745,40484
+59792,26262,30744,40166
+55385,26263,30743,40566
+58106,26264,30742,40474
+58723,26265,30741,40138
+58200,26266,30740,40747
+50586,26267,30739,40357
+59732,26268,30738,40725
+50032,26269,30737,40074
+56315,26270,30736,40613
+57615,26271,30735,40716
+53849,26272,30734,40662
+50471,26273,30733,40497
+59427,26274,30732,40362
+56754,26275,30731,40912
+59220,26276,30730,40946
+59144,26277,30729,40631
+50687,26278,30728,40778
+52798,26279,30727,40347
+52899,26280,30726,40364
+54602,26281,30725,40300
+51740,26282,30724,40223
+55154,26283,30723,40500
+50006,26284,30722,40260
+59762,26285,30721,40797
+50573,26286,30720,40844
+50683,26287,30719,40876
+57024,26288,30718,40209
+54345,26289,30717,40830
+52644,26290,30716,40352
+50861,26291,30715,40613
+53044,26292,30714,40904
+51161,26293,30713,40886
+59413,26294,30712,40946
+51195,26295,30711,40779
+57928,26296,30710,40444
+55664,26297,30709,40552
+53209,26298,30708,40644
+54074,26299,30707,40604
+53877,26300,30706,40029
+57292,26301,30705,40566
+57128,26302,30704,40057
+54178,26303,30703,40113
+59028,26304,30702,40356
+59277,26305,30701,40286
+52765,26306,30700,40640
+53724,26307,30699,40565
+58610,26308,30698,40074
+57798,26309,30697,40959
+54278,26310,30696,40497
+50612,26311,30695,40786
+52924,26312,30694,40270
+50684,26313,30693,40634
+55841,26314,30692,40516
+58200,26315,30691,40257
+54307,26316,30690,40061
+50708,26317,30689,40131
+56016,26318,30688,40743
+59142,26319,30687,40656
+55241,26320,30686,40554
+57145,26321,30685,40830
+59205,26322,30684,40694
+55425,26323,30683,40539
+57506,26324,30682,40645
+50355,26325,30681,40972
+59510,26326,30680,40858
+56317,26327,30679,40789
+55860,26328,30678,40461
+54832,26329,30677,40326
+53849,26330,30676,40629
+50622,26331,30675,40181
+56118,26332,30674,40880
+52046,26333,30673,40422
+53001,26334,30672,40674
+52557,26335,30671,40796
+50164,26336,30670,40303
+57982,26337,30669,40178
+54287,26338,30668,40823
+50034,26339,30667,40099
+50232,26340,30666,40637
+53219,26341,30665,40810
+57145,26342,30664,40979
+52186,26343,30663,40327
+54914,26344,30662,40014
+55920,26345,30661,40119
+52773,26346,30660,40434
+56434,26347,30659,40823
+51873,26348,30658,40330
+51709,26349,30657,40452
+50902,26350,30656,40814
+55961,26351,30655,40593
+54952,26352,30654,40081
+58643,26353,30653,40774
+52526,26354,30652,40253
+55570,26355,30651,40610
+58905,26356,30650,40510
+55530,26357,30649,40845
+54066,26358,30648,40160
+50847,26359,30647,40997
+53516,26360,30646,40237
+52689,26361,30645,40779
+50441,26362,30644,40047
+54710,26363,30643,40000
+50633,26364,30642,40486
+52838,26365,30641,40617
+50636,26366,30640,40831
+50816,26367,30639,40324
+51641,26368,30638,40210
+52265,26369,30637,40398
+52375,26370,30636,40926
+50756,26371,30635,40987
+53864,26372,30634,40403
+59245,26373,30633,40393
+55206,26374,30632,40171
+58896,26375,30631,40170
+54613,26376,30630,40189
+53751,26377,30629,40740
+50489,26378,30628,40984
+57248,26379,30627,40798
+56928,26380,30626,40689
+50284,26381,30625,40305
+50143,26382,30624,40625
+52548,26383,30623,40049
+53867,26384,30622,40854
+53223,26385,30621,40286
+54142,26386,30620,40565
+50382,26387,30619,40723
+51418,26388,30618,40552
+51648,26389,30617,40626
+53785,26390,30616,40585
+56993,26391,30615,40858
+55058,26392,30614,40563
+53045,26393,30613,40471
+54600,26394,30612,40329
+54243,26395,30611,40301
+58140,26396,30610,40551
+57652,26397,30609,40192
+53155,26398,30608,40681
+57691,26399,30607,40458
+53203,26400,30606,40715
+54766,26401,30605,40882
+53079,26402,30604,40925
+57443,26403,30603,40061
+55361,26404,30602,40507
+56966,26405,30601,40492
+51231,26406,30600,40967
+52198,26407,30599,40418
+58534,26408,30598,40011
+57840,26409,30597,40248
+53659,26410,30596,40751
+52002,26411,30595,40487
+50209,26412,30594,40083
+57058,26413,30593,40266
+58556,26414,30592,40026
+54167,26415,30591,40661
+53060,26416,30590,40619
+54853,26417,30589,40637
+58517,26418,30588,40749
+58486,26419,30587,40446
+55987,26420,30586,40870
+51620,26421,30585,40627
+50491,26422,30584,40640
+53555,26423,30583,40750
+51301,26424,30582,40800
+52299,26425,30581,40372
+58694,26426,30580,40725
+57603,26427,30579,40515
+54449,26428,30578,40618
+51673,26429,30577,40650
+59453,26430,30576,40199
+58446,26431,30575,40486
+57266,26432,30574,40578
+54542,26433,30573,40909
+51949,26434,30572,40998
+52909,26435,30571,40713
+52051,26436,30570,40085
+51149,26437,30569,40923
+52802,26438,30568,40840
+50732,26439,30567,40928
+59059,26440,30566,40791
+51834,26441,30565,40557
+57772,26442,30564,40423
+50297,26443,30563,40450
+53778,26444,30562,40567
+59917,26445,30561,40566
+56619,26446,30560,40406
+59362,26447,30559,40368
+50674,26448,30558,40854
+55645,26449,30557,40252
+51155,26450,30556,40213
+57339,26451,30555,40554
+58342,26452,30554,40566
+50901,26453,30553,40847
+51368,26454,30552,40147
+54965,26455,30551,40798
+50405,26456,30550,40516
+50195,26457,30549,40175
+51875,26458,30548,40753
+57848,26459,30547,40858
+58023,26460,30546,40551
+58787,26461,30545,40217
+57994,26462,30544,40915
+54345,26463,30543,40519
+53622,26464,30542,40313
+56359,26465,30541,40648
+53147,26466,30540,40391
+52212,26467,30539,40838
+59855,26468,30538,40987
+55698,26469,30537,40309
+58497,26470,30536,40373
+51299,26471,30535,40828
+53166,26472,30534,40154
+53815,26473,30533,40448
+55901,26474,30532,40415
+58803,26475,30531,40023
+54181,26476,30530,40902
+54494,26477,30529,40038
+59726,26478,30528,40880
+56774,26479,30527,40623
+57492,26480,30526,40151
+54256,26481,30525,40939
+58696,26482,30524,40679
+50603,26483,30523,40848
+59810,26484,30522,40096
+51237,26485,30521,40383
+54241,26486,30520,40418
+51240,26487,30519,40981
+59218,26488,30518,40742
+55133,26489,30517,40003
+59471,26490,30516,40109
+53344,26491,30515,40132
+56702,26492,30514,40000
+55635,26493,30513,40706
+53403,26494,30512,40178
+50333,26495,30511,40406
+56816,26496,30510,40726
+57387,26497,30509,40202
+57568,26498,30508,40544
+52902,26499,30507,40861
+50499,26500,30506,40987
+50149,26501,30505,40418
+58491,26502,30504,40007
+56841,26503,30503,40021
+52998,26504,30502,40684
+54230,26505,30501,40740
+59212,26506,30500,40111
+59122,26507,30499,40693
+58136,26508,30498,40112
+53798,26509,30497,40265
+50228,26510,30496,40662
+52737,26511,30495,40541
+53420,26512,30494,40632
+58028,26513,30493,40569
+57909,26514,30492,40465
+55624,26515,30491,40864
+58103,26516,30490,40766
+57892,26517,30489,40986
+57284,26518,30488,40039
+59658,26519,30487,40313
+58493,26520,30486,40199
+53913,26521,30485,40246
+50842,26522,30484,40298
+52029,26523,30483,40375
+50742,26524,30482,40291
+50346,26525,30481,40312
+52206,26526,30480,40775
+52680,26527,30479,40738
+57134,26528,30478,40489
+59256,26529,30477,40036
+50124,26530,30476,40491
+55125,26531,30475,40853
+54928,26532,30474,40917
+52358,26533,30473,40468
+58912,26534,30472,40397
+55600,26535,30471,40644
+52763,26536,30470,40316
+51224,26537,30469,40223
+55563,26538,30468,40481
+53845,26539,30467,40501
+50221,26540,30466,40956
+54164,26541,30465,40001
+50397,26542,30464,40129
+52629,26543,30463,40696
+53930,26544,30462,40142
+55923,26545,30461,40431
+59368,26546,30460,40875
+50865,26547,30459,40569
+56309,26548,30458,40486
+51568,26549,30457,40410
+55187,26550,30456,40743
+58429,26551,30455,40474
+53348,26552,30454,40671
+56167,26553,30453,40329
+55188,26554,30452,40360
+52926,26555,30451,40547
+58656,26556,30450,40536
+56647,26557,30449,40353
+51124,26558,30448,40608
+57548,26559,30447,40996
+56162,26560,30446,40803
+52928,26561,30445,40902
+58879,26562,30444,40334
+55456,26563,30443,40623
+59639,26564,30442,40763
+53173,26565,30441,40576
+58865,26566,30440,40689
+54566,26567,30439,40744
+55725,26568,30438,40163
+54043,26569,30437,40057
+52649,26570,30436,40675
+51772,26571,30435,40534
+54979,26572,30434,40963
+51175,26573,30433,40491
+59945,26574,30432,40024
+58529,26575,30431,40681
+58364,26576,30430,40964
+53428,26577,30429,40085
+50163,26578,30428,40662
+53318,26579,30427,40491
+59964,26580,30426,40152
+50298,26581,30425,40492
+58925,26582,30424,40733
+50192,26583,30423,40641
+55721,26584,30422,40109
+50519,26585,30421,40195
+58398,26586,30420,40622
+52227,26587,30419,40713
+50186,26588,30418,40296
+50778,26589,30417,40125
+51934,26590,30416,40246
+58696,26591,30415,40788
+51483,26592,30414,40844
+51634,26593,30413,40327
+54025,26594,30412,40345
+59767,26595,30411,40122
+57447,26596,30410,40018
+51891,26597,30409,40249
+55660,26598,30408,40048
+56813,26599,30407,40802
+51036,26600,30406,40728
+53840,26601,30405,40723
+57606,26602,30404,40174
+53911,26603,30403,40505
+50890,26604,30402,40077
+59329,26605,30401,40018
+57976,26606,30400,40988
+56853,26607,30399,40440
+51859,26608,30398,40072
+57311,26609,30397,40890
+51157,26610,30396,40754
+51415,26611,30395,40000
+53404,26612,30394,40933
+56707,26613,30393,40459
+58210,26614,30392,40079
+52915,26615,30391,40766
+58538,26616,30390,40481
+58954,26617,30389,40133
+52566,26618,30388,40013
+52988,26619,30387,40137
+54918,26620,30386,40099
+56483,26621,30385,40885
+57115,26622,30384,40672
+53276,26623,30383,40649
+59465,26624,30382,40195
+51214,26625,30381,40424
+54378,26626,30380,40861
+57946,26627,30379,40064
+58224,26628,30378,40501
+50862,26629,30377,40357
+50019,26630,30376,40117
+56772,26631,30375,40402
+59848,26632,30374,40500
+50664,26633,30373,40645
+54690,26634,30372,40025
+54166,26635,30371,40478
+56194,26636,30370,40865
+55185,26637,30369,40452
+57866,26638,30368,40293
+55137,26639,30367,40052
+51824,26640,30366,40667
+57764,26641,30365,40095
+55166,26642,30364,40375
+51557,26643,30363,40902
+58661,26644,30362,40469
+59017,26645,30361,40404
+54034,26646,30360,40566
+52518,26647,30359,40993
+58105,26648,30358,40701
+59390,26649,30357,40439
+51022,26650,30356,40637
+56682,26651,30355,40755
+50723,26652,30354,40097
+53746,26653,30353,40225
+52762,26654,30352,40007
+54072,26655,30351,40276
+50844,26656,30350,40121
+52180,26657,30349,40257
+53587,26658,30348,40794
+59530,26659,30347,40039
+50674,26660,30346,40234
+53720,26661,30345,40797
+56772,26662,30344,40014
+51147,26663,30343,40759
+57617,26664,30342,40455
+54321,26665,30341,40079
+54813,26666,30340,40293
+50810,26667,30339,40804
+50091,26668,30338,40343
+51560,26669,30337,40970
+50096,26670,30336,40490
+55141,26671,30335,40133
+54385,26672,30334,40668
+58627,26673,30333,40723
+59694,26674,30332,40596
+54606,26675,30331,40198
+50808,26676,30330,40280
+57438,26677,30329,40714
+50237,26678,30328,40865
+58297,26679,30327,40465
+52805,26680,30326,40647
+57285,26681,30325,40472
+56021,26682,30324,40205
+51067,26683,30323,40334
+55952,26684,30322,40001
+52903,26685,30321,40121
+54301,26686,30320,40194
+54412,26687,30319,40796
+52628,26688,30318,40286
+56340,26689,30317,40769
+57506,26690,30316,40815
+55653,26691,30315,40663
+51245,26692,30314,40546
+55456,26693,30313,40240
+54593,26694,30312,40260
+55202,26695,30311,40626
+56504,26696,30310,40768
+52253,26697,30309,40800
+55992,26698,30308,40233
+51697,26699,30307,40196
+59446,26700,30306,40305
+58036,26701,30305,40027
+58952,26702,30304,40436
+54161,26703,30303,40149
+56344,26704,30302,40068
+51356,26705,30301,40247
+52341,26706,30300,40745
+52484,26707,30299,40207
+51508,26708,30298,40845
+56049,26709,30297,40259
+55043,26710,30296,40905
+53724,26711,30295,40159
+59198,26712,30294,40919
+50030,26713,30293,40737
+58299,26714,30292,40382
+58977,26715,30291,40418
+57007,26716,30290,40859
+50610,26717,30289,40596
+58515,26718,30288,40787
+58584,26719,30287,40418
+55506,26720,30286,40385
+56223,26721,30285,40625
+52361,26722,30284,40693
+50157,26723,30283,40577
+55841,26724,30282,40612
+51525,26725,30281,40671
+54873,26726,30280,40662
+56314,26727,30279,40403
+52331,26728,30278,40440
+51275,26729,30277,40760
+55649,26730,30276,40270
+50269,26731,30275,40021
+50993,26732,30274,40021
+57798,26733,30273,40008
+53540,26734,30272,40946
+59013,26735,30271,40338
+59796,26736,30270,40195
+54368,26737,30269,40651
+59955,26738,30268,40080
+51893,26739,30267,40146
+57945,26740,30266,40188
+54658,26741,30265,40400
+53637,26742,30264,40926
+57344,26743,30263,40922
+54308,26744,30262,40391
+56729,26745,30261,40170
+52944,26746,30260,40559
+52602,26747,30259,40833
+52351,26748,30258,40929
+53076,26749,30257,40891
+59197,26750,30256,40807
+52221,26751,30255,40286
+57572,26752,30254,40803
+55283,26753,30253,40368
+51927,26754,30252,40540
+59713,26755,30251,40765
+52446,26756,30250,40489
+52458,26757,30249,40962
+51516,26758,30248,40765
+59377,26759,30247,40753
+53276,26760,30246,40183
+59163,26761,30245,40417
+57354,26762,30244,40588
+59885,26763,30243,40904
+50686,26764,30242,40043
+56319,26765,30241,40264
+59010,26766,30240,40405
+57410,26767,30239,40556
+55909,26768,30238,40296
+50589,26769,30237,40926
+57625,26770,30236,40311
+55443,26771,30235,40782
+54550,26772,30234,40023
+55217,26773,30233,40417
+57873,26774,30232,40559
+55395,26775,30231,40131
+59896,26776,30230,40781
+59014,26777,30229,40690
+51716,26778,30228,40617
+54104,26779,30227,40281
+58257,26780,30226,40217
+56556,26781,30225,40541
+59241,26782,30224,40121
+53593,26783,30223,40952
+56268,26784,30222,40660
+52802,26785,30221,40383
+57644,26786,30220,40731
+56752,26787,30219,40026
+50069,26788,30218,40481
+55724,26789,30217,40936
+58810,26790,30216,40801
+53129,26791,30215,40189
+52298,26792,30214,40924
+59037,26793,30213,40857
+53186,26794,30212,40987
+51648,26795,30211,40253
+54076,26796,30210,40085
+55550,26797,30209,40644
+50340,26798,30208,40301
+56471,26799,30207,40620
+54075,26800,30206,40276
+58629,26801,30205,40961
+59653,26802,30204,40510
+54493,26803,30203,40484
+50109,26804,30202,40323
+59150,26805,30201,40221
+55145,26806,30200,40525
+57239,26807,30199,40554
+57512,26808,30198,40329
+51701,26809,30197,40666
+59799,26810,30196,40116
+57130,26811,30195,40378
+50995,26812,30194,40883
+52409,26813,30193,40187
+59880,26814,30192,40655
+58526,26815,30191,40703
+56632,26816,30190,40347
+52267,26817,30189,40230
+56680,26818,30188,40979
+57772,26819,30187,40065
+50484,26820,30186,40347
+58524,26821,30185,40066
+57635,26822,30184,40425
+58193,26823,30183,40030
+57653,26824,30182,40224
+51010,26825,30181,40896
+57705,26826,30180,40946
+55976,26827,30179,40276
+52074,26828,30178,40345
+52546,26829,30177,40116
+57145,26830,30176,40194
+50387,26831,30175,40026
+58455,26832,30174,40806
+53095,26833,30173,40155
+55913,26834,30172,40406
+59395,26835,30171,40996
+55902,26836,30170,40243
+58297,26837,30169,40020
+50860,26838,30168,40480
+59803,26839,30167,40182
+56358,26840,30166,40900
+59895,26841,30165,40585
+52650,26842,30164,40991
+51952,26843,30163,40541
+58054,26844,30162,40257
+53069,26845,30161,40773
+51899,26846,30160,40057
+58107,26847,30159,40211
+58714,26848,30158,40105
+57150,26849,30157,40780
+52923,26850,30156,40428
+55380,26851,30155,40250
+57117,26852,30154,40792
+57834,26853,30153,40360
+52455,26854,30152,40086
+57113,26855,30151,40349
+59853,26856,30150,40842
+53800,26857,30149,40736
+58495,26858,30148,40361
+50485,26859,30147,40092
+53163,26860,30146,40511
+50893,26861,30145,40656
+54818,26862,30144,40891
+59944,26863,30143,40822
+57063,26864,30142,40535
+52995,26865,30141,40597
+50868,26866,30140,40463
+58629,26867,30139,40774
+50039,26868,30138,40397
+54067,26869,30137,40856
+53722,26870,30136,40183
+51183,26871,30135,40864
+54752,26872,30134,40942
+57946,26873,30133,40456
+55515,26874,30132,40923
+54346,26875,30131,40026
+57563,26876,30130,40760
+55271,26877,30129,40066
+56689,26878,30128,40745
+51776,26879,30127,40686
+52241,26880,30126,40325
+50299,26881,30125,40730
+57814,26882,30124,40197
+50185,26883,30123,40172
+50495,26884,30122,40847
+56775,26885,30121,40993
+57463,26886,30120,40583
+50577,26887,30119,40402
+58594,26888,30118,40331
+51857,26889,30117,40934
+50196,26890,30116,40786
+57845,26891,30115,40165
+59084,26892,30114,40264
+57631,26893,30113,40399
+54685,26894,30112,40536
+51061,26895,30111,40799
+57437,26896,30110,40023
+57529,26897,30109,40391
+53638,26898,30108,40464
+59107,26899,30107,40121
+50137,26900,30106,40474
+53850,26901,30105,40087
+54499,26902,30104,40874
+54211,26903,30103,40423
+52520,26904,30102,40190
+50144,26905,30101,40543
+58460,26906,30100,40096
+57017,26907,30099,40401
+59569,26908,30098,40051
+55959,26909,30097,40203
+51050,26910,30096,40973
+58621,26911,30095,40397
+53021,26912,30094,40656
+57762,26913,30093,40208
+52215,26914,30092,40950
+58313,26915,30091,40120
+58010,26916,30090,40593
+54722,26917,30089,40909
+59986,26918,30088,40077
+53737,26919,30087,40212
+58709,26920,30086,40243
+58795,26921,30085,40133
+51336,26922,30084,40930
+56441,26923,30083,40945
+56174,26924,30082,40804
+50022,26925,30081,40113
+55211,26926,30080,40554
+58842,26927,30079,40134
+59431,26928,30078,40477
+56313,26929,30077,40712
+58847,26930,30076,40603
+53486,26931,30075,40813
+57717,26932,30074,40585
+54704,26933,30073,40809
+53127,26934,30072,40619
+59991,26935,30071,40708
+56989,26936,30070,40917
+53256,26937,30069,40293
+56764,26938,30068,40853
+50299,26939,30067,40113
+55399,26940,30066,40824
+50174,26941,30065,40327
+51958,26942,30064,40117
+53234,26943,30063,40603
+54031,26944,30062,40482
+55539,26945,30061,40813
+56368,26946,30060,40025
+59717,26947,30059,40277
+50254,26948,30058,40958
+50090,26949,30057,40334
+57511,26950,30056,40248
+53441,26951,30055,40068
+58100,26952,30054,40081
+53640,26953,30053,40963
+57483,26954,30052,40732
+57346,26955,30051,40173
+53825,26956,30050,40826
+58899,26957,30049,40472
+53981,26958,30048,40321
+50648,26959,30047,40866
+58935,26960,30046,40514
+53347,26961,30045,40017
+53919,26962,30044,40941
+56756,26963,30043,40356
+59866,26964,30042,40051
+57940,26965,30041,40055
+51190,26966,30040,40856
+51531,26967,30039,40183
+59903,26968,30038,40571
+58225,26969,30037,40659
+57771,26970,30036,40875
+53650,26971,30035,40378
+54415,26972,30034,40184
+56890,26973,30033,40551
+58597,26974,30032,40570
+52479,26975,30031,40598
+50933,26976,30030,40202
+55230,26977,30029,40050
+58048,26978,30028,40116
+58334,26979,30027,40020
+57195,26980,30026,40669
+51201,26981,30025,40194
+52920,26982,30024,40999
+51785,26983,30023,40282
+58225,26984,30022,40334
+53255,26985,30021,40945
+53550,26986,30020,40790
+57614,26987,30019,40004
+53046,26988,30018,40579
+59958,26989,30017,40994
+59777,26990,30016,40285
+59261,26991,30015,40012
+57084,26992,30014,40070
+55416,26993,30013,40810
+56342,26994,30012,40681
+51863,26995,30011,40724
+50999,26996,30010,40224
+51926,26997,30009,40497
+55313,26998,30008,40922
+51807,26999,30007,40967
+59838,27000,30006,40679
+56301,27001,30005,40680
+54057,27002,30004,40305
+55340,27003,30003,40630
+59785,27004,30002,40106
+54140,27005,30001,40827
+55303,27006,30000,40196
+54933,27007,31000,40167
+50400,27008,30999,40185
+59780,27009,30998,40134
+53526,27010,30997,40231
+50451,27011,30996,40181
+58098,27012,30995,40175
+50070,27013,30994,40865
+59684,27014,30993,40739
+50578,27015,30992,40861
+58315,27016,30991,40362
+59592,27017,30990,40758
+51471,27018,30989,40201
+51919,27019,30988,40076
+53099,27020,30987,40311
+55280,27021,30986,40847
+50041,27022,30985,40053
+58731,27023,30984,40403
+57686,27024,30983,40280
+50582,27025,30982,40138
+51844,27026,30981,40500
+55286,27027,30980,40853
+52702,27028,30979,40151
+57558,27029,30978,40823
+57814,27030,30977,40016
+59220,27031,30976,40029
+54587,27032,30975,40074
+54315,27033,30974,40150
+53862,27034,30973,40625
+51307,27035,30972,40579
+51523,27036,30971,40095
+53157,27037,30970,40378
+51772,27038,30969,40347
+53122,27039,30968,40405
+57457,27040,30967,40871
+58372,27041,30966,40418
+56612,27042,30965,40204
+59396,27043,30964,40279
+54113,27044,30963,40772
+59689,27045,30962,40013
+56219,27046,30961,40148
+58602,27047,30960,40797
+56866,27048,30959,40252
+53003,27049,30958,40427
+57040,27050,30957,40764
+52132,27051,30956,40918
+51890,27052,30955,40149
+57392,27053,30954,40264
+54772,27054,30953,40394
+53824,27055,30952,40497
+55429,27056,30951,40977
+50303,27057,30950,40186
+58890,27058,30949,40297
+59073,27059,30948,40152
+56366,27060,30947,40929
+55588,27061,30946,40856
+50167,27062,30945,40603
+59241,27063,30944,40217
+58916,27064,30943,40619
+59116,27065,30942,40314
+56082,27066,30941,40024
+54782,27067,30940,40841
+55038,27068,30939,40822
+51540,27069,30938,40805
+58773,27070,30937,40198
+53032,27071,30936,40105
+54679,27072,30935,40603
+50221,27073,30934,40558
+56841,27074,30933,40521
+55729,27075,30932,40259
+59082,27076,30931,40699
+59515,27077,30930,40156
+55614,27078,30929,40801
+53315,27079,30928,40564
+52150,27080,30927,40125
+56721,27081,30926,40768
+56704,27082,30925,40456
+53485,27083,30924,40089
+51722,27084,30923,40134
+54374,27085,30922,40925
+51218,27086,30921,40851
+54046,27087,30920,40627
+58777,27088,30919,40632
+53266,27089,30918,40482
+54215,27090,30917,40294
+57304,27091,30916,40925
+53912,27092,30915,40134
+52161,27093,30914,40756
+50883,27094,30913,40962
+59933,27095,30912,40330
+54315,27096,30911,40964
+55486,27097,30910,40255
+50766,27098,30909,40217
+57909,27099,30908,40417
+55994,27100,30907,40553
+54761,27101,30906,40636
+52880,27102,30905,40589
+55334,27103,30904,40643
+56897,27104,30903,40909
+58593,27105,30902,40761
+53990,27106,30901,40310
+52810,27107,30900,40489
+56652,27108,30899,40797
+52810,27109,30898,40094
+57211,27110,30897,40299
+58988,27111,30896,40816
+53767,27112,30895,40460
+51176,27113,30894,40340
+56223,27114,30893,40405
+58810,27115,30892,40462
+50563,27116,30891,40810
+55745,27117,30890,40438
+50547,27118,30889,40217
+54100,27119,30888,40588
+58243,27120,30887,40790
+51215,27121,30886,40777
+57258,27122,30885,40364
+57243,27123,30884,40651
+53194,27124,30883,40997
+56075,27125,30882,40373
+57146,27126,30881,40654
+55426,27127,30880,40105
+52222,27128,30879,40682
+58755,27129,30878,40579
+56744,27130,30877,40690
+56848,27131,30876,40603
+56088,27132,30875,40365
+53860,27133,30874,40550
+53071,27134,30873,40538
+54012,27135,30872,40186
+56689,27136,30871,40047
+51951,27137,30870,40596
+55385,27138,30869,40292
+56111,27139,30868,40080
+50198,27140,30867,40073
+57608,27141,30866,40300
+56403,27142,30865,40065
+58469,27143,30864,40178
+53093,27144,30863,40746
+56530,27145,30862,40072
+58349,27146,30861,40778
+52747,27147,30860,40189
+59629,27148,30859,40784
+53988,27149,30858,40898
+51656,27150,30857,40619
+50546,27151,30856,40522
+51921,27152,30855,40779
+58262,27153,30854,40431
+53877,27154,30853,40554
+56682,27155,30852,40531
+57845,27156,30851,40367
+50740,27157,30850,40969
+58172,27158,30849,40247
+54071,27159,30848,40630
+50063,27160,30847,40320
+59129,27161,30846,40270
+54898,27162,30845,40339
+56032,27163,30844,40887
+54327,27164,30843,40645
+54181,27165,30842,40164
+50929,27166,30841,40547
+51306,27167,30840,40575
+53607,27168,30839,40919
+52375,27169,30838,40599
+54947,27170,30837,40051
+50608,27171,30836,40417
+55418,27172,30835,40481
+51283,27173,30834,40054
+54391,27174,30833,40412
+52206,27175,30832,40769
+51248,27176,30831,40519
+54349,27177,30830,40443
+56954,27178,30829,40400
+59697,27179,30828,40805
+52825,27180,30827,40609
+51308,27181,30826,40953
+50479,27182,30825,40171
+54088,27183,30824,40214
+50181,27184,30823,40614
+55703,27185,30822,40710
+54487,27186,30821,40050
+56917,27187,30820,40349
+56266,27188,30819,40115
+58666,27189,30818,40992
+51369,27190,30817,40309
+59277,27191,30816,40856
+54541,27192,30815,40596
+55186,27193,30814,40450
+58555,27194,30813,40493
+57923,27195,30812,40388
+53687,27196,30811,40535
+59484,27197,30810,40077
+52710,27198,30809,40944
+55328,27199,30808,40186
+50673,27200,30807,40093
+51370,27201,30806,40313
+55029,27202,30805,40402
+50974,27203,30804,40836
+56535,27204,30803,40714
+50637,27205,30802,40123
+57910,27206,30801,40052
+57239,27207,30800,40350
+50425,27208,30799,40778
+59259,27209,30798,40174
+52887,27210,30797,40759
+56202,27211,30796,40431
+59797,27212,30795,40700
+51098,27213,30794,40133
+53865,27214,30793,40639
+52646,27215,30792,40511
+56670,27216,30791,40352
+55043,27217,30790,40988
+56769,27218,30789,40409
+59213,27219,30788,40488
+59639,27220,30787,40514
+55247,27221,30786,40979
+52113,27222,30785,40311
+52194,27223,30784,40023
+58552,27224,30783,40326
+52733,27225,30782,40157
+53043,27226,30781,40368
+53562,27227,30780,40930
+50568,27228,30779,40604
+51562,27229,30778,40131
+54078,27230,30777,40722
+57774,27231,30776,40338
+53552,27232,30775,40139
+56454,27233,30774,40371
+50926,27234,30773,40434
+50791,27235,30772,40088
+56254,27236,30771,40519
+50997,27237,30770,40076
+55408,27238,30769,40392
+56289,27239,30768,40840
+50030,27240,30767,40668
+59611,27241,30766,40641
+52626,27242,30765,40390
+50728,27243,30764,40586
+56258,27244,30763,40746
+59358,27245,30762,40787
+59767,27246,30761,40159
+57229,27247,30760,40846
+59292,27248,30759,40649
+55576,27249,30758,40337
+53873,27250,30757,40354
+51361,27251,30756,40917
+50990,27252,30755,40468
+56018,27253,30754,40554
+52549,27254,30753,40741
+50727,27255,30752,40109
+59008,27256,30751,40456
+57418,27257,30750,40231
+57914,27258,30749,40380
+54150,27259,30748,40603
+56922,27260,30747,40771
+59821,27261,30746,40331
+57641,27262,30745,40753
+57013,27263,30744,40152
+56981,27264,30743,40223
+56909,27265,30742,40412
+50894,27266,30741,40515
+55517,27267,30740,40197
+52845,27268,30739,40846
+58123,27269,30738,40673
+59541,27270,30737,40577
+54922,27271,30736,40585
+50785,27272,30735,40717
+59197,27273,30734,40091
+54205,27274,30733,40470
+59626,27275,30732,40328
+53457,27276,30731,40507
+52441,27277,30730,40134
+53880,27278,30729,40788
+54123,27279,30728,40075
+51279,27280,30727,40400
+50056,27281,30726,40799
+51656,27282,30725,40938
+54208,27283,30724,40788
+56782,27284,30723,40768
+56723,27285,30722,40101
+54131,27286,30721,40592
+59453,27287,30720,40720
+50166,27288,30719,40799
+52713,27289,30718,40921
+50208,27290,30717,40016
+58006,27291,30716,40709
+57885,27292,30715,40797
+55431,27293,30714,40584
+54919,27294,30713,40396
+59379,27295,30712,40907
+55886,27296,30711,40625
+59485,27297,30710,40285
+53429,27298,30709,40804
+58927,27299,30708,40693
+50038,27300,30707,40503
+59135,27301,30706,40465
+55045,27302,30705,40847
+50580,27303,30704,40638
+58097,27304,30703,40876
+59745,27305,30702,40477
+58503,27306,30701,40966
+59679,27307,30700,40182
+52081,27308,30699,40342
+53946,27309,30698,40836
+53420,27310,30697,40686
+58002,27311,30696,40229
+51287,27312,30695,40422
+50449,27313,30694,40844
+52359,27314,30693,40052
+58156,27315,30692,40540
+53207,27316,30691,40151
+54291,27317,30690,40995
+52845,27318,30689,40536
+55042,27319,30688,40072
+58680,27320,30687,40310
+58810,27321,30686,40699
+50261,27322,30685,40621
+58442,27323,30684,40856
+55729,27324,30683,40385
+52726,27325,30682,40736
+56649,27326,30681,40837
+52330,27327,30680,40836
+58109,27328,30679,40754
+59624,27329,30678,40438
+59923,27330,30677,40270
+56957,27331,30676,40046
+55111,27332,30675,40379
+52594,27333,30674,40661
+55888,27334,30673,40865
+57101,27335,30672,40970
+55278,27336,30671,40346
+59864,27337,30670,40903
+51584,27338,30669,40463
+53193,27339,30668,40111
+57202,27340,30667,40244
+57937,27341,30666,40222
+52883,27342,30665,40196
+54177,27343,30664,40773
+59812,27344,30663,40562
+50965,27345,30662,40768
+52716,27346,30661,40878
+52452,27347,30660,40326
+59737,27348,30659,40025
+54128,27349,30658,40340
+54386,27350,30657,40941
+50712,27351,30656,40063
+58760,27352,30655,40475
+56077,27353,30654,40876
+53146,27354,30653,40146
+58082,27355,30652,40325
+54727,27356,30651,40553
+58828,27357,30650,40539
+56573,27358,30649,40625
+56617,27359,30648,40363
+50049,27360,30647,40186
+52637,27361,30646,40406
+52008,27362,30645,40601
+55159,27363,30644,40672
+59666,27364,30643,40210
+57602,27365,30642,40197
+54022,27366,30641,40496
+57335,27367,30640,40163
+56856,27368,30639,40198
+56656,27369,30638,40280
+51464,27370,30637,40433
+57768,27371,30636,40321
+54004,27372,30635,40132
+57662,27373,30634,40875
+53293,27374,30633,40207
+56231,27375,30632,40642
+56647,27376,30631,40965
+51043,27377,30630,40332
+58521,27378,30629,40809
+53512,27379,30628,40289
+57109,27380,30627,40461
+50219,27381,30626,40786
+59442,27382,30625,40982
+53470,27383,30624,40658
+56340,27384,30623,40983
+54019,27385,30622,40005
+52653,27386,30621,40408
+53022,27387,30620,40684
+55165,27388,30619,40146
+55928,27389,30618,40226
+51685,27390,30617,40786
+50320,27391,30616,40959
+52154,27392,30615,40968
+52156,27393,30614,40566
+52388,27394,30613,40932
+55181,27395,30612,40844
+54950,27396,30611,40148
+55712,27397,30610,40368
+57416,27398,30609,40307
+51249,27399,30608,40146
+56447,27400,30607,40265
+50139,27401,30606,40249
+50383,27402,30605,40883
+55914,27403,30604,40052
+53607,27404,30603,40711
+52353,27405,30602,40140
+57898,27406,30601,40703
+58214,27407,30600,40213
+55470,27408,30599,40655
+58315,27409,30598,40088
+53741,27410,30597,40084
+55097,27411,30596,40814
+56345,27412,30595,40932
+56005,27413,30594,40139
+53405,27414,30593,40372
+56740,27415,30592,40556
+57822,27416,30591,40185
+58315,27417,30590,40726
+51856,27418,30589,40070
+52622,27419,30588,40741
+52741,27420,30587,40095
+59082,27421,30586,40237
+53376,27422,30585,40296
+51087,27423,30584,40265
+52744,27424,30583,40670
+52197,27425,30582,40558
+50360,27426,30581,40527
+50399,27427,30580,40922
+50803,27428,30579,40832
+55501,27429,30578,40247
+54276,27430,30577,40875
+50491,27431,30576,40422
+59060,27432,30575,40235
+52087,27433,30574,40009
+51637,27434,30573,40214
+57600,27435,30572,40866
+58859,27436,30571,40293
+55066,27437,30570,40537
+58779,27438,30569,40595
+58686,27439,30568,40574
+54891,27440,30567,40864
+56253,27441,30566,40027
+55016,27442,30565,40585
+54805,27443,30564,40389
+50455,27444,30563,40885
+53597,27445,30562,40839
+55235,27446,30561,40998
+55786,27447,30560,40043
+59717,27448,30559,40826
+50660,27449,30558,40822
+52562,27450,30557,40022
+52787,27451,30556,40519
+56158,27452,30555,40158
+54915,27453,30554,40564
+51262,27454,30553,40195
+56454,27455,30552,40824
+58027,27456,30551,40505
+56616,27457,30550,40805
+50818,27458,30549,40148
+59758,27459,30548,40146
+51993,27460,30547,40191
+58056,27461,30546,40025
+52908,27462,30545,40112
+52790,27463,30544,40624
+51733,27464,30543,40771
+58630,27465,30542,40121
+55434,27466,30541,40225
+58043,27467,30540,40544
+53745,27468,30539,40533
+58601,27469,30538,40441
+58830,27470,30537,40508
+59325,27471,30536,40372
+50397,27472,30535,40902
+57066,27473,30534,40846
+51658,27474,30533,40237
+59726,27475,30532,40213
+59741,27476,30531,40657
+57427,27477,30530,40567
+54532,27478,30529,40477
+55809,27479,30528,40480
+50153,27480,30527,40794
+53168,27481,30526,40062
+58515,27482,30525,40125
+50283,27483,30524,40146
+52936,27484,30523,40184
+56641,27485,30522,40822
+52511,27486,30521,40946
+52502,27487,30520,40621
+51395,27488,30519,40731
+53814,27489,30518,40764
+57956,27490,30517,40145
+59689,27491,30516,40227
+56152,27492,30515,40448
+55265,27493,30514,40255
+55055,27494,30513,40745
+53744,27495,30512,40863
+51316,27496,30511,40490
+59159,27497,30510,40753
+58495,27498,30509,40910
+54777,27499,30508,40321
+54647,27500,30507,40024
+57871,27501,30506,40269
+53845,27502,30505,40489
+57972,27503,30504,40744
+58035,27504,30503,40020
+53872,27505,30502,40713
+50212,27506,30501,40043
+52994,27507,30500,40943
+51677,27508,30499,40520
+57047,27509,30498,40891
+57633,27510,30497,40673
+54599,27511,30496,40042
+53050,27512,30495,40728
+54494,27513,30494,40726
+55859,27514,30493,40534
+57848,27515,30492,40304
+52483,27516,30491,40148
+54598,27517,30490,40012
+51139,27518,30489,40190
+53293,27519,30488,40048
+57969,27520,30487,40132
+51661,27521,30486,40730
+59741,27522,30485,40062
+59775,27523,30484,40078
+55584,27524,30483,40636
+58888,27525,30482,40073
+58397,27526,30481,40119
+54938,27527,30480,40812
+52336,27528,30479,40496
+55210,27529,30478,40360
+51731,27530,30477,40338
+53628,27531,30476,40603
+56991,27532,30475,40206
+50346,27533,30474,40741
+50911,27534,30473,40396
+59060,27535,30472,40562
+51839,27536,30471,40576
+53665,27537,30470,40763
+51684,27538,30469,40248
+51139,27539,30468,40883
+56278,27540,30467,40825
+58953,27541,30466,40883
+55560,27542,30465,40073
+57054,27543,30464,40266
+53209,27544,30463,40133
+53358,27545,30462,40213
+53570,27546,30461,40559
+59385,27547,30460,40287
+51839,27548,30459,40235
+53088,27549,30458,40653
+59163,27550,30457,40110
+58525,27551,30456,40265
+59363,27552,30455,40588
+52271,27553,30454,40096
+56141,27554,30453,40093
+57442,27555,30452,40522
+51618,27556,30451,40427
+57021,27557,30450,40910
+50902,27558,30449,40202
+52965,27559,30448,40231
+50249,27560,30447,40476
+55791,27561,30446,40463
+56133,27562,30445,40112
+54899,27563,30444,40615
+50215,27564,30443,40486
+56604,27565,30442,40480
+57723,27566,30441,40706
+54955,27567,30440,40721
+57569,27568,30439,40947
+53068,27569,30438,40933
+51580,27570,30437,40959
+53408,27571,30436,40848
+56856,27572,30435,40117
+52153,27573,30434,40406
+50187,27574,30433,40782
+56004,27575,30432,40376
+53252,27576,30431,40216
+56497,27577,30430,40156
+54800,27578,30429,40513
+54190,27579,30428,40575
+57868,27580,30427,40996
+55441,27581,30426,40890
+57717,27582,30425,40067
+50324,27583,30424,40879
+56411,27584,30423,40738
+52267,27585,30422,40742
+52724,27586,30421,40668
+50849,27587,30420,40200
+52701,27588,30419,40580
+52288,27589,30418,40599
+56906,27590,30417,40209
+59360,27591,30416,40576
+51135,27592,30415,40231
+50790,27593,30414,40150
+57815,27594,30413,40886
+51166,27595,30412,40652
+55297,27596,30411,40273
+56244,27597,30410,40807
+58219,27598,30409,40974
+50795,27599,30408,40510
+56423,27600,30407,40469
+59472,27601,30406,40687
+58954,27602,30405,40374
+52825,27603,30404,40423
+56275,27604,30403,40155
+58167,27605,30402,40205
+59417,27606,30401,40071
+50315,27607,30400,40707
+55133,27608,30399,40530
+53061,27609,30398,40090
+50078,27610,30397,40051
+57623,27611,30396,40073
+52123,27612,30395,40313
+52680,27613,30394,40691
+59091,27614,30393,40437
+54484,27615,30392,40706
+51160,27616,30391,40431
+58561,27617,30390,40819
+59762,27618,30389,40035
+59052,27619,30388,40864
+51520,27620,30387,40192
+50130,27621,30386,40147
+55994,27622,30385,40733
+51451,27623,30384,40998
+56770,27624,30383,40480
+58128,27625,30382,40268
+58424,27626,30381,40349
+53405,27627,30380,40659
+53308,27628,30379,40327
+57554,27629,30378,40921
+58303,27630,30377,40054
+55891,27631,30376,40888
+55202,27632,30375,40053
+59567,27633,30374,40432
+50032,27634,30373,40630
+51806,27635,30372,40815
+50472,27636,30371,40557
+50180,27637,30370,40015
+50578,27638,30369,40198
+58482,27639,30368,40748
+52418,27640,30367,40931
+55339,27641,30366,40152
+53178,27642,30365,40190
+58916,27643,30364,40317
+50266,27644,30363,40671
+51349,27645,30362,40423
+53315,27646,30361,40475
+50028,27647,30360,40459
+54250,27648,30359,40210
+59505,27649,30358,40856
+54008,27650,30357,40619
+56723,27651,30356,40289
+58573,27652,30355,40670
+54225,27653,30354,40131
+51844,27654,30353,40301
+54118,27655,30352,40369
+58850,27656,30351,40568
+51691,27657,30350,40514
+57416,27658,30349,40108
+58848,27659,30348,40844
+59353,27660,30347,40263
+50662,27661,30346,40759
+50597,27662,30345,40194
+57683,27663,30344,40920
+53603,27664,30343,40827
+52296,27665,30342,40200
+50973,27666,30341,40214
+57743,27667,30340,40881
+58293,27668,30339,40640
+53102,27669,30338,40125
+52966,27670,30337,40480
+52201,27671,30336,40090
+51861,27672,30335,40395
+54036,27673,30334,40926
+53446,27674,30333,40122
+50965,27675,30332,40315
+52411,27676,30331,40009
+55768,27677,30330,40991
+54727,27678,30329,40334
+54972,27679,30328,40978
+58609,27680,30327,40124
+56955,27681,30326,40181
+53268,27682,30325,40828
+55191,27683,30324,40508
+53811,27684,30323,40863
+56607,27685,30322,40943
+59290,27686,30321,40922
+55169,27687,30320,40275
+59535,27688,30319,40207
+50313,27689,30318,40799
+55240,27690,30317,40252
+51640,27691,30316,40430
+56598,27692,30315,40674
+51728,27693,30314,40353
+56462,27694,30313,40979
+53083,27695,30312,40174
+57145,27696,30311,40817
+50972,27697,30310,40705
+52369,27698,30309,40928
+55981,27699,30308,40397
+55084,27700,30307,40839
+53607,27701,30306,40366
+55765,27702,30305,40592
+50848,27703,30304,40054
+52101,27704,30303,40857
+50815,27705,30302,40555
+58897,27706,30301,40758
+50907,27707,30300,40746
+58013,27708,30299,40109
+51329,27709,30298,40178
+51326,27710,30297,40990
+50420,27711,30296,40855
+53245,27712,30295,40965
+55727,27713,30294,40793
+57304,27714,30293,40885
+59926,27715,30292,40544
+54738,27716,30291,40747
+54091,27717,30290,40895
+54987,27718,30289,40172
+58924,27719,30288,40098
+56554,27720,30287,40484
+54825,27721,30286,40461
+59190,27722,30285,40468
+58867,27723,30284,40266
+53796,27724,30283,40567
+56160,27725,30282,40862
+53515,27726,30281,40353
+57899,27727,30280,40145
+53587,27728,30279,40349
+51705,27729,30278,40249
+50188,27730,30277,40177
+55334,27731,30276,40074
+54347,27732,30275,40480
+52014,27733,30274,40500
+56879,27734,30273,40812
+55928,27735,30272,40760
+53282,27736,30271,40149
+59052,27737,30270,40824
+56715,27738,30269,40779
+58900,27739,30268,40626
+59792,27740,30267,40324
+52225,27741,30266,40257
+52216,27742,30265,40842
+52741,27743,30264,40785
+52415,27744,30263,40028
+55597,27745,30262,40965
+52338,27746,30261,40519
+56114,27747,30260,40470
+53530,27748,30259,40841
+55493,27749,30258,40810
+54407,27750,30257,40914
+53415,27751,30256,40190
+52865,27752,30255,40952
+50619,27753,30254,40801
+50307,27754,30253,40965
+59395,27755,30252,40879
+52084,27756,30251,40255
+56391,27757,30250,40167
+53114,27758,30249,40036
+54596,27759,30248,40666
+57555,27760,30247,40023
+50992,27761,30246,40792
+57502,27762,30245,40060
+58300,27763,30244,40897
+56676,27764,30243,40259
+56851,27765,30242,40324
+57269,27766,30241,40492
+59263,27767,30240,40168
+56764,27768,30239,40312
+53611,27769,30238,40444
+56420,27770,30237,40888
+52418,27771,30236,40573
+58692,27772,30235,40797
+51974,27773,30234,40916
+55141,27774,30233,40763
+50301,27775,30232,40721
+52997,27776,30231,40586
+52582,27777,30230,40584
+54724,27778,30229,40210
+54313,27779,30228,40966
+53308,27780,30227,40314
+59069,27781,30226,40544
+56222,27782,30225,40509
+55795,27783,30224,40488
+53590,27784,30223,40811
+57471,27785,30222,40126
+50112,27786,30221,40823
+59015,27787,30220,40375
+53380,27788,30219,40674
+50563,27789,30218,40668
+55117,27790,30217,40259
+58948,27791,30216,40561
+52552,27792,30215,40944
+55970,27793,30214,40448
+52792,27794,30213,40534
+58526,27795,30212,40665
+59718,27796,30211,40610
+55862,27797,30210,40593
+59845,27798,30209,40490
+51031,27799,30208,40789
+52916,27800,30207,40762
+54364,27801,30206,40905
+50300,27802,30205,40742
+50500,27803,30204,40105
+51385,27804,30203,40657
+53922,27805,30202,40282
+59398,27806,30201,40576
+55710,27807,30200,40638
+57377,27808,30199,40583
+59113,27809,30198,40283
+59981,27810,30197,40461
+54205,27811,30196,40551
+57078,27812,30195,40630
+59795,27813,30194,40346
+57472,27814,30193,40117
+50414,27815,30192,40504
+52904,27816,30191,40300
+50277,27817,30190,40866
+54732,27818,30189,40146
+56510,27819,30188,40959
+59014,27820,30187,40391
+58486,27821,30186,40052
+53539,27822,30185,40879
+55037,27823,30184,40933
+55522,27824,30183,40752
+56526,27825,30182,40422
+51608,27826,30181,40626
+59068,27827,30180,40452
+50689,27828,30179,40995
+55095,27829,30178,40001
+51857,27830,30177,40748
+59774,27831,30176,40143
+55352,27832,30175,40075
+54143,27833,30174,40692
+58403,27834,30173,40029
+57978,27835,30172,40326
+54351,27836,30171,40712
+59939,27837,30170,40732
+57897,27838,30169,40282
+58022,27839,30168,40921
+54401,27840,30167,40410
+59845,27841,30166,40958
+55581,27842,30165,40521
+55725,27843,30164,40674
+58538,27844,30163,40828
+57559,27845,30162,40664
+55181,27846,30161,40969
+57002,27847,30160,40118
+53735,27848,30159,40580
+57053,27849,30158,40325
+50759,27850,30157,40125
+50634,27851,30156,40962
+50889,27852,30155,40455
+59439,27853,30154,40568
+57724,27854,30153,40527
+55614,27855,30152,40183
+59746,27856,30151,40087
+57380,27857,30150,40877
+56704,27858,30149,40478
+59926,27859,30148,40727
+54517,27860,30147,40694
+55383,27861,30146,40221
+54792,27862,30145,40610
+59558,27863,30144,40647
+59610,27864,30143,40682
+56788,27865,30142,40504
+51738,27866,30141,40980
+51556,27867,30140,40054
+53207,27868,30139,40929
+58155,27869,30138,40442
+55552,27870,30137,40601
+52215,27871,30136,40362
+59495,27872,30135,40595
+55548,27873,30134,40720
+50869,27874,30133,40070
+54633,27875,30132,40359
+53783,27876,30131,40885
+55450,27877,30130,40274
+53349,27878,30129,40610
+56918,27879,30128,40337
+52835,27880,30127,40890
+51496,27881,30126,40336
+51201,27882,30125,40776
+58939,27883,30124,40942
+50045,27884,30123,40225
+58062,27885,30122,40023
+56184,27886,30121,40492
+57975,27887,30120,40638
+55814,27888,30119,40500
+58371,27889,30118,40507
+56111,27890,30117,40027
+51772,27891,30116,40941
+59122,27892,30115,40684
+52308,27893,30114,40716
+51290,27894,30113,40581
+56270,27895,30112,40949
+50152,27896,30111,40072
+56924,27897,30110,40403
+58143,27898,30109,40868
+54098,27899,30108,40374
+54015,27900,30107,40845
+55369,27901,30106,40259
+59514,27902,30105,40484
+59568,27903,30104,40256
+50274,27904,30103,40996
+51549,27905,30102,40242
+55315,27906,30101,40283
+53586,27907,30100,40877
+56637,27908,30099,40280
+53418,27909,30098,40615
+53689,27910,30097,40583
+51266,27911,30096,40673
+57708,27912,30095,40551
+51860,27913,30094,40939
+54106,27914,30093,40477
+52927,27915,30092,40123
+56946,27916,30091,40694
+52227,27917,30090,40882
+51352,27918,30089,40750
+55208,27919,30088,40987
+51760,27920,30087,40688
+51070,27921,30086,40521
+52261,27922,30085,40232
+59774,27923,30084,40468
+53497,27924,30083,40245
+57216,27925,30082,40951
+58631,27926,30081,40701
+55782,27927,30080,40233
+54516,27928,30079,40068
+59829,27929,30078,40557
+57316,27930,30077,40341
+50644,27931,30076,40897
+52681,27932,30075,40571
+51069,27933,30074,40840
+55734,27934,30073,40724
+52609,27935,30072,40580
+52946,27936,30071,40067
+51688,27937,30070,40180
+59320,27938,30069,40528
+52358,27939,30068,40573
+57596,27940,30067,40830
+56505,27941,30066,40392
+50791,27942,30065,40554
+50862,27943,30064,40376
+58099,27944,30063,40707
+52643,27945,30062,40750
+53858,27946,30061,40873
+53175,27947,30060,40739
+55512,27948,30059,40254
+53454,27949,30058,40681
+58890,27950,30057,40372
+54833,27951,30056,40326
+53150,27952,30055,40874
+55474,27953,30054,40990
+54722,27954,30053,40575
+57389,27955,30052,40012
+55752,27956,30051,40017
+51833,27957,30050,40381
+59359,27958,30049,40593
+57486,27959,30048,40434
+52957,27960,30047,40503
+55745,27961,30046,40112
+58504,27962,30045,40094
+51819,27963,30044,40837
+52548,27964,30043,40436
+51672,27965,30042,40048
+57382,27966,30041,40049
+57322,27967,30040,40670
+56104,27968,30039,40825
+56702,27969,30038,40101
+57941,27970,30037,40311
+55206,27971,30036,40598
+58276,27972,30035,40431
+57622,27973,30034,40380
+51041,27974,30033,40520
+52522,27975,30032,40980
+51094,27976,30031,40797
+58932,27977,30030,40816
+50810,27978,30029,40895
+51844,27979,30028,40579
+51584,27980,30027,40025
+59311,27981,30026,40707
+55216,27982,30025,40791
+58226,27983,30024,40230
+51394,27984,30023,40627
+51152,27985,30022,40115
+51958,27986,30021,40109
+55655,27987,30020,40143
+53025,27988,30019,40023
+57329,27989,30018,40544
+52002,27990,30017,40018
+59686,27991,30016,40073
+53603,27992,30015,40025
+52511,27993,30014,40506
+52763,27994,30013,40088
+58021,27995,30012,40499
+50208,27996,30011,40408
+51896,27997,30010,40861
+54169,27998,30009,40258
+56268,27999,30008,40869
+59890,28000,30007,40885
+54589,28001,30006,40326
+54457,28002,30005,40198
+55619,28003,30004,40321
+59477,28004,30003,40307
+51248,28005,30002,40612
+55014,28006,30001,40300
+59911,28007,30000,40309
+56199,28008,31000,40070
+51671,28009,30999,40625
+53456,28010,30998,40541
+55028,28011,30997,40222
+56888,28012,30996,40262
+55655,28013,30995,40945
+57688,28014,30994,40510
+56491,28015,30993,40276
+54715,28016,30992,40066
+50233,28017,30991,40640
+54129,28018,30990,40493
+56926,28019,30989,40755
+57699,28020,30988,40959
+51703,28021,30987,40405
+53521,28022,30986,40632
+55903,28023,30985,40941
+52982,28024,30984,40537
+50751,28025,30983,40875
+50717,28026,30982,40394
+51012,28027,30981,40370
+58573,28028,30980,40966
+55295,28029,30979,40884
+54352,28030,30978,40976
+52613,28031,30977,40033
+50673,28032,30976,40768
+57175,28033,30975,40438
+54036,28034,30974,40125
+57388,28035,30973,40886
+52863,28036,30972,40337
+57097,28037,30971,40656
+56454,28038,30970,40084
+50707,28039,30969,40612
+50501,28040,30968,40933
+54780,28041,30967,40619
+58950,28042,30966,40716
+57097,28043,30965,40698
+55810,28044,30964,40805
+54500,28045,30963,40114
+56660,28046,30962,40306
+59949,28047,30961,40824
+56521,28048,30960,40745
+56843,28049,30959,40996
+58436,28050,30958,40212
+55985,28051,30957,40136
+51838,28052,30956,40747
+58907,28053,30955,40799
+58202,28054,30954,40770
+59383,28055,30953,40111
+57463,28056,30952,40184
+53745,28057,30951,40140
+55694,28058,30950,40478
+57223,28059,30949,40539
+52619,28060,30948,40591
+52859,28061,30947,40552
+55881,28062,30946,40130
+54135,28063,30945,40596
+59334,28064,30944,40945
+55002,28065,30943,40838
+50070,28066,30942,40670
+55649,28067,30941,40797
+50851,28068,30940,40185
+59281,28069,30939,40567
+52849,28070,30938,40040
+51168,28071,30937,40737
+54624,28072,30936,40052
+52243,28073,30935,40724
+55338,28074,30934,40435
+58514,28075,30933,40896
+51997,28076,30932,40255
+50792,28077,30931,40904
+54787,28078,30930,40579
+57591,28079,30929,40529
+56316,28080,30928,40141
+50040,28081,30927,40544
+57389,28082,30926,40733
+53839,28083,30925,40311
+55234,28084,30924,40288
+52525,28085,30923,40484
+54639,28086,30922,40737
+55696,28087,30921,40743
+55732,28088,30920,40429
+57269,28089,30919,40525
+59663,28090,30918,40273
+59278,28091,30917,40669
+55227,28092,30916,40923
+54558,28093,30915,40463
+57525,28094,30914,40177
+50016,28095,30913,40708
+53168,28096,30912,40377
+53016,28097,30911,40190
+51423,28098,30910,40711
+54369,28099,30909,40244
+54703,28100,30908,40340
+58150,28101,30907,40402
+57991,28102,30906,40937
+54172,28103,30905,40979
+52424,28104,30904,40704
+55164,28105,30903,40508
+58366,28106,30902,40299
+51055,28107,30901,40759
+56009,28108,30900,40356
+58450,28109,30899,40536
+57253,28110,30898,40800
+55688,28111,30897,40896
+59892,28112,30896,40665
+56831,28113,30895,40891
+50574,28114,30894,40902
+59722,28115,30893,40872
+55311,28116,30892,40497
+57662,28117,30891,40405
+54687,28118,30890,40061
+57305,28119,30889,40972
+59231,28120,30888,40641
+54812,28121,30887,40990
+54542,28122,30886,40350
+55721,28123,30885,40834
+51099,28124,30884,40073
+58823,28125,30883,40585
+51845,28126,30882,40293
+59378,28127,30881,40074
+55998,28128,30880,40088
+58096,28129,30879,40797
+52479,28130,30878,40584
+51919,28131,30877,40926
+53413,28132,30876,40122
+57641,28133,30875,40344
+51673,28134,30874,40327
+57369,28135,30873,40548
+59709,28136,30872,40820
+59864,28137,30871,40781
+50129,28138,30870,40764
+51104,28139,30869,40864
+52934,28140,30868,40644
+56258,28141,30867,40365
+55610,28142,30866,40154
+58130,28143,30865,40908
+50306,28144,30864,40267
+53408,28145,30863,40311
+55558,28146,30862,40976
+54818,28147,30861,40906
+53349,28148,30860,40580
+59148,28149,30859,40041
+54026,28150,30858,40309
+57042,28151,30857,40247
+52806,28152,30856,40164
+56532,28153,30855,40406
+53613,28154,30854,40107
+54360,28155,30853,40607
+53485,28156,30852,40202
+54022,28157,30851,40943
+57192,28158,30850,40589
+58956,28159,30849,40565
+56498,28160,30848,40470
+53042,28161,30847,40468
+57622,28162,30846,40922
+53080,28163,30845,40545
+51860,28164,30844,40251
+58939,28165,30843,40635
+54069,28166,30842,40346
+52230,28167,30841,40881
+58463,28168,30840,40862
+51221,28169,30839,40508
+50403,28170,30838,40493
+59397,28171,30837,40035
+53620,28172,30836,40598
+57496,28173,30835,40373
+57122,28174,30834,40921
+57796,28175,30833,40871
+57060,28176,30832,40732
+54459,28177,30831,40121
+58773,28178,30830,40146
+55269,28179,30829,40543
+58917,28180,30828,40890
+54313,28181,30827,40377
+55648,28182,30826,40410
+50859,28183,30825,40279
+56155,28184,30824,40429
+58191,28185,30823,40573
+51002,28186,30822,40352
+56523,28187,30821,40247
+59103,28188,30820,40005
+55307,28189,30819,40937
+55214,28190,30818,40074
+51694,28191,30817,40638
+51884,28192,30816,40386
+57694,28193,30815,40883
+59461,28194,30814,40277
+53741,28195,30813,40046
+56108,28196,30812,40974
+53291,28197,30811,40632
+54557,28198,30810,40721
+51187,28199,30809,40885
+56446,28200,30808,40575
+52597,28201,30807,40356
+50773,28202,30806,40103
+50735,28203,30805,40503
+53171,28204,30804,40171
+56836,28205,30803,40593
+55838,28206,30802,40420
+57774,28207,30801,40046
+54444,28208,30800,40651
+51632,28209,30799,40100
+56491,28210,30798,40093
+55844,28211,30797,40812
+57917,28212,30796,40386
+51252,28213,30795,40133
+54129,28214,30794,40107
+57716,28215,30793,40021
+57117,28216,30792,40389
+51804,28217,30791,40220
+51271,28218,30790,40142
+58579,28219,30789,40792
+56563,28220,30788,40875
+59670,28221,30787,40447
+52919,28222,30786,40988
+51657,28223,30785,40660
+51706,28224,30784,40004
+50148,28225,30783,40761
+52426,28226,30782,40192
+50143,28227,30781,40053
+55965,28228,30780,40572
+56552,28229,30779,40570
+59680,28230,30778,40340
+53105,28231,30777,40181
+54187,28232,30776,40102
+59239,28233,30775,40779
+53189,28234,30774,40338
+50012,28235,30773,40349
+58595,28236,30772,40522
+58022,28237,30771,40080
+54700,28238,30770,40684
+54561,28239,30769,40084
+51593,28240,30768,40593
+50207,28241,30767,40334
+58344,28242,30766,40985
+58416,28243,30765,40667
+58847,28244,30764,40309
+56124,28245,30763,40556
+51757,28246,30762,40049
+54973,28247,30761,40329
+57644,28248,30760,40212
+51930,28249,30759,40491
+58019,28250,30758,40829
+57070,28251,30757,40771
+50408,28252,30756,40156
+58264,28253,30755,40792
+50115,28254,30754,40290
+53499,28255,30753,40968
+56077,28256,30752,40021
+57941,28257,30751,40519
+50935,28258,30750,40596
+53628,28259,30749,40869
+54935,28260,30748,40492
+50180,28261,30747,40327
+50141,28262,30746,40549
+55886,28263,30745,40468
+52757,28264,30744,40316
+53547,28265,30743,40330
+59573,28266,30742,40397
+54702,28267,30741,40780
+51829,28268,30740,40204
+55423,28269,30739,40507
+58446,28270,30738,40521
+57451,28271,30737,40887
+55355,28272,30736,40014
+52089,28273,30735,40639
+53772,28274,30734,40853
+57448,28275,30733,40020
+56314,28276,30732,40819
+55011,28277,30731,40535
+52869,28278,30730,40068
+54695,28279,30729,40170
+57160,28280,30728,40889
+54507,28281,30727,40530
+59758,28282,30726,40021
+50302,28283,30725,40887
+59598,28284,30724,40831
+59631,28285,30723,40549
+54242,28286,30722,40580
+57445,28287,30721,40906
+58582,28288,30720,40669
+52847,28289,30719,40668
+53051,28290,30718,40345
+50774,28291,30717,40742
+51899,28292,30716,40791
+52190,28293,30715,40471
+55190,28294,30714,40130
+51883,28295,30713,40536
+55291,28296,30712,40648
+59489,28297,30711,40928
+56787,28298,30710,40825
+52707,28299,30709,40454
+51912,28300,30708,40185
+55293,28301,30707,40665
+52003,28302,30706,40418
+57214,28303,30705,40103
+52178,28304,30704,40433
+59708,28305,30703,40409
+59256,28306,30702,40530
+53003,28307,30701,40527
+51822,28308,30700,40491
+51273,28309,30699,40509
+50855,28310,30698,40091
+54512,28311,30697,40409
+54002,28312,30696,40757
+55775,28313,30695,40328
+59528,28314,30694,40101
+51650,28315,30693,40141
+58789,28316,30692,40253
+56557,28317,30691,40037
+59636,28318,30690,40004
+59640,28319,30689,40057
+56323,28320,30688,40820
+54836,28321,30687,40425
+54118,28322,30686,40550
+58076,28323,30685,40450
+56407,28324,30684,40148
+51734,28325,30683,40212
+54139,28326,30682,40723
+54217,28327,30681,40562
+52943,28328,30680,40244
+56041,28329,30679,40046
+58881,28330,30678,40069
+56669,28331,30677,40224
+59835,28332,30676,40208
+51858,28333,30675,40179
+54395,28334,30674,40023
+50952,28335,30673,40525
+59150,28336,30672,40635
+54490,28337,30671,40201
+56458,28338,30670,40384
+52651,28339,30669,40211
+51819,28340,30668,40134
+51932,28341,30667,40999
+57029,28342,30666,40951
+54810,28343,30665,40894
+51416,28344,30664,40928
+59412,28345,30663,40598
+54354,28346,30662,40421
+54730,28347,30661,40692
+57694,28348,30660,40006
+55713,28349,30659,40369
+58650,28350,30658,40936
+51651,28351,30657,40279
+59126,28352,30656,40761
+58691,28353,30655,40806
+56712,28354,30654,40596
+59438,28355,30653,40975
+59485,28356,30652,40613
+55533,28357,30651,40499
+57978,28358,30650,40097
+58471,28359,30649,40953
+55520,28360,30648,40077
+57891,28361,30647,40100
+52126,28362,30646,40250
+56347,28363,30645,40349
+56192,28364,30644,40234
+55933,28365,30643,40185
+50726,28366,30642,40930
+59771,28367,30641,40401
+52303,28368,30640,40907
+52578,28369,30639,40954
+50468,28370,30638,40134
+57111,28371,30637,40073
+53953,28372,30636,40108
+55917,28373,30635,40861
+57694,28374,30634,40823
+57764,28375,30633,40768
+51772,28376,30632,40281
+52699,28377,30631,40387
+52090,28378,30630,40351
+55830,28379,30629,40158
+55127,28380,30628,40161
+58090,28381,30627,40057
+55570,28382,30626,40529
+59941,28383,30625,40233
+54238,28384,30624,40885
+54252,28385,30623,40002
+53393,28386,30622,40827
+55388,28387,30621,40917
+59931,28388,30620,40368
+54273,28389,30619,40215
+54612,28390,30618,40332
+55429,28391,30617,40054
+51966,28392,30616,40878
+55296,28393,30615,40161
+51786,28394,30614,40784
+56509,28395,30613,40299
+51494,28396,30612,40382
+56350,28397,30611,40275
+50612,28398,30610,40329
+55590,28399,30609,40067
+51238,28400,30608,40474
+55124,28401,30607,40506
+58821,28402,30606,40821
+52223,28403,30605,40128
+50592,28404,30604,40964
+52269,28405,30603,40582
+59297,28406,30602,40156
+59998,28407,30601,40272
+55324,28408,30600,40776
+57237,28409,30599,40480
+59186,28410,30598,40881
+57254,28411,30597,40352
+52898,28412,30596,40967
+51418,28413,30595,40828
+51326,28414,30594,40870
+52608,28415,30593,40657
+51819,28416,30592,40175
+50399,28417,30591,40950
+59426,28418,30590,40962
+55094,28419,30589,40110
+54115,28420,30588,40485
+57220,28421,30587,40446
+59466,28422,30586,40620
+58243,28423,30585,40521
+59178,28424,30584,40365
+56606,28425,30583,40174
+54389,28426,30582,40030
+53972,28427,30581,40250
+50109,28428,30580,40846
+58645,28429,30579,40592
+52933,28430,30578,40212
+55097,28431,30577,40674
+55601,28432,30576,40968
+58544,28433,30575,40690
+54268,28434,30574,40036
+59124,28435,30573,40953
+55961,28436,30572,40647
+51146,28437,30571,40307
+51921,28438,30570,40286
+53448,28439,30569,40638
+50714,28440,30568,40919
+54224,28441,30567,40541
+51928,28442,30566,40851
+55226,28443,30565,40357
+55325,28444,30564,40871
+54177,28445,30563,40875
+57341,28446,30562,40592
+58915,28447,30561,40423
+55117,28448,30560,40480
+59820,28449,30559,40234
+55465,28450,30558,40793
+53756,28451,30557,40735
+56804,28452,30556,40517
+50436,28453,30555,40849
+54426,28454,30554,40216
+51457,28455,30553,40961
+51602,28456,30552,40699
+59951,28457,30551,40786
+58207,28458,30550,40445
+52718,28459,30549,40668
+54294,28460,30548,40599
+51954,28461,30547,40952
+52441,28462,30546,40669
+51092,28463,30545,40350
+55186,28464,30544,40516
+56630,28465,30543,40569
+55627,28466,30542,40772
+51684,28467,30541,40492
+55130,28468,30540,40900
+50214,28469,30539,40308
+52603,28470,30538,40133
+58898,28471,30537,40769
+53797,28472,30536,40244
+57950,28473,30535,40929
+57918,28474,30534,40791
+57763,28475,30533,40787
+57658,28476,30532,40442
+57133,28477,30531,40085
+54895,28478,30530,40966
+56911,28479,30529,40627
+51428,28480,30528,40528
+52931,28481,30527,40829
+50344,28482,30526,40074
+59151,28483,30525,40179
+58318,28484,30524,40536
+58627,28485,30523,40343
+57574,28486,30522,40120
+54591,28487,30521,40023
+57287,28488,30520,40303
+59346,28489,30519,40502
+56409,28490,30518,40700
+57229,28491,30517,40615
+55359,28492,30516,40252
+57637,28493,30515,40173
+51745,28494,30514,40145
+50151,28495,30513,40809
+58232,28496,30512,40855
+53574,28497,30511,40408
+56018,28498,30510,40477
+53921,28499,30509,40128
+55685,28500,30508,40348
+59232,28501,30507,40592
+52416,28502,30506,40456
+52316,28503,30505,40201
+52669,28504,30504,40663
+51950,28505,30503,40077
+53731,28506,30502,40906
+58047,28507,30501,40331
+50120,28508,30500,40272
+57423,28509,30499,40707
+53508,28510,30498,40986
+52898,28511,30497,40180
+59504,28512,30496,40262
+50004,28513,30495,40479
+50989,28514,30494,40203
+56381,28515,30493,40024
+56242,28516,30492,40085
+57072,28517,30491,40705
+53821,28518,30490,40910
+56637,28519,30489,40915
+52954,28520,30488,40026
+52791,28521,30487,40282
+57473,28522,30486,40365
+58607,28523,30485,40219
+53178,28524,30484,40875
+51647,28525,30483,40957
+57980,28526,30482,40275
+53907,28527,30481,40921
+56994,28528,30480,40889
+52369,28529,30479,40791
+51887,28530,30478,40726
+50201,28531,30477,40022
+52185,28532,30476,40079
+55560,28533,30475,40200
+53504,28534,30474,40671
+53863,28535,30473,40664
+51026,28536,30472,40018
+56080,28537,30471,40481
+58087,28538,30470,40159
+56832,28539,30469,40900
+52743,28540,30468,40608
+54488,28541,30467,40629
+55644,28542,30466,40929
+57357,28543,30465,40963
+56930,28544,30464,40795
+55968,28545,30463,40618
+59738,28546,30462,40828
+59768,28547,30461,40263
+56158,28548,30460,40805
+50699,28549,30459,40143
+59295,28550,30458,40128
+56876,28551,30457,40964
+55514,28552,30456,40195
+52910,28553,30455,40165
+59353,28554,30454,40085
+55142,28555,30453,40753
+52821,28556,30452,40028
+56862,28557,30451,40400
+52025,28558,30450,40313
+54886,28559,30449,40558
+53282,28560,30448,40273
+50597,28561,30447,40709
+52131,28562,30446,40456
+53534,28563,30445,40950
+59654,28564,30444,40788
+57870,28565,30443,40819
+59740,28566,30442,40875
+55105,28567,30441,40268
+58661,28568,30440,40651
+50036,28569,30439,40548
+53226,28570,30438,40020
+56693,28571,30437,40991
+52999,28572,30436,40195
+58806,28573,30435,40050
+57047,28574,30434,40030
+59761,28575,30433,40999
+52056,28576,30432,40511
+52470,28577,30431,40620
+50670,28578,30430,40711
+51311,28579,30429,40583
+56131,28580,30428,40912
+56900,28581,30427,40304
+59212,28582,30426,40406
+53898,28583,30425,40019
+58023,28584,30424,40603
+59575,28585,30423,40456
+53472,28586,30422,40911
+52961,28587,30421,40681
+57650,28588,30420,40713
+54452,28589,30419,40111
+56058,28590,30418,40436
+57858,28591,30417,40345
+55848,28592,30416,40856
+57956,28593,30415,40507
+53535,28594,30414,40107
+51247,28595,30413,40815
+58924,28596,30412,40004
+55234,28597,30411,40482
+59235,28598,30410,40836
+53500,28599,30409,40373
+52052,28600,30408,40138
+54954,28601,30407,40359
+53549,28602,30406,40722
+58142,28603,30405,40659
+59753,28604,30404,40321
+56955,28605,30403,40587
+55030,28606,30402,40399
+50909,28607,30401,40489
+58612,28608,30400,40545
+56389,28609,30399,40142
+51449,28610,30398,40201
+57106,28611,30397,40401
+56562,28612,30396,40181
+58315,28613,30395,40029
+56788,28614,30394,40682
+53785,28615,30393,40183
+50321,28616,30392,40682
+53314,28617,30391,40763
+57061,28618,30390,40593
+56656,28619,30389,40858
+55072,28620,30388,40623
+55328,28621,30387,40778
+50126,28622,30386,40003
+59254,28623,30385,40562
+58690,28624,30384,40381
+51981,28625,30383,40496
+59826,28626,30382,40086
+58748,28627,30381,40722
+59243,28628,30380,40351
+53604,28629,30379,40837
+54116,28630,30378,40898
+54135,28631,30377,40533
+52225,28632,30376,40336
+56867,28633,30375,40801
+50574,28634,30374,40248
+54763,28635,30373,40706
+52715,28636,30372,40241
+52038,28637,30371,40880
+59687,28638,30370,40980
+53216,28639,30369,40345
+56757,28640,30368,40417
+51023,28641,30367,40255
+54188,28642,30366,40525
+59393,28643,30365,40213
+54602,28644,30364,40773
+54641,28645,30363,40238
+53880,28646,30362,40321
+51768,28647,30361,40929
+55864,28648,30360,40128
+52464,28649,30359,40357
+56282,28650,30358,40675
+52579,28651,30357,40701
+58751,28652,30356,40281
+54274,28653,30355,40873
+55989,28654,30354,40791
+50337,28655,30353,40185
+52033,28656,30352,40401
+58336,28657,30351,40429
+50827,28658,30350,40034
+55141,28659,30349,40902
+51795,28660,30348,40892
+56725,28661,30347,40075
+52977,28662,30346,40500
+59681,28663,30345,40065
+57978,28664,30344,40285
+58365,28665,30343,40507
+57055,28666,30342,40999
+59260,28667,30341,40769
+57279,28668,30340,40996
+58807,28669,30339,40760
+51308,28670,30338,40599
+50284,28671,30337,40510
+57755,28672,30336,40225
+56291,28673,30335,40312
+51813,28674,30334,40897
+50332,28675,30333,40914
+56779,28676,30332,40279
+59124,28677,30331,40802
+54422,28678,30330,40390
+55879,28679,30329,40177
+50540,28680,30328,40732
+53807,28681,30327,40503
+59439,28682,30326,40031
+52234,28683,30325,40970
+55247,28684,30324,40997
+58634,28685,30323,40987
+55520,28686,30322,40320
+58453,28687,30321,40909
+56102,28688,30320,40916
+52013,28689,30319,40530
+55100,28690,30318,40406
+53402,28691,30317,40724
+53189,28692,30316,40628
+52547,28693,30315,40085
+59769,28694,30314,40499
+53964,28695,30313,40716
+56513,28696,30312,40432
+56586,28697,30311,40645
+50110,28698,30310,40311
+52305,28699,30309,40451
+52148,28700,30308,40361
+50510,28701,30307,40092
+57060,28702,30306,40382
+59009,28703,30305,40913
+54022,28704,30304,40911
+50929,28705,30303,40496
+55349,28706,30302,40518
+57636,28707,30301,40977
+51589,28708,30300,40020
+58550,28709,30299,40609
+59600,28710,30298,40839
+58678,28711,30297,40470
+53213,28712,30296,40238
+59793,28713,30295,40824
+59218,28714,30294,40771
+54694,28715,30293,40056
+53456,28716,30292,40996
+57912,28717,30291,40712
+54605,28718,30290,40909
+52018,28719,30289,40367
+54864,28720,30288,40405
+54852,28721,30287,40847
+59972,28722,30286,40675
+55116,28723,30285,40332
+58095,28724,30284,40902
+57856,28725,30283,40091
+59021,28726,30282,40214
+56440,28727,30281,40276
+52133,28728,30280,40714
+59400,28729,30279,40198
+58109,28730,30278,40647
+50411,28731,30277,40526
+57881,28732,30276,40193
+50299,28733,30275,40115
+50223,28734,30274,40097
+58269,28735,30273,40369
+51974,28736,30272,40932
+50874,28737,30271,40273
+54813,28738,30270,40515
+53815,28739,30269,40833
+53944,28740,30268,40676
+53318,28741,30267,40021
+53108,28742,30266,40541
+53541,28743,30265,40177
+57180,28744,30264,40760
+56405,28745,30263,40809
+53140,28746,30262,40311
+50869,28747,30261,40296
+54963,28748,30260,40445
+59692,28749,30259,40618
+54190,28750,30258,40239
+57238,28751,30257,40270
+53390,28752,30256,40885
+51270,28753,30255,40811
+55187,28754,30254,40033
+56937,28755,30253,40533
+54563,28756,30252,40187
+58267,28757,30251,40444
+54667,28758,30250,40334
+50291,28759,30249,40010
+53158,28760,30248,40843
+52521,28761,30247,40527
+54683,28762,30246,40944
+51064,28763,30245,40891
+54464,28764,30244,40796
+59903,28765,30243,40407
+52221,28766,30242,40233
+51921,28767,30241,40656
+56732,28768,30240,40295
+51388,28769,30239,40760
+51020,28770,30238,40929
+59047,28771,30237,40283
+52292,28772,30236,40350
+54788,28773,30235,40644
+59092,28774,30234,40175
+59157,28775,30233,40602
+52536,28776,30232,40048
+52202,28777,30231,40170
+52214,28778,30230,40996
+54105,28779,30229,40730
+54358,28780,30228,40457
+52951,28781,30227,40078
+50593,28782,30226,40877
+56690,28783,30225,40803
+53426,28784,30224,40228
+59489,28785,30223,40062
+55578,28786,30222,40415
+50803,28787,30221,40376
+58480,28788,30220,40611
+59268,28789,30219,40977
+54388,28790,30218,40619
+56974,28791,30217,40341
+57988,28792,30216,40730
+56572,28793,30215,40471
+52840,28794,30214,40273
+54314,28795,30213,40305
+53432,28796,30212,40201
+50152,28797,30211,40693
+57454,28798,30210,40918
+55661,28799,30209,40179
+51254,28800,30208,40809
+55513,28801,30207,40952
+54658,28802,30206,40608
+50482,28803,30205,40759
+51241,28804,30204,40787
+54352,28805,30203,40615
+57343,28806,30202,40862
+51284,28807,30201,40008
+51344,28808,30200,40678
+55315,28809,30199,40058
+58614,28810,30198,40506
+52624,28811,30197,40237
+54123,28812,30196,40240
+57559,28813,30195,40839
+52024,28814,30194,40368
+51511,28815,30193,40053
+59867,28816,30192,40689
+50855,28817,30191,40481
+57479,28818,30190,40326
+50457,28819,30189,40044
+52469,28820,30188,40565
+53451,28821,30187,40025
+57561,28822,30186,40796
+57898,28823,30185,40566
+52896,28824,30184,40840
+59665,28825,30183,40087
+56340,28826,30182,40715
+55136,28827,30181,40401
+53746,28828,30180,40953
+55128,28829,30179,40104
+56211,28830,30178,40936
+53730,28831,30177,40397
+55574,28832,30176,40185
+54367,28833,30175,40319
+50863,28834,30174,40479
+55994,28835,30173,40148
+57800,28836,30172,40851
+53546,28837,30171,40013
+57653,28838,30170,40689
+50836,28839,30169,40148
+58998,28840,30168,40315
+56061,28841,30167,40546
+59814,28842,30166,40545
+52892,28843,30165,40177
+50417,28844,30164,40625
+55774,28845,30163,40075
+51538,28846,30162,40176
+53683,28847,30161,40062
+50857,28848,30160,40997
+56534,28849,30159,40596
+55932,28850,30158,40150
+51595,28851,30157,40019
+59477,28852,30156,40706
+58784,28853,30155,40309
+53177,28854,30154,40090
+59846,28855,30153,40661
+52616,28856,30152,40380
+59945,28857,30151,40279
+56448,28858,30150,40494
+53194,28859,30149,40418
+53852,28860,30148,40268
+59918,28861,30147,40694
+54842,28862,30146,40950
+50175,28863,30145,40140
+58698,28864,30144,40520
+59184,28865,30143,40502
+58227,28866,30142,40327
+50718,28867,30141,40939
+52175,28868,30140,40565
+59538,28869,30139,40911
+55895,28870,30138,40170
+55699,28871,30137,40165
+56745,28872,30136,40311
+50361,28873,30135,40426
+50789,28874,30134,40368
+52940,28875,30133,40648
+59784,28876,30132,40453
+54507,28877,30131,40371
+55503,28878,30130,40004
+59928,28879,30129,40539
+52445,28880,30128,40463
+51360,28881,30127,40679
+59271,28882,30126,40390
+50217,28883,30125,40090
+51725,28884,30124,40089
+54148,28885,30123,40373
+59619,28886,30122,40829
+57665,28887,30121,40465
+56318,28888,30120,40648
+59042,28889,30119,40007
+59722,28890,30118,40025
+55272,28891,30117,40630
+52087,28892,30116,40641
+52937,28893,30115,40223
+50629,28894,30114,40160
+54804,28895,30113,40912
+58724,28896,30112,40926
+51675,28897,30111,40444
+57086,28898,30110,40775
+54510,28899,30109,40868
+52192,28900,30108,40474
+55177,28901,30107,40500
+56547,28902,30106,40886
+52403,28903,30105,40356
+56633,28904,30104,40794
+52295,28905,30103,40865
+57267,28906,30102,40055
+52960,28907,30101,40352
+55824,28908,30100,40811
+56750,28909,30099,40366
+58255,28910,30098,40621
+57701,28911,30097,40687
+56853,28912,30096,40489
+58708,28913,30095,40681
+59098,28914,30094,40455
+58243,28915,30093,40108
+52350,28916,30092,40108
+52949,28917,30091,40992
+55353,28918,30090,40556
+58685,28919,30089,40019
+54608,28920,30088,40929
+59149,28921,30087,40686
+56596,28922,30086,40933
+54026,28923,30085,40546
+51508,28924,30084,40185
+59163,28925,30083,40620
+59186,28926,30082,40296
+53441,28927,30081,40451
+51155,28928,30080,40835
+56944,28929,30079,40498
+55760,28930,30078,40297
+59370,28931,30077,40886
+54970,28932,30076,40562
+52653,28933,30075,40837
+51546,28934,30074,40832
+57059,28935,30073,40137
+58567,28936,30072,40147
+58520,28937,30071,40032
+56112,28938,30070,40215
+55993,28939,30069,40921
+57721,28940,30068,40653
+51207,28941,30067,40797
+59682,28942,30066,40539
+54554,28943,30065,40440
+56259,28944,30064,40256
+57106,28945,30063,40563
+54804,28946,30062,40832
+53689,28947,30061,40759
+52664,28948,30060,40928
+51429,28949,30059,40199
+53226,28950,30058,40519
+55465,28951,30057,40922
+55440,28952,30056,40082
+59243,28953,30055,40388
+56396,28954,30054,40218
+57776,28955,30053,40225
+51170,28956,30052,40355
+55220,28957,30051,40765
+51971,28958,30050,40148
+59350,28959,30049,40139
+55303,28960,30048,40046
+58383,28961,30047,40847
+53370,28962,30046,40018
+57779,28963,30045,40986
+56813,28964,30044,40676
+56320,28965,30043,40682
+50045,28966,30042,40526
+58851,28967,30041,40979
+59905,28968,30040,40458
+57115,28969,30039,40518
+55804,28970,30038,40728
+55749,28971,30037,40669
+53331,28972,30036,40856
+53654,28973,30035,40320
+53593,28974,30034,40829
+56775,28975,30033,40167
+59237,28976,30032,40359
+52346,28977,30031,40348
+59957,28978,30030,40938
+50811,28979,30029,40139
+54668,28980,30028,40016
+54098,28981,30027,40106
+59335,28982,30026,40552
+51093,28983,30025,40551
+52941,28984,30024,40849
+53979,28985,30023,40955
+58692,28986,30022,40743
+56491,28987,30021,40297
+52785,28988,30020,40168
+52546,28989,30019,40186
+55639,28990,30018,40388
+56259,28991,30017,40392
+52673,28992,30016,40263
+59704,28993,30015,40029
+58842,28994,30014,40557
+55591,28995,30013,40695
+54953,28996,30012,40976
+58177,28997,30011,40418
+52731,28998,30010,40881
+54564,28999,30009,40315
+59624,29000,30008,40049
+56527,29001,30007,40141
+53240,29002,30006,40514
+58182,29003,30005,40842
+54942,29004,30004,40023
+58395,29005,30003,40894
+57938,29006,30002,40220
+56108,29007,30001,40979
+55421,29008,30000,40964
+53213,29009,31000,40925
+52301,29010,30999,40173
+52106,29011,30998,40418
+50953,29012,30997,40802
+56574,29013,30996,40344
+54558,29014,30995,40677
+50114,29015,30994,40778
+54327,29016,30993,40374
+57149,29017,30992,40720
+55547,29018,30991,40989
+52457,29019,30990,40451
+57472,29020,30989,40104
+54310,29021,30988,40074
+53134,29022,30987,40873
+56072,29023,30986,40603
+54239,29024,30985,40319
+58012,29025,30984,40881
+57314,29026,30983,40408
+52200,29027,30982,40671
+53726,29028,30981,40944
+50624,29029,30980,40478
+59764,29030,30979,40736
+52132,29031,30978,40889
+53517,29032,30977,40982
+54486,29033,30976,40650
+58738,29034,30975,40972
+52074,29035,30974,40384
+53243,29036,30973,40712
+51447,29037,30972,40856
+58655,29038,30971,40680
+55626,29039,30970,40309
+51066,29040,30969,40375
+58121,29041,30968,40334
+59133,29042,30967,40648
+50277,29043,30966,40170
+59166,29044,30965,40326
+56488,29045,30964,40938
+50784,29046,30963,40278
+56590,29047,30962,40771
+50430,29048,30961,40995
+56049,29049,30960,40806
+51985,29050,30959,40603
+53628,29051,30958,40582
+50594,29052,30957,40442
+58641,29053,30956,40046
+56632,29054,30955,40043
+59254,29055,30954,40292
+59714,29056,30953,40534
+54009,29057,30952,40246
+52849,29058,30951,40787
+55253,29059,30950,40719
+53549,29060,30949,40186
+52013,29061,30948,40974
+53272,29062,30947,40660
+56052,29063,30946,40780
+54639,29064,30945,40009
+54175,29065,30944,40600
+50076,29066,30943,40167
+55139,29067,30942,40720
+58553,29068,30941,40355
+57464,29069,30940,40942
+59007,29070,30939,40504
+51919,29071,30938,40101
+50940,29072,30937,40561
+52672,29073,30936,40334
+58307,29074,30935,40956
+58411,29075,30934,40979
+58205,29076,30933,40734
+54496,29077,30932,40807
+56030,29078,30931,40773
+54971,29079,30930,40649
+54435,29080,30929,40813
+56759,29081,30928,40963
+50070,29082,30927,40090
+53283,29083,30926,40211
+52084,29084,30925,40399
+51825,29085,30924,40389
+55990,29086,30923,40698
+52475,29087,30922,40634
+52187,29088,30921,40116
+53249,29089,30920,40682
+58423,29090,30919,40608
+58989,29091,30918,40626
+59044,29092,30917,40030
+50280,29093,30916,40649
+54407,29094,30915,40309
+56405,29095,30914,40693
+53982,29096,30913,40723
+52267,29097,30912,40173
+54853,29098,30911,40978
+52784,29099,30910,40154
+58503,29100,30909,40378
+54752,29101,30908,40887
+51602,29102,30907,40612
+51853,29103,30906,40506
+57603,29104,30905,40354
+57168,29105,30904,40518
+58145,29106,30903,40833
+50126,29107,30902,40190
+53557,29108,30901,40032
+53367,29109,30900,40000
+51543,29110,30899,40008
+58215,29111,30898,40940
+53412,29112,30897,40839
+53983,29113,30896,40448
+50251,29114,30895,40130
+50734,29115,30894,40641
+58647,29116,30893,40527
+54312,29117,30892,40000
+51622,29118,30891,40221
+51769,29119,30890,40438
+50025,29120,30889,40936
+55459,29121,30888,40806
+53172,29122,30887,40548
+53490,29123,30886,40590
+54309,29124,30885,40566
+53969,29125,30884,40800
+53073,29126,30883,40642
+51494,29127,30882,40155
+50042,29128,30881,40916
+55976,29129,30880,40337
+54461,29130,30879,40982
+52385,29131,30878,40957
+58263,29132,30877,40579
+59422,29133,30876,40460
+54968,29134,30875,40518
+50227,29135,30874,40549
+59583,29136,30873,40338
+54235,29137,30872,40517
+58373,29138,30871,40102
+51565,29139,30870,40331
+54503,29140,30869,40172
+59867,29141,30868,40245
+53338,29142,30867,40759
+58596,29143,30866,40765
+56995,29144,30865,40919
+53561,29145,30864,40988
+58206,29146,30863,40443
+57277,29147,30862,40666
+54972,29148,30861,40433
+58437,29149,30860,40047
+54778,29150,30859,40532
+57407,29151,30858,40792
+56988,29152,30857,40219
+52079,29153,30856,40369
+51496,29154,30855,40746
+59427,29155,30854,40523
+58271,29156,30853,40943
+50392,29157,30852,40939
+54102,29158,30851,40516
+55991,29159,30850,40167
+52131,29160,30849,40964
+55623,29161,30848,40797
+50980,29162,30847,40315
+52118,29163,30846,40410
+50040,29164,30845,40430
+52226,29165,30844,40460
+55633,29166,30843,40444
+55649,29167,30842,40405
+59777,29168,30841,40688
+57216,29169,30840,40772
+57393,29170,30839,40700
+57017,29171,30838,40746
+52097,29172,30837,40635
+54861,29173,30836,40653
+51129,29174,30835,40195
+55889,29175,30834,40580
+53438,29176,30833,40110
+54134,29177,30832,40832
+53425,29178,30831,40636
+59664,29179,30830,40603
+52209,29180,30829,40710
+56248,29181,30828,40489
+56833,29182,30827,40433
+59821,29183,30826,40215
+50547,29184,30825,40436
+53893,29185,30824,40772
+55780,29186,30823,40800
+53601,29187,30822,40426
+50017,29188,30821,40244
+51182,29189,30820,40537
+58807,29190,30819,40778
+59048,29191,30818,40337
+51897,29192,30817,40250
+59049,29193,30816,40561
+51210,29194,30815,40376
+51519,29195,30814,40713
+54314,29196,30813,40988
+51836,29197,30812,40056
+54637,29198,30811,40118
+55431,29199,30810,40835
+59095,29200,30809,40476
+51375,29201,30808,40323
+56804,29202,30807,40962
+51792,29203,30806,40842
+58196,29204,30805,40629
+54190,29205,30804,40929
+50739,29206,30803,40068
+52702,29207,30802,40777
+53767,29208,30801,40731
+58772,29209,30800,40188
+50578,29210,30799,40541
+51527,29211,30798,40959
+57496,29212,30797,40717
+50001,29213,30796,40863
+53146,29214,30795,40137
+55363,29215,30794,40484
+55585,29216,30793,40468
+53909,29217,30792,40430
+53670,29218,30791,40879
+56458,29219,30790,40859
+52333,29220,30789,40067
+50514,29221,30788,40571
+58387,29222,30787,40573
+53321,29223,30786,40112
+51980,29224,30785,40782
+56813,29225,30784,40533
+57151,29226,30783,40334
+52140,29227,30782,40089
+51038,29228,30781,40957
+50796,29229,30780,40664
+58732,29230,30779,40070
+53285,29231,30778,40287
+59395,29232,30777,40632
+52361,29233,30776,40106
+56846,29234,30775,40417
+56019,29235,30774,40251
+57153,29236,30773,40850
+56941,29237,30772,40713
+50265,29238,30771,40185
+59899,29239,30770,40603
+56957,29240,30769,40967
+52117,29241,30768,40447
+59107,29242,30767,40887
+50435,29243,30766,40860
+58720,29244,30765,40391
+58976,29245,30764,40032
+50535,29246,30763,40447
+56753,29247,30762,40959
+54531,29248,30761,40379
+54111,29249,30760,40001
+54384,29250,30759,40606
+53103,29251,30758,40326
+57907,29252,30757,40813
+54243,29253,30756,40440
+54601,29254,30755,40317
+51258,29255,30754,40869
+56653,29256,30753,40850
+52082,29257,30752,40200
+59155,29258,30751,40485
+57230,29259,30750,40931
+57282,29260,30749,40832
+57794,29261,30748,40867
+53311,29262,30747,40945
+56043,29263,30746,40208
+51652,29264,30745,40586
+59228,29265,30744,40160
+57343,29266,30743,40489
+53939,29267,30742,40847
+57478,29268,30741,40209
+57338,29269,30740,40372
+59849,29270,30739,40779
+51049,29271,30738,40587
+58608,29272,30737,40042
+58296,29273,30736,40506
+53587,29274,30735,40810
+52082,29275,30734,40779
+52217,29276,30733,40556
+57334,29277,30732,40389
+53895,29278,30731,40641
+56199,29279,30730,40231
+50186,29280,30729,40344
+52051,29281,30728,40626
+58752,29282,30727,40694
+51697,29283,30726,40386
+51041,29284,30725,40801
+59955,29285,30724,40689
+55003,29286,30723,40233
+59595,29287,30722,40784
+52299,29288,30721,40996
+51306,29289,30720,40291
+54583,29290,30719,40340
+50564,29291,30718,40783
+51207,29292,30717,40142
+57327,29293,30716,40714
+52597,29294,30715,40578
+53177,29295,30714,40975
+54362,29296,30713,40538
+56489,29297,30712,40805
+56727,29298,30711,40577
+54926,29299,30710,40723
+52661,29300,30709,40770
+55653,29301,30708,40947
+51428,29302,30707,40266
+56966,29303,30706,40928
+53360,29304,30705,40939
+50829,29305,30704,40461
+58786,29306,30703,40110
+55526,29307,30702,40443
+57603,29308,30701,40555
+59105,29309,30700,40134
+51851,29310,30699,40696
+52134,29311,30698,40981
+55799,29312,30697,40308
+52946,29313,30696,40692
+58296,29314,30695,40895
+59935,29315,30694,40512
+53928,29316,30693,40728
+50625,29317,30692,40694
+58136,29318,30691,40098
+54765,29319,30690,40448
+57173,29320,30689,40260
+52276,29321,30688,40397
+55482,29322,30687,40868
+51597,29323,30686,40956
+53294,29324,30685,40951
+53793,29325,30684,40813
+55341,29326,30683,40535
+55344,29327,30682,40324
+50837,29328,30681,40689
+56257,29329,30680,40730
+54807,29330,30679,40882
+50966,29331,30678,40802
+51949,29332,30677,40895
+51918,29333,30676,40637
+57627,29334,30675,40621
+57396,29335,30674,40155
+56334,29336,30673,40571
+53316,29337,30672,40047
+56449,29338,30671,40770
+51902,29339,30670,40085
+54462,29340,30669,40389
+59635,29341,30668,40002
+54257,29342,30667,40539
+59564,29343,30666,40096
+52536,29344,30665,40439
+51739,29345,30664,40837
+51559,29346,30663,40568
+57977,29347,30662,40375
+51945,29348,30661,40498
+51086,29349,30660,40614
+51536,29350,30659,40304
+54622,29351,30658,40806
+56413,29352,30657,40683
+53921,29353,30656,40511
+52715,29354,30655,40644
+51531,29355,30654,40578
+54838,29356,30653,40442
+57450,29357,30652,40155
+54137,29358,30651,40953
+54783,29359,30650,40510
+51189,29360,30649,40383
+52115,29361,30648,40287
+57261,29362,30647,40604
+59498,29363,30646,40986
+52046,29364,30645,40109
+50549,29365,30644,40163
+52381,29366,30643,40429
+50081,29367,30642,40396
+51016,29368,30641,40857
+55629,29369,30640,40865
+50286,29370,30639,40798
+51482,29371,30638,40232
+58186,29372,30637,40466
+56987,29373,30636,40666
+59674,29374,30635,40443
+55115,29375,30634,40649
+58559,29376,30633,40904
+56205,29377,30632,40238
+55005,29378,30631,40458
+53103,29379,30630,40472
+54450,29380,30629,40726
+53512,29381,30628,40850
+53220,29382,30627,40722
+57440,29383,30626,40705
+54434,29384,30625,40961
+50655,29385,30624,40712
+59116,29386,30623,40269
+50285,29387,30622,40203
+59247,29388,30621,40896
+53161,29389,30620,40333
+55640,29390,30619,40319
+56629,29391,30618,40566
+58746,29392,30617,40756
+59438,29393,30616,40700
+53883,29394,30615,40350
+56362,29395,30614,40772
+56072,29396,30613,40079
+59402,29397,30612,40429
+54361,29398,30611,40730
+58863,29399,30610,40690
+55232,29400,30609,40366
+54665,29401,30608,40106
+59178,29402,30607,40980
+57866,29403,30606,40726
+53896,29404,30605,40969
+50617,29405,30604,40398
+55639,29406,30603,40456
+53666,29407,30602,40248
+54667,29408,30601,40504
+58010,29409,30600,40150
+53067,29410,30599,40380
+59057,29411,30598,40642
+59205,29412,30597,40285
+59232,29413,30596,40577
+56098,29414,30595,40001
+55586,29415,30594,40250
+52836,29416,30593,40829
+52239,29417,30592,40178
+53548,29418,30591,40511
+51683,29419,30590,40269
+57769,29420,30589,40281
+58990,29421,30588,40946
+50762,29422,30587,40482
+50196,29423,30586,40875
+50213,29424,30585,40055
+51092,29425,30584,40240
+56656,29426,30583,40457
+50470,29427,30582,40751
+57084,29428,30581,40187
+51758,29429,30580,40410
+57665,29430,30579,40835
+50925,29431,30578,40550
+58964,29432,30577,40330
+57766,29433,30576,40379
+59398,29434,30575,40778
+53214,29435,30574,40036
+53158,29436,30573,40787
+51901,29437,30572,40829
+56071,29438,30571,40025
+54273,29439,30570,40540
+58028,29440,30569,40566
+51281,29441,30568,40958
+52623,29442,30567,40824
+54287,29443,30566,40519
+58641,29444,30565,40574
+55223,29445,30564,40972
+54939,29446,30563,40252
+50804,29447,30562,40177
+58148,29448,30561,40545
+56306,29449,30560,40858
+51419,29450,30559,40025
+52254,29451,30558,40184
+55316,29452,30557,40220
+53129,29453,30556,40587
+51312,29454,30555,40794
+58459,29455,30554,40366
+57556,29456,30553,40225
+58629,29457,30552,40668
+50764,29458,30551,40220
+50335,29459,30550,40636
+53448,29460,30549,40146
+52607,29461,30548,40939
+56116,29462,30547,40409
+52123,29463,30546,40799
+51334,29464,30545,40813
+58950,29465,30544,40428
+59633,29466,30543,40264
+50614,29467,30542,40461
+50678,29468,30541,40295
+50983,29469,30540,40496
+50666,29470,30539,40199
+54168,29471,30538,40334
+56818,29472,30537,40219
+56434,29473,30536,40005
+50282,29474,30535,40163
+59465,29475,30534,40516
+55158,29476,30533,40706
+55035,29477,30532,40571
+50720,29478,30531,40764
+51478,29479,30530,40493
+53312,29480,30529,40146
+59094,29481,30528,40203
+55034,29482,30527,40380
+59996,29483,30526,40022
+59904,29484,30525,40113
+53498,29485,30524,40368
+51064,29486,30523,40814
+54810,29487,30522,40559
+51753,29488,30521,40585
+53455,29489,30520,40119
+54495,29490,30519,40744
+52451,29491,30518,40587
+51358,29492,30517,40051
+52463,29493,30516,40472
+57538,29494,30515,40755
+53353,29495,30514,40198
+56597,29496,30513,40438
+54152,29497,30512,40484
+52330,29498,30511,40904
+56547,29499,30510,40559
+52839,29500,30509,40870
+55479,29501,30508,40182
+55072,29502,30507,40074
+50967,29503,30506,40277
+58477,29504,30505,40295
+52334,29505,30504,40357
+54221,29506,30503,40496
+52144,29507,30502,40164
+55572,29508,30501,40841
+57934,29509,30500,40430
+57648,29510,30499,40273
+55272,29511,30498,40919
+58329,29512,30497,40355
+54397,29513,30496,40659
+53223,29514,30495,40767
+52302,29515,30494,40421
+57470,29516,30493,40114
+53665,29517,30492,40248
+53284,29518,30491,40519
+54370,29519,30490,40229
+54739,29520,30489,40000
+50803,29521,30488,40130
+50185,29522,30487,40645
+51411,29523,30486,40115
+50268,29524,30485,40407
+52765,29525,30484,40640
+55420,29526,30483,40130
+57703,29527,30482,40219
+56465,29528,30481,40085
+58331,29529,30480,40996
+59603,29530,30479,40303
+50611,29531,30478,40931
+51625,29532,30477,40848
+59149,29533,30476,40825
+53865,29534,30475,40225
+59438,29535,30474,40155
+51709,29536,30473,40185
+59235,29537,30472,40099
+51399,29538,30471,40917
+51506,29539,30470,40509
+52827,29540,30469,40250
+51647,29541,30468,40196
+58395,29542,30467,40202
+50923,29543,30466,40350
+56144,29544,30465,40713
+50582,29545,30464,40275
+54467,29546,30463,40924
+53845,29547,30462,40405
+57724,29548,30461,40181
+55215,29549,30460,40164
+54562,29550,30459,40124
+52561,29551,30458,40633
+54033,29552,30457,40029
+50834,29553,30456,40293
+58778,29554,30455,40426
+59168,29555,30454,40533
+57236,29556,30453,40118
+52329,29557,30452,40956
+54064,29558,30451,40446
+54054,29559,30450,40085
+50762,29560,30449,40203
+53489,29561,30448,40323
+53691,29562,30447,40824
+57928,29563,30446,40720
+54988,29564,30445,40044
+55130,29565,30444,40494
+56197,29566,30443,40869
+50942,29567,30442,40389
+59267,29568,30441,40922
+59123,29569,30440,40154
+58520,29570,30439,40530
+54849,29571,30438,40384
+50379,29572,30437,40830
+51566,29573,30436,40797
+55649,29574,30435,40000
+59053,29575,30434,40532
+52404,29576,30433,40367
+58657,29577,30432,40378
+52815,29578,30431,40536
+55376,29579,30430,40167
+51740,29580,30429,40235
+54546,29581,30428,40995
+54164,29582,30427,40878
+56361,29583,30426,40995
+52870,29584,30425,40274
+55370,29585,30424,40863
+56115,29586,30423,40008
+52505,29587,30422,40441
+59589,29588,30421,40631
+54707,29589,30420,40131
+50192,29590,30419,40406
+50644,29591,30418,40485
+54273,29592,30417,40272
+56650,29593,30416,40531
+53747,29594,30415,40801
+50689,29595,30414,40975
+57184,29596,30413,40115
+50076,29597,30412,40720
+59652,29598,30411,40285
+50261,29599,30410,40222
+52589,29600,30409,40801
+51420,29601,30408,40528
+53775,29602,30407,40961
+50508,29603,30406,40580
+55733,29604,30405,40285
+50731,29605,30404,40099
+53762,29606,30403,40611
+53904,29607,30402,40638
+58709,29608,30401,40211
+59358,29609,30400,40649
+53770,29610,30399,40990
+52495,29611,30398,40355
+50445,29612,30397,40627
+58012,29613,30396,40927
+56041,29614,30395,40709
+50232,29615,30394,40610
+52991,29616,30393,40405
+59885,29617,30392,40889
+53906,29618,30391,40575
+59534,29619,30390,40182
+55679,29620,30389,40755
+59979,29621,30388,40902
+55450,29622,30387,40147
+59555,29623,30386,40619
+50980,29624,30385,40588
+56052,29625,30384,40553
+50825,29626,30383,40387
+53326,29627,30382,40271
+52725,29628,30381,40437
+59221,29629,30380,40293
+55959,29630,30379,40690
+55925,29631,30378,40684
+59245,29632,30377,40399
+52320,29633,30376,40949
+53279,29634,30375,40127
+59811,29635,30374,40994
+51240,29636,30373,40518
+54503,29637,30372,40651
+53886,29638,30371,40986
+55154,29639,30370,40479
+54067,29640,30369,40718
+57502,29641,30368,40328
+58388,29642,30367,40188
+57218,29643,30366,40032
+54974,29644,30365,40934
+50540,29645,30364,40069
+58113,29646,30363,40489
+54608,29647,30362,40511
+53901,29648,30361,40870
+52058,29649,30360,40178
+51497,29650,30359,40334
+54399,29651,30358,40325
+51138,29652,30357,40978
+51234,29653,30356,40285
+55248,29654,30355,40732
+55746,29655,30354,40550
+50981,29656,30353,40310
+55552,29657,30352,40576
+54734,29658,30351,40893
+57647,29659,30350,40954
+55836,29660,30349,40937
+58692,29661,30348,40821
+54780,29662,30347,40244
+50621,29663,30346,40543
+53858,29664,30345,40556
+54206,29665,30344,40171
+54629,29666,30343,40343
+51830,29667,30342,40922
+58515,29668,30341,40642
+55072,29669,30340,40528
+57445,29670,30339,40326
+53237,29671,30338,40197
+50910,29672,30337,40447
+50895,29673,30336,40054
+58285,29674,30335,40826
+59513,29675,30334,40143
+50641,29676,30333,40139
+50706,29677,30332,40760
+55292,29678,30331,40857
+59056,29679,30330,40633
+50192,29680,30329,40566
+53825,29681,30328,40883
+50423,29682,30327,40134
+53836,29683,30326,40885
+58751,29684,30325,40811
+52774,29685,30324,40520
+59441,29686,30323,40786
+56065,29687,30322,40431
+54658,29688,30321,40663
+57103,29689,30320,40374
+59907,29690,30319,40104
+59828,29691,30318,40525
+51635,29692,30317,40912
+51449,29693,30316,40305
+58549,29694,30315,40986
+51581,29695,30314,40217
+51020,29696,30313,40328
+56806,29697,30312,40552
+56930,29698,30311,40151
+52565,29699,30310,40675
+50120,29700,30309,40137
+53368,29701,30308,40008
+54040,29702,30307,40526
+56632,29703,30306,40353
+56614,29704,30305,40047
+53811,29705,30304,40308
+59669,29706,30303,40098
+54212,29707,30302,40648
+55462,29708,30301,40251
+55774,29709,30300,40006
+59642,29710,30299,40309
+55416,29711,30298,40009
+58969,29712,30297,40661
+55282,29713,30296,40364
+56236,29714,30295,40398
+52757,29715,30294,40631
+52884,29716,30293,40231
+59241,29717,30292,40093
+53933,29718,30291,40053
+52816,29719,30290,40510
+53309,29720,30289,40862
+51567,29721,30288,40459
+53127,29722,30287,40148
+51965,29723,30286,40419
+51068,29724,30285,40625
+56020,29725,30284,40217
+52673,29726,30283,40781
+59173,29727,30282,40536
+52635,29728,30281,40146
+57496,29729,30280,40300
+54423,29730,30279,40304
+59760,29731,30278,40462
+53094,29732,30277,40146
+55517,29733,30276,40124
+55797,29734,30275,40835
+51707,29735,30274,40506
+59451,29736,30273,40294
+54072,29737,30272,40425
+58412,29738,30271,40434
+52611,29739,30270,40458
+57707,29740,30269,40774
+58592,29741,30268,40608
+59692,29742,30267,40846
+56312,29743,30266,40575
+52647,29744,30265,40794
+50445,29745,30264,40356
+59352,29746,30263,40271
+55339,29747,30262,40835
+57539,29748,30261,40901
+53881,29749,30260,40327
+54948,29750,30259,40853
+59183,29751,30258,40402
+59605,29752,30257,40196
+59378,29753,30256,40193
+57577,29754,30255,40392
+56299,29755,30254,40266
+59446,29756,30253,40005
+59669,29757,30252,40605
+58416,29758,30251,40725
+51813,29759,30250,40020
+53471,29760,30249,40538
+54276,29761,30248,40880
+57378,29762,30247,40129
+52024,29763,30246,40178
+59757,29764,30245,40701
+52801,29765,30244,40875
+53454,29766,30243,40125
+54543,29767,30242,40305
+50990,29768,30241,40619
+54704,29769,30240,40454
+51315,29770,30239,40626
+58503,29771,30238,40694
+51793,29772,30237,40387
+59864,29773,30236,40133
+57952,29774,30235,40940
+51056,29775,30234,40370
+53609,29776,30233,40318
+52978,29777,30232,40772
+56939,29778,30231,40377
+56076,29779,30230,40048
+59898,29780,30229,40382
+58552,29781,30228,40460
+52455,29782,30227,40169
+50030,29783,30226,40811
+50986,29784,30225,40313
+58583,29785,30224,40622
+57931,29786,30223,40381
+54940,29787,30222,40787
+52129,29788,30221,40308
+58957,29789,30220,40292
+51410,29790,30219,40505
+56381,29791,30218,40990
+55865,29792,30217,40170
+56955,29793,30216,40020
+53909,29794,30215,40627
+55468,29795,30214,40000
+54830,29796,30213,40196
+59939,29797,30212,40006
+55234,29798,30211,40779
+54482,29799,30210,40413
+53054,29800,30209,40623
+59194,29801,30208,40459
+56648,29802,30207,40915
+50655,29803,30206,40908
+55769,29804,30205,40299
+50519,29805,30204,40510
+54760,29806,30203,40359
+55985,29807,30202,40523
+57073,29808,30201,40729
+58242,29809,30200,40580
+55589,29810,30199,40139
+59994,29811,30198,40679
+59259,29812,30197,40586
+57461,29813,30196,40061
+50852,29814,30195,40633
+59150,29815,30194,40331
+55596,29816,30193,40236
+55566,29817,30192,40954
+57036,29818,30191,40202
+52274,29819,30190,40533
+50429,29820,30189,40453
+52026,29821,30188,40807
+58119,29822,30187,40098
+51310,29823,30186,40958
+57911,29824,30185,40031
+53835,29825,30184,40408
+53631,29826,30183,40366
+56539,29827,30182,40408
+58055,29828,30181,40245
+56995,29829,30180,40693
+50289,29830,30179,40790
+52246,29831,30178,40312
+50058,29832,30177,40370
+54966,29833,30176,40383
+57245,29834,30175,40117
+59722,29835,30174,40459
+52673,29836,30173,40046
+59451,29837,30172,40826
+57619,29838,30171,40198
+52185,29839,30170,40442
+52974,29840,30169,40082
+52731,29841,30168,40131
+52198,29842,30167,40366
+54580,29843,30166,40648
+57813,29844,30165,40512
+55344,29845,30164,40346
+59629,29846,30163,40791
+52829,29847,30162,40336
+59260,29848,30161,40534
+53975,29849,30160,40574
+56477,29850,30159,40008
+57452,29851,30158,40953
+57354,29852,30157,40753
+50046,29853,30156,40517
+59823,29854,30155,40599
+51526,29855,30154,40507
+58235,29856,30153,40308
+56486,29857,30152,40763
+59408,29858,30151,40401
+54951,29859,30150,40361
+56793,29860,30149,40760
+58417,29861,30148,40087
+56571,29862,30147,40766
+52601,29863,30146,40399
+53204,29864,30145,40483
+56820,29865,30144,40347
+54624,29866,30143,40295
+52554,29867,30142,40155
+57147,29868,30141,40371
+57718,29869,30140,40746
+58625,29870,30139,40586
+54010,29871,30138,40750
+57008,29872,30137,40546
+57951,29873,30136,40866
+54943,29874,30135,40690
+53578,29875,30134,40490
+56936,29876,30133,40713
+59385,29877,30132,40410
+54928,29878,30131,40792
+53389,29879,30130,40258
+54184,29880,30129,40046
+59970,29881,30128,40746
+56028,29882,30127,40207
+54970,29883,30126,40070
+50616,29884,30125,40128
+57672,29885,30124,40215
+57259,29886,30123,40041
+51607,29887,30122,40862
+56003,29888,30121,40151
+57307,29889,30120,40061
+50136,29890,30119,40997
+54687,29891,30118,40388
+51413,29892,30117,40454
+55005,29893,30116,40885
+56096,29894,30115,40554
+50653,29895,30114,40437
+57467,29896,30113,40619
+55877,29897,30112,40631
+53091,29898,30111,40363
+59166,29899,30110,40918
+59430,29900,30109,40907
+57159,29901,30108,40190
+59282,29902,30107,40002
+57683,29903,30106,40416
+57992,29904,30105,40048
+59789,29905,30104,40225
+51391,29906,30103,40030
+59932,29907,30102,40725
+52500,29908,30101,40373
+55569,29909,30100,40669
+53919,29910,30099,40303
+55517,29911,30098,40011
+50466,29912,30097,40933
+59328,29913,30096,40341
+54154,29914,30095,40158
+53452,29915,30094,40966
+57624,29916,30093,40657
+53608,29917,30092,40597
+59129,29918,30091,40048
+50656,29919,30090,40351
+58337,29920,30089,40609
+58010,29921,30088,40784
+57959,29922,30087,40721
+53347,29923,30086,40585
+56874,29924,30085,40439
+52072,29925,30084,40750
+53825,29926,30083,40075
+56337,29927,30082,40952
+54783,29928,30081,40385
+53895,29929,30080,40738
+53797,29930,30079,40222
+50630,29931,30078,40834
+52978,29932,30077,40470
+55470,29933,30076,40926
+55286,29934,30075,40287
+52579,29935,30074,40919
+50724,29936,30073,40597
+57861,29937,30072,40063
+59413,29938,30071,40915
+56810,29939,30070,40843
+51097,29940,30069,40187
+59718,29941,30068,40917
+56297,29942,30067,40962
+58925,29943,30066,40784
+56616,29944,30065,40635
+52751,29945,30064,40184
+53880,29946,30063,40078
+50882,29947,30062,40222
+55720,29948,30061,40621
+59246,29949,30060,40614
+55867,29950,30059,40531
+50929,29951,30058,40741
+58628,29952,30057,40577
+57123,29953,30056,40314
+55581,29954,30055,40695
+57683,29955,30054,40583
+51832,29956,30053,40851
+58640,29957,30052,40300
+58268,29958,30051,40472
+52533,29959,30050,40616
+56513,29960,30049,40526
+54988,29961,30048,40909
+56070,29962,30047,40306
+54227,29963,30046,40351
+59343,29964,30045,40915
+58539,29965,30044,40017
+53037,29966,30043,40248
+58012,29967,30042,40445
+50582,29968,30041,40457
+57655,29969,30040,40797
+52659,29970,30039,40712
+55076,29971,30038,40898
+58120,29972,30037,40972
+51082,29973,30036,40344
+51422,29974,30035,40466
+58269,29975,30034,40188
+52892,29976,30033,40279
+55617,29977,30032,40547
+50517,29978,30031,40375
+53324,29979,30030,40345
+56170,29980,30029,40875
+56458,29981,30028,40019
+55376,29982,30027,40523
+58186,29983,30026,40529
+56588,29984,30025,40960
+57590,29985,30024,40307
+52603,29986,30023,40156
+50176,29987,30022,40727
+58399,29988,30021,40569
+51079,29989,30020,40404
+53139,29990,30019,40418
+54089,29991,30018,40831
+57631,29992,30017,40186
+54466,29993,30016,40415
+55191,29994,30015,40727
+56803,29995,30014,40215
+50317,29996,30013,40723
+56398,29997,30012,40239
+54488,29998,30011,40495
+55390,29999,30010,40223
+52796,30000,30009,40939
+54403,30001,30008,40994
+52344,30002,30007,40774
+52490,30003,30006,40675
+53103,30004,30005,40663
+52492,30005,30004,40718
+50702,30006,30003,40558
+58573,30007,30002,40685
+53645,30008,30001,40634
+56996,30009,30000,40669
+51249,30010,31000,40376
+58331,30011,30999,40381
+59997,30012,30998,40420
+59004,30013,30997,40411
+50504,30014,30996,40696
+51195,30015,30995,40155
+52042,30016,30994,40446
+59747,30017,30993,40324
+55472,30018,30992,40187
+58280,30019,30991,40217
+54831,30020,30990,40722
+56828,30021,30989,40705
+50725,30022,30988,40737
+57757,30023,30987,40867
+53594,30024,30986,40156
+50491,30025,30985,40193
+55542,30026,30984,40805
+58103,30027,30983,40368
+53693,30028,30982,40213
+53735,30029,30981,40818
+59125,30030,30980,40162
+50471,30031,30979,40960
+58872,30032,30978,40164
+59753,30033,30977,40991
+57253,30034,30976,40197
+57597,30035,30975,40645
+51392,30036,30974,40805
+55771,30037,30973,40880
+59258,30038,30972,40879
+59677,30039,30971,40556
+54235,30040,30970,40284
+56634,30041,30969,40375
+51851,30042,30968,40082
+52781,30043,30967,40107
+58257,30044,30966,40304
+52148,30045,30965,40143
+54723,30046,30964,40609
+54853,30047,30963,40410
+57740,30048,30962,40138
+56154,30049,30961,40669
+58475,30050,30960,40450
+57293,30051,30959,40348
+56296,30052,30958,40637
+53503,30053,30957,40977
+56177,30054,30956,40645
+59393,30055,30955,40388
+50947,30056,30954,40669
+51222,30057,30953,40259
+54677,30058,30952,40283
+55952,30059,30951,40178
+59117,30060,30950,40954
+56727,30061,30949,40552
+55110,30062,30948,40977
+58296,30063,30947,40294
+54857,30064,30946,40268
+53818,30065,30945,40363
+57352,30066,30944,40140
+52508,30067,30943,40065
+54954,30068,30942,40726
+55481,30069,30941,40446
+55664,30070,30940,40189
+52542,30071,30939,40344
+59798,30072,30938,40644
+52146,30073,30937,40825
+59046,30074,30936,40772
+57172,30075,30935,40474
+59702,30076,30934,40254
+58595,30077,30933,40895
+54129,30078,30932,40223
+53270,30079,30931,40887
+53091,30080,30930,40101
+56396,30081,30929,40846
+54574,30082,30928,40849
+58717,30083,30927,40592
+59258,30084,30926,40707
+57152,30085,30925,40932
+58249,30086,30924,40084
+56662,30087,30923,40011
+57897,30088,30922,40173
+52755,30089,30921,40572
+52882,30090,30920,40034
+51405,30091,30919,40166
+54735,30092,30918,40970
+56316,30093,30917,40081
+54969,30094,30916,40053
+58082,30095,30915,40697
+56063,30096,30914,40986
+54612,30097,30913,40256
+55618,30098,30912,40060
+58614,30099,30911,40110
+56813,30100,30910,40441
+51686,30101,30909,40480
+56258,30102,30908,40335
+53758,30103,30907,40162
+51279,30104,30906,40835
+52412,30105,30905,40954
+56335,30106,30904,40583
+56223,30107,30903,40027
+59228,30108,30902,40167
+56280,30109,30901,40754
+53187,30110,30900,40153
+55664,30111,30899,40097
+57040,30112,30898,40575
+51032,30113,30897,40870
+55384,30114,30896,40634
+55898,30115,30895,40663
+59606,30116,30894,40276
+54127,30117,30893,40356
+57165,30118,30892,40497
+56170,30119,30891,40648
+53916,30120,30890,40483
+57154,30121,30889,40925
+55423,30122,30888,40857
+58462,30123,30887,40292
+50274,30124,30886,40804
+53224,30125,30885,40894
+51264,30126,30884,40590
+54563,30127,30883,40170
+58922,30128,30882,40095
+50985,30129,30881,40140
+50772,30130,30880,40557
+52141,30131,30879,40797
+51801,30132,30878,40767
+59984,30133,30877,40669
+56702,30134,30876,40006
+55439,30135,30875,40258
+50484,30136,30874,40057
+57417,30137,30873,40995
+53591,30138,30872,40643
+57694,30139,30871,40821
+57364,30140,30870,40211
+52482,30141,30869,40969
+53959,30142,30868,40768
+57617,30143,30867,40561
+51730,30144,30866,40103
+50925,30145,30865,40957
+53961,30146,30864,40226
+52323,30147,30863,40415
+57914,30148,30862,40327
+50220,30149,30861,40079
+53531,30150,30860,40794
+52591,30151,30859,40651
+59436,30152,30858,40960
+54100,30153,30857,40686
+53931,30154,30856,40897
+57395,30155,30855,40915
+52806,30156,30854,40017
+53050,30157,30853,40359
+53162,30158,30852,40379
+52013,30159,30851,40961
+50234,30160,30850,40170
+56584,30161,30849,40023
+59036,30162,30848,40782
+57088,30163,30847,40590
+50110,30164,30846,40047
+53400,30165,30845,40525
+56358,30166,30844,40226
+58848,30167,30843,40439
+51591,30168,30842,40285
+55351,30169,30841,40851
+55662,30170,30840,40823
+50225,30171,30839,40666
+57836,30172,30838,40193
+50739,30173,30837,40500
+50031,30174,30836,40324
+55147,30175,30835,40991
+51801,30176,30834,40406
+56669,30177,30833,40309
+58369,30178,30832,40324
+56376,30179,30831,40384
+53048,30180,30830,40141
+59768,30181,30829,40293
+56276,30182,30828,40728
+53972,30183,30827,40867
+50921,30184,30826,40315
+56151,30185,30825,40102
+59526,30186,30824,40674
+57228,30187,30823,40964
+55307,30188,30822,40368
+58514,30189,30821,40964
+57576,30190,30820,40389
+59167,30191,30819,40382
+54496,30192,30818,40092
+55553,30193,30817,40514
+53292,30194,30816,40056
+58959,30195,30815,40441
+59868,30196,30814,40683
+50591,30197,30813,40386
+56137,30198,30812,40522
+51521,30199,30811,40170
+52614,30200,30810,40935
+59687,30201,30809,40387
+55541,30202,30808,40644
+54920,30203,30807,40863
+57942,30204,30806,40730
+56778,30205,30805,40946
+55801,30206,30804,40068
+56845,30207,30803,40962
+52580,30208,30802,40035
+51495,30209,30801,40268
+52641,30210,30800,40734
+57818,30211,30799,40163
+58303,30212,30798,40917
+54849,30213,30797,40295
+54201,30214,30796,40507
+57145,30215,30795,40776
+56035,30216,30794,40669
+55869,30217,30793,40630
+59680,30218,30792,40379
+56297,30219,30791,40367
+56945,30220,30790,40786
+53680,30221,30789,40468
+53912,30222,30788,40796
+58713,30223,30787,40704
+58618,30224,30786,40741
+51028,30225,30785,40669
+59363,30226,30784,40154
+50475,30227,30783,40685
+58868,30228,30782,40736
+55907,30229,30781,40082
+52935,30230,30780,40654
+50962,30231,30779,40466
+50499,30232,30778,40625
+59050,30233,30777,40627
+52091,30234,30776,40850
+52307,30235,30775,40687
+50640,30236,30774,40901
+59015,30237,30773,40684
+56695,30238,30772,40017
+54359,30239,30771,40867
+53910,30240,30770,40535
+57115,30241,30769,40986
+58519,30242,30768,40297
+58007,30243,30767,40872
+56831,30244,30766,40848
+59865,30245,30765,40789
+57624,30246,30764,40746
+51342,30247,30763,40908
+58650,30248,30762,40182
+51956,30249,30761,40654
+54105,30250,30760,40325
+52906,30251,30759,40987
+56168,30252,30758,40079
+58964,30253,30757,40319
+57979,30254,30756,40039
+54043,30255,30755,40855
+50138,30256,30754,40268
+59009,30257,30753,40935
+53345,30258,30752,40453
+53474,30259,30751,40554
+52612,30260,30750,40641
+50374,30261,30749,40525
+54720,30262,30748,40960
+52643,30263,30747,40928
+58061,30264,30746,40009
+51519,30265,30745,40922
+58688,30266,30744,40952
+57507,30267,30743,40184
+54044,30268,30742,40835
+59963,30269,30741,40153
+56632,30270,30740,40882
+55417,30271,30739,40409
+57452,30272,30738,40218
+52106,30273,30737,40056
+54597,30274,30736,40364
+50912,30275,30735,40303
+56904,30276,30734,40566
+59727,30277,30733,40638
+53523,30278,30732,40333
+56243,30279,30731,40149
+56783,30280,30730,40989
+51854,30281,30729,40397
+54258,30282,30728,40064
+55952,30283,30727,40585
+54704,30284,30726,40079
+54350,30285,30725,40983
+50322,30286,30724,40975
+53269,30287,30723,40085
+53759,30288,30722,40970
+55033,30289,30721,40305
+59718,30290,30720,40985
+58214,30291,30719,40138
+59630,30292,30718,40628
+55335,30293,30717,40092
+55845,30294,30716,40846
+57284,30295,30715,40348
+57252,30296,30714,40775
+56326,30297,30713,40273
+55476,30298,30712,40167
+51718,30299,30711,40531
+55337,30300,30710,40188
+56788,30301,30709,40227
+58300,30302,30708,40969
+56132,30303,30707,40141
+55047,30304,30706,40739
+56738,30305,30705,40702
+57178,30306,30704,40080
+57862,30307,30703,40919
+50922,30308,30702,40798
+50850,30309,30701,40699
+52649,30310,30700,40430
+54713,30311,30699,40507
+56721,30312,30698,40307
+52410,30313,30697,40763
+52937,30314,30696,40419
+51865,30315,30695,40985
+58698,30316,30694,40585
+51759,30317,30693,40825
+51947,30318,30692,40515
+55096,30319,30691,40888
+52329,30320,30690,40495
+53514,30321,30689,40307
+50499,30322,30688,40165
+54389,30323,30687,40475
+55937,30324,30686,40017
+52550,30325,30685,40791
+50436,30326,30684,40017
+56213,30327,30683,40918
+57928,30328,30682,40549
+57279,30329,30681,40376
+52535,30330,30680,40135
+57454,30331,30679,40719
+54791,30332,30678,40312
+59011,30333,30677,40671
+58803,30334,30676,40571
+55846,30335,30675,40375
+54200,30336,30674,40755
+56021,30337,30673,40476
+51237,30338,30672,40797
+54968,30339,30671,40417
+54390,30340,30670,40937
+53061,30341,30669,40156
+53786,30342,30668,40451
+56318,30343,30667,40319
+53599,30344,30666,40488
+58888,30345,30665,40370
+51024,30346,30664,40079
+58007,30347,30663,40154
+51013,30348,30662,40813
+52789,30349,30661,40328
+58136,30350,30660,40253
+51010,30351,30659,40496
+56024,30352,30658,40370
+52598,30353,30657,40220
+58822,30354,30656,40914
+53201,30355,30655,40341
+58525,30356,30654,40306
+52371,30357,30653,40470
+55759,30358,30652,40144
+58583,30359,30651,40477
+56586,30360,30650,40817
+54394,30361,30649,40710
+57609,30362,30648,40791
+58202,30363,30647,40677
+51510,30364,30646,40501
+57241,30365,30645,40992
+56222,30366,30644,40459
+50321,30367,30643,40247
+51901,30368,30642,40729
+58113,30369,30641,40441
+54872,30370,30640,40122
+58758,30371,30639,40660
+51210,30372,30638,40519
+59573,30373,30637,40472
+54454,30374,30636,40188
+50004,30375,30635,40001
+50187,30376,30634,40059
+50168,30377,30633,40024
+56198,30378,30632,40747
+54137,30379,30631,40845
+55952,30380,30630,40036
+58322,30381,30629,40442
+52344,30382,30628,40218
+58389,30383,30627,40574
+50684,30384,30626,40852
+58971,30385,30625,40189
+54385,30386,30624,40264
+59257,30387,30623,40521
+50706,30388,30622,40389
+52014,30389,30621,40104
+58372,30390,30620,40916
+59497,30391,30619,40025
+56008,30392,30618,40899
+54679,30393,30617,40171
+57450,30394,30616,40151
+53594,30395,30615,40189
+51975,30396,30614,40694
+58229,30397,30613,40476
+52619,30398,30612,40391
+55699,30399,30611,40452
+55592,30400,30610,40006
+55544,30401,30609,40285
+56675,30402,30608,40808
+50831,30403,30607,40065
+59080,30404,30606,40615
+59779,30405,30605,40144
+54285,30406,30604,40322
+58618,30407,30603,40489
+53748,30408,30602,40493
+50394,30409,30601,40469
+50554,30410,30600,40912
+51279,30411,30599,40257
+59741,30412,30598,40798
+59029,30413,30597,40328
+59131,30414,30596,40741
+55456,30415,30595,40586
+57877,30416,30594,40220
+52651,30417,30593,40374
+57090,30418,30592,40710
+59869,30419,30591,40415
+51072,30420,30590,40530
+50151,30421,30589,40589
+56613,30422,30588,40180
+55402,30423,30587,40907
+52180,30424,30586,40164
+54646,30425,30585,40281
+54667,30426,30584,40713
+51838,30427,30583,40078
+54904,30428,30582,40833
+52599,30429,30581,40395
+54188,30430,30580,40072
+53518,30431,30579,40894
+53272,30432,30578,40498
+56393,30433,30577,40978
+54708,30434,30576,40086
+59910,30435,30575,40952
+55662,30436,30574,40842
+57535,30437,30573,40644
+53904,30438,30572,40013
+56927,30439,30571,40646
+53002,30440,30570,40477
+55183,30441,30569,40394
+51150,30442,30568,40934
+51369,30443,30567,40743
+55307,30444,30566,40323
+56798,30445,30565,40481
+51649,30446,30564,40676
+56211,30447,30563,40372
+57764,30448,30562,40804
+55509,30449,30561,40534
+55259,30450,30560,40533
+53373,30451,30559,40078
+57959,30452,30558,40002
+58767,30453,30557,40799
+58577,30454,30556,40169
+55804,30455,30555,40555
+56700,30456,30554,40764
+57384,30457,30553,40934
+59946,30458,30552,40763
+54006,30459,30551,40713
+54331,30460,30550,40447
+56319,30461,30549,40990
+59727,30462,30548,40253
+56497,30463,30547,40431
+54525,30464,30546,40581
+59092,30465,30545,40758
+54764,30466,30544,40974
+55608,30467,30543,40512
+52158,30468,30542,40927
+53882,30469,30541,40235
+57421,30470,30540,40349
+53509,30471,30539,40640
+51406,30472,30538,40916
+58741,30473,30537,40351
+50386,30474,30536,40172
+50647,30475,30535,40999
+57162,30476,30534,40018
+58829,30477,30533,40350
+56395,30478,30532,40087
+56270,30479,30531,40227
+50983,30480,30530,40801
+50585,30481,30529,40301
+55563,30482,30528,40084
+56601,30483,30527,40119
+58344,30484,30526,40933
+59572,30485,30525,40844
+58184,30486,30524,40875
+58718,30487,30523,40681
+54578,30488,30522,40266
+52428,30489,30521,40899
+53433,30490,30520,40253
+53698,30491,30519,40365
+59934,30492,30518,40191
+53901,30493,30517,40141
+58966,30494,30516,40751
+56188,30495,30515,40452
+59751,30496,30514,40902
+59081,30497,30513,40265
+59404,30498,30512,40429
+53126,30499,30511,40986
+58599,30500,30510,40488
+59859,30501,30509,40038
+58550,30502,30508,40593
+56939,30503,30507,40529
+54990,30504,30506,40562
+55422,30505,30505,40940
+56484,30506,30504,40955
+58426,30507,30503,40352
+52731,30508,30502,40948
+51826,30509,30501,40991
+57111,30510,30500,40303
+53265,30511,30499,40768
+56992,30512,30498,40782
+54199,30513,30497,40941
+52114,30514,30496,40228
+50434,30515,30495,40858
+51425,30516,30494,40067
+51336,30517,30493,40068
+54986,30518,30492,40985
+50561,30519,30491,40308
+54821,30520,30490,40383
+54657,30521,30489,40279
+52423,30522,30488,40937
+55859,30523,30487,40131
+50995,30524,30486,40899
+56161,30525,30485,40528
+52864,30526,30484,40781
+57071,30527,30483,40160
+53830,30528,30482,40157
+52728,30529,30481,40412
+52498,30530,30480,40486
+59793,30531,30479,40045
+58297,30532,30478,40530
+54697,30533,30477,40188
+57626,30534,30476,40625
+53026,30535,30475,40750
+53819,30536,30474,40950
+54201,30537,30473,40848
+52997,30538,30472,40586
+51360,30539,30471,40667
+52572,30540,30470,40216
+51566,30541,30469,40803
+50422,30542,30468,40828
+54508,30543,30467,40482
+54126,30544,30466,40329
+51680,30545,30465,40500
+57408,30546,30464,40101
+57334,30547,30463,40638
+53940,30548,30462,40195
+50826,30549,30461,40080
+57592,30550,30460,40548
+55187,30551,30459,40437
+50053,30552,30458,40371
+54105,30553,30457,40243
+55508,30554,30456,40820
+50151,30555,30455,40523
+52201,30556,30454,40944
+54427,30557,30453,40126
+51053,30558,30452,40856
+51435,30559,30451,40561
+58122,30560,30450,40780
+54866,30561,30449,40983
+58336,30562,30448,40639
+55223,30563,30447,40610
+53205,30564,30446,40648
+56814,30565,30445,40910
+57784,30566,30444,40311
+53896,30567,30443,40946
+55302,30568,30442,40505
+58406,30569,30441,40620
+58406,30570,30440,40435
+57691,30571,30439,40567
+59887,30572,30438,40615
+54319,30573,30437,40011
+53321,30574,30436,40166
+58271,30575,30435,40658
+58816,30576,30434,40066
+54946,30577,30433,40042
+52293,30578,30432,40782
+54135,30579,30431,40572
+52736,30580,30430,40333
+55610,30581,30429,40864
+58816,30582,30428,40324
+50197,30583,30427,40195
+56226,30584,30426,40229
+50323,30585,30425,40310
+57504,30586,30424,40017
+52416,30587,30423,40620
+59424,30588,30422,40473
+54588,30589,30421,40488
+51134,30590,30420,40480
+53406,30591,30419,40976
+56900,30592,30418,40128
+54533,30593,30417,40978
+57045,30594,30416,40448
+53338,30595,30415,40297
+53705,30596,30414,40344
+57473,30597,30413,40852
+52866,30598,30412,40057
+50575,30599,30411,40219
+53822,30600,30410,40243
+57429,30601,30409,40167
+56501,30602,30408,40115
+53338,30603,30407,40461
+54450,30604,30406,40481
+51895,30605,30405,40927
+56159,30606,30404,40204
+56923,30607,30403,40927
+52073,30608,30402,40482
+58175,30609,30401,40329
+54146,30610,30400,40528
+50452,30611,30399,40732
+57445,30612,30398,40683
+56824,30613,30397,40748
+51876,30614,30396,40976
+54152,30615,30395,40087
+53117,30616,30394,40574
+58607,30617,30393,40201
+54475,30618,30392,40022
+58109,30619,30391,40671
+53705,30620,30390,40292
+53123,30621,30389,40173
+58065,30622,30388,40629
+54842,30623,30387,40508
+59282,30624,30386,40004
+57043,30625,30385,40036
+51873,30626,30384,40403
+58809,30627,30383,40705
+51334,30628,30382,40866
+59954,30629,30381,40741
+51767,30630,30380,40063
+57541,30631,30379,40426
+57635,30632,30378,40515
+55574,30633,30377,40898
+59572,30634,30376,40757
+50669,30635,30375,40533
+54759,30636,30374,40813
+51236,30637,30373,40958
+57368,30638,30372,40975
+57327,30639,30371,40611
+52278,30640,30370,40425
+54501,30641,30369,40111
+52309,30642,30368,40267
+52477,30643,30367,40957
+50291,30644,30366,40978
+55556,30645,30365,40635
+58306,30646,30364,40953
+57403,30647,30363,40913
+58564,30648,30362,40773
+57949,30649,30361,40708
+55077,30650,30360,40612
+58286,30651,30359,40124
+51554,30652,30358,40921
+57567,30653,30357,40798
+56103,30654,30356,40678
+51997,30655,30355,40670
+54790,30656,30354,40774
+56139,30657,30353,40717
+50084,30658,30352,40045
+51844,30659,30351,40859
+59453,30660,30350,40487
+51394,30661,30349,40842
+50872,30662,30348,40354
+53484,30663,30347,40078
+56805,30664,30346,40130
+51945,30665,30345,40628
+53442,30666,30344,40359
+56420,30667,30343,40406
+59325,30668,30342,40420
+51288,30669,30341,40641
+54628,30670,30340,40158
+53914,30671,30339,40379
+58088,30672,30338,40399
+50806,30673,30337,40748
+58096,30674,30336,40654
+56738,30675,30335,40224
+56938,30676,30334,40925
+59327,30677,30333,40094
+57740,30678,30332,40799
+53837,30679,30331,40192
+51737,30680,30330,40588
+55438,30681,30329,40015
+56463,30682,30328,40018
+50275,30683,30327,40514
+58060,30684,30326,40609
+59778,30685,30325,40291
+56369,30686,30324,40891
+51481,30687,30323,40790
+53455,30688,30322,40655
+56940,30689,30321,40343
+58414,30690,30320,40313
+57403,30691,30319,40799
+57789,30692,30318,40975
+56479,30693,30317,40082
+57523,30694,30316,40610
+52275,30695,30315,40992
+59092,30696,30314,40215
+51119,30697,30313,40423
+54239,30698,30312,40825
+59583,30699,30311,40201
+58756,30700,30310,40397
+58066,30701,30309,40879
+57763,30702,30308,40610
+55502,30703,30307,40538
+51167,30704,30306,40939
+55297,30705,30305,40737
+55698,30706,30304,40098
+50307,30707,30303,40684
+51770,30708,30302,40130
+52055,30709,30301,40172
+54001,30710,30300,40913
+58128,30711,30299,40909
+58250,30712,30298,40128
+56976,30713,30297,40337
+52039,30714,30296,40817
+53724,30715,30295,40562
+59016,30716,30294,40260
+51867,30717,30293,40467
+58846,30718,30292,40342
+57805,30719,30291,40755
+50493,30720,30290,40606
+58041,30721,30289,40392
+52143,30722,30288,40893
+53566,30723,30287,40308
+58582,30724,30286,40754
+52059,30725,30285,40094
+57930,30726,30284,40207
+57306,30727,30283,40072
+52416,30728,30282,40571
+57558,30729,30281,40624
+57019,30730,30280,40613
+55614,30731,30279,40726
+51580,30732,30278,40801
+52402,30733,30277,40041
+55381,30734,30276,40451
+53834,30735,30275,40454
+55500,30736,30274,40141
+50470,30737,30273,40278
+53067,30738,30272,40638
+50032,30739,30271,40858
+56916,30740,30270,40954
+52366,30741,30269,40958
+51297,30742,30268,40632
+51059,30743,30267,40519
+56201,30744,30266,40732
+56213,30745,30265,40178
+53663,30746,30264,40281
+58505,30747,30263,40531
+50309,30748,30262,40274
+53050,30749,30261,40407
+59886,30750,30260,40109
+51161,30751,30259,40542
+53648,30752,30258,40548
+59103,30753,30257,40984
+51432,30754,30256,40626
+56221,30755,30255,40683
+53768,30756,30254,40031
+54420,30757,30253,40054
+56923,30758,30252,40688
+56993,30759,30251,40837
+58105,30760,30250,40062
+55408,30761,30249,40634
+59894,30762,30248,40825
+54385,30763,30247,40060
+50037,30764,30246,40768
+59544,30765,30245,40191
+56891,30766,30244,40064
+53038,30767,30243,40368
+51950,30768,30242,40343
+58870,30769,30241,40794
+51947,30770,30240,40443
+52682,30771,30239,40794
+59350,30772,30238,40916
+56322,30773,30237,40811
+54844,30774,30236,40867
+53190,30775,30235,40407
+54869,30776,30234,40982
+53436,30777,30233,40684
+51222,30778,30232,40273
+56235,30779,30231,40558
+58354,30780,30230,40946
+59296,30781,30229,40966
+57920,30782,30228,40784
+56797,30783,30227,40191
+52392,30784,30226,40785
+50191,30785,30225,40992
+57752,30786,30224,40447
+58313,30787,30223,40227
+56723,30788,30222,40247
+50316,30789,30221,40658
+53224,30790,30220,40926
+50120,30791,30219,40624
+54240,30792,30218,40383
+50140,30793,30217,40355
+53537,30794,30216,40333
+57161,30795,30215,40516
+56784,30796,30214,40786
+56965,30797,30213,40544
+51779,30798,30212,40362
+58235,30799,30211,40295
+51156,30800,30210,40991
+52463,30801,30209,40326
+52338,30802,30208,40217
+51535,30803,30207,40083
+50802,30804,30206,40129
+52021,30805,30205,40917
+56994,30806,30204,40715
+57418,30807,30203,40773
+58926,30808,30202,40166
+59029,30809,30201,40180
+52188,30810,30200,40298
+59056,30811,30199,40835
+58481,30812,30198,40244
+51051,30813,30197,40569
+56311,30814,30196,40496
+50026,30815,30195,40805
+52630,30816,30194,40580
+52953,30817,30193,40048
+50158,30818,30192,40317
+54592,30819,30191,40197
+51143,30820,30190,40433
+58770,30821,30189,40137
+52784,30822,30188,40064
+59391,30823,30187,40993
+58836,30824,30186,40335
+51114,30825,30185,40852
+55242,30826,30184,40514
+52596,30827,30183,40454
+51563,30828,30182,40510
+54839,30829,30181,40581
+58991,30830,30180,40960
+57759,30831,30179,40012
+50864,30832,30178,40797
+55634,30833,30177,40170
+55296,30834,30176,40927
+50113,30835,30175,40902
+51271,30836,30174,40881
+59686,30837,30173,40577
+58868,30838,30172,40216
+50813,30839,30171,40320
+57204,30840,30170,40774
+55136,30841,30169,40301
+50193,30842,30168,40811
+51226,30843,30167,40409
+57185,30844,30166,40218
+51069,30845,30165,40840
+52768,30846,30164,40125
+55662,30847,30163,40353
+58536,30848,30162,40166
+50014,30849,30161,40592
+55308,30850,30160,40820
+54452,30851,30159,40473
+55556,30852,30158,40318
+50278,30853,30157,40595
+53171,30854,30156,40911
+57783,30855,30155,40103
+53252,30856,30154,40118
+54106,30857,30153,40620
+59443,30858,30152,40784
+52020,30859,30151,40361
+56088,30860,30150,40452
+51088,30861,30149,40030
+53438,30862,30148,40859
+50764,30863,30147,40487
+59697,30864,30146,40027
+58144,30865,30145,40797
+57462,30866,30144,40574
+59647,30867,30143,40555
+53460,30868,30142,40779
+58901,30869,30141,40186
+56416,30870,30140,40081
+56305,30871,30139,40561
+51432,30872,30138,40419
+53645,30873,30137,40701
+50997,30874,30136,40389
+57832,30875,30135,40545
+55171,30876,30134,40354
+53956,30877,30133,40738
+51324,30878,30132,40375
+53915,30879,30131,40842
+56384,30880,30130,40199
+57036,30881,30129,40831
+59329,30882,30128,40139
+58500,30883,30127,40382
+54971,30884,30126,40973
+59496,30885,30125,40665
+54145,30886,30124,40946
+56623,30887,30123,40947
+50850,30888,30122,40019
+52988,30889,30121,40772
+55355,30890,30120,40454
+59016,30891,30119,40198
+54422,30892,30118,40270
+58270,30893,30117,40746
+50847,30894,30116,40555
+58560,30895,30115,40039
+50903,30896,30114,40582
+56424,30897,30113,40968
+54610,30898,30112,40798
+52872,30899,30111,40929
+56236,30900,30110,40624
+56067,30901,30109,40566
+57528,30902,30108,40447
+59044,30903,30107,40032
+54936,30904,30106,40280
+58732,30905,30105,40357
+52888,30906,30104,40624
+58202,30907,30103,40051
+56709,30908,30102,40220
+51234,30909,30101,40496
+58044,30910,30100,40645
+51648,30911,30099,40649
+52796,30912,30098,40830
+57589,30913,30097,40837
+55885,30914,30096,40723
+54160,30915,30095,40493
+51674,30916,30094,40392
+50857,30917,30093,40019
+54810,30918,30092,40003
+59174,30919,30091,40900
+50333,30920,30090,40647
+59488,30921,30089,40724
+59314,30922,30088,40501
+59232,30923,30087,40652
+54149,30924,30086,40737
+53155,30925,30085,40338
+51508,30926,30084,40543
+55933,30927,30083,40837
+56067,30928,30082,40361
+56473,30929,30081,40882
+50838,30930,30080,40582
+56030,30931,30079,40921
+52764,30932,30078,40376
+51566,30933,30077,40006
+56267,30934,30076,40257
+50856,30935,30075,40420
+55094,30936,30074,40379
+54909,30937,30073,40756
+52587,30938,30072,40059
+57392,30939,30071,40758
+57336,30940,30070,40185
+54598,30941,30069,40986
+52110,30942,30068,40631
+51852,30943,30067,40814
+56622,30944,30066,40396
+59736,30945,30065,40572
+54464,30946,30064,40083
+52184,30947,30063,40614
+53299,30948,30062,40309
+55783,30949,30061,40385
+52367,30950,30060,40039
+54190,30951,30059,40723
+59877,30952,30058,40737
+53991,30953,30057,40315
+50600,30954,30056,40597
+57324,30955,30055,40604
+54933,30956,30054,40696
+58142,30957,30053,40997
+51533,30958,30052,40848
+58770,30959,30051,40992
+55267,30960,30050,40922
+55093,30961,30049,40653
+53937,30962,30048,40575
+53537,30963,30047,40066
+55797,30964,30046,40442
+59047,30965,30045,40961
+55449,30966,30044,40953
+52892,30967,30043,40606
+59264,30968,30042,40901
+53153,30969,30041,40427
+59356,30970,30040,40967
+50474,30971,30039,40731
+58685,30972,30038,40641
+54560,30973,30037,40943
+59716,30974,30036,40487
+58625,30975,30035,40248
+56437,30976,30034,40960
+56373,30977,30033,40450
+57563,30978,30032,40989
+51372,30979,30031,40056
+54501,30980,30030,40956
+52835,30981,30029,40346
+56494,30982,30028,40297
+52262,30983,30027,40344
+56891,30984,30026,40450
+50767,30985,30025,40266
+55597,30986,30024,40701
+54812,30987,30023,40262
+57046,30988,30022,40017
+53599,30989,30021,40116
+51128,30990,30020,40462
+59903,30991,30019,40233
+51673,30992,30018,40018
+51083,30993,30017,40150
+58163,30994,30016,40672
+51807,30995,30015,40789
+55751,30996,30014,40175
+59007,30997,30013,40488
+59058,30998,30012,40324
+56496,30999,30011,40374
+55416,31000,30010,40413
+53478,31001,30009,40546
+51789,31002,30008,40756
+55848,31003,30007,40569
+58819,31004,30006,40604
+59184,31005,30005,40640
+55213,31006,30004,40778
+54814,31007,30003,40448
+58705,31008,30002,40179
+52700,31009,30001,40303
+51612,31010,30000,40734
+51501,31011,31000,40516
+57792,31012,30999,40950
+54046,31013,30998,40535
+51817,31014,30997,40303
+56606,31015,30996,40738
+55725,31016,30995,40578
+56010,31017,30994,40120
+56385,31018,30993,40510
+55329,31019,30992,40449
+56216,31020,30991,40965
+50166,31021,30990,40943
+56742,31022,30989,40288
+54100,31023,30988,40712
+56227,31024,30987,40328
+55468,31025,30986,40192
+52380,31026,30985,40514
+53049,31027,30984,40802
+56051,31028,30983,40742
+54357,31029,30982,40010
+58406,31030,30981,40029
+54036,31031,30980,40163
+54477,31032,30979,40188
+51134,31033,30978,40696
+50093,31034,30977,40569
+57707,31035,30976,40385
+53482,31036,30975,40649
+54550,31037,30974,40396
+53993,31038,30973,40531
+56932,31039,30972,40989
+55219,31040,30971,40077
+52030,31041,30970,40012
+55173,31042,30969,40217
+50681,31043,30968,40810
+52452,31044,30967,40387
+52295,31045,30966,40439
+58574,31046,30965,40498
+53942,31047,30964,40426
+51964,31048,30963,40792
+51239,31049,30962,40286
+50244,31050,30961,40984
+52409,31051,30960,40214
+56683,31052,30959,40987
+59268,31053,30958,40965
+59400,31054,30957,40815
+53841,31055,30956,40020
+58659,31056,30955,40869
+52758,31057,30954,40153
+56412,31058,30953,40189
+50912,31059,30952,40640
+57496,31060,30951,40392
+57721,31061,30950,40169
+52853,31062,30949,40674
+50649,31063,30948,40365
+58191,31064,30947,40714
+56408,31065,30946,40894
+54700,31066,30945,40575
+51169,31067,30944,40691
+54609,31068,30943,40744
+54371,31069,30942,40725
+52760,31070,30941,40008
+52865,31071,30940,40008
+56857,31072,30939,40269
+56025,31073,30938,40049
+50741,31074,30937,40028
+53762,31075,30936,40633
+57185,31076,30935,40738
+56766,31077,30934,40643
+54207,31078,30933,40713
+58991,31079,30932,40584
+53849,31080,30931,40437
+51398,31081,30930,40340
+50199,31082,30929,40970
+56566,31083,30928,40644
+53408,31084,30927,40012
+50887,31085,30926,40095
+53132,31086,30925,40883
+58839,31087,30924,40006
+59400,31088,30923,40665
+55706,31089,30922,40849
+52899,31090,30921,40960
+57858,31091,30920,40184
+50965,31092,30919,40445
+56939,31093,30918,40930
+54016,31094,30917,40339
+58781,31095,30916,40198
+52350,31096,30915,40782
+53588,31097,30914,40340
+56730,31098,30913,40037
+52924,31099,30912,40846
+59755,31100,30911,40715
+55925,31101,30910,40368
+58826,31102,30909,40634
+54022,31103,30908,40465
+56372,31104,30907,40676
+53150,31105,30906,40525
+52875,31106,30905,40295
+59750,31107,30904,40713
+51800,31108,30903,40236
+52147,31109,30902,40699
+53147,31110,30901,40325
+54343,31111,30900,40715
+50147,31112,30899,40657
+58199,31113,30898,40705
+50808,31114,30897,40445
+54903,31115,30896,40652
+57766,31116,30895,40600
+58609,31117,30894,40228
+59732,31118,30893,40771
+58228,31119,30892,40022
+59714,31120,30891,40867
+50153,31121,30890,40514
+53586,31122,30889,40444
+52641,31123,30888,40257
+52668,31124,30887,40492
+57835,31125,30886,40448
+52901,31126,30885,40714
+58930,31127,30884,40172
+51067,31128,30883,40473
+53410,31129,30882,40801
+58423,31130,30881,40724
+59131,31131,30880,40123
+53365,31132,30879,40840
+55150,31133,30878,40864
+51263,31134,30877,40600
+54868,31135,30876,40912
+51269,31136,30875,40569
+52622,31137,30874,40618
+53721,31138,30873,40660
+53799,31139,30872,40100
+56151,31140,30871,40185
+55910,31141,30870,40486
+51986,31142,30869,40261
+58672,31143,30868,40940
+53402,31144,30867,40547
+50042,31145,30866,40782
+59593,31146,30865,40128
+53660,31147,30864,40910
+52524,31148,30863,40822
+56512,31149,30862,40606
+54415,31150,30861,40255
+58266,31151,30860,40267
+54497,31152,30859,40252
+57138,31153,30858,40145
+53943,31154,30857,40457
+55036,31155,30856,40141
+53132,31156,30855,40454
+52489,31157,30854,40753
+50534,31158,30853,40621
+54246,31159,30852,40199
+51828,31160,30851,40234
+52082,31161,30850,40838
+51779,31162,30849,40564
+58702,31163,30848,40626
+52496,31164,30847,40903
+57100,31165,30846,40163
+53784,31166,30845,40203
+57834,31167,30844,40228
+58411,31168,30843,40405
+50088,31169,30842,40141
+55662,31170,30841,40344
+56452,31171,30840,40248
+58240,31172,30839,40816
+53799,31173,30838,40965
+52037,31174,30837,40275
+51129,31175,30836,40434
+55038,31176,30835,40628
+55423,31177,30834,40987
+57481,31178,30833,40084
+52357,31179,30832,40513
+50308,31180,30831,40943
+55094,31181,30830,40194
+50637,31182,30829,40761
+51752,31183,30828,40806
+54229,31184,30827,40275
+54981,31185,30826,40073
+50340,31186,30825,40478
+55955,31187,30824,40937
+54406,31188,30823,40363
+59237,31189,30822,40086
+51378,31190,30821,40684
+52155,31191,30820,40129
+57003,31192,30819,40522
+56501,31193,30818,40826
+54463,31194,30817,40876
+54917,31195,30816,40826
+57213,31196,30815,40857
+59217,31197,30814,40369
+55955,31198,30813,40035
+53735,31199,30812,40157
+58726,31200,30811,40104
+53404,31201,30810,40097
+57723,31202,30809,40846
+53805,31203,30808,40965
+55922,31204,30807,40594
+51767,31205,30806,40063
+59026,31206,30805,40125
+56785,31207,30804,40598
+52319,31208,30803,40355
+59143,31209,30802,40463
+55576,31210,30801,40217
+54514,31211,30800,40390
+57565,31212,30799,40010
+59986,31213,30798,40171
+59342,31214,30797,40351
+58493,31215,30796,40494
+55951,31216,30795,40985
+58626,31217,30794,40480
+52840,31218,30793,40860
+56710,31219,30792,40560
+58040,31220,30791,40594
+59120,31221,30790,40414
+53081,31222,30789,40736
+58080,31223,30788,40727
+53422,31224,30787,40409
+55913,31225,30786,40302
+52969,31226,30785,40093
+53232,31227,30784,40772
+52714,31228,30783,40454
+57260,31229,30782,40075
+57906,31230,30781,40439
+57565,31231,30780,40221
+59990,31232,30779,40002
+57434,31233,30778,40067
+50990,31234,30777,40884
+52226,31235,30776,40141
+53996,31236,30775,40078
+55044,31237,30774,40606
+53834,31238,30773,40675
+57450,31239,30772,40024
+58925,31240,30771,40177
+58480,31241,30770,40707
+58374,31242,30769,40158
+53619,31243,30768,40141
+59447,31244,30767,40473
+59700,31245,30766,40535
+51955,31246,30765,40297
+59259,31247,30764,40060
+57981,31248,30763,40254
+55433,31249,30762,40087
+58786,31250,30761,40818
+52838,31251,30760,40272
+54115,31252,30759,40660
+55000,31253,30758,40994
+58075,31254,30757,40305
+51589,31255,30756,40044
+51247,31256,30755,40159
+52608,31257,30754,40754
+54003,31258,30753,40927
+57423,31259,30752,40687
+59673,31260,30751,40030
+56749,31261,30750,40441
+55124,31262,30749,40640
+58027,31263,30748,40537
+56059,31264,30747,40306
+50706,31265,30746,40646
+55897,31266,30745,40971
+56692,31267,30744,40805
+58688,31268,30743,40109
+50960,31269,30742,40263
+57662,31270,30741,40310
+51636,31271,30740,40431
+59960,31272,30739,40814
+56636,31273,30738,40694
+56004,31274,30737,40012
+55672,31275,30736,40325
+56547,31276,30735,40297
+59820,31277,30734,40119
+58746,31278,30733,40249
+57122,31279,30732,40264
+59073,31280,30731,40569
+55402,31281,30730,40797
+52836,31282,30729,40281
+55185,31283,30728,40201
+59288,31284,30727,40840
+58895,31285,30726,40306
+58792,31286,30725,40494
+51002,31287,30724,40270
+59837,31288,30723,40899
+59992,31289,30722,40789
+51228,31290,30721,40265
+56514,31291,30720,40004
+51833,31292,30719,40312
+59781,31293,30718,40835
+55929,31294,30717,40206
+52450,31295,30716,40396
+55125,31296,30715,40520
+52036,31297,30714,40656
+53114,31298,30713,40834
+53681,31299,30712,40336
+57502,31300,30711,40199
+50778,31301,30710,40421
+50941,31302,30709,40798
+57197,31303,30708,40516
+50833,31304,30707,40193
+55322,31305,30706,40625
+58320,31306,30705,40291
+54274,31307,30704,40015
+59711,31308,30703,40653
+51627,31309,30702,40562
+56356,31310,30701,40676
+58768,31311,30700,40857
+52122,31312,30699,40739
+56901,31313,30698,40315
+55070,31314,30697,40537
+59081,31315,30696,40357
+55750,31316,30695,40631
+50212,31317,30694,40493
+57440,31318,30693,40944
+59599,31319,30692,40574
+58440,31320,30691,40077
+51459,31321,30690,40568
+56708,31322,30689,40108
+54506,31323,30688,40195
+58170,31324,30687,40436
+55586,31325,30686,40256
+57153,31326,30685,40112
+59904,31327,30684,40012
+54420,31328,30683,40486
+51713,31329,30682,40050
+58067,31330,30681,40810
+54987,31331,30680,40100
+54580,31332,30679,40187
+54338,31333,30678,40894
+53031,31334,30677,40826
+54886,31335,30676,40711
+56791,31336,30675,40907
+56378,31337,30674,40636
+59915,31338,30673,40778
+52159,31339,30672,40723
+55901,31340,30671,40555
+54463,31341,30670,40609
+58930,31342,30669,40165
+58486,31343,30668,40456
+55272,31344,30667,40156
+51547,31345,30666,40265
+53790,31346,30665,40961
+56428,31347,30664,40018
+52044,31348,30663,40905
+51255,31349,30662,40068
+52100,31350,30661,40647
+53813,31351,30660,40731
+58480,31352,30659,40662
+53808,31353,30658,40978
+59483,31354,30657,40288
+51014,31355,30656,40923
+52316,31356,30655,40116
+50118,31357,30654,40097
+57084,31358,30653,40031
+57942,31359,30652,40330
+55705,31360,30651,40569
+54535,31361,30650,40955
+55539,31362,30649,40035
+54193,31363,30648,40370
+51443,31364,30647,40458
+52776,31365,30646,40026
+56604,31366,30645,40507
+52000,31367,30644,40207
+58089,31368,30643,40563
+54300,31369,30642,40015
+57798,31370,30641,40233
+50945,31371,30640,40016
+54827,31372,30639,40075
+50778,31373,30638,40907
+51923,31374,30637,40738
+51301,31375,30636,40381
+55563,31376,30635,40731
+56218,31377,30634,40278
+51813,31378,30633,40954
+56700,31379,30632,40520
+59260,31380,30631,40349
+52696,31381,30630,40638
+59266,31382,30629,40618
+58554,31383,30628,40190
+54182,31384,30627,40662
+51711,31385,30626,40357
+53297,31386,30625,40636
+50934,31387,30624,40053
+59247,31388,30623,40238
+59247,31389,30622,40925
+56618,31390,30621,40514
+52133,31391,30620,40085
+59090,31392,30619,40532
+58232,31393,30618,40805
+57770,31394,30617,40124
+57842,31395,30616,40201
+55116,31396,30615,40828
+51340,31397,30614,40883
+56460,31398,30613,40329
+55865,31399,30612,40485
+50537,31400,30611,40235
+51568,31401,30610,40736
+58224,31402,30609,40138
+56247,31403,30608,40975
+55691,31404,30607,40223
+51130,31405,30606,40425
+56007,31406,30605,40916
+53371,31407,30604,40759
+54563,31408,30603,40577
+50708,31409,30602,40587
+52852,31410,30601,40972
+54376,31411,30600,40176
+54613,31412,30599,40524
+52791,31413,30598,40732
+58124,31414,30597,40946
+56288,31415,30596,40583
+57624,31416,30595,40613
+52017,31417,30594,40430
+57942,31418,30593,40880
+53257,31419,30592,40681
+53827,31420,30591,40970
+57471,31421,30590,40589
+56732,31422,30589,40218
+50028,31423,30588,40598
+56705,31424,30587,40893
+53731,31425,30586,40501
+59944,31426,30585,40486
+58208,31427,30584,40226
+53389,31428,30583,40528
+58526,31429,30582,40110
+52903,31430,30581,40779
+54723,31431,30580,40920
+50598,31432,30579,40464
+59350,31433,30578,40506
+58089,31434,30577,40571
+52897,31435,30576,40391
+59292,31436,30575,40860
+56593,31437,30574,40779
+57294,31438,30573,40922
+53306,31439,30572,40839
+52080,31440,30571,40821
+53320,31441,30570,40343
+58810,31442,30569,40102
+51888,31443,30568,40586
+58519,31444,30567,40997
+59141,31445,30566,40238
+53025,31446,30565,40047
+55126,31447,30564,40853
+53896,31448,30563,40514
+59908,31449,30562,40512
+59755,31450,30561,40230
+51746,31451,30560,40262
+58377,31452,30559,40313
+51173,31453,30558,40473
+56865,31454,30557,40391
+57931,31455,30556,40661
+51045,31456,30555,40029
+58231,31457,30554,40669
+55645,31458,30553,40824
+59930,31459,30552,40301
+51128,31460,30551,40919
+58831,31461,30550,40774
+58861,31462,30549,40234
+58573,31463,30548,40489
+58704,31464,30547,40301
+59119,31465,30546,40875
+50753,31466,30545,40429
+54591,31467,30544,40585
+53886,31468,30543,40061
+54638,31469,30542,40825
+50232,31470,30541,40269
+56145,31471,30540,40508
+58083,31472,30539,40168
+54934,31473,30538,40332
+51510,31474,30537,40569
+50644,31475,30536,40029
+53651,31476,30535,40578
+57024,31477,30534,40219
+52417,31478,30533,40664
+55132,31479,30532,40649
+57972,31480,30531,40494
+55162,31481,30530,40025
+53838,31482,30529,40796
+55438,31483,30528,40413
+55743,31484,30527,40139
+53617,31485,30526,40016
+56104,31486,30525,40848
+55874,31487,30524,40745
+55054,31488,30523,40955
+58118,31489,30522,40757
+58633,31490,30521,40923
+52849,31491,30520,40473
+58300,31492,30519,40001
+54653,31493,30518,40225
+56847,31494,30517,40349
+50935,31495,30516,40088
+57944,31496,30515,40832
+57065,31497,30514,40996
+53848,31498,30513,40127
+53633,31499,30512,40189
+54529,31500,30511,40368
+52417,31501,30510,40106
+54245,31502,30509,40381
+53792,31503,30508,40191
+52826,31504,30507,40778
+50078,31505,30506,40519
+57354,31506,30505,40428
+58500,31507,30504,40955
+53831,31508,30503,40456
+53930,31509,30502,40180
+57612,31510,30501,40470
+51890,31511,30500,40978
+58046,31512,30499,40976
+52833,31513,30498,40465
+51793,31514,30497,40762
+55742,31515,30496,40325
+50583,31516,30495,40032
+53683,31517,30494,40851
+55155,31518,30493,40470
+50673,31519,30492,40475
+53116,31520,30491,40248
+53856,31521,30490,40757
+57421,31522,30489,40679
+56691,31523,30488,40159
+52569,31524,30487,40005
+50874,31525,30486,40872
+59660,31526,30485,40477
+54225,31527,30484,40524
+53788,31528,30483,40690
+50246,31529,30482,40621
+53278,31530,30481,40754
+54296,31531,30480,40434
+53461,31532,30479,40862
+59066,31533,30478,40326
+52102,31534,30477,40057
+50823,31535,30476,40192
+59655,31536,30475,40762
+52688,31537,30474,40891
+52629,31538,30473,40304
+55313,31539,30472,40489
+50325,31540,30471,40082
+54177,31541,30470,40889
+56242,31542,30469,40888
+50984,31543,30468,40721
+55173,31544,30467,40465
+53099,31545,30466,40810
+59411,31546,30465,40169
+56066,31547,30464,40039
+58720,31548,30463,40927
+52683,31549,30462,40793
+52993,31550,30461,40649
+56376,31551,30460,40955
+58495,31552,30459,40116
+52963,31553,30458,40076
+55223,31554,30457,40764
+52209,31555,30456,40526
+51039,31556,30455,40675
+57636,31557,30454,40226
+54707,31558,30453,40882
+53339,31559,30452,40958
+57309,31560,30451,40706
+52821,31561,30450,40381
+55555,31562,30449,40077
+52498,31563,30448,40113
+56811,31564,30447,40950
+55274,31565,30446,40327
+50137,31566,30445,40978
+53094,31567,30444,40062
+56953,31568,30443,40481
+59132,31569,30442,40062
+53639,31570,30441,40557
+56825,31571,30440,40809
+57147,31572,30439,40126
+53511,31573,30438,40985
+58387,31574,30437,40703
+50686,31575,30436,40462
+55593,31576,30435,40833
+57415,31577,30434,40890
+54468,31578,30433,40017
+58843,31579,30432,40331
+57546,31580,30431,40296
+51274,31581,30430,40563
+54306,31582,30429,40072
+52791,31583,30428,40064
+55207,31584,30427,40030
+57552,31585,30426,40435
+53657,31586,30425,40073
+50396,31587,30424,40311
+56430,31588,30423,40981
+53101,31589,30422,40736
+51942,31590,30421,40923
+59205,31591,30420,40422
+55788,31592,30419,40649
+59429,31593,30418,40908
+54568,31594,30417,40393
+50049,31595,30416,40745
+50321,31596,30415,40201
+59295,31597,30414,40267
+53040,31598,30413,40583
+52670,31599,30412,40178
+57360,31600,30411,40919
+52147,31601,30410,40512
+51137,31602,30409,40138
+55694,31603,30408,40883
+58235,31604,30407,40420
+57307,31605,30406,40925
+56320,31606,30405,40283
+57317,31607,30404,40177
+55034,31608,30403,40687
+57477,31609,30402,40441
+54356,31610,30401,40226
+55224,31611,30400,40104
+53557,31612,30399,40557
+56334,31613,30398,40545
+56086,31614,30397,40588
+50463,31615,30396,40018
+55397,31616,30395,40975
+53720,31617,30394,40025
+56274,31618,30393,40918
+59907,31619,30392,40266
+51591,31620,30391,40385
+51700,31621,30390,40474
+52708,31622,30389,40561
+51488,31623,30388,40027
+52344,31624,30387,40054
+50553,31625,30386,40052
+53852,31626,30385,40302
+58036,31627,30384,40955
+53417,31628,30383,40423
+57403,31629,30382,40242
+52250,31630,30381,40105
+50013,31631,30380,40172
+52364,31632,30379,40451
+52851,31633,30378,40506
+56737,31634,30377,40344
+50719,31635,30376,40551
+54580,31636,30375,40566
+57933,31637,30374,40004
+58807,31638,30373,40491
+59755,31639,30372,40944
+54617,31640,30371,40780
+52261,31641,30370,40544
+59347,31642,30369,40803
+52041,31643,30368,40215
+51873,31644,30367,40372
+56873,31645,30366,40330
+56078,31646,30365,40868
+53726,31647,30364,40083
+52215,31648,30363,40663
+50905,31649,30362,40415
+51378,31650,30361,40356
+59990,31651,30360,40502
+56552,31652,30359,40919
+59658,31653,30358,40031
+56797,31654,30357,40364
+55723,31655,30356,40718
+57425,31656,30355,40731
+58654,31657,30354,40003
+57070,31658,30353,40972
+55594,31659,30352,40519
+55810,31660,30351,40559
+54428,31661,30350,40885
+50679,31662,30349,40976
+54389,31663,30348,40387
+54285,31664,30347,40102
+57454,31665,30346,40908
+51065,31666,30345,40439
+58876,31667,30344,40796
+56870,31668,30343,40930
+54781,31669,30342,40887
+54570,31670,30341,40848
+55682,31671,30340,40702
+56036,31672,30339,40847
+51511,31673,30338,40791
+58401,31674,30337,40741
+58861,31675,30336,40984
+57358,31676,30335,40053
+50025,31677,30334,40655
+52008,31678,30333,40745
+52102,31679,30332,40657
+58079,31680,30331,40549
+58517,31681,30330,40645
+53737,31682,30329,40499
+58789,31683,30328,40764
+57571,31684,30327,40337
+59533,31685,30326,40032
+50417,31686,30325,40019
+56085,31687,30324,40950
+52967,31688,30323,40413
+52533,31689,30322,40449
+50224,31690,30321,40498
+50533,31691,30320,40860
+58795,31692,30319,40650
+56980,31693,30318,40804
+51175,31694,30317,40035
+51525,31695,30316,40192
+56339,31696,30315,40783
+52979,31697,30314,40222
+51047,31698,30313,40286
+56373,31699,30312,40707
+52561,31700,30311,40080
+55212,31701,30310,40806
+50168,31702,30309,40617
+52505,31703,30308,40185
+51181,31704,30307,40657
+56491,31705,30306,40013
+56959,31706,30305,40007
+53943,31707,30304,40138
+53261,31708,30303,40169
+52432,31709,30302,40120
+59158,31710,30301,40995
+54230,31711,30300,40573
+54780,31712,30299,40170
+58059,31713,30298,40954
+52871,31714,30297,40315
+58096,31715,30296,40948
+57894,31716,30295,40004
+52158,31717,30294,40261
+55889,31718,30293,40335
+57056,31719,30292,40268
+53002,31720,30291,40559
+54478,31721,30290,40879
+51249,31722,30289,40344
+50869,31723,30288,40036
+51926,31724,30287,40608
+51218,31725,30286,40823
+50362,31726,30285,40355
+52189,31727,30284,40906
+59150,31728,30283,40671
+52962,31729,30282,40639
+56237,31730,30281,40446
+56970,31731,30280,40219
+55878,31732,30279,40721
+51000,31733,30278,40208
+50689,31734,30277,40031
+55424,31735,30276,40210
+54546,31736,30275,40154
+53410,31737,30274,40275
+52811,31738,30273,40154
+54052,31739,30272,40910
+51452,31740,30271,40880
+51709,31741,30270,40004
+58710,31742,30269,40257
+56494,31743,30268,40542
+57214,31744,30267,40797
+57931,31745,30266,40943
+56589,31746,30265,40227
+59831,31747,30264,40549
+59443,31748,30263,40939
+55710,31749,30262,40701
+58731,31750,30261,40223
+52475,31751,30260,40568
+55994,31752,30259,40415
+55498,31753,30258,40149
+54049,31754,30257,40880
+59976,31755,30256,40996
+55187,31756,30255,40184
+57894,31757,30254,40021
+56096,31758,30253,40032
+51677,31759,30252,40531
+58991,31760,30251,40382
+59595,31761,30250,40904
+51578,31762,30249,40926
+59451,31763,30248,40957
+58467,31764,30247,40245
+53811,31765,30246,40134
+54997,31766,30245,40314
+50999,31767,30244,40385
+53733,31768,30243,40089
+55051,31769,30242,40313
+53831,31770,30241,40949
+56837,31771,30240,40099
+51005,31772,30239,40074
+56001,31773,30238,40052
+55258,31774,30237,40348
+54318,31775,30236,40389
+59909,31776,30235,40839
+59141,31777,30234,40292
+51913,31778,30233,40924
+56266,31779,30232,40008
+56345,31780,30231,40468
+58523,31781,30230,40376
+58369,31782,30229,40172
+58214,31783,30228,40329
+57271,31784,30227,40230
+58760,31785,30226,40427
+58977,31786,30225,40880
+59951,31787,30224,40990
+53840,31788,30223,40738
+52896,31789,30222,40318
+51369,31790,30221,40481
+51973,31791,30220,40312
+58821,31792,30219,40412
+57598,31793,30218,40720
+54606,31794,30217,40271
+54120,31795,30216,40689
+51226,31796,30215,40732
+56172,31797,30214,40336
+53652,31798,30213,40026
+59870,31799,30212,40917
+57333,31800,30211,40455
+57909,31801,30210,40462
+59314,31802,30209,40162
+59642,31803,30208,40994
+51199,31804,30207,40822
+55386,31805,30206,40871
+52513,31806,30205,40188
+57078,31807,30204,40265
+53393,31808,30203,40320
+51114,31809,30202,40984
+53277,31810,30201,40189
+57386,31811,30200,40980
+55736,31812,30199,40088
+58231,31813,30198,40976
+53266,31814,30197,40054
+55761,31815,30196,40224
+55890,31816,30195,40019
+53127,31817,30194,40768
+55983,31818,30193,40296
+53995,31819,30192,40170
+56078,31820,30191,40032
+55873,31821,30190,40340
+59942,31822,30189,40055
+54949,31823,30188,40892
+53887,31824,30187,40191
+59947,31825,30186,40664
+50041,31826,30185,40617
+54802,31827,30184,40102
+54073,31828,30183,40141
+58770,31829,30182,40237
+58686,31830,30181,40318
+52965,31831,30180,40799
+55490,31832,30179,40716
+52002,31833,30178,40275
+56828,31834,30177,40551
+52884,31835,30176,40259
+51922,31836,30175,40636
+58793,31837,30174,40839
+57099,31838,30173,40752
+52166,31839,30172,40411
+57139,31840,30171,40159
+50040,31841,30170,40556
+57976,31842,30169,40647
+59285,31843,30168,40506
+51233,31844,30167,40062
+55457,31845,30166,40515
+54398,31846,30165,40068
+58477,31847,30164,40765
+57946,31848,30163,40783
+50185,31849,30162,40941
+56925,31850,30161,40345
+53482,31851,30160,40056
+58750,31852,30159,40408
+56247,31853,30158,40356
+58422,31854,30157,40514
+52701,31855,30156,40045
+54463,31856,30155,40706
+52755,31857,30154,40304
+57052,31858,30153,40419
+54456,31859,30152,40789
+51362,31860,30151,40136
+54199,31861,30150,40526
+50644,31862,30149,40990
+53341,31863,30148,40902
+50428,31864,30147,40583
+51585,31865,30146,40216
+57141,31866,30145,40588
+58802,31867,30144,40789
+51206,31868,30143,40862
+55343,31869,30142,40377
+59737,31870,30141,40473
+54431,31871,30140,40955
+55049,31872,30139,40829
+52864,31873,30138,40439
+57945,31874,30137,40602
+58893,31875,30136,40514
+52850,31876,30135,40695
+50185,31877,30134,40785
+54101,31878,30133,40543
+57944,31879,30132,40735
+58094,31880,30131,40153
+57709,31881,30130,40641
+59593,31882,30129,40635
+54878,31883,30128,40263
+54126,31884,30127,40410
+50964,31885,30126,40849
+54664,31886,30125,40169
+57038,31887,30124,40651
+55156,31888,30123,40427
+50933,31889,30122,40287
+56485,31890,30121,40574
+51705,31891,30120,40438
+50772,31892,30119,40343
+51483,31893,30118,40530
+52884,31894,30117,40627
+58804,31895,30116,40467
+52484,31896,30115,40721
+50434,31897,30114,40452
+50189,31898,30113,40978
+59363,31899,30112,40313
+56507,31900,30111,40050
+56730,31901,30110,40984
+58153,31902,30109,40014
+58415,31903,30108,40329
+51045,31904,30107,40049
+53939,31905,30106,40794
+53394,31906,30105,40309
+50417,31907,30104,40404
+55948,31908,30103,40107
+58223,31909,30102,40196
+54887,31910,30101,40183
+58404,31911,30100,40774
+50912,31912,30099,40139
+54052,31913,30098,40621
+58197,31914,30097,40781
+56742,31915,30096,40552
+57456,31916,30095,40256
+54086,31917,30094,40457
+55198,31918,30093,40828
+52583,31919,30092,40319
+59470,31920,30091,40939
+51248,31921,30090,40286
+57254,31922,30089,40082
+54631,31923,30088,40640
+54191,31924,30087,40399
+57996,31925,30086,40394
+56800,31926,30085,40941
+50044,31927,30084,40485
+57048,31928,30083,40586
+54390,31929,30082,40432
+59125,31930,30081,40441
+55081,31931,30080,40423
+53249,31932,30079,40472
+58636,31933,30078,40330
+52796,31934,30077,40792
+53272,31935,30076,40356
+58261,31936,30075,40380
+59760,31937,30074,40677
+54642,31938,30073,40753
+51073,31939,30072,40079
+59328,31940,30071,40329
+53988,31941,30070,40338
+51060,31942,30069,40099
+51280,31943,30068,40468
+59775,31944,30067,40538
+53436,31945,30066,40840
+53808,31946,30065,40176
+59410,31947,30064,40730
+50963,31948,30063,40160
+58255,31949,30062,40773
+50983,31950,30061,40357
+59031,31951,30060,40177
+59054,31952,30059,40514
+59814,31953,30058,40964
+55389,31954,30057,40934
+55473,31955,30056,40668
+51139,31956,30055,40043
+53115,31957,30054,40926
+58887,31958,30053,40916
+58724,31959,30052,40422
+54186,31960,30051,40417
+52107,31961,30050,40203
+56219,31962,30049,40464
+57443,31963,30048,40406
+51449,31964,30047,40082
+52100,31965,30046,40871
+57923,31966,30045,40631
+53177,31967,30044,40844
+55515,31968,30043,40094
+59335,31969,30042,40287
+53930,31970,30041,40389
+52061,31971,30040,40059
+52349,31972,30039,40962
+58662,31973,30038,40115
+57596,31974,30037,40292
+51435,31975,30036,40863
+59099,31976,30035,40977
+56630,31977,30034,40364
+58426,31978,30033,40204
+51122,31979,30032,40822
+57815,31980,30031,40394
+59630,31981,30030,40172
+51251,31982,30029,40962
+58784,31983,30028,40209
+51970,31984,30027,40263
+54262,31985,30026,40746
+55900,31986,30025,40603
+53543,31987,30024,40753
+57825,31988,30023,40041
+51017,31989,30022,40653
+59545,31990,30021,40327
+54731,31991,30020,40252
+54696,31992,30019,40198
+59426,31993,30018,40998
+53329,31994,30017,40912
+52258,31995,30016,40103
+53378,31996,30015,40789
+54050,31997,30014,40368
+51837,31998,30013,40162
+54066,31999,30012,40738
+51974,32000,30011,40071
+55336,32001,30010,40017
+57192,32002,30009,40282
+54229,32003,30008,40689
+56611,32004,30007,40389
+53047,32005,30006,40968
+57104,32006,30005,40449
+53527,32007,30004,40496
+54101,32008,30003,40326
+52659,32009,30002,40799
+50899,32010,30001,40704
+58582,32011,30000,40092
+52412,32012,31000,40735
+56707,32013,30999,40671
+53667,32014,30998,40819
+59137,32015,30997,40094
+55532,32016,30996,40666
+59408,32017,30995,40552
+50002,32018,30994,40512
+58106,32019,30993,40632
+52391,32020,30992,40042
+57092,32021,30991,40440
+51974,32022,30990,40879
+52451,32023,30989,40471
+56964,32024,30988,40827
+56412,32025,30987,40897
+58677,32026,30986,40305
+56612,32027,30985,40857
+50251,32028,30984,40660
+51446,32029,30983,40021
+50745,32030,30982,40546
+52227,32031,30981,40350
+54502,32032,30980,40422
+56163,32033,30979,40227
+58638,32034,30978,40840
+53705,32035,30977,40665
+50434,32036,30976,40963
+58485,32037,30975,40641
+58766,32038,30974,40307
+59140,32039,30973,40862
+57691,32040,30972,40673
+50354,32041,30971,40609
+50107,32042,30970,40957
+57615,32043,30969,40147
+51714,32044,30968,40343
+57086,32045,30967,40286
+55525,32046,30966,40130
+53626,32047,30965,40200
+57338,32048,30964,40181
+53012,32049,30963,40446
+50559,32050,30962,40886
+52168,32051,30961,40057
+58502,32052,30960,40037
+54334,32053,30959,40367
+51038,32054,30958,40027
+57948,32055,30957,40608
+59437,32056,30956,40542
+59080,32057,30955,40827
+59145,32058,30954,40546
+51338,32059,30953,40197
+59830,32060,30952,40605
+59040,32061,30951,40110
+59881,32062,30950,40454
+51916,32063,30949,40928
+50508,32064,30948,40451
+58104,32065,30947,40468
+57994,32066,30946,40423
+50202,32067,30945,40827
+58714,32068,30944,40044
+53298,32069,30943,40229
+54424,32070,30942,40514
+58608,32071,30941,40730
+57342,32072,30940,40276
+52834,32073,30939,40323
+54467,32074,30938,40267
+52843,32075,30937,40436
+50058,32076,30936,40089
+56224,32077,30935,40472
+53035,32078,30934,40084
+55100,32079,30933,40009
+55342,32080,30932,40070
+58803,32081,30931,40251
+59379,32082,30930,40861
+50835,32083,30929,40219
+57007,32084,30928,40979
+52518,32085,30927,40281
+52097,32086,30926,40052
+59892,32087,30925,40759
+58355,32088,30924,40623
+58262,32089,30923,40701
+53468,32090,30922,40676
+53445,32091,30921,40075
+57634,32092,30920,40105
+52641,32093,30919,40770
+50125,32094,30918,40264
+50365,32095,30917,40359
+51702,32096,30916,40696
+58280,32097,30915,40325
+52318,32098,30914,40142
+54471,32099,30913,40806
+56395,32100,30912,40655
+51885,32101,30911,40013
+50157,32102,30910,40277
+54572,32103,30909,40843
+50287,32104,30908,40885
+57237,32105,30907,40133
+56937,32106,30906,40529
+53820,32107,30905,40470
+53122,32108,30904,40558
+50615,32109,30903,40606
+58803,32110,30902,40932
+51589,32111,30901,40279
+50711,32112,30900,40189
+51227,32113,30899,40319
+50830,32114,30898,40100
+53273,32115,30897,40623
+53105,32116,30896,40884
+59116,32117,30895,40391
+58048,32118,30894,40410
+56684,32119,30893,40219
+58406,32120,30892,40296
+53030,32121,30891,40055
+59870,32122,30890,40886
+55812,32123,30889,40906
+58255,32124,30888,40839
+57751,32125,30887,40346
+52707,32126,30886,40805
+59927,32127,30885,40620
+51477,32128,30884,40522
+54033,32129,30883,40573
+56009,32130,30882,40109
+56671,32131,30881,40346
+52981,32132,30880,40757
+50278,32133,30879,40793
+59725,32134,30878,40701
+52793,32135,30877,40243
+58557,32136,30876,40044
+51834,32137,30875,40574
+54591,32138,30874,40880
+54641,32139,30873,40850
+57404,32140,30872,40254
+50133,32141,30871,40136
+57535,32142,30870,40472
+55799,32143,30869,40186
+57013,32144,30868,40811
+55838,32145,30867,40563
+56369,32146,30866,40732
+50188,32147,30865,40444
+53484,32148,30864,40863
+56656,32149,30863,40048
+57273,32150,30862,40868
+58261,32151,30861,40493
+53223,32152,30860,40951
+55160,32153,30859,40488
+59327,32154,30858,40985
+55094,32155,30857,40987
+53805,32156,30856,40064
+54660,32157,30855,40933
+57660,32158,30854,40496
+53960,32159,30853,40654
+52250,32160,30852,40940
+59589,32161,30851,40197
+59851,32162,30850,40653
+56468,32163,30849,40129
+59593,32164,30848,40374
+53421,32165,30847,40784
+56088,32166,30846,40498
+58469,32167,30845,40199
+57768,32168,30844,40778
+57714,32169,30843,40247
+53556,32170,30842,40551
+55545,32171,30841,40312
+57767,32172,30840,40368
+54920,32173,30839,40405
+54398,32174,30838,40353
+54560,32175,30837,40483
+51256,32176,30836,40008
+57626,32177,30835,40919
+53626,32178,30834,40141
+50315,32179,30833,40580
+58016,32180,30832,40997
+56402,32181,30831,40165
+52510,32182,30830,40443
+59630,32183,30829,40213
+55972,32184,30828,40580
+52492,32185,30827,40240
+53264,32186,30826,40293
+59597,32187,30825,40470
+55165,32188,30824,40179
+59847,32189,30823,40780
+53993,32190,30822,40911
+51746,32191,30821,40998
+59252,32192,30820,40668
+59027,32193,30819,40261
+50148,32194,30818,40446
+55109,32195,30817,40209
+58005,32196,30816,40985
+58898,32197,30815,40539
+53966,32198,30814,40667
+57122,32199,30813,40122
+57632,32200,30812,40415
+55835,32201,30811,40672
+50407,32202,30810,40935
+53542,32203,30809,40336
+58739,32204,30808,40221
+54962,32205,30807,40278
+59840,32206,30806,40742
+50146,32207,30805,40754
+50304,32208,30804,40820
+56881,32209,30803,40387
+50059,32210,30802,40726
+55126,32211,30801,40716
+59661,32212,30800,40006
+59298,32213,30799,40174
+50622,32214,30798,40654
+57206,32215,30797,40210
+56271,32216,30796,40768
+54274,32217,30795,40513
+52159,32218,30794,40281
+59000,32219,30793,40898
+52381,32220,30792,40338
+57359,32221,30791,40487
+53608,32222,30790,40683
+55252,32223,30789,40780
+56051,32224,30788,40475
+51114,32225,30787,40564
+55940,32226,30786,40784
+50050,32227,30785,40370
+55067,32228,30784,40831
+58742,32229,30783,40314
+50480,32230,30782,40217
+53551,32231,30781,40554
+56213,32232,30780,40859
+52294,32233,30779,40592
+59882,32234,30778,40968
+55704,32235,30777,40201
+50240,32236,30776,40501
+54210,32237,30775,40674
+58837,32238,30774,40984
+56140,32239,30773,40343
+58334,32240,30772,40758
+52994,32241,30771,40670
+52915,32242,30770,40928
+55389,32243,30769,40079
+57043,32244,30768,40833
+57825,32245,30767,40266
+58697,32246,30766,40695
+51471,32247,30765,40613
+51654,32248,30764,40299
+50939,32249,30763,40457
+56014,32250,30762,40294
+54056,32251,30761,40656
+52207,32252,30760,40018
+53934,32253,30759,40637
+53307,32254,30758,40899
+55870,32255,30757,40815
+52871,32256,30756,40776
+58700,32257,30755,40635
+59963,32258,30754,40706
+56971,32259,30753,40094
+53434,32260,30752,40240
+55927,32261,30751,40108
+53996,32262,30750,40427
+50892,32263,30749,40608
+52263,32264,30748,40181
+56388,32265,30747,40192
+52557,32266,30746,40313
+57965,32267,30745,40531
+50702,32268,30744,40879
+50216,32269,30743,40482
+51181,32270,30742,40559
+51390,32271,30741,40841
+57776,32272,30740,40743
+56605,32273,30739,40801
+57410,32274,30738,40754
+56752,32275,30737,40507
+59827,32276,30736,40255
+57005,32277,30735,40474
+55647,32278,30734,40803
+53240,32279,30733,40797
+59963,32280,30732,40028
+53973,32281,30731,40377
+51131,32282,30730,40364
+58139,32283,30729,40623
+52081,32284,30728,40222
+51252,32285,30727,40471
+57697,32286,30726,40288
+53008,32287,30725,40896
+51232,32288,30724,40796
+57129,32289,30723,40154
+50394,32290,30722,40989
+51761,32291,30721,40674
+53346,32292,30720,40995
+57063,32293,30719,40266
+57540,32294,30718,40394
+57881,32295,30717,40491
+50358,32296,30716,40365
+59459,32297,30715,40631
+57214,32298,30714,40331
+56726,32299,30713,40879
+51917,32300,30712,40076
+59140,32301,30711,40566
+52553,32302,30710,40266
+53240,32303,30709,40785
+57656,32304,30708,40894
+55362,32305,30707,40251
+53112,32306,30706,40981
+57184,32307,30705,40728
+52171,32308,30704,40879
+53496,32309,30703,40494
+57424,32310,30702,40557
+51794,32311,30701,40956
+58989,32312,30700,40612
+53874,32313,30699,40840
+57000,32314,30698,40050
+51474,32315,30697,40912
+55029,32316,30696,40572
+50991,32317,30695,40396
+59498,32318,30694,40329
+52650,32319,30693,40302
+52206,32320,30692,40639
+59103,32321,30691,40824
+59983,32322,30690,40440
+59762,32323,30689,40073
+54243,32324,30688,40488
+57061,32325,30687,40339
+54055,32326,30686,40414
+52310,32327,30685,40708
+55059,32328,30684,40429
+53845,32329,30683,40198
+51166,32330,30682,40382
+50292,32331,30681,40333
+52094,32332,30680,40595
+54503,32333,30679,40093
+58391,32334,30678,40898
+55066,32335,30677,40430
+57301,32336,30676,40532
+51410,32337,30675,40677
+52997,32338,30674,40801
+52619,32339,30673,40320
+59933,32340,30672,40930
+51121,32341,30671,40237
+54092,32342,30670,40475
+51211,32343,30669,40815
+54332,32344,30668,40555
+54545,32345,30667,40111
+53012,32346,30666,40137
+59918,32347,30665,40347
+55466,32348,30664,40333
+59040,32349,30663,40781
+53334,32350,30662,40616
+57772,32351,30661,40211
+58417,32352,30660,40568
+54927,32353,30659,40678
+50572,32354,30658,40180
+55936,32355,30657,40209
+58365,32356,30656,40321
+57297,32357,30655,40456
+52452,32358,30654,40111
+54708,32359,30653,40818
+57869,32360,30652,40043
+55433,32361,30651,40501
+50798,32362,30650,40617
+51349,32363,30649,40685
+57119,32364,30648,40077
+55775,32365,30647,40832
+55977,32366,30646,40006
+54037,32367,30645,40809
+59207,32368,30644,40144
+55323,32369,30643,40855
+58009,32370,30642,40309
+55283,32371,30641,40801
+56888,32372,30640,40480
+56376,32373,30639,40128
+52276,32374,30638,40133
+58436,32375,30637,40201
+58984,32376,30636,40520
+51314,32377,30635,40029
+50815,32378,30634,40000
+53838,32379,30633,40855
+56736,32380,30632,40928
+59544,32381,30631,40290
+52325,32382,30630,40898
+54135,32383,30629,40492
+59530,32384,30628,40368
+51898,32385,30627,40419
+59456,32386,30626,40593
+55929,32387,30625,40579
+54193,32388,30624,40276
+52107,32389,30623,40769
+51997,32390,30622,40883
+51857,32391,30621,40110
+50515,32392,30620,40071
+56191,32393,30619,40203
+59244,32394,30618,40812
+55771,32395,30617,40862
+51794,32396,30616,40746
+52606,32397,30615,40303
+54590,32398,30614,40751
+56007,32399,30613,40089
+54574,32400,30612,40972
+57311,32401,30611,40360
+50645,32402,30610,40922
+56567,32403,30609,40067
+52812,32404,30608,40000
+57549,32405,30607,40126
+59462,32406,30606,40301
+57790,32407,30605,40464
+58546,32408,30604,40036
+57973,32409,30603,40323
+56505,32410,30602,40073
+59537,32411,30601,40997
+56983,32412,30600,40894
+54581,32413,30599,40962
+52887,32414,30598,40514
+53465,32415,30597,40540
+57831,32416,30596,40391
+53662,32417,30595,40496
+53019,32418,30594,40190
+55359,32419,30593,40838
+53629,32420,30592,40041
+55123,32421,30591,40293
+54770,32422,30590,40621
+53119,32423,30589,40888
+59143,32424,30588,40074
+58551,32425,30587,40449
+52282,32426,30586,40281
+55029,32427,30585,40217
+51973,32428,30584,40641
+56178,32429,30583,40433
+52118,32430,30582,40437
+55922,32431,30581,40160
+53402,32432,30580,40389
+59325,32433,30579,40569
+55927,32434,30578,40632
+55415,32435,30577,40959
+50585,32436,30576,40475
+58400,32437,30575,40960
+54756,32438,30574,40465
+52743,32439,30573,40868
+54392,32440,30572,40798
+51210,32441,30571,40313
+53859,32442,30570,40385
+56313,32443,30569,40125
+58512,32444,30568,40166
+50041,32445,30567,40206
+54909,32446,30566,40958
+57428,32447,30565,40593
+58378,32448,30564,40740
+54141,32449,30563,40381
+50333,32450,30562,40092
+54465,32451,30561,40779
+56451,32452,30560,40713
+53001,32453,30559,40829
+59041,32454,30558,40971
+53068,32455,30557,40616
+59416,32456,30556,40076
+59762,32457,30555,40607
+53367,32458,30554,40934
+54115,32459,30553,40837
+54922,32460,30552,40299
+50669,32461,30551,40703
+58929,32462,30550,40240
+51829,32463,30549,40995
+57387,32464,30548,40670
+56196,32465,30547,40921
+59521,32466,30546,40710
+52083,32467,30545,40981
+57039,32468,30544,40073
+54642,32469,30543,40918
+52173,32470,30542,40849
+53073,32471,30541,40607
+53420,32472,30540,40200
+53122,32473,30539,40399
+55643,32474,30538,40915
+57941,32475,30537,40974
+50889,32476,30536,40266
+53475,32477,30535,40462
+58373,32478,30534,40134
+55335,32479,30533,40387
+57218,32480,30532,40848
+58710,32481,30531,40705
+59558,32482,30530,40418
+50159,32483,30529,40146
+50129,32484,30528,40692
+51822,32485,30527,40328
+52747,32486,30526,40822
+52346,32487,30525,40677
+52534,32488,30524,40077
+54446,32489,30523,40684
+56755,32490,30522,40316
+52133,32491,30521,40659
+59632,32492,30520,40343
+53726,32493,30519,40568
+58063,32494,30518,40567
+53455,32495,30517,40293
+52658,32496,30516,40893
+55926,32497,30515,40633
+58642,32498,30514,40381
+51051,32499,30513,40229
+51886,32500,30512,40433
+56609,32501,30511,40551
+56865,32502,30510,40822
+52107,32503,30509,40246
+54924,32504,30508,40287
+52875,32505,30507,40532
+54083,32506,30506,40893
+50614,32507,30505,40113
+53076,32508,30504,40351
+56387,32509,30503,40369
+55361,32510,30502,40812
+59938,32511,30501,40956
+53359,32512,30500,40426
+59001,32513,30499,40773
+53367,32514,30498,40129
+58858,32515,30497,40490
+50697,32516,30496,40010
+55344,32517,30495,40287
+53153,32518,30494,40102
+54008,32519,30493,40399
+59458,32520,30492,40756
+54429,32521,30491,40841
+50718,32522,30490,40842
+54747,32523,30489,40547
+51703,32524,30488,40750
+54213,32525,30487,40077
+54576,32526,30486,40962
+58994,32527,30485,40844
+59085,32528,30484,40883
+51826,32529,30483,40601
+54826,32530,30482,40700
+53903,32531,30481,40694
+59447,32532,30480,40868
+54863,32533,30479,40700
+53888,32534,30478,40328
+56645,32535,30477,40036
+56634,32536,30476,40694
+53751,32537,30475,40288
+59529,32538,30474,40735
+55306,32539,30473,40653
+53759,32540,30472,40584
+54729,32541,30471,40312
+51200,32542,30470,40342
+50613,32543,30469,40858
+53409,32544,30468,40979
+50502,32545,30467,40577
+58926,32546,30466,40353
+55760,32547,30465,40404
+59802,32548,30464,40050
+58641,32549,30463,40994
+56737,32550,30462,40194
+57014,32551,30461,40640
+53210,32552,30460,40865
+55821,32553,30459,40573
+57080,32554,30458,40009
+50246,32555,30457,40692
+59136,32556,30456,40672
+57346,32557,30455,40230
+57556,32558,30454,40642
+57262,32559,30453,40049
+51198,32560,30452,40239
+52745,32561,30451,40528
+53833,32562,30450,40756
+56768,32563,30449,40312
+59400,32564,30448,40825
+55626,32565,30447,40669
+56250,32566,30446,40526
+52383,32567,30445,40415
+50709,32568,30444,40011
+58517,32569,30443,40164
+59919,32570,30442,40122
+57538,32571,30441,40227
+52604,32572,30440,40059
+57605,32573,30439,40650
+52882,32574,30438,40884
+53680,32575,30437,40045
+54424,32576,30436,40171
+54590,32577,30435,40668
+59741,32578,30434,40402
+54098,32579,30433,40255
+51044,32580,30432,40701
+55472,32581,30431,40745
+53389,32582,30430,40098
+58052,32583,30429,40334
+50463,32584,30428,40646
+50672,32585,30427,40161
+57074,32586,30426,40456
+50164,32587,30425,40815
+59033,32588,30424,40708
+52374,32589,30423,40954
+56700,32590,30422,40167
+59009,32591,30421,40171
+57428,32592,30420,40450
+56089,32593,30419,40007
+58479,32594,30418,40725
+52316,32595,30417,40149
+50339,32596,30416,40561
+53620,32597,30415,40721
+55503,32598,30414,40345
+57418,32599,30413,40552
+55245,32600,30412,40782
+53700,32601,30411,40712
+54916,32602,30410,40398
+56159,32603,30409,40523
+52410,32604,30408,40937
+58648,32605,30407,40441
+50151,32606,30406,40063
+57411,32607,30405,40678
+53076,32608,30404,40217
+57951,32609,30403,40463
+56514,32610,30402,40867
+56241,32611,30401,40540
+59800,32612,30400,40306
+51037,32613,30399,40859
+53524,32614,30398,40376
+51443,32615,30397,40497
+58370,32616,30396,40801
+55877,32617,30395,40029
+54586,32618,30394,40352
+50823,32619,30393,40627
+54276,32620,30392,40667
+59535,32621,30391,40050
+53447,32622,30390,40209
+50092,32623,30389,40971
+55679,32624,30388,40860
+54467,32625,30387,40822
+56716,32626,30386,40838
+50789,32627,30385,40914
+55991,32628,30384,40724
+55676,32629,30383,40137
+52432,32630,30382,40297
+54084,32631,30381,40270
+51154,32632,30380,40817
+54902,32633,30379,40828
+59080,32634,30378,40587
+56688,32635,30377,40676
+54889,32636,30376,40961
+54784,32637,30375,40250
+53039,32638,30374,40015
+50527,32639,30373,40461
+56678,32640,30372,40078
+50090,32641,30371,40233
+50815,32642,30370,40572
+56524,32643,30369,40555
+55844,32644,30368,40586
+56584,32645,30367,40480
+59638,32646,30366,40538
+57249,32647,30365,40010
+58786,32648,30364,40738
+53004,32649,30363,40905
+52898,32650,30362,40929
+51181,32651,30361,40321
+59343,32652,30360,40365
+59567,32653,30359,40989
+50324,32654,30358,40695
+55424,32655,30357,40785
+58863,32656,30356,40084
+59592,32657,30355,40204
+50752,32658,30354,40376
+52058,32659,30353,40626
+56004,32660,30352,40257
+55394,32661,30351,40828
+55947,32662,30350,40613
+50639,32663,30349,40522
+58197,32664,30348,40409
+53018,32665,30347,40247
+55003,32666,30346,40912
+59220,32667,30345,40743
+51002,32668,30344,40652
+51199,32669,30343,40352
+51130,32670,30342,40108
+51731,32671,30341,40873
+52412,32672,30340,40969
+57280,32673,30339,40587
+55059,32674,30338,40382
+54179,32675,30337,40811
+55946,32676,30336,40413
+57633,32677,30335,40713
+53363,32678,30334,40112
+57600,32679,30333,40793
+59759,32680,30332,40018
+58085,32681,30331,40116
+52457,32682,30330,40395
+54440,32683,30329,40561
+52460,32684,30328,40153
+51540,32685,30327,40389
+55669,32686,30326,40810
+50315,32687,30325,40332
+52292,32688,30324,40323
+58039,32689,30323,40666
+52492,32690,30322,40347
+56586,32691,30321,40059
+51290,32692,30320,40217
+56823,32693,30319,40627
+57362,32694,30318,40070
+53977,32695,30317,40140
+51924,32696,30316,40017
+54897,32697,30315,40770
+52894,32698,30314,40230
+56301,32699,30313,40268
+58555,32700,30312,40057
+50980,32701,30311,40657
+56569,32702,30310,40629
+55566,32703,30309,40851
+55592,32704,30308,40844
+59184,32705,30307,40933
+59758,32706,30306,40033
+59877,32707,30305,40589
+54664,32708,30304,40962
+55816,32709,30303,40598
+51547,32710,30302,40859
+56623,32711,30301,40358
+53337,32712,30300,40417
+51505,32713,30299,40222
+54031,32714,30298,40997
+52183,32715,30297,40683
+57706,32716,30296,40544
+53376,32717,30295,40223
+57497,32718,30294,40849
+51791,32719,30293,40973
+51982,32720,30292,40192
+56032,32721,30291,40616
+58127,32722,30290,40710
+56616,32723,30289,40766
+50281,32724,30288,40331
+54457,32725,30287,40318
+51108,32726,30286,40060
+56788,32727,30285,40315
+56945,32728,30284,40303
+52329,32729,30283,40524
+56479,32730,30282,40188
+55354,32731,30281,40170
+58013,32732,30280,40474
+56495,32733,30279,40733
+51498,32734,30278,40218
+57298,32735,30277,40731
+58825,32736,30276,40908
+50042,32737,30275,40838
+55329,32738,30274,40525
+57653,32739,30273,40825
+59712,32740,30272,40610
+57165,32741,30271,40423
+53375,32742,30270,40011
+50350,32743,30269,40568
+58880,32744,30268,40255
+58670,32745,30267,40784
+50068,32746,30266,40055
+52394,32747,30265,40862
+57225,32748,30264,40618
+51378,32749,30263,40534
+50992,32750,30262,40805
+58923,32751,30261,40339
+51981,32752,30260,40860
+53379,32753,30259,40589
+58920,32754,30258,40642
+52593,32755,30257,40114
+57857,32756,30256,40105
+56985,32757,30255,40157
+59888,32758,30254,40294
+58447,32759,30253,40400
+55690,32760,30252,40452
+54532,32761,30251,40366
+51584,32762,30250,40075
+52972,32763,30249,40898
+51921,32764,30248,40612
+57216,32765,30247,40325
+51951,32766,30246,40085
+54238,32767,30245,40400
+54927,32768,30244,40340
+53470,32769,30243,40671
+53050,32770,30242,40313
+51504,32771,30241,40838
+52689,32772,30240,40852
+55777,32773,30239,40819
+57325,32774,30238,40072
+53069,32775,30237,40663
+59988,32776,30236,40283
+53486,32777,30235,40578
+59672,32778,30234,40851
+51778,32779,30233,40602
+55161,32780,30232,40174
+56218,32781,30231,40168
+51499,32782,30230,40693
+50559,32783,30229,40175
+59090,32784,30228,40815
+58320,32785,30227,40466
+53161,32786,30226,40062
+52534,32787,30225,40128
+50723,32788,30224,40904
+52760,32789,30223,40078
+57225,32790,30222,40237
+54938,32791,30221,40802
+59857,32792,30220,40335
+51664,32793,30219,40728
+58447,32794,30218,40450
+54577,32795,30217,40731
+50646,32796,30216,40850
+52919,32797,30215,40841
+50759,32798,30214,40724
+57986,32799,30213,40123
+56970,32800,30212,40738
+57356,32801,30211,40672
+56810,32802,30210,40286
+58199,32803,30209,40453
+56267,32804,30208,40897
+51091,32805,30207,40715
+55527,32806,30206,40713
+51756,32807,30205,40437
+50453,32808,30204,40837
+56383,32809,30203,40428
+52731,32810,30202,40182
+51005,32811,30201,40024
+53450,32812,30200,40501
+57129,32813,30199,40198
+54648,32814,30198,40667
+53835,32815,30197,40625
+58364,32816,30196,40008
+56807,32817,30195,40342
+55121,32818,30194,40187
+55687,32819,30193,40704
+55453,32820,30192,40779
+50773,32821,30191,40636
+52668,32822,30190,40744
+56037,32823,30189,40906
+57349,32824,30188,40593
+59135,32825,30187,40321
+59422,32826,30186,40635
+54186,32827,30185,40462
+58219,32828,30184,40330
+55092,32829,30183,40993
+56914,32830,30182,40143
+54750,32831,30181,40924
+51130,32832,30180,40494
+54948,32833,30179,40584
+57650,32834,30178,40427
+59355,32835,30177,40437
+51342,32836,30176,40593
+55266,32837,30175,40382
+54427,32838,30174,40862
+53307,32839,30173,40544
+58548,32840,30172,40199
+51379,32841,30171,40301
+55773,32842,30170,40083
+54397,32843,30169,40064
+58525,32844,30168,40870
+56877,32845,30167,40303
+59417,32846,30166,40517
+53727,32847,30165,40712
+55066,32848,30164,40560
+51064,32849,30163,40274
+56945,32850,30162,40289
+54661,32851,30161,40273
+57457,32852,30160,40480
+51600,32853,30159,40789
+54282,32854,30158,40672
+55297,32855,30157,40848
+52838,32856,30156,40626
+52944,32857,30155,40034
+51228,32858,30154,40995
+52248,32859,30153,40931
+54498,32860,30152,40221
+56551,32861,30151,40756
+50304,32862,30150,40111
+54003,32863,30149,40826
+53152,32864,30148,40890
+51575,32865,30147,40144
+53809,32866,30146,40321
+59706,32867,30145,40560
+51450,32868,30144,40139
+53232,32869,30143,40381
+50027,32870,30142,40481
+59034,32871,30141,40800
+52096,32872,30140,40270
+55929,32873,30139,40965
+56975,32874,30138,40552
+56864,32875,30137,40402
+52576,32876,30136,40722
+50487,32877,30135,40308
+56428,32878,30134,40203
+52444,32879,30133,40101
+53886,32880,30132,40079
+51636,32881,30131,40102
+52263,32882,30130,40607
+50911,32883,30129,40140
+54494,32884,30128,40751
+56553,32885,30127,40610
+52035,32886,30126,40285
+51756,32887,30125,40907
+55588,32888,30124,40961
+55811,32889,30123,40403
+59065,32890,30122,40373
+52634,32891,30121,40177
+59050,32892,30120,40698
+54673,32893,30119,40003
+51009,32894,30118,40964
+53327,32895,30117,40414
+53018,32896,30116,40228
+52147,32897,30115,40239
+58855,32898,30114,40344
+58182,32899,30113,40795
+58095,32900,30112,40360
+52700,32901,30111,40784
+58486,32902,30110,40656
+58186,32903,30109,40262
+50808,32904,30108,40413
+53457,32905,30107,40966
+58431,32906,30106,40798
+54600,32907,30105,40133
+50197,32908,30104,40929
+50280,32909,30103,40332
+51891,32910,30102,40590
+50800,32911,30101,40696
+57175,32912,30100,40836
+51289,32913,30099,40649
+57076,32914,30098,40342
+50452,32915,30097,40729
+51108,32916,30096,40826
+56800,32917,30095,40658
+53048,32918,30094,40038
+50807,32919,30093,40516
+51840,32920,30092,40311
+50696,32921,30091,40867
+50693,32922,30090,40543
+55135,32923,30089,40393
+54324,32924,30088,40138
+54592,32925,30087,40469
+51398,32926,30086,40621
+52548,32927,30085,40895
+59594,32928,30084,40404
+51466,32929,30083,40222
+58020,32930,30082,40175
+54478,32931,30081,40617
+54286,32932,30080,40480
+53335,32933,30079,40126
+58716,32934,30078,40371
+52505,32935,30077,40514
+52366,32936,30076,40002
+54091,32937,30075,40750
+55695,32938,30074,40740
+50196,32939,30073,40170
+58763,32940,30072,40266
+52767,32941,30071,40208
+55021,32942,30070,40150
+57677,32943,30069,40336
+55989,32944,30068,40856
+50777,32945,30067,40204
+58841,32946,30066,40094
+56710,32947,30065,40335
+53971,32948,30064,40591
+51637,32949,30063,40103
+53160,32950,30062,40868
+55027,32951,30061,40067
+58104,32952,30060,40467
+57807,32953,30059,40939
+56833,32954,30058,40410
+54153,32955,30057,40955
+55006,32956,30056,40461
+51558,32957,30055,40267
+51709,32958,30054,40219
+56752,32959,30053,40542
+52526,32960,30052,40979
+57195,32961,30051,40485
+54632,32962,30050,40983
+58306,32963,30049,40968
+54390,32964,30048,40162
+54943,32965,30047,40624
+51166,32966,30046,40371
+54281,32967,30045,40755
+56577,32968,30044,40142
+59960,32969,30043,40709
+58222,32970,30042,40075
+59620,32971,30041,40440
+51490,32972,30040,40094
+58219,32973,30039,40160
+50154,32974,30038,40170
+57682,32975,30037,40758
+52041,32976,30036,40026
+57267,32977,30035,40542
+50116,32978,30034,40622
+51572,32979,30033,40607
+59899,32980,30032,40558
+58488,32981,30031,40326
+56706,32982,30030,40094
+56863,32983,30029,40447
+51726,32984,30028,40534
+54537,32985,30027,40578
+58955,32986,30026,40349
+51255,32987,30025,40606
+52889,32988,30024,40906
+53250,32989,30023,40381
+58568,32990,30022,40358
+56709,32991,30021,40227
+53304,32992,30020,40161
+55637,32993,30019,40763
+53683,32994,30018,40815
+57666,32995,30017,40728
+55846,32996,30016,40927
+59413,32997,30015,40597
+56698,32998,30014,40344
+50841,32999,30013,40323
+51393,33000,30012,40880
+56806,33001,30011,40301
+57084,33002,30010,40715
+57400,33003,30009,40704
+54106,33004,30008,40412
+54798,33005,30007,40396
+52346,33006,30006,40635
+52431,33007,30005,40873
+51377,33008,30004,40011
+54631,33009,30003,40992
+52183,33010,30002,40167
+52894,33011,30001,40120
+57612,33012,30000,40856
+53002,33013,31000,40170
+52616,33014,30999,40293
+57643,33015,30998,40729
+58704,33016,30997,40944
+59700,33017,30996,40425
+51509,33018,30995,40577
+55690,33019,30994,40963
+58182,33020,30993,40202
+56660,33021,30992,40002
+55093,33022,30991,40501
+54786,33023,30990,40037
+58983,33024,30989,40349
+51946,33025,30988,40833
+54549,33026,30987,40088
+52750,33027,30986,40922
+51792,33028,30985,40662
+53434,33029,30984,40807
+52738,33030,30983,40426
+50905,33031,30982,40599
+50783,33032,30981,40209
+56519,33033,30980,40851
+50889,33034,30979,40619
+59072,33035,30978,40690
+57027,33036,30977,40253
+54535,33037,30976,40555
+50365,33038,30975,40961
+53684,33039,30974,40610
+59103,33040,30973,40483
+54645,33041,30972,40282
+56865,33042,30971,40328
+55493,33043,30970,40979
+53926,33044,30969,40373
+50873,33045,30968,40657
+53305,33046,30967,40561
+51143,33047,30966,40473
+52432,33048,30965,40273
+58946,33049,30964,40408
+52349,33050,30963,40966
+56880,33051,30962,40817
+57956,33052,30961,40164
+50348,33053,30960,40499
+53942,33054,30959,40753
+56119,33055,30958,40637
+53179,33056,30957,40056
+55018,33057,30956,40527
+51919,33058,30955,40914
+56038,33059,30954,40598
+52707,33060,30953,40049
+54012,33061,30952,40699
+52084,33062,30951,40504
+51721,33063,30950,40362
+54329,33064,30949,40513
+59753,33065,30948,40858
+58022,33066,30947,40356
+52653,33067,30946,40437
+57501,33068,30945,40321
+52935,33069,30944,40125
+56004,33070,30943,40291
+53745,33071,30942,40905
+52177,33072,30941,40857
+54509,33073,30940,40371
+58322,33074,30939,40322
+55879,33075,30938,40272
+59098,33076,30937,40277
+53415,33077,30936,40931
+59894,33078,30935,40322
+51715,33079,30934,40646
+54285,33080,30933,40023
+54465,33081,30932,40854
+52325,33082,30931,40883
+59594,33083,30930,40761
+51987,33084,30929,40689
+58682,33085,30928,40927
+56538,33086,30927,40730
+53652,33087,30926,40354
+58917,33088,30925,40922
+53171,33089,30924,40716
+53983,33090,30923,40821
+57899,33091,30922,40916
+58159,33092,30921,40117
+54909,33093,30920,40628
+51826,33094,30919,40547
+57241,33095,30918,40273
+55984,33096,30917,40575
+55694,33097,30916,40264
+54385,33098,30915,40870
+57501,33099,30914,40442
+52664,33100,30913,40264
+58540,33101,30912,40180
+53357,33102,30911,40170
+53126,33103,30910,40125
+56777,33104,30909,40406
+57167,33105,30908,40476
+59402,33106,30907,40277
+55206,33107,30906,40494
+50375,33108,30905,40631
+59315,33109,30904,40585
+57389,33110,30903,40767
+53327,33111,30902,40345
+56887,33112,30901,40325
+53043,33113,30900,40220
+50236,33114,30899,40241
+55433,33115,30898,40131
+56675,33116,30897,40473
+58623,33117,30896,40879
+50641,33118,30895,40215
+53962,33119,30894,40126
+56035,33120,30893,40050
+59926,33121,30892,40505
+59655,33122,30891,40144
+59579,33123,30890,40499
+53924,33124,30889,40389
+55740,33125,30888,40624
+56683,33126,30887,40378
+57082,33127,30886,40591
+54677,33128,30885,40316
+58261,33129,30884,40878
+57467,33130,30883,40808
+51266,33131,30882,40640
+53509,33132,30881,40321
+51655,33133,30880,40502
+54554,33134,30879,40605
+54531,33135,30878,40887
+54806,33136,30877,40114
+56060,33137,30876,40189
+53498,33138,30875,40256
+59344,33139,30874,40182
+57276,33140,30873,40035
+53917,33141,30872,40531
+59967,33142,30871,40728
+54924,33143,30870,40945
+55582,33144,30869,40129
+55015,33145,30868,40673
+56002,33146,30867,40159
+59820,33147,30866,40102
+59060,33148,30865,40939
+51193,33149,30864,40411
+51403,33150,30863,40857
+50343,33151,30862,40646
+53974,33152,30861,40257
+51508,33153,30860,40277
+54934,33154,30859,40923
+54152,33155,30858,40404
+58259,33156,30857,40959
+53881,33157,30856,40327
+56095,33158,30855,40182
+57466,33159,30854,40594
+56647,33160,30853,40568
+52286,33161,30852,40643
+58613,33162,30851,40115
+52330,33163,30850,40376
+57059,33164,30849,40588
+52089,33165,30848,40851
+59914,33166,30847,40918
+50585,33167,30846,40951
+59432,33168,30845,40020
+51858,33169,30844,40509
+50832,33170,30843,40900
+53655,33171,30842,40752
+50619,33172,30841,40883
+55509,33173,30840,40209
+57037,33174,30839,40759
+51288,33175,30838,40647
+50819,33176,30837,40492
+50365,33177,30836,40702
+52984,33178,30835,40134
+57202,33179,30834,40699
+59862,33180,30833,40784
+51034,33181,30832,40474
+59817,33182,30831,40943
+57757,33183,30830,40901
+54655,33184,30829,40583
+59851,33185,30828,40825
+53836,33186,30827,40879
+57681,33187,30826,40294
+57194,33188,30825,40163
+55396,33189,30824,40051
+53038,33190,30823,40158
+57419,33191,30822,40763
+55990,33192,30821,40690
+54249,33193,30820,40120
+53502,33194,30819,40238
+52231,33195,30818,40746
+51217,33196,30817,40378
+56049,33197,30816,40620
+58205,33198,30815,40682
+52343,33199,30814,40916
+58831,33200,30813,40901
+55395,33201,30812,40845
+59438,33202,30811,40161
+55834,33203,30810,40771
+55971,33204,30809,40004
+54539,33205,30808,40522
+59398,33206,30807,40361
+51173,33207,30806,40098
+54148,33208,30805,40598
+56806,33209,30804,40523
+57436,33210,30803,40230
+51629,33211,30802,40122
+57433,33212,30801,40320
+58753,33213,30800,40965
+51129,33214,30799,40978
+55978,33215,30798,40433
+50733,33216,30797,40715
+54765,33217,30796,40517
+59289,33218,30795,40076
+57526,33219,30794,40826
+59931,33220,30793,40730
+58766,33221,30792,40530
+55640,33222,30791,40500
+54235,33223,30790,40977
+55298,33224,30789,40467
+53926,33225,30788,40229
+54358,33226,30787,40842
+50872,33227,30786,40262
+55121,33228,30785,40771
+58414,33229,30784,40921
+55923,33230,30783,40996
+51548,33231,30782,40183
+54492,33232,30781,40620
+57038,33233,30780,40277
+59618,33234,30779,40414
+55209,33235,30778,40215
+54501,33236,30777,40513
+54560,33237,30776,40334
+56313,33238,30775,40361
+51746,33239,30774,40820
+53379,33240,30773,40510
+51344,33241,30772,40160
+59951,33242,30771,40422
+52828,33243,30770,40578
+54510,33244,30769,40842
+52302,33245,30768,40628
+58637,33246,30767,40634
+58128,33247,30766,40346
+55712,33248,30765,40270
+52281,33249,30764,40896
+57645,33250,30763,40722
+56151,33251,30762,40407
+58661,33252,30761,40732
+56693,33253,30760,40052
+58766,33254,30759,40869
+59740,33255,30758,40333
+58843,33256,30757,40602
+53302,33257,30756,40553
+52253,33258,30755,40895
+52525,33259,30754,40907
+59458,33260,30753,40743
+56050,33261,30752,40417
+50316,33262,30751,40298
+58266,33263,30750,40287
+55837,33264,30749,40211
+57923,33265,30748,40028
+53102,33266,30747,40132
+51060,33267,30746,40214
+58255,33268,30745,40976
+53138,33269,30744,40767
+50664,33270,30743,40901
+52191,33271,30742,40595
+55806,33272,30741,40373
+57484,33273,30740,40732
+56738,33274,30739,40263
+56848,33275,30738,40492
+52099,33276,30737,40615
+52097,33277,30736,40686
+52164,33278,30735,40094
+51998,33279,30734,40866
+52264,33280,30733,40447
+53231,33281,30732,40435
+52345,33282,30731,40360
+58548,33283,30730,40463
+57543,33284,30729,40561
+55307,33285,30728,40442
+54622,33286,30727,40204
+55693,33287,30726,40759
+55041,33288,30725,40440
+52743,33289,30724,40725
+52083,33290,30723,40500
+56519,33291,30722,40329
+58644,33292,30721,40837
+54604,33293,30720,40534
+54952,33294,30719,40868
+57664,33295,30718,40981
+54003,33296,30717,40287
+56562,33297,30716,40794
+50273,33298,30715,40114
+59147,33299,30714,40590
+59186,33300,30713,40618
+57459,33301,30712,40627
+55138,33302,30711,40934
+54572,33303,30710,40127
+51543,33304,30709,40986
+50379,33305,30708,40359
+55588,33306,30707,40105
+53015,33307,30706,40103
+59002,33308,30705,40950
+53274,33309,30704,40868
+50029,33310,30703,40887
+57700,33311,30702,40695
+58629,33312,30701,40079
+53423,33313,30700,40902
+50251,33314,30699,40143
+51518,33315,30698,40200
+53539,33316,30697,40944
+55086,33317,30696,40068
+52650,33318,30695,40071
+52814,33319,30694,40598
+59700,33320,30693,40877
+51739,33321,30692,40792
+55752,33322,30691,40514
+55060,33323,30690,40073
+59320,33324,30689,40174
+59053,33325,30688,40165
+54568,33326,30687,40363
+52174,33327,30686,40633
+54639,33328,30685,40603
+50011,33329,30684,40209
+58315,33330,30683,40633
+52799,33331,30682,40616
+56255,33332,30681,40542
+55169,33333,30680,40530
+55560,33334,30679,40211
+56425,33335,30678,40257
+55869,33336,30677,40830
+52204,33337,30676,40219
+56276,33338,30675,40531
+55458,33339,30674,40500
+53874,33340,30673,40720
+57097,33341,30672,40157
+52058,33342,30671,40109
+57170,33343,30670,40161
+53351,33344,30669,40870
+58506,33345,30668,40584
+52549,33346,30667,40902
+59267,33347,30666,40162
+55321,33348,30665,40154
+56352,33349,30664,40297
+53712,33350,30663,40591
+57072,33351,30662,40515
+54773,33352,30661,40720
+54193,33353,30660,40019
+52620,33354,30659,40552
+58454,33355,30658,40891
+57052,33356,30657,40001
+53267,33357,30656,40712
+50698,33358,30655,40191
+57397,33359,30654,40946
+53504,33360,30653,40989
+50542,33361,30652,40583
+53100,33362,30651,40193
+56670,33363,30650,40491
+58981,33364,30649,40237
+58158,33365,30648,40749
+53496,33366,30647,40781
+58038,33367,30646,40385
+57088,33368,30645,40873
+54671,33369,30644,40678
+53764,33370,30643,40814
+50130,33371,30642,40507
+59275,33372,30641,40613
+58814,33373,30640,40403
+53370,33374,30639,40319
+56345,33375,30638,40345
+57150,33376,30637,40267
+56094,33377,30636,40242
+50197,33378,30635,40246
+53573,33379,30634,40174
+55927,33380,30633,40606
+53904,33381,30632,40922
+57218,33382,30631,40619
+57825,33383,30630,40198
+54371,33384,30629,40666
+59097,33385,30628,40246
+54512,33386,30627,40080
+59339,33387,30626,40911
+50306,33388,30625,40175
+51439,33389,30624,40615
+52816,33390,30623,40576
+56083,33391,30622,40777
+57446,33392,30621,40733
+59580,33393,30620,40844
+54564,33394,30619,40201
+57898,33395,30618,40568
+50019,33396,30617,40383
+57113,33397,30616,40972
+59523,33398,30615,40508
+51460,33399,30614,40566
+50704,33400,30613,40714
+59748,33401,30612,40528
+51363,33402,30611,40465
+53558,33403,30610,40723
+59205,33404,30609,40468
+51706,33405,30608,40891
+52862,33406,30607,40294
+56487,33407,30606,40807
+53947,33408,30605,40711
+59628,33409,30604,40155
+57597,33410,30603,40103
+54850,33411,30602,40660
+50798,33412,30601,40373
+50686,33413,30600,40551
+51727,33414,30599,40900
+58642,33415,30598,40086
+58045,33416,30597,40891
+59539,33417,30596,40808
+59747,33418,30595,40943
+57845,33419,30594,40012
+51672,33420,30593,40776
+56770,33421,30592,40562
+57672,33422,30591,40757
+54922,33423,30590,40734
+50881,33424,30589,40348
+59715,33425,30588,40218
+59476,33426,30587,40146
+59794,33427,30586,40358
+53243,33428,30585,40235
+53346,33429,30584,40215
+57658,33430,30583,40503
+50417,33431,30582,40169
+59102,33432,30581,40162
+58587,33433,30580,40594
+51170,33434,30579,40222
+50749,33435,30578,40645
+54032,33436,30577,40062
+59467,33437,30576,40024
+58820,33438,30575,40857
+58190,33439,30574,40186
+52796,33440,30573,40201
+52325,33441,30572,40788
+57036,33442,30571,40584
+55084,33443,30570,40115
+51311,33444,30569,40126
+52362,33445,30568,40855
+56599,33446,30567,40370
+53620,33447,30566,40548
+58629,33448,30565,40794
+57903,33449,30564,40666
+50916,33450,30563,40232
+54947,33451,30562,40875
+50960,33452,30561,40227
+51067,33453,30560,40496
+50372,33454,30559,40231
+50344,33455,30558,40816
+54129,33456,30557,40178
+51413,33457,30556,40519
+55930,33458,30555,40500
+51174,33459,30554,40230
+55436,33460,30553,40498
+55305,33461,30552,40405
+55036,33462,30551,40359
+54799,33463,30550,40682
+57544,33464,30549,40664
+54068,33465,30548,40436
+57295,33466,30547,40418
+54009,33467,30546,40937
+54444,33468,30545,40267
+50323,33469,30544,40472
+50045,33470,30543,40697
+58173,33471,30542,40995
+53443,33472,30541,40723
+50152,33473,30540,40477
+59714,33474,30539,40170
+52482,33475,30538,40548
+50984,33476,30537,40419
+53748,33477,30536,40027
+51723,33478,30535,40132
+53670,33479,30534,40191
+54634,33480,30533,40120
+53869,33481,30532,40204
+54389,33482,30531,40745
+58501,33483,30530,40179
+54601,33484,30529,40033
+51609,33485,30528,40706
+59187,33486,30527,40717
+50783,33487,30526,40432
+58886,33488,30525,40935
+55089,33489,30524,40123
+50648,33490,30523,40774
+51062,33491,30522,40536
+55254,33492,30521,40940
+57631,33493,30520,40174
+52632,33494,30519,40812
+56989,33495,30518,40248
+51088,33496,30517,40357
+51551,33497,30516,40837
+52587,33498,30515,40448
+51721,33499,30514,40262
+56314,33500,30513,40602
+52993,33501,30512,40435
+53418,33502,30511,40966
+54163,33503,30510,40562
+56431,33504,30509,40114
+50309,33505,30508,40917
+57424,33506,30507,40396
+55284,33507,30506,40591
+58139,33508,30505,40394
+52509,33509,30504,40019
+52745,33510,30503,40291
+51016,33511,30502,40456
+51676,33512,30501,40228
+52510,33513,30500,40237
+55317,33514,30499,40618
+56667,33515,30498,40477
+55971,33516,30497,40369
+51630,33517,30496,40655
+59398,33518,30495,40280
+50654,33519,30494,40677
+59329,33520,30493,40778
+55047,33521,30492,40735
+59885,33522,30491,40137
+53362,33523,30490,40596
+50394,33524,30489,40736
+53954,33525,30488,40620
+52304,33526,30487,40445
+59931,33527,30486,40708
+53348,33528,30485,40913
+55949,33529,30484,40334
+53687,33530,30483,40718
+58990,33531,30482,40141
+58360,33532,30481,40276
+53573,33533,30480,40458
+51306,33534,30479,40143
+56226,33535,30478,40534
+53184,33536,30477,40433
+59676,33537,30476,40917
+51748,33538,30475,40399
+57426,33539,30474,40299
+52983,33540,30473,40411
+50115,33541,30472,40062
+59922,33542,30471,40398
+53658,33543,30470,40103
+53996,33544,30469,40391
+53984,33545,30468,40878
+53556,33546,30467,40621
+54821,33547,30466,40921
+57566,33548,30465,40349
+57241,33549,30464,40840
+50747,33550,30463,40472
+57141,33551,30462,40473
+56704,33552,30461,40491
+55143,33553,30460,40123
+56665,33554,30459,40348
+57034,33555,30458,40537
+58374,33556,30457,40319
+50612,33557,30456,40917
+50147,33558,30455,40882
+58937,33559,30454,40845
+54294,33560,30453,40467
+59708,33561,30452,40879
+58754,33562,30451,40468
+53654,33563,30450,40309
+50600,33564,30449,40672
+55087,33565,30448,40917
+56365,33566,30447,40941
+51705,33567,30446,40648
+56182,33568,30445,40110
+55117,33569,30444,40117
+55807,33570,30443,40539
+52285,33571,30442,40441
+59388,33572,30441,40711
+55480,33573,30440,40381
+51527,33574,30439,40166
+58044,33575,30438,40387
+57704,33576,30437,40129
+57109,33577,30436,40774
+58611,33578,30435,40360
+55268,33579,30434,40606
+56794,33580,30433,40434
+55147,33581,30432,40557
+52291,33582,30431,40684
+59652,33583,30430,40905
+50373,33584,30429,40220
+55489,33585,30428,40766
+59868,33586,30427,40960
+54479,33587,30426,40863
+50940,33588,30425,40432
+53028,33589,30424,40849
+59954,33590,30423,40317
+55223,33591,30422,40365
+51781,33592,30421,40348
+57246,33593,30420,40487
+51515,33594,30419,40723
+56994,33595,30418,40287
+59241,33596,30417,40047
+50784,33597,30416,40420
+57251,33598,30415,40400
+53785,33599,30414,40816
+52625,33600,30413,40969
+52808,33601,30412,40916
+52450,33602,30411,40768
+52867,33603,30410,40528
+50154,33604,30409,40326
+53275,33605,30408,40999
+50871,33606,30407,40683
+59006,33607,30406,40965
+56833,33608,30405,40737
+55468,33609,30404,40502
+51024,33610,30403,40939
+56102,33611,30402,40829
+51736,33612,30401,40883
+50450,33613,30400,40173
+57087,33614,30399,40597
+58427,33615,30398,40008
+52463,33616,30397,40227
+58182,33617,30396,40473
+55174,33618,30395,40965
+51647,33619,30394,40416
+57610,33620,30393,40564
+53953,33621,30392,40361
+58395,33622,30391,40468
+54949,33623,30390,40961
+51169,33624,30389,40476
+53810,33625,30388,40783
+53524,33626,30387,40171
+54275,33627,30386,40323
+52491,33628,30385,40601
+51862,33629,30384,40710
+52412,33630,30383,40023
+54608,33631,30382,40365
+51841,33632,30381,40620
+59422,33633,30380,40371
+56060,33634,30379,40077
+59270,33635,30378,40929
+59665,33636,30377,40975
+58444,33637,30376,40165
+52482,33638,30375,40356
+50608,33639,30374,40597
+54760,33640,30373,40560
+57269,33641,30372,40870
+53741,33642,30371,40103
+53565,33643,30370,40549
+52406,33644,30369,40334
+54431,33645,30368,40777
+56973,33646,30367,40226
+54875,33647,30366,40868
+56142,33648,30365,40160
+52996,33649,30364,40796
+54087,33650,30363,40874
+57308,33651,30362,40762
+55554,33652,30361,40235
+58973,33653,30360,40853
+55711,33654,30359,40399
+51875,33655,30358,40364
+52202,33656,30357,40914
+59848,33657,30356,40119
+59212,33658,30355,40292
+55383,33659,30354,40488
+57278,33660,30353,40701
+57058,33661,30352,40188
+50328,33662,30351,40883
+59311,33663,30350,40248
+59959,33664,30349,40645
+52461,33665,30348,40593
+51629,33666,30347,40042
+57861,33667,30346,40372
+51922,33668,30345,40283
+55403,33669,30344,40687
+52712,33670,30343,40602
+50611,33671,30342,40170
+53020,33672,30341,40174
+52464,33673,30340,40597
+57080,33674,30339,40420
+59079,33675,30338,40528
+55638,33676,30337,40315
+59587,33677,30336,40273
+59215,33678,30335,40788
+52946,33679,30334,40581
+55746,33680,30333,40284
+58589,33681,30332,40822
+53056,33682,30331,40248
+50516,33683,30330,40685
+59427,33684,30329,40084
+56568,33685,30328,40557
+53663,33686,30327,40248
+54578,33687,30326,40044
+56929,33688,30325,40391
+59936,33689,30324,40646
+58084,33690,30323,40117
+52736,33691,30322,40039
+54050,33692,30321,40076
+58677,33693,30320,40805
+54311,33694,30319,40451
+50023,33695,30318,40456
+54546,33696,30317,40820
+57735,33697,30316,40415
+56629,33698,30315,40096
+52447,33699,30314,40528
+53131,33700,30313,40314
+58968,33701,30312,40940
+57737,33702,30311,40557
+54085,33703,30310,40989
+57261,33704,30309,40594
+57949,33705,30308,40388
+58275,33706,30307,40786
+51670,33707,30306,40963
+59927,33708,30305,40828
+56525,33709,30304,40500
+59332,33710,30303,40219
+54002,33711,30302,40836
+53227,33712,30301,40567
+58255,33713,30300,40120
+56207,33714,30299,40895
+57113,33715,30298,40106
+51309,33716,30297,40096
+50848,33717,30296,40416
+53450,33718,30295,40861
+54577,33719,30294,40673
+51108,33720,30293,40083
+52618,33721,30292,40261
+57946,33722,30291,40478
+52797,33723,30290,40392
+59911,33724,30289,40214
+52110,33725,30288,40650
+52682,33726,30287,40712
+52663,33727,30286,40462
+56513,33728,30285,40186
+51515,33729,30284,40781
+52724,33730,30283,40247
+50694,33731,30282,40454
+57102,33732,30281,40963
+57139,33733,30280,40621
+56933,33734,30279,40756
+51464,33735,30278,40917
+52347,33736,30277,40581
+51100,33737,30276,40731
+59213,33738,30275,40248
+54937,33739,30274,40049
+56728,33740,30273,40318
+56236,33741,30272,40859
+55100,33742,30271,40974
+51770,33743,30270,40041
+56457,33744,30269,40737
+51068,33745,30268,40946
+57110,33746,30267,40870
+57130,33747,30266,40689
+58720,33748,30265,40919
+54390,33749,30264,40175
+59135,33750,30263,40998
+50783,33751,30262,40308
+59365,33752,30261,40661
+59271,33753,30260,40443
+59871,33754,30259,40945
+52158,33755,30258,40010
+56190,33756,30257,40730
+59101,33757,30256,40259
+55391,33758,30255,40578
+52766,33759,30254,40754
+51174,33760,30253,40975
+54424,33761,30252,40826
+53114,33762,30251,40471
+58057,33763,30250,40886
+58713,33764,30249,40615
+51149,33765,30248,40577
+58108,33766,30247,40504
+57627,33767,30246,40139
+53910,33768,30245,40078
+54094,33769,30244,40905
+50502,33770,30243,40630
+52240,33771,30242,40840
+57131,33772,30241,40628
+56114,33773,30240,40957
+57559,33774,30239,40262
+51865,33775,30238,40844
+53852,33776,30237,40601
+53107,33777,30236,40531
+50628,33778,30235,40206
+59592,33779,30234,40585
+59085,33780,30233,40448
+52795,33781,30232,40193
+55632,33782,30231,40204
+50398,33783,30230,40837
+54770,33784,30229,40141
+53061,33785,30228,40193
+58660,33786,30227,40244
+53587,33787,30226,40854
+58805,33788,30225,40945
+53682,33789,30224,40543
+52376,33790,30223,40472
+52222,33791,30222,40747
+57376,33792,30221,40970
+56240,33793,30220,40335
+57959,33794,30219,40311
+51433,33795,30218,40559
+59246,33796,30217,40357
+56420,33797,30216,40793
+56351,33798,30215,40840
+53662,33799,30214,40239
+58798,33800,30213,40286
+54673,33801,30212,40255
+56446,33802,30211,40182
+54556,33803,30210,40040
+55792,33804,30209,40604
+50398,33805,30208,40003
+57653,33806,30207,40723
+55745,33807,30206,40972
+55445,33808,30205,40816
+51364,33809,30204,40947
+50153,33810,30203,40142
+59857,33811,30202,40275
+53218,33812,30201,40436
+51101,33813,30200,40303
+51116,33814,30199,40183
+54237,33815,30198,40116
+53117,33816,30197,40517
+56934,33817,30196,40231
+59624,33818,30195,40079
+53933,33819,30194,40035
+56718,33820,30193,40074
+54557,33821,30192,40167
+55652,33822,30191,40219
+51510,33823,30190,40850
+57679,33824,30189,40260
+56195,33825,30188,40812
+54364,33826,30187,40152
+51268,33827,30186,40386
+56268,33828,30185,40462
+51738,33829,30184,40989
+57436,33830,30183,40733
+54001,33831,30182,40672
+54803,33832,30181,40925
+55943,33833,30180,40316
+52821,33834,30179,40317
+51131,33835,30178,40235
+58613,33836,30177,40815
+58017,33837,30176,40333
+59191,33838,30175,40244
+59346,33839,30174,40873
+52035,33840,30173,40493
+53967,33841,30172,40924
+57637,33842,30171,40271
+54358,33843,30170,40023
+57025,33844,30169,40490
+54766,33845,30168,40152
+50483,33846,30167,40438
+53101,33847,30166,40352
+51621,33848,30165,40075
+59801,33849,30164,40404
+58171,33850,30163,40620
+56586,33851,30162,40554
+52360,33852,30161,40278
+50974,33853,30160,40650
+55843,33854,30159,40915
+50967,33855,30158,40902
+56611,33856,30157,40309
+56895,33857,30156,40112
+50919,33858,30155,40881
+52571,33859,30154,40247
+51731,33860,30153,40538
+50142,33861,30152,40132
+54853,33862,30151,40759
+58244,33863,30150,40375
+52118,33864,30149,40594
+58015,33865,30148,40562
+59084,33866,30147,40558
+59166,33867,30146,40902
+50267,33868,30145,40044
+51209,33869,30144,40626
+50827,33870,30143,40354
+59802,33871,30142,40901
+56197,33872,30141,40441
+58741,33873,30140,40348
+52648,33874,30139,40241
+52600,33875,30138,40708
+57752,33876,30137,40211
+56957,33877,30136,40942
+50534,33878,30135,40178
+54924,33879,30134,40474
+53130,33880,30133,40503
+52701,33881,30132,40529
+51116,33882,30131,40192
+55676,33883,30130,40075
+50963,33884,30129,40381
+56933,33885,30128,40921
+52453,33886,30127,40471
+54847,33887,30126,40713
+57133,33888,30125,40700
+51861,33889,30124,40524
+59514,33890,30123,40043
+51423,33891,30122,40285
+51796,33892,30121,40720
+57215,33893,30120,40920
+58681,33894,30119,40285
+54712,33895,30118,40820
+55702,33896,30117,40046
+54217,33897,30116,40166
+50673,33898,30115,40335
+58998,33899,30114,40828
+57347,33900,30113,40476
+50599,33901,30112,40203
+55443,33902,30111,40861
+57674,33903,30110,40662
+58797,33904,30109,40187
+57012,33905,30108,40739
+55986,33906,30107,40187
+56421,33907,30106,40699
+53865,33908,30105,40719
+55002,33909,30104,40830
+55251,33910,30103,40131
+58379,33911,30102,40281
+50300,33912,30101,40795
+51412,33913,30100,40565
+50798,33914,30099,40654
+57649,33915,30098,40725
+55519,33916,30097,40094
+53475,33917,30096,40948
+52218,33918,30095,40423
+58063,33919,30094,40672
+58011,33920,30093,40978
+50326,33921,30092,40631
+58996,33922,30091,40030
+50639,33923,30090,40228
+56749,33924,30089,40649
+54752,33925,30088,40406
+54440,33926,30087,40568
+58555,33927,30086,40686
+58562,33928,30085,40777
+56325,33929,30084,40467
+56694,33930,30083,40105
+59250,33931,30082,40643
+54059,33932,30081,40311
+51180,33933,30080,40880
+50302,33934,30079,40236
+59139,33935,30078,40380
+50874,33936,30077,40805
+53054,33937,30076,40691
+52451,33938,30075,40502
+51935,33939,30074,40452
+59475,33940,30073,40557
+58565,33941,30072,40203
+58220,33942,30071,40920
+52795,33943,30070,40582
+58924,33944,30069,40412
+59838,33945,30068,40578
+55502,33946,30067,40204
+58235,33947,30066,40217
+57932,33948,30065,40914
+56471,33949,30064,40058
+51202,33950,30063,40653
+57480,33951,30062,40118
+59407,33952,30061,40807
+54832,33953,30060,40682
+53269,33954,30059,40001
+54459,33955,30058,40918
+59951,33956,30057,40005
+58974,33957,30056,40904
+52850,33958,30055,40816
+57135,33959,30054,40502
+50540,33960,30053,40178
+59891,33961,30052,40255
+54041,33962,30051,40387
+56546,33963,30050,40004
+53557,33964,30049,40236
+54615,33965,30048,40680
+50956,33966,30047,40263
+56387,33967,30046,40128
+54684,33968,30045,40460
+57954,33969,30044,40457
+51692,33970,30043,40794
+50613,33971,30042,40670
+59686,33972,30041,40510
+52510,33973,30040,40011
+59727,33974,30039,40114
+53579,33975,30038,40577
+56632,33976,30037,40781
+57145,33977,30036,40493
+58109,33978,30035,40931
+57609,33979,30034,40711
+56747,33980,30033,40113
+54384,33981,30032,40748
+54030,33982,30031,40367
+57769,33983,30030,40065
+58423,33984,30029,40450
+50565,33985,30028,40109
+57725,33986,30027,40309
+51443,33987,30026,40337
+52283,33988,30025,40787
+50618,33989,30024,40606
+50503,33990,30023,40052
+54546,33991,30022,40083
+55171,33992,30021,40250
+54947,33993,30020,40466
+57852,33994,30019,40294
+52486,33995,30018,40958
+52752,33996,30017,40108
+52455,33997,30016,40812
+54173,33998,30015,40656
+52588,33999,30014,40602
+51060,34000,30013,40011
+59935,34001,30012,40576
+59148,34002,30011,40534
+54935,34003,30010,40026
+54962,34004,30009,40513
+52309,34005,30008,40751
+58302,34006,30007,40369
+56666,34007,30006,40577
+54402,34008,30005,40516
+54870,34009,30004,40481
+51995,34010,30003,40802
+59179,34011,30002,40758
+55530,34012,30001,40866
+55184,34013,30000,40290
+55100,34014,31000,40886
+54530,34015,30999,40505
+51925,34016,30998,40052
+53828,34017,30997,40183
+53756,34018,30996,40241
+57335,34019,30995,40077
+59264,34020,30994,40084
+54036,34021,30993,40248
+51448,34022,30992,40357
+57878,34023,30991,40294
+53943,34024,30990,40576
+51270,34025,30989,40485
+50974,34026,30988,40802
+57886,34027,30987,40410
+57228,34028,30986,40247
+51214,34029,30985,40259
+55425,34030,30984,40525
+52221,34031,30983,40642
+54976,34032,30982,40320
+56407,34033,30981,40038
+56018,34034,30980,40083
+50357,34035,30979,40439
+56536,34036,30978,40056
+57787,34037,30977,40334
+58696,34038,30976,40300
+54826,34039,30975,40103
+57080,34040,30974,40554
+50342,34041,30973,40236
+50553,34042,30972,40626
+50225,34043,30971,40695
+58779,34044,30970,40422
+57686,34045,30969,40899
+50590,34046,30968,40209
+57378,34047,30967,40912
+51208,34048,30966,40532
+56845,34049,30965,40009
+51475,34050,30964,40214
+57079,34051,30963,40491
+57326,34052,30962,40232
+55453,34053,30961,40853
+56979,34054,30960,40314
+58970,34055,30959,40183
+54985,34056,30958,40861
+59146,34057,30957,40472
+58128,34058,30956,40750
+53466,34059,30955,40200
+56121,34060,30954,40052
+53755,34061,30953,40342
+50936,34062,30952,40822
+51510,34063,30951,40071
+55682,34064,30950,40844
+55161,34065,30949,40916
+51088,34066,30948,40272
+50896,34067,30947,40382
+53542,34068,30946,40968
+59534,34069,30945,40649
+58893,34070,30944,40508
+54874,34071,30943,40426
+59420,34072,30942,40446
+58311,34073,30941,40942
+58566,34074,30940,40683
+58860,34075,30939,40409
+53068,34076,30938,40940
+53938,34077,30937,40168
+57715,34078,30936,40329
+55836,34079,30935,40707
+53936,34080,30934,40803
+58064,34081,30933,40595
+59696,34082,30932,40748
+53155,34083,30931,40331
+53701,34084,30930,40998
+57540,34085,30929,40982
+54458,34086,30928,40668
+58030,34087,30927,40045
+52791,34088,30926,40785
+59169,34089,30925,40195
+54496,34090,30924,40291
+53082,34091,30923,40391
+50616,34092,30922,40011
+52543,34093,30921,40317
+59699,34094,30920,40341
+58823,34095,30919,40394
+52200,34096,30918,40070
+53782,34097,30917,40473
+56165,34098,30916,40902
+51158,34099,30915,40631
+52830,34100,30914,40244
+55296,34101,30913,40087
+56269,34102,30912,40624
+59450,34103,30911,40899
+53021,34104,30910,40313
+52683,34105,30909,40273
+59906,34106,30908,40804
+56490,34107,30907,40965
+56543,34108,30906,40005
+53333,34109,30905,40168
+55035,34110,30904,40565
+50589,34111,30903,40301
+55102,34112,30902,40489
+52430,34113,30901,40484
+56837,34114,30900,40834
+54828,34115,30899,40725
+55333,34116,30898,40678
+59496,34117,30897,40890
+58924,34118,30896,40947
+58035,34119,30895,40793
+50498,34120,30894,40715
+53882,34121,30893,40365
+59823,34122,30892,40361
+50831,34123,30891,40640
+50647,34124,30890,40902
+57448,34125,30889,40223
+58288,34126,30888,40271
+54096,34127,30887,40218
+53524,34128,30886,40519
+57132,34129,30885,40670
+52434,34130,30884,40445
+50104,34131,30883,40633
+54398,34132,30882,40706
+51856,34133,30881,40154
+59129,34134,30880,40161
+54171,34135,30879,40730
+55947,34136,30878,40779
+59841,34137,30877,40523
+54381,34138,30876,40126
+59309,34139,30875,40083
+50131,34140,30874,40529
+51277,34141,30873,40708
+53542,34142,30872,40055
+57558,34143,30871,40324
+52672,34144,30870,40261
+51621,34145,30869,40245
+50960,34146,30868,40385
+54656,34147,30867,40640
+56126,34148,30866,40346
+52507,34149,30865,40306
+53737,34150,30864,40998
+57463,34151,30863,40899
+58731,34152,30862,40980
+53989,34153,30861,40462
+59774,34154,30860,40007
+52456,34155,30859,40898
+52519,34156,30858,40323
+54343,34157,30857,40253
+50113,34158,30856,40087
+55650,34159,30855,40016
+51349,34160,30854,40711
+54940,34161,30853,40635
+55013,34162,30852,40420
+53959,34163,30851,40253
+59384,34164,30850,40594
+53246,34165,30849,40999
+55631,34166,30848,40623
+58523,34167,30847,40614
+51494,34168,30846,40875
+53145,34169,30845,40252
+53607,34170,30844,40992
+59875,34171,30843,40994
+59672,34172,30842,40735
+50667,34173,30841,40852
+59604,34174,30840,40260
+57336,34175,30839,40193
+59431,34176,30838,40088
+54390,34177,30837,40277
+56963,34178,30836,40721
+54161,34179,30835,40797
+59440,34180,30834,40824
+52588,34181,30833,40822
+57665,34182,30832,40734
+54763,34183,30831,40826
+50641,34184,30830,40812
+57234,34185,30829,40488
+53611,34186,30828,40982
+56980,34187,30827,40903
+53785,34188,30826,40554
+56621,34189,30825,40774
+58405,34190,30824,40923
+56027,34191,30823,40779
+58443,34192,30822,40749
+55703,34193,30821,40751
+57487,34194,30820,40572
+59090,34195,30819,40555
+55435,34196,30818,40174
+58106,34197,30817,40913
+59857,34198,30816,40993
+55788,34199,30815,40800
+59631,34200,30814,40970
+51731,34201,30813,40781
+52384,34202,30812,40939
+55838,34203,30811,40099
+54494,34204,30810,40000
+56848,34205,30809,40408
+58822,34206,30808,40695
+53777,34207,30807,40227
+55860,34208,30806,40689
+54978,34209,30805,40773
+58445,34210,30804,40249
+51214,34211,30803,40787
+53516,34212,30802,40816
+53275,34213,30801,40085
+50052,34214,30800,40491
+50080,34215,30799,40263
+50692,34216,30798,40146
+59137,34217,30797,40905
+55624,34218,30796,40949
+54533,34219,30795,40164
+58092,34220,30794,40141
+52905,34221,30793,40284
+51883,34222,30792,40284
+56861,34223,30791,40253
+55094,34224,30790,40410
+58796,34225,30789,40659
+56401,34226,30788,40773
+53097,34227,30787,40074
+57463,34228,30786,40084
+59716,34229,30785,40734
+54317,34230,30784,40320
+54812,34231,30783,40656
+54884,34232,30782,40791
+52239,34233,30781,40302
+56392,34234,30780,40775
+50637,34235,30779,40998
+54070,34236,30778,40900
+51851,34237,30777,40215
+54442,34238,30776,40142
+53507,34239,30775,40035
+55686,34240,30774,40722
+50680,34241,30773,40109
+51310,34242,30772,40482
+52935,34243,30771,40571
+50384,34244,30770,40300
+53945,34245,30769,40368
+59730,34246,30768,40004
+56133,34247,30767,40969
+55172,34248,30766,40983
+51938,34249,30765,40385
+57332,34250,30764,40197
+56970,34251,30763,40876
+53776,34252,30762,40162
+51211,34253,30761,40788
+54981,34254,30760,40249
+53015,34255,30759,40411
+59175,34256,30758,40141
+56555,34257,30757,40462
+56794,34258,30756,40726
+55868,34259,30755,40723
+51196,34260,30754,40476
+59645,34261,30753,40111
+53381,34262,30752,40323
+53517,34263,30751,40333
+52311,34264,30750,40532
+54974,34265,30749,40014
+51744,34266,30748,40979
+57696,34267,30747,40914
+56526,34268,30746,40930
+57532,34269,30745,40921
+55746,34270,30744,40905
+50797,34271,30743,40049
+55997,34272,30742,40648
+55736,34273,30741,40572
+52871,34274,30740,40132
+53148,34275,30739,40181
+54878,34276,30738,40177
+57972,34277,30737,40638
+54715,34278,30736,40594
+58122,34279,30735,40833
+50097,34280,30734,40714
+56214,34281,30733,40762
+59791,34282,30732,40022
+54003,34283,30731,40725
+51349,34284,30730,40585
+56722,34285,30729,40345
+58035,34286,30728,40651
+54681,34287,30727,40280
+59948,34288,30726,40087
+57267,34289,30725,40202
+53929,34290,30724,40936
+52491,34291,30723,40941
+55754,34292,30722,40487
+58787,34293,30721,40443
+56664,34294,30720,40804
+57086,34295,30719,40328
+58966,34296,30718,40046
+53678,34297,30717,40007
+55446,34298,30716,40312
+51063,34299,30715,40312
+58072,34300,30714,40083
+54339,34301,30713,40983
+58284,34302,30712,40674
+53487,34303,30711,40140
+54617,34304,30710,40398
+51666,34305,30709,40579
+50918,34306,30708,40302
+57101,34307,30707,40632
+56309,34308,30706,40850
+54638,34309,30705,40953
+52944,34310,30704,40852
+59135,34311,30703,40303
+58955,34312,30702,40012
+50699,34313,30701,40705
+54921,34314,30700,40023
+59169,34315,30699,40031
+58507,34316,30698,40123
+56833,34317,30697,40913
+52979,34318,30696,40866
+58755,34319,30695,40418
+53606,34320,30694,40616
+50023,34321,30693,40644
+56064,34322,30692,40642
+51542,34323,30691,40425
+55378,34324,30690,40253
+51896,34325,30689,40259
+52390,34326,30688,40480
+54239,34327,30687,40673
+57235,34328,30686,40728
+52705,34329,30685,40510
+52188,34330,30684,40094
+53315,34331,30683,40087
+51570,34332,30682,40686
+57216,34333,30681,40617
+51386,34334,30680,40423
+58745,34335,30679,40037
+57450,34336,30678,40522
+58390,34337,30677,40986
+50060,34338,30676,40169
+51562,34339,30675,40397
+54709,34340,30674,40749
+54645,34341,30673,40294
+58383,34342,30672,40467
+56468,34343,30671,40562
+52434,34344,30670,40037
+59379,34345,30669,40153
+52338,34346,30668,40849
+59666,34347,30667,40684
+53090,34348,30666,40056
+58037,34349,30665,40477
+59958,34350,30664,40461
+54645,34351,30663,40026
+52422,34352,30662,40055
+59524,34353,30661,40879
+52590,34354,30660,40317
+58062,34355,30659,40751
+53826,34356,30658,40910
+54906,34357,30657,40799
+58143,34358,30656,40522
+55360,34359,30655,40151
+59885,34360,30654,40584
+54965,34361,30653,40365
+52977,34362,30652,40636
+56130,34363,30651,40360
+54958,34364,30650,40156
+57955,34365,30649,40791
+51035,34366,30648,40160
+54459,34367,30647,40004
+59568,34368,30646,40268
+55444,34369,30645,40664
+57210,34370,30644,40292
+53252,34371,30643,40603
+57982,34372,30642,40971
+51019,34373,30641,40552
+54410,34374,30640,40383
+53321,34375,30639,40731
+59879,34376,30638,40036
+56781,34377,30637,40048
+56155,34378,30636,40723
+56855,34379,30635,40097
+54379,34380,30634,40615
+55503,34381,30633,40908
+56605,34382,30632,40757
+51935,34383,30631,40607
+51291,34384,30630,40453
+58171,34385,30629,40873
+51331,34386,30628,40073
+53665,34387,30627,40982
+50636,34388,30626,40969
+57771,34389,30625,40398
+58756,34390,30624,40560
+53298,34391,30623,40482
+55220,34392,30622,40638
+59980,34393,30621,40692
+51045,34394,30620,40643
+50966,34395,30619,40251
+51643,34396,30618,40309
+50220,34397,30617,40776
+57439,34398,30616,40414
+59525,34399,30615,40136
+57083,34400,30614,40862
+59517,34401,30613,40750
+56095,34402,30612,40360
+54410,34403,30611,40979
+53616,34404,30610,40395
+58335,34405,30609,40829
+59984,34406,30608,40391
+51534,34407,30607,40625
+58009,34408,30606,40082
+50536,34409,30605,40111
+52690,34410,30604,40719
+53329,34411,30603,40737
+53412,34412,30602,40225
+50847,34413,30601,40727
+51878,34414,30600,40705
+56257,34415,30599,40490
+58709,34416,30598,40490
+52701,34417,30597,40410
+52160,34418,30596,40277
+54865,34419,30595,40767
+57343,34420,30594,40662
+54426,34421,30593,40305
+53347,34422,30592,40394
+51555,34423,30591,40716
+53960,34424,30590,40835
+56357,34425,30589,40883
+58664,34426,30588,40170
+57805,34427,30587,40679
+53577,34428,30586,40048
+50076,34429,30585,40429
+54326,34430,30584,40745
+57064,34431,30583,40922
+58172,34432,30582,40510
+51135,34433,30581,40099
+53627,34434,30580,40549
+56571,34435,30579,40173
+54012,34436,30578,40438
+58145,34437,30577,40541
+53905,34438,30576,40876
+57812,34439,30575,40486
+50551,34440,30574,40607
+59319,34441,30573,40997
+50041,34442,30572,40106
+53471,34443,30571,40786
+51519,34444,30570,40174
+58118,34445,30569,40437
+59059,34446,30568,40570
+51484,34447,30567,40075
+53252,34448,30566,40413
+56328,34449,30565,40956
+51523,34450,30564,40546
+51485,34451,30563,40121
+55563,34452,30562,40383
+54743,34453,30561,40887
+57870,34454,30560,40482
+59621,34455,30559,40123
+52570,34456,30558,40559
+52608,34457,30557,40857
+53930,34458,30556,40850
+57657,34459,30555,40781
+59704,34460,30554,40094
+56482,34461,30553,40269
+51857,34462,30552,40353
+51122,34463,30551,40953
+51361,34464,30550,40508
+52140,34465,30549,40896
+59886,34466,30548,40922
+54441,34467,30547,40224
+59072,34468,30546,40348
+54497,34469,30545,40962
+59165,34470,30544,40822
+51929,34471,30543,40799
+57971,34472,30542,40554
+52759,34473,30541,40187
+56671,34474,30540,40103
+53169,34475,30539,40197
+57866,34476,30538,40917
+52451,34477,30537,40647
+57763,34478,30536,40392
+52831,34479,30535,40769
+50024,34480,30534,40423
+55326,34481,30533,40820
+51534,34482,30532,40832
+59509,34483,30531,40646
+52336,34484,30530,40201
+52675,34485,30529,40977
+58435,34486,30528,40869
+54742,34487,30527,40823
+59256,34488,30526,40714
+55102,34489,30525,40415
+50347,34490,30524,40379
+58537,34491,30523,40181
+59097,34492,30522,40143
+56349,34493,30521,40233
+55038,34494,30520,40821
+51589,34495,30519,40014
+58380,34496,30518,40313
+51490,34497,30517,40341
+56312,34498,30516,40968
+54195,34499,30515,40709
+57006,34500,30514,40540
+57664,34501,30513,40785
+53620,34502,30512,40492
+53839,34503,30511,40559
+57036,34504,30510,40092
+57561,34505,30509,40729
+52803,34506,30508,40692
+52327,34507,30507,40945
+55351,34508,30506,40279
+54230,34509,30505,40805
+54166,34510,30504,40082
+51817,34511,30503,40469
+52609,34512,30502,40079
+58398,34513,30501,40669
+53615,34514,30500,40830
+52137,34515,30499,40891
+55438,34516,30498,40839
+56380,34517,30497,40903
+52980,34518,30496,40454
+54616,34519,30495,40562
+54532,34520,30494,40123
+55165,34521,30493,40428
+51788,34522,30492,40213
+53437,34523,30491,40174
+54874,34524,30490,40178
+53215,34525,30489,40224
+50351,34526,30488,40253
+52848,34527,30487,40377
+56972,34528,30486,40581
+58211,34529,30485,40489
+52921,34530,30484,40286
+56345,34531,30483,40092
+56037,34532,30482,40551
+56892,34533,30481,40205
+54279,34534,30480,40503
+50053,34535,30479,40372
+52370,34536,30478,40588
+57357,34537,30477,40246
+50840,34538,30476,40657
+56273,34539,30475,40551
+51733,34540,30474,40340
+56862,34541,30473,40031
+54844,34542,30472,40361
+57548,34543,30471,40318
+58640,34544,30470,40270
+55208,34545,30469,40285
+52120,34546,30468,40005
+56345,34547,30467,40699
+56901,34548,30466,40359
+56435,34549,30465,40952
+52768,34550,30464,40387
+59098,34551,30463,40746
+59036,34552,30462,40787
+59806,34553,30461,40313
+55659,34554,30460,40231
+56373,34555,30459,40881
+50925,34556,30458,40400
+50471,34557,30457,40692
+54099,34558,30456,40787
+54979,34559,30455,40897
+51921,34560,30454,40063
+54430,34561,30453,40301
+50209,34562,30452,40893
+51403,34563,30451,40636
+52436,34564,30450,40364
+50422,34565,30449,40892
+57092,34566,30448,40030
+58107,34567,30447,40375
+54198,34568,30446,40147
+58884,34569,30445,40155
+51275,34570,30444,40768
+53743,34571,30443,40751
+53701,34572,30442,40291
+51822,34573,30441,40620
+52470,34574,30440,40197
+56199,34575,30439,40661
+51683,34576,30438,40036
+54748,34577,30437,40365
+51560,34578,30436,40524
+59413,34579,30435,40740
+50150,34580,30434,40258
+57657,34581,30433,40933
+58583,34582,30432,40917
+51068,34583,30431,40751
+55922,34584,30430,40993
+57938,34585,30429,40398
+57930,34586,30428,40905
+59346,34587,30427,40605
+58593,34588,30426,40416
+59157,34589,30425,40059
+59913,34590,30424,40901
+53294,34591,30423,40599
+50635,34592,30422,40712
+51092,34593,30421,40472
+56305,34594,30420,40677
+58218,34595,30419,40934
+56765,34596,30418,40132
+51951,34597,30417,40303
+59421,34598,30416,40467
+57345,34599,30415,40575
+54422,34600,30414,40169
+59069,34601,30413,40551
+55885,34602,30412,40687
+52001,34603,30411,40248
+53477,34604,30410,40317
+58243,34605,30409,40164
+51474,34606,30408,40045
+58714,34607,30407,40330
+55932,34608,30406,40416
+58394,34609,30405,40812
+54879,34610,30404,40348
+55421,34611,30403,40326
+59935,34612,30402,40631
+51021,34613,30401,40870
+58649,34614,30400,40323
+54558,34615,30399,40194
+55093,34616,30398,40510
+59272,34617,30397,40922
+50824,34618,30396,40548
+58450,34619,30395,40459
+57925,34620,30394,40643
+59370,34621,30393,40773
+50819,34622,30392,40107
+51489,34623,30391,40946
+54106,34624,30390,40216
+55523,34625,30389,40196
+57047,34626,30388,40933
+59572,34627,30387,40697
+59263,34628,30386,40679
+59766,34629,30385,40893
+51649,34630,30384,40691
+51687,34631,30383,40997
+51731,34632,30382,40653
+53975,34633,30381,40307
+57230,34634,30380,40742
+51112,34635,30379,40211
+51062,34636,30378,40921
+53438,34637,30377,40473
+51921,34638,30376,40170
+56582,34639,30375,40068
+56266,34640,30374,40099
+51986,34641,30373,40355
+56029,34642,30372,40989
+53602,34643,30371,40779
+53058,34644,30370,40903
+58255,34645,30369,40274
+54759,34646,30368,40118
+52848,34647,30367,40947
+58229,34648,30366,40282
+53184,34649,30365,40526
+52373,34650,30364,40544
+55367,34651,30363,40307
+56187,34652,30362,40321
+58932,34653,30361,40323
+50320,34654,30360,40790
+51932,34655,30359,40491
+51614,34656,30358,40222
+56268,34657,30357,40518
+54149,34658,30356,40702
+52347,34659,30355,40406
+56027,34660,30354,40044
+55851,34661,30353,40135
+51471,34662,30352,40553
+57646,34663,30351,40792
+56670,34664,30350,40880
+53124,34665,30349,40023
+54607,34666,30348,40846
+58179,34667,30347,40551
+56313,34668,30346,40224
+57024,34669,30345,40569
+59700,34670,30344,40116
+58649,34671,30343,40348
+55709,34672,30342,40093
+56117,34673,30341,40258
+54659,34674,30340,40238
+52195,34675,30339,40169
+59182,34676,30338,40002
+56399,34677,30337,40426
+59311,34678,30336,40566
+55715,34679,30335,40993
+52146,34680,30334,40220
+55914,34681,30333,40074
+59585,34682,30332,40477
+51647,34683,30331,40396
+54578,34684,30330,40793
+53925,34685,30329,40941
+53585,34686,30328,40718
+52528,34687,30327,40674
+57160,34688,30326,40291
+53296,34689,30325,40689
+59674,34690,30324,40664
+59533,34691,30323,40629
+51631,34692,30322,40460
+54142,34693,30321,40261
+56393,34694,30320,40722
+51620,34695,30319,40562
+52214,34696,30318,40298
+53907,34697,30317,40298
+54430,34698,30316,40255
+59793,34699,30315,40894
+54764,34700,30314,40578
+59607,34701,30313,40788
+59514,34702,30312,40498
+56021,34703,30311,40618
+55429,34704,30310,40605
+53681,34705,30309,40816
+57807,34706,30308,40936
+55791,34707,30307,40104
+56211,34708,30306,40671
+51517,34709,30305,40821
+53267,34710,30304,40058
+58405,34711,30303,40973
+50237,34712,30302,40597
+59366,34713,30301,40725
+58876,34714,30300,40890
+55676,34715,30299,40887
+50331,34716,30298,40292
+55531,34717,30297,40906
+55764,34718,30296,40650
+52475,34719,30295,40759
+55991,34720,30294,40759
+50189,34721,30293,40523
+55206,34722,30292,40114
+57628,34723,30291,40846
+52669,34724,30290,40887
+58751,34725,30289,40806
+50783,34726,30288,40001
+59739,34727,30287,40640
+55259,34728,30286,40322
+51381,34729,30285,40621
+58013,34730,30284,40904
+58100,34731,30283,40973
+57073,34732,30282,40574
+52389,34733,30281,40547
+58659,34734,30280,40300
+57048,34735,30279,40293
+57666,34736,30278,40724
+57400,34737,30277,40302
+50882,34738,30276,40987
+58312,34739,30275,40593
+52801,34740,30274,40175
+59673,34741,30273,40868
+58657,34742,30272,40459
+53416,34743,30271,40103
+52116,34744,30270,40604
+59188,34745,30269,40231
+52006,34746,30268,40535
+50541,34747,30267,40772
+56196,34748,30266,40098
+55096,34749,30265,40131
+59594,34750,30264,40436
+56367,34751,30263,40697
+54618,34752,30262,40401
+52533,34753,30261,40741
+54505,34754,30260,40333
+57767,34755,30259,40535
+51251,34756,30258,40588
+50398,34757,30257,40531
+59247,34758,30256,40940
+54273,34759,30255,40893
+54429,34760,30254,40456
+59912,34761,30253,40408
+51067,34762,30252,40939
+51419,34763,30251,40543
+50183,34764,30250,40893
+51321,34765,30249,40022
+50079,34766,30248,40788
+59460,34767,30247,40774
+55796,34768,30246,40937
+52841,34769,30245,40227
+59421,34770,30244,40246
+52173,34771,30243,40116
+50152,34772,30242,40378
+56931,34773,30241,40271
+56475,34774,30240,40953
+59531,34775,30239,40302
+51481,34776,30238,40218
+50932,34777,30237,40370
+54688,34778,30236,40310
+52002,34779,30235,40354
+58014,34780,30234,40850
+50091,34781,30233,40824
+51892,34782,30232,40441
+59067,34783,30231,40832
+56467,34784,30230,40178
+57335,34785,30229,40046
+56259,34786,30228,40977
+53054,34787,30227,40235
+59427,34788,30226,40604
+59164,34789,30225,40788
+58218,34790,30224,40324
+51656,34791,30223,40879
+57036,34792,30222,40806
+55741,34793,30221,40306
+59656,34794,30220,40093
+58486,34795,30219,40538
+57774,34796,30218,40838
+50291,34797,30217,40684
+50670,34798,30216,40736
+53528,34799,30215,40724
+56251,34800,30214,40990
+58708,34801,30213,40112
+57295,34802,30212,40954
+50382,34803,30211,40608
+57288,34804,30210,40623
+52156,34805,30209,40789
+55600,34806,30208,40915
+54478,34807,30207,40235
+50282,34808,30206,40991
+54423,34809,30205,40636
+59752,34810,30204,40325
+55355,34811,30203,40106
+57786,34812,30202,40421
+53700,34813,30201,40360
+53793,34814,30200,40716
+54335,34815,30199,40054
+58305,34816,30198,40777
+56883,34817,30197,40399
+56805,34818,30196,40677
+53410,34819,30195,40448
+55452,34820,30194,40911
+52743,34821,30193,40912
+53041,34822,30192,40377
+55183,34823,30191,40470
+57768,34824,30190,40130
+55861,34825,30189,40228
+53284,34826,30188,40472
+51846,34827,30187,40454
+55958,34828,30186,40306
+50755,34829,30185,40807
+51568,34830,30184,40112
+55204,34831,30183,40856
+58648,34832,30182,40816
+54393,34833,30181,40573
+51240,34834,30180,40318
+59292,34835,30179,40599
+59138,34836,30178,40287
+58018,34837,30177,40911
+56160,34838,30176,40938
+58497,34839,30175,40317
+57465,34840,30174,40682
+59035,34841,30173,40334
+52176,34842,30172,40428
+59817,34843,30171,40675
+57655,34844,30170,40514
+58963,34845,30169,40716
+55553,34846,30168,40762
+54235,34847,30167,40968
+51236,34848,30166,40127
+52697,34849,30165,40511
+59165,34850,30164,40962
+57425,34851,30163,40175
+51790,34852,30162,40956
+57377,34853,30161,40823
+55842,34854,30160,40010
+51334,34855,30159,40492
+51121,34856,30158,40181
+53343,34857,30157,40717
+59967,34858,30156,40572
+52025,34859,30155,40101
+50015,34860,30154,40793
+54041,34861,30153,40580
+52870,34862,30152,40430
+58280,34863,30151,40626
+52234,34864,30150,40578
+50200,34865,30149,40322
+56296,34866,30148,40251
+52517,34867,30147,40830
+50932,34868,30146,40297
+55495,34869,30145,40973
+54747,34870,30144,40821
+55172,34871,30143,40526
+53205,34872,30142,40580
+56242,34873,30141,40151
+53127,34874,30140,40989
+53562,34875,30139,40214
+56564,34876,30138,40714
+56536,34877,30137,40515
+50372,34878,30136,40710
+56879,34879,30135,40881
+58140,34880,30134,40764
+55801,34881,30133,40648
+59485,34882,30132,40623
+57148,34883,30131,40056
+57801,34884,30130,40995
+57929,34885,30129,40131
+54447,34886,30128,40447
+52414,34887,30127,40807
+55560,34888,30126,40174
+54394,34889,30125,40510
+59386,34890,30124,40937
+50048,34891,30123,40092
+53802,34892,30122,40594
+59157,34893,30121,40741
+54798,34894,30120,40063
+59348,34895,30119,40657
+54192,34896,30118,40474
+52257,34897,30117,40078
+59455,34898,30116,40537
+51051,34899,30115,40223
+59056,34900,30114,40046
+51290,34901,30113,40766
+51490,34902,30112,40829
+57767,34903,30111,40635
+54878,34904,30110,40661
+50363,34905,30109,40832
+55869,34906,30108,40645
+58992,34907,30107,40211
+50659,34908,30106,40797
+57956,34909,30105,40638
+50074,34910,30104,40966
+54955,34911,30103,40735
+59194,34912,30102,40575
+57908,34913,30101,40999
+54243,34914,30100,40813
+50948,34915,30099,40314
+55917,34916,30098,40806
+50639,34917,30097,40666
+50311,34918,30096,40170
+56453,34919,30095,40895
+56828,34920,30094,40062
+54568,34921,30093,40079
+59651,34922,30092,40190
+50732,34923,30091,40726
+57365,34924,30090,40342
+59624,34925,30089,40810
+54250,34926,30088,40318
+56377,34927,30087,40558
+51564,34928,30086,40600
+53621,34929,30085,40337
+59629,34930,30084,40447
+55228,34931,30083,40562
+55381,34932,30082,40382
+57803,34933,30081,40835
+52499,34934,30080,40154
+50939,34935,30079,40855
+52180,34936,30078,40380
+58342,34937,30077,40112
+53522,34938,30076,40567
+57833,34939,30075,40289
+57172,34940,30074,40362
+58869,34941,30073,40149
+55347,34942,30072,40971
+57056,34943,30071,40027
+51948,34944,30070,40993
+59864,34945,30069,40551
+52977,34946,30068,40644
+58232,34947,30067,40373
+57914,34948,30066,40899
+55027,34949,30065,40116
+57370,34950,30064,40249
+54508,34951,30063,40061
+58524,34952,30062,40828
+51288,34953,30061,40663
+51468,34954,30060,40332
+55132,34955,30059,40409
+52770,34956,30058,40213
+52160,34957,30057,40243
+57785,34958,30056,40921
+58197,34959,30055,40465
+57725,34960,30054,40941
+59122,34961,30053,40812
+53123,34962,30052,40326
+54906,34963,30051,40687
+57235,34964,30050,40512
+54138,34965,30049,40499
+56064,34966,30048,40587
+55246,34967,30047,40480
+56808,34968,30046,40602
+50563,34969,30045,40624
+58550,34970,30044,40081
+59047,34971,30043,40273
+56645,34972,30042,40674
+58589,34973,30041,40616
+55007,34974,30040,40212
+50732,34975,30039,40220
+55156,34976,30038,40801
+58311,34977,30037,40742
+51910,34978,30036,40403
+54136,34979,30035,40147
+51153,34980,30034,40002
+53877,34981,30033,40979
+54504,34982,30032,40764
+50500,34983,30031,40073
+58058,34984,30030,40530
+58235,34985,30029,40343
+53076,34986,30028,40834
+53904,34987,30027,40302
+57056,34988,30026,40806
+59935,34989,30025,40903
+52699,34990,30024,40969
+53962,34991,30023,40441
+58835,34992,30022,40993
+58426,34993,30021,40079
+51775,34994,30020,40382
+57844,34995,30019,40005
+50858,34996,30018,40061
+53431,34997,30017,40327
+55975,34998,30016,40625
+51634,34999,30015,40814
+51160,35000,30014,40679
+56914,35001,30013,40064
+58697,35002,30012,40756
+53993,35003,30011,40811
+53239,35004,30010,40709
+52442,35005,30009,40326
+52655,35006,30008,40914
+50241,35007,30007,40779
+58785,35008,30006,40969
+59600,35009,30005,40101
+55794,35010,30004,40627
+59715,35011,30003,40658
+59377,35012,30002,40804
+57909,35013,30001,40661
+51745,35014,30000,40372
+59388,35015,31000,40011
+59989,35016,30999,40061
+56991,35017,30998,40543
+57051,35018,30997,40169
+54280,35019,30996,40220
+59071,35020,30995,40418
+50372,35021,30994,40687
+55488,35022,30993,40245
+58102,35023,30992,40087
+58495,35024,30991,40284
+58924,35025,30990,40786
+53907,35026,30989,40854
+58874,35027,30988,40565
+51306,35028,30987,40445
+59203,35029,30986,40095
+55898,35030,30985,40259
+59526,35031,30984,40594
+52550,35032,30983,40202
+50372,35033,30982,40160
+52892,35034,30981,40799
+58863,35035,30980,40206
+53242,35036,30979,40522
+57742,35037,30978,40079
+53169,35038,30977,40525
+51422,35039,30976,40464
+56018,35040,30975,40574
+52739,35041,30974,40675
+52421,35042,30973,40491
+58460,35043,30972,40858
+59833,35044,30971,40961
+54884,35045,30970,40722
+56326,35046,30969,40545
+58020,35047,30968,40730
+59341,35048,30967,40840
+55013,35049,30966,40334
+56257,35050,30965,40137
+54308,35051,30964,40156
+53311,35052,30963,40801
+50095,35053,30962,40058
+58392,35054,30961,40990
+57963,35055,30960,40818
+50607,35056,30959,40581
+52504,35057,30958,40828
+50638,35058,30957,40519
+58348,35059,30956,40334
+52323,35060,30955,40159
+52748,35061,30954,40026
+55464,35062,30953,40850
+53130,35063,30952,40084
+55616,35064,30951,40796
+55831,35065,30950,40015
+53469,35066,30949,40019
+55321,35067,30948,40923
+50847,35068,30947,40586
+53368,35069,30946,40468
+51191,35070,30945,40876
+53027,35071,30944,40482
+52393,35072,30943,40355
+54320,35073,30942,40305
+58048,35074,30941,40116
+53712,35075,30940,40133
+53195,35076,30939,40961
+56912,35077,30938,40099
+53413,35078,30937,40315
+52300,35079,30936,40774
+52051,35080,30935,40337
+53694,35081,30934,40653
+57878,35082,30933,40049
+58948,35083,30932,40414
+51280,35084,30931,40258
+57392,35085,30930,40257
+51443,35086,30929,40396
+51528,35087,30928,40059
+51269,35088,30927,40828
+50270,35089,30926,40734
+57712,35090,30925,40792
+59482,35091,30924,40097
+50773,35092,30923,40980
+58612,35093,30922,40169
+51873,35094,30921,40502
+59598,35095,30920,40972
+52824,35096,30919,40748
+59037,35097,30918,40661
+58494,35098,30917,40400
+50168,35099,30916,40945
+56725,35100,30915,40108
+50264,35101,30914,40013
+54032,35102,30913,40904
+50720,35103,30912,40666
+55973,35104,30911,40030
+52343,35105,30910,40174
+52913,35106,30909,40078
+54304,35107,30908,40880
+55206,35108,30907,40603
+58683,35109,30906,40112
+51280,35110,30905,40577
+57156,35111,30904,40425
+50229,35112,30903,40529
+53560,35113,30902,40243
+53898,35114,30901,40346
+55192,35115,30900,40489
+54450,35116,30899,40174
+55335,35117,30898,40716
+55425,35118,30897,40198
+58638,35119,30896,40234
+56159,35120,30895,40827
+58302,35121,30894,40015
+58233,35122,30893,40706
+54037,35123,30892,40966
+58337,35124,30891,40723
+58265,35125,30890,40628
+55109,35126,30889,40703
+51618,35127,30888,40697
+58701,35128,30887,40443
+53717,35129,30886,40172
+52158,35130,30885,40237
+55098,35131,30884,40753
+50092,35132,30883,40305
+52847,35133,30882,40154
+56444,35134,30881,40521
+57525,35135,30880,40152
+57367,35136,30879,40151
+53784,35137,30878,40444
+50250,35138,30877,40536
+51101,35139,30876,40254
+58426,35140,30875,40800
+52131,35141,30874,40568
+59269,35142,30873,40475
+52828,35143,30872,40433
+59844,35144,30871,40311
+51461,35145,30870,40938
+54253,35146,30869,40604
+57281,35147,30868,40086
+52853,35148,30867,40748
+53236,35149,30866,40386
+57507,35150,30865,40941
+56891,35151,30864,40782
+52495,35152,30863,40048
+55273,35153,30862,40024
+58073,35154,30861,40429
+53371,35155,30860,40748
+53174,35156,30859,40621
+59293,35157,30858,40002
+54068,35158,30857,40932
+58342,35159,30856,40468
+54047,35160,30855,40297
+56798,35161,30854,40385
+55277,35162,30853,40187
+54545,35163,30852,40803
+52756,35164,30851,40657
+55915,35165,30850,40858
+52651,35166,30849,40974
+57762,35167,30848,40267
+55373,35168,30847,40664
+50372,35169,30846,40552
+59034,35170,30845,40219
+56213,35171,30844,40213
+52999,35172,30843,40527
+50010,35173,30842,40300
+53974,35174,30841,40179
+59364,35175,30840,40486
+56865,35176,30839,40180
+50292,35177,30838,40127
+59136,35178,30837,40270
+50941,35179,30836,40581
+50810,35180,30835,40237
+54424,35181,30834,40276
+54264,35182,30833,40856
+52928,35183,30832,40792
+52708,35184,30831,40899
+58957,35185,30830,40129
+57728,35186,30829,40232
+56766,35187,30828,40932
+58980,35188,30827,40906
+59007,35189,30826,40500
+57247,35190,30825,40717
+58859,35191,30824,40412
+56462,35192,30823,40125
+57370,35193,30822,40847
+53867,35194,30821,40225
+59963,35195,30820,40382
+50301,35196,30819,40539
+59646,35197,30818,40902
+56589,35198,30817,40544
+52968,35199,30816,40851
+59768,35200,30815,40279
+52441,35201,30814,40805
+50352,35202,30813,40487
+55018,35203,30812,40086
+55630,35204,30811,40337
+56031,35205,30810,40092
+52546,35206,30809,40327
+53789,35207,30808,40289
+51355,35208,30807,40359
+58005,35209,30806,40277
+56674,35210,30805,40257
+51246,35211,30804,40749
+50048,35212,30803,40513
+54929,35213,30802,40306
+50895,35214,30801,40952
+51918,35215,30800,40728
+55518,35216,30799,40550
+53916,35217,30798,40745
+53199,35218,30797,40020
+52541,35219,30796,40582
+59290,35220,30795,40713
+51807,35221,30794,40793
+53396,35222,30793,40179
+52239,35223,30792,40474
+56379,35224,30791,40608
+54665,35225,30790,40708
+57816,35226,30789,40872
+54090,35227,30788,40971
+57015,35228,30787,40304
+58326,35229,30786,40893
+58466,35230,30785,40206
+53189,35231,30784,40562
+53241,35232,30783,40156
+59241,35233,30782,40609
+53742,35234,30781,40731
+57195,35235,30780,40800
+52173,35236,30779,40302
+52147,35237,30778,40983
+54345,35238,30777,40095
+54661,35239,30776,40386
+53554,35240,30775,40362
+52808,35241,30774,40402
+52466,35242,30773,40092
+59619,35243,30772,40518
+50902,35244,30771,40943
+53089,35245,30770,40427
+53370,35246,30769,40605
+50659,35247,30768,40589
+56029,35248,30767,40422
+50772,35249,30766,40097
+57780,35250,30765,40585
+53608,35251,30764,40497
+59306,35252,30763,40491
+53805,35253,30762,40418
+56741,35254,30761,40206
+52836,35255,30760,40334
+53060,35256,30759,40017
+50601,35257,30758,40545
+58580,35258,30757,40032
+55165,35259,30756,40162
+56194,35260,30755,40496
+50821,35261,30754,40728
+53179,35262,30753,40433
+57307,35263,30752,40014
+50591,35264,30751,40864
+50146,35265,30750,40885
+55380,35266,30749,40560
+56920,35267,30748,40268
+52086,35268,30747,40941
+51332,35269,30746,40430
+51612,35270,30745,40412
+52830,35271,30744,40009
+58982,35272,30743,40402
+55758,35273,30742,40278
+50978,35274,30741,40688
+55404,35275,30740,40058
+55374,35276,30739,40312
+52759,35277,30738,40840
+52168,35278,30737,40879
+58897,35279,30736,40816
+53090,35280,30735,40198
+56535,35281,30734,40256
+53009,35282,30733,40550
+51359,35283,30732,40230
+52020,35284,30731,40079
+52826,35285,30730,40399
+50401,35286,30729,40414
+50265,35287,30728,40887
+59755,35288,30727,40321
+57835,35289,30726,40325
+58736,35290,30725,40984
+57604,35291,30724,40954
+58655,35292,30723,40458
+56538,35293,30722,40142
+59638,35294,30721,40865
+55337,35295,30720,40024
+58842,35296,30719,40715
+50310,35297,30718,40072
+56240,35298,30717,40542
+59377,35299,30716,40146
+55544,35300,30715,40343
+51283,35301,30714,40137
+56457,35302,30713,40550
+50519,35303,30712,40301
+52567,35304,30711,40618
+52373,35305,30710,40282
+53249,35306,30709,40373
+54569,35307,30708,40167
+59335,35308,30707,40875
+57443,35309,30706,40696
+52601,35310,30705,40349
+51543,35311,30704,40624
+58507,35312,30703,40188
+53940,35313,30702,40191
+58629,35314,30701,40508
+58450,35315,30700,40476
+57270,35316,30699,40591
+54852,35317,30698,40119
+59784,35318,30697,40610
+55709,35319,30696,40582
+53278,35320,30695,40088
+58592,35321,30694,40809
+55146,35322,30693,40753
+59807,35323,30692,40092
+55804,35324,30691,40263
+55957,35325,30690,40937
+59125,35326,30689,40933
+54223,35327,30688,40759
+57322,35328,30687,40420
+51940,35329,30686,40179
+54819,35330,30685,40044
+59126,35331,30684,40337
+57011,35332,30683,40971
+58081,35333,30682,40279
+58898,35334,30681,40813
+55792,35335,30680,40470
+52053,35336,30679,40656
+59759,35337,30678,40830
+54575,35338,30677,40834
+50394,35339,30676,40429
+51873,35340,30675,40921
+59925,35341,30674,40196
+54974,35342,30673,40191
+53573,35343,30672,40735
+52585,35344,30671,40457
+56299,35345,30670,40278
+57119,35346,30669,40490
+51809,35347,30668,40564
+51585,35348,30667,40960
+55853,35349,30666,40531
+50538,35350,30665,40227
+58195,35351,30664,40778
+58287,35352,30663,40653
+51830,35353,30662,40693
+53044,35354,30661,40627
+53951,35355,30660,40535
+50923,35356,30659,40550
+51126,35357,30658,40815
+53735,35358,30657,40118
+55485,35359,30656,40972
+56895,35360,30655,40844
+59069,35361,30654,40797
+54414,35362,30653,40859
+57495,35363,30652,40634
+55943,35364,30651,40850
+59004,35365,30650,40569
+50133,35366,30649,40304
+55225,35367,30648,40542
+55712,35368,30647,40885
+53690,35369,30646,40690
+58637,35370,30645,40920
+56832,35371,30644,40486
+56621,35372,30643,40482
+51869,35373,30642,40300
+57541,35374,30641,40478
+54075,35375,30640,40178
+58813,35376,30639,40260
+55020,35377,30638,40468
+50082,35378,30637,40465
+53597,35379,30636,40614
+59878,35380,30635,40765
+53700,35381,30634,40091
+56471,35382,30633,40256
+56360,35383,30632,40260
+51389,35384,30631,40201
+58495,35385,30630,40424
+51021,35386,30629,40941
+56562,35387,30628,40380
+57579,35388,30627,40633
+59773,35389,30626,40968
+51148,35390,30625,40340
+52019,35391,30624,40862
+59161,35392,30623,40779
+57823,35393,30622,40736
+57663,35394,30621,40978
+52773,35395,30620,40044
+56772,35396,30619,40012
+53430,35397,30618,40006
+56996,35398,30617,40496
+52898,35399,30616,40442
+52985,35400,30615,40696
+59320,35401,30614,40893
+57945,35402,30613,40054
+57241,35403,30612,40712
+52921,35404,30611,40478
+52443,35405,30610,40167
+50384,35406,30609,40498
+51209,35407,30608,40260
+51222,35408,30607,40862
+52632,35409,30606,40090
+50939,35410,30605,40801
+51444,35411,30604,40961
+51700,35412,30603,40836
+59112,35413,30602,40254
+56623,35414,30601,40858
+50205,35415,30600,40420
+50888,35416,30599,40516
+54970,35417,30598,40227
+57877,35418,30597,40184
+54602,35419,30596,40747
+54620,35420,30595,40157
+54923,35421,30594,40501
+55716,35422,30593,40446
+50274,35423,30592,40542
+51976,35424,30591,40984
+52203,35425,30590,40857
+50120,35426,30589,40874
+57662,35427,30588,40552
+57786,35428,30587,40040
+54766,35429,30586,40711
+50702,35430,30585,40412
+53132,35431,30584,40821
+53199,35432,30583,40890
+58328,35433,30582,40983
+50612,35434,30581,40678
+53223,35435,30580,40203
+58039,35436,30579,40252
+54336,35437,30578,40934
+59245,35438,30577,40875
+52544,35439,30576,40427
+56308,35440,30575,40391
+53069,35441,30574,40320
+55851,35442,30573,40233
+50424,35443,30572,40142
+51114,35444,30571,40309
+57515,35445,30570,40773
+52307,35446,30569,40473
+53746,35447,30568,40185
+56368,35448,30567,40181
+58391,35449,30566,40798
+51196,35450,30565,40603
+55157,35451,30564,40875
+53710,35452,30563,40685
+55422,35453,30562,40838
+58703,35454,30561,40021
+51576,35455,30560,40428
+54125,35456,30559,40380
+55234,35457,30558,40891
+52152,35458,30557,40915
+52116,35459,30556,40678
+58345,35460,30555,40103
+56820,35461,30554,40204
+56643,35462,30553,40077
+59413,35463,30552,40661
+51553,35464,30551,40802
+53210,35465,30550,40642
+57215,35466,30549,40077
+57065,35467,30548,40152
+57445,35468,30547,40244
+58794,35469,30546,40625
+53934,35470,30545,40304
+59134,35471,30544,40519
+55863,35472,30543,40070
+58285,35473,30542,40677
+57298,35474,30541,40450
+58632,35475,30540,40732
+51130,35476,30539,40714
+58157,35477,30538,40775
+57761,35478,30537,40900
+59528,35479,30536,40844
+58374,35480,30535,40805
+56390,35481,30534,40096
+51426,35482,30533,40079
+56997,35483,30532,40541
+56961,35484,30531,40582
+51848,35485,30530,40618
+59438,35486,30529,40752
+52487,35487,30528,40064
+50902,35488,30527,40503
+54674,35489,30526,40092
+54335,35490,30525,40116
+53558,35491,30524,40709
+50530,35492,30523,40025
+56616,35493,30522,40764
+56035,35494,30521,40578
+56255,35495,30520,40923
+58849,35496,30519,40921
+53550,35497,30518,40516
+55737,35498,30517,40302
+51455,35499,30516,40783
+57909,35500,30515,40999
+54283,35501,30514,40215
+51282,35502,30513,40314
+58629,35503,30512,40113
+58015,35504,30511,40012
+56710,35505,30510,40210
+52353,35506,30509,40180
+51923,35507,30508,40565
+55395,35508,30507,40489
+56028,35509,30506,40293
+51723,35510,30505,40940
+52158,35511,30504,40445
+54408,35512,30503,40452
+58823,35513,30502,40570
+58759,35514,30501,40668
+52089,35515,30500,40977
+54533,35516,30499,40075
+53448,35517,30498,40338
+52783,35518,30497,40416
+54301,35519,30496,40857
+52373,35520,30495,40040
+55181,35521,30494,40173
+51997,35522,30493,40678
+58836,35523,30492,40086
+53587,35524,30491,40677
+55291,35525,30490,40184
+50110,35526,30489,40581
+53592,35527,30488,40038
+54702,35528,30487,40839
+59429,35529,30486,40317
+56610,35530,30485,40519
+51865,35531,30484,40071
+51494,35532,30483,40309
+51586,35533,30482,40313
+59017,35534,30481,40712
+51988,35535,30480,40129
+50940,35536,30479,40272
+53578,35537,30478,40223
+56199,35538,30477,40710
+52267,35539,30476,40038
+53510,35540,30475,40010
+59161,35541,30474,40999
+58144,35542,30473,40648
+57347,35543,30472,40627
+50440,35544,30471,40985
+56635,35545,30470,40419
+50796,35546,30469,40889
+50737,35547,30468,40758
+53147,35548,30467,40554
+57629,35549,30466,40606
+51172,35550,30465,40365
+59820,35551,30464,40967
+50777,35552,30463,40760
+54898,35553,30462,40549
+57026,35554,30461,40616
+54382,35555,30460,40105
+59294,35556,30459,40044
+50027,35557,30458,40247
+58180,35558,30457,40971
+52226,35559,30456,40023
+57208,35560,30455,40876
+50333,35561,30454,40127
+59482,35562,30453,40456
+52679,35563,30452,40097
+50344,35564,30451,40102
+56991,35565,30450,40638
+52307,35566,30449,40875
+56651,35567,30448,40871
+52079,35568,30447,40115
+56616,35569,30446,40227
+54931,35570,30445,40153
+59712,35571,30444,40514
+51841,35572,30443,40822
+56837,35573,30442,40372
+52288,35574,30441,40450
+59161,35575,30440,40423
+57780,35576,30439,40676
+59872,35577,30438,40348
+59359,35578,30437,40095
+55869,35579,30436,40957
+59153,35580,30435,40564
+59673,35581,30434,40477
+51195,35582,30433,40742
+56435,35583,30432,40867
+53314,35584,30431,40702
+50257,35585,30430,40741
+50590,35586,30429,40419
+54580,35587,30428,40589
+55394,35588,30427,40733
+54074,35589,30426,40066
+50950,35590,30425,40495
+54636,35591,30424,40094
+54464,35592,30423,40783
+58632,35593,30422,40015
+52785,35594,30421,40235
+55993,35595,30420,40885
+58043,35596,30419,40057
+50978,35597,30418,40991
+52160,35598,30417,40778
+51434,35599,30416,40698
+54614,35600,30415,40893
+52656,35601,30414,40903
+55495,35602,30413,40676
+51616,35603,30412,40503
+57957,35604,30411,40189
+50590,35605,30410,40909
+58683,35606,30409,40567
+54399,35607,30408,40449
+52766,35608,30407,40671
+59320,35609,30406,40357
+51874,35610,30405,40990
+53646,35611,30404,40797
+58873,35612,30403,40936
+57692,35613,30402,40072
+57462,35614,30401,40148
+58370,35615,30400,40813
+55968,35616,30399,40155
+50874,35617,30398,40036
+57714,35618,30397,40559
+59159,35619,30396,40955
+54671,35620,30395,40458
+51318,35621,30394,40033
+59459,35622,30393,40575
+54950,35623,30392,40010
+50516,35624,30391,40376
+53956,35625,30390,40880
+53271,35626,30389,40278
+53087,35627,30388,40394
+59871,35628,30387,40727
+55115,35629,30386,40556
+56625,35630,30385,40703
+57873,35631,30384,40538
+52979,35632,30383,40485
+51635,35633,30382,40226
+55714,35634,30381,40000
+51980,35635,30380,40181
+51379,35636,30379,40559
+56950,35637,30378,40114
+57281,35638,30377,40314
+53558,35639,30376,40176
+50053,35640,30375,40176
+59985,35641,30374,40261
+59060,35642,30373,40821
+58626,35643,30372,40335
+55250,35644,30371,40600
+52276,35645,30370,40734
+54660,35646,30369,40215
+57945,35647,30368,40173
+57171,35648,30367,40559
+55229,35649,30366,40531
+57229,35650,30365,40323
+54325,35651,30364,40410
+52917,35652,30363,40128
+57313,35653,30362,40684
+56570,35654,30361,40828
+57937,35655,30360,40516
+55061,35656,30359,40713
+50259,35657,30358,40536
+59165,35658,30357,40729
+50586,35659,30356,40721
+59509,35660,30355,40511
+58922,35661,30354,40636
+51828,35662,30353,40088
+59648,35663,30352,40201
+56693,35664,30351,40060
+53135,35665,30350,40141
+57941,35666,30349,40861
+56965,35667,30348,40350
+59833,35668,30347,40587
+55676,35669,30346,40530
+57698,35670,30345,40345
+54332,35671,30344,40764
+58585,35672,30343,40103
+53613,35673,30342,40013
+54485,35674,30341,40021
+52555,35675,30340,40643
+51900,35676,30339,40481
+58818,35677,30338,40232
+51349,35678,30337,40262
+52345,35679,30336,40018
+58859,35680,30335,40305
+51851,35681,30334,40562
+59934,35682,30333,40096
+58336,35683,30332,40956
+55236,35684,30331,40514
+57434,35685,30330,40010
+50445,35686,30329,40301
+53499,35687,30328,40804
+54868,35688,30327,40360
+55668,35689,30326,40024
+55169,35690,30325,40429
+51864,35691,30324,40322
+53947,35692,30323,40418
+56704,35693,30322,40840
+57707,35694,30321,40413
+52103,35695,30320,40541
+53100,35696,30319,40282
+55861,35697,30318,40976
+57134,35698,30317,40922
+56041,35699,30316,40584
+50073,35700,30315,40017
+58912,35701,30314,40854
+58143,35702,30313,40124
+54274,35703,30312,40695
+54954,35704,30311,40678
+56699,35705,30310,40396
+51381,35706,30309,40630
+54055,35707,30308,40552
+50647,35708,30307,40550
+58855,35709,30306,40742
+51145,35710,30305,40290
+52240,35711,30304,40389
+53849,35712,30303,40282
+52354,35713,30302,40161
+55911,35714,30301,40070
+52566,35715,30300,40590
+50429,35716,30299,40806
+58785,35717,30298,40667
+56091,35718,30297,40867
+55734,35719,30296,40537
+56240,35720,30295,40989
+52311,35721,30294,40879
+58389,35722,30293,40914
+55979,35723,30292,40388
+51075,35724,30291,40808
+50827,35725,30290,40935
+52757,35726,30289,40287
+58087,35727,30288,40358
+52088,35728,30287,40598
+55062,35729,30286,40586
+55518,35730,30285,40064
+53930,35731,30284,40259
+57714,35732,30283,40995
+54898,35733,30282,40253
+59718,35734,30281,40653
+58953,35735,30280,40831
+57545,35736,30279,40898
+53067,35737,30278,40524
+54020,35738,30277,40999
+51128,35739,30276,40775
+59155,35740,30275,40597
+50689,35741,30274,40520
+50225,35742,30273,40514
+50256,35743,30272,40357
+50476,35744,30271,40977
+52564,35745,30270,40479
+54503,35746,30269,40700
+54441,35747,30268,40955
+58174,35748,30267,40748
+58422,35749,30266,40760
+59716,35750,30265,40342
+50064,35751,30264,40380
+52729,35752,30263,40417
+58812,35753,30262,40670
+56919,35754,30261,40511
+58194,35755,30260,40543
+59116,35756,30259,40976
+50381,35757,30258,40267
+56994,35758,30257,40446
+58351,35759,30256,40093
+58889,35760,30255,40625
+57473,35761,30254,40300
+52486,35762,30253,40231
+51687,35763,30252,40880
+59876,35764,30251,40062
+52466,35765,30250,40248
+53401,35766,30249,40106
+56431,35767,30248,40580
+56180,35768,30247,40734
+59118,35769,30246,40650
+57170,35770,30245,40152
+58077,35771,30244,40176
+52176,35772,30243,40990
+57970,35773,30242,40949
+53851,35774,30241,40831
+59655,35775,30240,40763
+55443,35776,30239,40664
+55616,35777,30238,40660
+54270,35778,30237,40539
+59384,35779,30236,40657
+56185,35780,30235,40733
+56137,35781,30234,40705
+53865,35782,30233,40293
+52363,35783,30232,40252
+54015,35784,30231,40484
+57476,35785,30230,40037
+51241,35786,30229,40165
+57901,35787,30228,40397
+52390,35788,30227,40483
+58037,35789,30226,40377
+57586,35790,30225,40665
+58270,35791,30224,40194
+52383,35792,30223,40906
+58805,35793,30222,40477
+52836,35794,30221,40078
+55056,35795,30220,40039
+56847,35796,30219,40949
+53056,35797,30218,40151
+58055,35798,30217,40795
+54422,35799,30216,40672
+50925,35800,30215,40760
+53634,35801,30214,40905
+54832,35802,30213,40963
+51590,35803,30212,40250
+50063,35804,30211,40690
+50249,35805,30210,40909
+59353,35806,30209,40825
+58279,35807,30208,40103
+56640,35808,30207,40972
+53958,35809,30206,40034
+53036,35810,30205,40923
+58275,35811,30204,40328
+57324,35812,30203,40753
+50859,35813,30202,40062
+58184,35814,30201,40985
+54262,35815,30200,40034
+51794,35816,30199,40918
+54013,35817,30198,40051
+54634,35818,30197,40412
+50156,35819,30196,40764
+52577,35820,30195,40448
+58771,35821,30194,40660
+54581,35822,30193,40233
+53718,35823,30192,40637
+53370,35824,30191,40826
+53092,35825,30190,40982
+59450,35826,30189,40406
+50085,35827,30188,40400
+56548,35828,30187,40446
+58797,35829,30186,40177
+50176,35830,30185,40577
+51181,35831,30184,40657
+58018,35832,30183,40417
+56745,35833,30182,40871
+55165,35834,30181,40630
+51379,35835,30180,40334
+58190,35836,30179,40529
+56214,35837,30178,40241
+52713,35838,30177,40167
+57167,35839,30176,40792
+57810,35840,30175,40848
+56038,35841,30174,40975
+52370,35842,30173,40571
+53930,35843,30172,40593
+58585,35844,30171,40376
+58906,35845,30170,40870
+53682,35846,30169,40473
+52209,35847,30168,40590
+52242,35848,30167,40574
+56553,35849,30166,40489
+52746,35850,30165,40379
+54479,35851,30164,40988
+57245,35852,30163,40408
+50311,35853,30162,40450
+55659,35854,30161,40446
+50550,35855,30160,40765
+51051,35856,30159,40841
+55774,35857,30158,40271
+55111,35858,30157,40038
+58711,35859,30156,40002
+54369,35860,30155,40962
+55399,35861,30154,40359
+54548,35862,30153,40932
+52724,35863,30152,40751
+57797,35864,30151,40880
+57905,35865,30150,40548
+58216,35866,30149,40793
+59845,35867,30148,40611
+57295,35868,30147,40995
+53241,35869,30146,40881
+59725,35870,30145,40282
+51855,35871,30144,40231
+56841,35872,30143,40706
+55664,35873,30142,40811
+53102,35874,30141,40156
+58622,35875,30140,40540
+55567,35876,30139,40465
+51272,35877,30138,40490
+59784,35878,30137,40614
+57436,35879,30136,40055
+52911,35880,30135,40292
+53542,35881,30134,40310
+52186,35882,30133,40531
+53623,35883,30132,40047
+58346,35884,30131,40547
+50977,35885,30130,40148
+54500,35886,30129,40847
+57710,35887,30128,40099
+54065,35888,30127,40918
+59329,35889,30126,40447
+58891,35890,30125,40559
+50788,35891,30124,40309
+55816,35892,30123,40632
+57600,35893,30122,40185
+50403,35894,30121,40659
+56363,35895,30120,40755
+58805,35896,30119,40142
+51148,35897,30118,40706
+54563,35898,30117,40670
+59130,35899,30116,40682
+51073,35900,30115,40095
+57986,35901,30114,40261
+51572,35902,30113,40441
+57287,35903,30112,40892
+50373,35904,30111,40858
+55924,35905,30110,40863
+54338,35906,30109,40021
+54141,35907,30108,40217
+54094,35908,30107,40037
+50996,35909,30106,40929
+55001,35910,30105,40336
+54007,35911,30104,40334
+51482,35912,30103,40258
+50934,35913,30102,40403
+52856,35914,30101,40788
+51490,35915,30100,40043
+57976,35916,30099,40617
+55601,35917,30098,40489
+51953,35918,30097,40549
+51247,35919,30096,40830
+53259,35920,30095,40946
+54542,35921,30094,40303
+54262,35922,30093,40889
+51502,35923,30092,40635
+58167,35924,30091,40049
+57399,35925,30090,40982
+56605,35926,30089,40375
+53395,35927,30088,40885
+56594,35928,30087,40059
+56777,35929,30086,40719
+51213,35930,30085,40210
+51668,35931,30084,40563
+54692,35932,30083,40557
+58578,35933,30082,40212
+53342,35934,30081,40494
+58232,35935,30080,40206
+55799,35936,30079,40794
+56723,35937,30078,40227
+53501,35938,30077,40983
+51145,35939,30076,40575
+56519,35940,30075,40451
+54974,35941,30074,40804
+54869,35942,30073,40822
+51554,35943,30072,40451
+57186,35944,30071,40974
+56504,35945,30070,40296
+52838,35946,30069,40965
+54674,35947,30068,40784
+57880,35948,30067,40697
+56952,35949,30066,40546
+56225,35950,30065,40759
+50183,35951,30064,40567
+55499,35952,30063,40293
+56198,35953,30062,40367
+51897,35954,30061,40435
+54275,35955,30060,40825
+51296,35956,30059,40437
+50508,35957,30058,40679
+53283,35958,30057,40680
+58106,35959,30056,40516
+55196,35960,30055,40606
+51144,35961,30054,40395
+58422,35962,30053,40952
+58113,35963,30052,40021
+55160,35964,30051,40871
+55271,35965,30050,40839
+58732,35966,30049,40782
+56016,35967,30048,40421
+50397,35968,30047,40184
+53227,35969,30046,40287
+58110,35970,30045,40170
+55633,35971,30044,40143
+59752,35972,30043,40740
+53158,35973,30042,40787
+51740,35974,30041,40665
+52446,35975,30040,40389
+56918,35976,30039,40152
+59676,35977,30038,40724
+57138,35978,30037,40607
+50310,35979,30036,40455
+56088,35980,30035,40291
+54420,35981,30034,40118
+55875,35982,30033,40407
+55063,35983,30032,40413
+52848,35984,30031,40094
+50631,35985,30030,40822
+55954,35986,30029,40438
+56357,35987,30028,40458
+58450,35988,30027,40315
+50879,35989,30026,40475
+54243,35990,30025,40645
+58642,35991,30024,40729
+54916,35992,30023,40048
+58281,35993,30022,40343
+52830,35994,30021,40294
+57011,35995,30020,40345
+59773,35996,30019,40715
+57171,35997,30018,40709
+57548,35998,30017,40404
+59825,35999,30016,40483
+50569,36000,30015,40932
+53470,36001,30014,40026
+54007,36002,30013,40645
+53636,36003,30012,40848
+51506,36004,30011,40735
+59484,36005,30010,40445
+50894,36006,30009,40465
+57161,36007,30008,40509
+56600,36008,30007,40217
+59554,36009,30006,40464
+51694,36010,30005,40489
+59438,36011,30004,40016
+51770,36012,30003,40561
+52679,36013,30002,40910
+57241,36014,30001,40306
+55648,36015,30000,40314
+55371,36016,31000,40934
+54195,36017,30999,40157
+51411,36018,30998,40818
+56259,36019,30997,40348
+54301,36020,30996,40852
+53568,36021,30995,40020
+58671,36022,30994,40287
+51481,36023,30993,40580
+52920,36024,30992,40720
+51628,36025,30991,40619
+50694,36026,30990,40285
+51849,36027,30989,40967
+52900,36028,30988,40473
+53765,36029,30987,40143
+56831,36030,30986,40302
+59849,36031,30985,40598
+50090,36032,30984,40711
+54026,36033,30983,40308
+50712,36034,30982,40129
+52356,36035,30981,40107
+52038,36036,30980,40462
+51008,36037,30979,40571
+57563,36038,30978,40213
+52469,36039,30977,40858
+55463,36040,30976,40017
+58330,36041,30975,40724
+51113,36042,30974,40796
+58570,36043,30973,40754
+51117,36044,30972,40944
+50432,36045,30971,40450
+56546,36046,30970,40259
+51281,36047,30969,40531
+56730,36048,30968,40660
+58985,36049,30967,40836
+52170,36050,30966,40118
+50349,36051,30965,40864
+51201,36052,30964,40240
+53792,36053,30963,40903
+54422,36054,30962,40638
+59537,36055,30961,40228
+50154,36056,30960,40024
+57361,36057,30959,40862
+55548,36058,30958,40076
+54307,36059,30957,40998
+58955,36060,30956,40434
+58060,36061,30955,40172
+50182,36062,30954,40349
+55818,36063,30953,40294
+53871,36064,30952,40833
+58497,36065,30951,40775
+56063,36066,30950,40093
+55728,36067,30949,40879
+56675,36068,30948,40439
+57955,36069,30947,40084
+59459,36070,30946,40422
+50385,36071,30945,40821
+53827,36072,30944,40800
+55733,36073,30943,40911
+53683,36074,30942,40744
+52042,36075,30941,40806
+52129,36076,30940,40571
+59951,36077,30939,40034
+59150,36078,30938,40688
+59466,36079,30937,40059
+52462,36080,30936,40945
+53129,36081,30935,40010
+59273,36082,30934,40995
+51987,36083,30933,40373
+55407,36084,30932,40686
+50669,36085,30931,40719
+59656,36086,30930,40801
+55753,36087,30929,40386
+50104,36088,30928,40085
+55241,36089,30927,40937
+53266,36090,30926,40772
+54081,36091,30925,40327
+53716,36092,30924,40534
+59002,36093,30923,40822
+54352,36094,30922,40208
+58323,36095,30921,40869
+59930,36096,30920,40609
+52306,36097,30919,40537
+53405,36098,30918,40954
+59145,36099,30917,40582
+53033,36100,30916,40658
+51971,36101,30915,40890
+51975,36102,30914,40515
+58173,36103,30913,40254
+57325,36104,30912,40397
+59665,36105,30911,40794
+52700,36106,30910,40172
+52702,36107,30909,40828
+54086,36108,30908,40056
+59412,36109,30907,40525
+58894,36110,30906,40969
+58059,36111,30905,40864
+52808,36112,30904,40813
+51043,36113,30903,40663
+55423,36114,30902,40336
+58181,36115,30901,40972
+52453,36116,30900,40207
+56345,36117,30899,40378
+54437,36118,30898,40329
+56673,36119,30897,40805
+56596,36120,30896,40226
+58355,36121,30895,40076
+54246,36122,30894,40413
+53012,36123,30893,40385
+56413,36124,30892,40789
+56431,36125,30891,40131
+55805,36126,30890,40056
+53764,36127,30889,40913
+58490,36128,30888,40104
+50302,36129,30887,40010
+53459,36130,30886,40696
+51437,36131,30885,40613
+59960,36132,30884,40656
+54354,36133,30883,40362
+54293,36134,30882,40301
+52777,36135,30881,40956
+57443,36136,30880,40784
+51536,36137,30879,40684
+50596,36138,30878,40457
+57974,36139,30877,40803
+53341,36140,30876,40272
+59791,36141,30875,40827
+54704,36142,30874,40783
+50260,36143,30873,40643
+52806,36144,30872,40135
+55543,36145,30871,40205
+58668,36146,30870,40814
+57844,36147,30869,40813
+56965,36148,30868,40794
+53314,36149,30867,40288
+57709,36150,30866,40621
+53273,36151,30865,40311
+58029,36152,30864,40389
+50254,36153,30863,40046
+50298,36154,30862,40406
+59434,36155,30861,40365
+59224,36156,30860,40694
+57081,36157,30859,40555
+51836,36158,30858,40508
+55614,36159,30857,40586
+56720,36160,30856,40910
+56083,36161,30855,40323
+56975,36162,30854,40942
+51421,36163,30853,40151
+54688,36164,30852,40257
+51535,36165,30851,40919
+54655,36166,30850,40484
+57109,36167,30849,40280
+53926,36168,30848,40739
+55201,36169,30847,40590
+57500,36170,30846,40815
+52243,36171,30845,40836
+55428,36172,30844,40194
+53488,36173,30843,40510
+58934,36174,30842,40730
+54037,36175,30841,40371
+51546,36176,30840,40336
+57221,36177,30839,40378
+52272,36178,30838,40941
+53790,36179,30837,40655
+50797,36180,30836,40088
+51722,36181,30835,40906
+52608,36182,30834,40774
+59058,36183,30833,40517
+53347,36184,30832,40190
+58412,36185,30831,40880
+52301,36186,30830,40632
+59669,36187,30829,40811
+50244,36188,30828,40945
+54201,36189,30827,40631
+54496,36190,30826,40517
+51183,36191,30825,40969
+54250,36192,30824,40834
+53186,36193,30823,40732
+55731,36194,30822,40349
+57114,36195,30821,40794
+54545,36196,30820,40988
+55130,36197,30819,40439
+57348,36198,30818,40903
+50082,36199,30817,40778
+50791,36200,30816,40516
+56692,36201,30815,40464
+58130,36202,30814,40797
+56627,36203,30813,40023
+53628,36204,30812,40439
+54936,36205,30811,40340
+53086,36206,30810,40450
+57055,36207,30809,40345
+58547,36208,30808,40656
+50142,36209,30807,40790
+59403,36210,30806,40036
+59301,36211,30805,40667
+58319,36212,30804,40313
+55808,36213,30803,40680
+50671,36214,30802,40396
+58250,36215,30801,40578
+51750,36216,30800,40822
+54940,36217,30799,40162
+57967,36218,30798,40994
+51363,36219,30797,40770
+52097,36220,30796,40016
+56192,36221,30795,40791
+50694,36222,30794,40475
+56600,36223,30793,40918
+54201,36224,30792,40093
+54476,36225,30791,40205
+58766,36226,30790,40853
+51395,36227,30789,40535
+54987,36228,30788,40186
+50195,36229,30787,40954
+52500,36230,30786,40279
+54590,36231,30785,40689
+58661,36232,30784,40091
+53225,36233,30783,40529
+58536,36234,30782,40420
+53192,36235,30781,40883
+50198,36236,30780,40767
+59157,36237,30779,40845
+58922,36238,30778,40118
+58048,36239,30777,40613
+52129,36240,30776,40326
+56688,36241,30775,40595
+54830,36242,30774,40220
+50056,36243,30773,40561
+57194,36244,30772,40476
+50324,36245,30771,40834
+58968,36246,30770,40982
+57474,36247,30769,40836
+56839,36248,30768,40629
+58222,36249,30767,40523
+55920,36250,30766,40349
+51398,36251,30765,40747
+50383,36252,30764,40345
+58044,36253,30763,40498
+56191,36254,30762,40934
+59866,36255,30761,40480
+51470,36256,30760,40527
+57855,36257,30759,40901
+52496,36258,30758,40237
+59400,36259,30757,40877
+54079,36260,30756,40883
+59802,36261,30755,40842
+59487,36262,30754,40572
+59371,36263,30753,40414
+57200,36264,30752,40553
+55696,36265,30751,40114
+50221,36266,30750,40427
+55601,36267,30749,40704
+51217,36268,30748,40369
+50553,36269,30747,40378
+55065,36270,30746,40039
+51690,36271,30745,40966
+55860,36272,30744,40771
+53325,36273,30743,40839
+52247,36274,30742,40149
+58871,36275,30741,40400
+54670,36276,30740,40834
+56919,36277,30739,40135
+53437,36278,30738,40613
+54827,36279,30737,40635
+57204,36280,30736,40427
+56402,36281,30735,40220
+59745,36282,30734,40697
+53757,36283,30733,40734
+52223,36284,30732,40388
+53695,36285,30731,40297
+53884,36286,30730,40293
+54004,36287,30729,40280
+58280,36288,30728,40967
+56541,36289,30727,40396
+56073,36290,30726,40206
+58713,36291,30725,40093
+57755,36292,30724,40902
+57823,36293,30723,40149
+53957,36294,30722,40957
+51640,36295,30721,40416
+50216,36296,30720,40926
+53977,36297,30719,40339
+53296,36298,30718,40707
+55150,36299,30717,40744
+57885,36300,30716,40086
+55171,36301,30715,40387
+59931,36302,30714,40260
+53547,36303,30713,40421
+52269,36304,30712,40959
+58084,36305,30711,40378
+52392,36306,30710,40774
+57698,36307,30709,40377
+59243,36308,30708,40915
+52375,36309,30707,40743
+59597,36310,30706,40152
+50676,36311,30705,40208
+59695,36312,30704,40663
+53545,36313,30703,40044
+54832,36314,30702,40469
+59256,36315,30701,40394
+51119,36316,30700,40338
+52808,36317,30699,40420
+58667,36318,30698,40528
+50181,36319,30697,40110
+58887,36320,30696,40278
+51189,36321,30695,40580
+57785,36322,30694,40480
+55878,36323,30693,40407
+58833,36324,30692,40108
+52626,36325,30691,40203
+55767,36326,30690,40955
+59700,36327,30689,40280
+53437,36328,30688,40441
+55829,36329,30687,40179
+58787,36330,30686,40620
+55951,36331,30685,40745
+55898,36332,30684,40165
+59427,36333,30683,40767
+55568,36334,30682,40398
+55013,36335,30681,40977
+52058,36336,30680,40609
+54501,36337,30679,40941
+56784,36338,30678,40368
+55001,36339,30677,40795
+56881,36340,30676,40454
+57765,36341,30675,40324
+55000,36342,30674,40768
+58557,36343,30673,40363
+58182,36344,30672,40266
+54001,36345,30671,40724
+55199,36346,30670,40250
+55239,36347,30669,40864
+50977,36348,30668,40995
+59543,36349,30667,40658
+57550,36350,30666,40873
+54447,36351,30665,40545
+54123,36352,30664,40899
+54695,36353,30663,40699
+50480,36354,30662,40042
+54767,36355,30661,40018
+59632,36356,30660,40380
+57092,36357,30659,40765
+58771,36358,30658,40581
+50241,36359,30657,40086
+53327,36360,30656,40428
+56722,36361,30655,40091
+52662,36362,30654,40589
+57380,36363,30653,40723
+53976,36364,30652,40712
+54201,36365,30651,40256
+50648,36366,30650,40781
+50087,36367,30649,40571
+57884,36368,30648,40185
+50112,36369,30647,40976
+58479,36370,30646,40106
+50524,36371,30645,40207
+53697,36372,30644,40892
+54866,36373,30643,40900
+55422,36374,30642,40605
+55094,36375,30641,40788
+52299,36376,30640,40811
+54988,36377,30639,40068
+59324,36378,30638,40296
+59835,36379,30637,40759
+50899,36380,30636,40015
+55756,36381,30635,40796
+59988,36382,30634,40606
+53617,36383,30633,40250
+59812,36384,30632,40711
+57907,36385,30631,40433
+58847,36386,30630,40653
+57620,36387,30629,40631
+59976,36388,30628,40391
+51428,36389,30627,40811
+56939,36390,30626,40060
+59118,36391,30625,40993
+56687,36392,30624,40108
+53282,36393,30623,40731
+59015,36394,30622,40311
+59198,36395,30621,40256
+56951,36396,30620,40916
+59029,36397,30619,40547
+58836,36398,30618,40336
+51186,36399,30617,40011
+59860,36400,30616,40577
+50325,36401,30615,40242
+54933,36402,30614,40805
+51787,36403,30613,40389
+55753,36404,30612,40188
+52541,36405,30611,40847
+51839,36406,30610,40479
+57183,36407,30609,40914
+53437,36408,30608,40363
+51833,36409,30607,40835
+57977,36410,30606,40318
+50770,36411,30605,40856
+58327,36412,30604,40253
+52407,36413,30603,40361
+57117,36414,30602,40911
+58300,36415,30601,40473
+56786,36416,30600,40106
+58137,36417,30599,40527
+53956,36418,30598,40032
+55238,36419,30597,40837
+59881,36420,30596,40997
+55009,36421,30595,40564
+58670,36422,30594,40336
+51292,36423,30593,40442
+56699,36424,30592,40372
+52228,36425,30591,40162
+51463,36426,30590,40465
+50411,36427,30589,40438
+58824,36428,30588,40332
+54375,36429,30587,40274
+59010,36430,30586,40616
+54100,36431,30585,40752
+57505,36432,30584,40559
+54148,36433,30583,40724
+51522,36434,30582,40441
+52439,36435,30581,40892
+52619,36436,30580,40903
+55926,36437,30579,40334
+51147,36438,30578,40373
+54680,36439,30577,40398
+56252,36440,30576,40114
+58728,36441,30575,40058
+53637,36442,30574,40941
+56862,36443,30573,40923
+52956,36444,30572,40837
+51881,36445,30571,40882
+57433,36446,30570,40605
+56142,36447,30569,40581
+58293,36448,30568,40619
+54810,36449,30567,40221
+51425,36450,30566,40788
+59843,36451,30565,40525
+52083,36452,30564,40949
+53725,36453,30563,40753
+52028,36454,30562,40368
+53561,36455,30561,40505
+58005,36456,30560,40689
+51992,36457,30559,40104
+57207,36458,30558,40804
+52573,36459,30557,40997
+54841,36460,30556,40278
+57648,36461,30555,40653
+57348,36462,30554,40268
+55630,36463,30553,40067
+56259,36464,30552,40355
+51774,36465,30551,40750
+53398,36466,30550,40061
+51519,36467,30549,40141
+50886,36468,30548,40021
+50989,36469,30547,40729
+55907,36470,30546,40719
+53058,36471,30545,40863
+51896,36472,30544,40639
+55801,36473,30543,40977
+56496,36474,30542,40900
+55949,36475,30541,40519
+57106,36476,30540,40393
+58512,36477,30539,40055
+54328,36478,30538,40648
+55736,36479,30537,40623
+57374,36480,30536,40531
+54388,36481,30535,40467
+57781,36482,30534,40833
+53453,36483,30533,40439
+52903,36484,30532,40056
+53639,36485,30531,40058
+58997,36486,30530,40173
+56199,36487,30529,40317
+58219,36488,30528,40062
+56190,36489,30527,40263
+55653,36490,30526,40688
+58579,36491,30525,40430
+52860,36492,30524,40642
+56350,36493,30523,40876
+56537,36494,30522,40773
+57665,36495,30521,40884
+57482,36496,30520,40870
+58481,36497,30519,40135
+50962,36498,30518,40778
+56338,36499,30517,40272
+51306,36500,30516,40557
+54594,36501,30515,40512
+58645,36502,30514,40662
+50619,36503,30513,40502
+52290,36504,30512,40123
+55340,36505,30511,40351
+50298,36506,30510,40028
+51857,36507,30509,40709
+55113,36508,30508,40527
+51790,36509,30507,40584
+52650,36510,30506,40512
+57099,36511,30505,40446
+56091,36512,30504,40730
+57849,36513,30503,40592
+50448,36514,30502,40134
+56211,36515,30501,40762
+56595,36516,30500,40070
+50160,36517,30499,40710
+50777,36518,30498,40887
+54804,36519,30497,40255
+57535,36520,30496,40761
+50710,36521,30495,40805
+57955,36522,30494,40941
+53324,36523,30493,40937
+53600,36524,30492,40410
+50515,36525,30491,40582
+57921,36526,30490,40091
+59339,36527,30489,40890
+58678,36528,30488,40128
+54863,36529,30487,40072
+50708,36530,30486,40152
+56274,36531,30485,40996
+50487,36532,30484,40672
+51829,36533,30483,40914
+58019,36534,30482,40106
+55685,36535,30481,40643
+54232,36536,30480,40733
+59745,36537,30479,40164
+54628,36538,30478,40037
+59561,36539,30477,40037
+55522,36540,30476,40991
+52354,36541,30475,40580
+52528,36542,30474,40516
+59286,36543,30473,40027
+55550,36544,30472,40727
+58248,36545,30471,40820
+52290,36546,30470,40548
+59285,36547,30469,40799
+57097,36548,30468,40726
+53526,36549,30467,40950
+57512,36550,30466,40981
+56594,36551,30465,40894
+54768,36552,30464,40155
+51712,36553,30463,40050
+51104,36554,30462,40266
+59738,36555,30461,40437
+53700,36556,30460,40988
+53086,36557,30459,40836
+59836,36558,30458,40981
+51998,36559,30457,40790
+56263,36560,30456,40977
+55383,36561,30455,40449
+59822,36562,30454,40006
+54766,36563,30453,40913
+54445,36564,30452,40341
+56950,36565,30451,40655
+50348,36566,30450,40915
+50223,36567,30449,40877
+59148,36568,30448,40078
+53224,36569,30447,40410
+50812,36570,30446,40510
+53973,36571,30445,40710
+55003,36572,30444,40290
+59724,36573,30443,40059
+56936,36574,30442,40964
+57161,36575,30441,40822
+56134,36576,30440,40402
+55553,36577,30439,40281
+59771,36578,30438,40210
+57731,36579,30437,40086
+52440,36580,30436,40401
+51562,36581,30435,40506
+58930,36582,30434,40856
+55712,36583,30433,40955
+53374,36584,30432,40923
+55485,36585,30431,40943
+55644,36586,30430,40687
+57613,36587,30429,40627
+51578,36588,30428,40856
+53231,36589,30427,40717
+56042,36590,30426,40772
+54121,36591,30425,40287
+57486,36592,30424,40760
+52185,36593,30423,40913
+55199,36594,30422,40749
+54934,36595,30421,40257
+50236,36596,30420,40294
+52534,36597,30419,40057
+54896,36598,30418,40136
+56969,36599,30417,40453
+55372,36600,30416,40998
+51570,36601,30415,40319
+55092,36602,30414,40624
+56933,36603,30413,40049
+56022,36604,30412,40608
+56476,36605,30411,40574
+59108,36606,30410,40010
+51779,36607,30409,40214
+51107,36608,30408,40141
+55092,36609,30407,40645
+52972,36610,30406,40433
+59805,36611,30405,40894
+56431,36612,30404,40975
+55229,36613,30403,40929
+58728,36614,30402,40231
+57822,36615,30401,40790
+53786,36616,30400,40354
+56379,36617,30399,40508
+54295,36618,30398,40678
+55107,36619,30397,40467
+53122,36620,30396,40479
+51382,36621,30395,40474
+50475,36622,30394,40504
+54460,36623,30393,40166
+57849,36624,30392,40790
+54533,36625,30391,40368
+59863,36626,30390,40900
+55163,36627,30389,40730
+54981,36628,30388,40912
+52243,36629,30387,40654
+53583,36630,30386,40266
+58531,36631,30385,40578
+56074,36632,30384,40548
+54733,36633,30383,40268
+59050,36634,30382,40315
+55113,36635,30381,40549
+52578,36636,30380,40057
+55090,36637,30379,40349
+58275,36638,30378,40963
+52631,36639,30377,40692
+55029,36640,30376,40806
+57857,36641,30375,40904
+55571,36642,30374,40273
+50707,36643,30373,40520
+58544,36644,30372,40423
+56845,36645,30371,40548
+57032,36646,30370,40211
+59948,36647,30369,40854
+53138,36648,30368,40088
+53084,36649,30367,40089
+58785,36650,30366,40717
+54813,36651,30365,40172
+59370,36652,30364,40777
+51539,36653,30363,40939
+56559,36654,30362,40872
+54199,36655,30361,40099
+50266,36656,30360,40138
+52683,36657,30359,40568
+57462,36658,30358,40996
+51131,36659,30357,40965
+50262,36660,30356,40646
+51807,36661,30355,40374
+57145,36662,30354,40352
+57277,36663,30353,40989
+53365,36664,30352,40673
+53019,36665,30351,40296
+52848,36666,30350,40118
+51538,36667,30349,40507
+51454,36668,30348,40668
+50302,36669,30347,40920
+56197,36670,30346,40986
+54306,36671,30345,40175
+56476,36672,30344,40997
+59832,36673,30343,40299
+59394,36674,30342,40953
+51830,36675,30341,40020
+52011,36676,30340,40097
+50988,36677,30339,40570
+52836,36678,30338,40148
+55946,36679,30337,40964
+52136,36680,30336,40236
+58383,36681,30335,40149
+59355,36682,30334,40468
+51676,36683,30333,40063
+55923,36684,30332,40749
+54486,36685,30331,40503
+53783,36686,30330,40549
+56506,36687,30329,40605
+57008,36688,30328,40749
+58204,36689,30327,40023
+53212,36690,30326,40589
+56872,36691,30325,40297
+55681,36692,30324,40678
+52665,36693,30323,40146
+53966,36694,30322,40843
+55319,36695,30321,40022
+58045,36696,30320,40957
+53243,36697,30319,40372
+51941,36698,30318,40162
+50718,36699,30317,40555
+51676,36700,30316,40465
+53184,36701,30315,40094
+51082,36702,30314,40925
+59749,36703,30313,40961
+50984,36704,30312,40500
+50474,36705,30311,40806
+58920,36706,30310,40962
+50101,36707,30309,40505
+53646,36708,30308,40143
+57369,36709,30307,40324
+52059,36710,30306,40937
+51816,36711,30305,40056
+54714,36712,30304,40303
+58068,36713,30303,40465
+54355,36714,30302,40348
+54526,36715,30301,40196
+59614,36716,30300,40884
+50160,36717,30299,40063
+53426,36718,30298,40599
+53222,36719,30297,40925
+57979,36720,30296,40646
+59670,36721,30295,40773
+52921,36722,30294,40270
+53088,36723,30293,40751
+56837,36724,30292,40976
+55713,36725,30291,40138
+51711,36726,30290,40403
+58930,36727,30289,40060
+51890,36728,30288,40527
+53101,36729,30287,40791
+53081,36730,30286,40887
+50239,36731,30285,40812
+52987,36732,30284,40338
+59913,36733,30283,40880
+52290,36734,30282,40125
+54161,36735,30281,40983
+55594,36736,30280,40825
+54805,36737,30279,40297
+54948,36738,30278,40892
+55279,36739,30277,40815
+50122,36740,30276,40303
+58964,36741,30275,40426
+53565,36742,30274,40707
+59351,36743,30273,40449
+54970,36744,30272,40481
+55355,36745,30271,40860
+53339,36746,30270,40525
+52997,36747,30269,40330
+50681,36748,30268,40662
+59039,36749,30267,40467
+54127,36750,30266,40621
+58937,36751,30265,40211
+50843,36752,30264,40519
+52923,36753,30263,40940
+59405,36754,30262,40904
+57333,36755,30261,40878
+55298,36756,30260,40030
+53127,36757,30259,40134
+53386,36758,30258,40287
+53996,36759,30257,40136
+51925,36760,30256,40520
+57393,36761,30255,40026
+50156,36762,30254,40564
+53124,36763,30253,40487
+59836,36764,30252,40285
+54023,36765,30251,40509
+51485,36766,30250,40793
+54532,36767,30249,40828
+56321,36768,30248,40372
+56427,36769,30247,40851
+57288,36770,30246,40118
+59366,36771,30245,40400
+54208,36772,30244,40217
+59421,36773,30243,40899
+58106,36774,30242,40530
+59621,36775,30241,40543
+52295,36776,30240,40393
+57028,36777,30239,40983
+54834,36778,30238,40014
+52768,36779,30237,40922
+54918,36780,30236,40674
+57157,36781,30235,40634
+54032,36782,30234,40455
+53157,36783,30233,40630
+57472,36784,30232,40157
+51417,36785,30231,40497
+56985,36786,30230,40179
+52390,36787,30229,40975
+55704,36788,30228,40014
+55691,36789,30227,40117
+57153,36790,30226,40392
+59559,36791,30225,40554
+55476,36792,30224,40328
+53271,36793,30223,40485
+59183,36794,30222,40729
+57597,36795,30221,40020
+52621,36796,30220,40114
+58764,36797,30219,40246
+51052,36798,30218,40986
+51856,36799,30217,40166
+59116,36800,30216,40150
+50490,36801,30215,40740
+50243,36802,30214,40409
+52073,36803,30213,40290
+58270,36804,30212,40647
+51253,36805,30211,40224
+57106,36806,30210,40045
+59396,36807,30209,40260
+51889,36808,30208,40512
+58735,36809,30207,40049
+54088,36810,30206,40264
+54303,36811,30205,40843
+50372,36812,30204,40009
+52145,36813,30203,40885
+50068,36814,30202,40983
+52553,36815,30201,40623
+58602,36816,30200,40811
+50066,36817,30199,40359
+58780,36818,30198,40902
+52739,36819,30197,40394
+51079,36820,30196,40520
+51038,36821,30195,40359
+52451,36822,30194,40750
+59554,36823,30193,40553
+54385,36824,30192,40774
+57344,36825,30191,40944
+51504,36826,30190,40288
+55866,36827,30189,40746
+52559,36828,30188,40810
+59415,36829,30187,40938
+56126,36830,30186,40448
+51540,36831,30185,40053
+53629,36832,30184,40815
+53124,36833,30183,40270
+59830,36834,30182,40485
+53816,36835,30181,40621
+53286,36836,30180,40942
+58586,36837,30179,40112
+52486,36838,30178,40997
+54563,36839,30177,40150
+54424,36840,30176,40837
+53901,36841,30175,40204
+58893,36842,30174,40315
+58978,36843,30173,40734
+54019,36844,30172,40955
+55475,36845,30171,40827
+50844,36846,30170,40763
+58597,36847,30169,40181
+53049,36848,30168,40855
+56469,36849,30167,40436
+50325,36850,30166,40730
+51116,36851,30165,40935
+54591,36852,30164,40789
+59150,36853,30163,40830
+53392,36854,30162,40048
+53709,36855,30161,40166
+51066,36856,30160,40187
+59098,36857,30159,40482
+50609,36858,30158,40823
+53819,36859,30157,40218
+52051,36860,30156,40757
+56520,36861,30155,40891
+50242,36862,30154,40770
+51904,36863,30153,40731
+50579,36864,30152,40271
+51640,36865,30151,40527
+57439,36866,30150,40801
+53139,36867,30149,40730
+52279,36868,30148,40209
+52718,36869,30147,40956
+51860,36870,30146,40273
+59314,36871,30145,40422
+57564,36872,30144,40025
+56252,36873,30143,40085
+50097,36874,30142,40206
+59641,36875,30141,40139
+51235,36876,30140,40743
+55739,36877,30139,40157
+56934,36878,30138,40363
+57049,36879,30137,40985
+53246,36880,30136,40618
+57522,36881,30135,40673
+54102,36882,30134,40750
+58689,36883,30133,40389
+59393,36884,30132,40909
+52429,36885,30131,40160
+53064,36886,30130,40654
+59224,36887,30129,40761
+51713,36888,30128,40510
+58071,36889,30127,40035
+58912,36890,30126,40766
+56224,36891,30125,40099
+56776,36892,30124,40987
+52735,36893,30123,40703
+53786,36894,30122,40057
+57364,36895,30121,40390
+50046,36896,30120,40069
+52134,36897,30119,40807
+57120,36898,30118,40821
+56811,36899,30117,40497
+53041,36900,30116,40134
+56687,36901,30115,40964
+54654,36902,30114,40675
+55041,36903,30113,40875
+58270,36904,30112,40652
+51502,36905,30111,40719
+57492,36906,30110,40426
+54917,36907,30109,40849
+53348,36908,30108,40837
+57305,36909,30107,40187
+59251,36910,30106,40713
+58357,36911,30105,40507
+52632,36912,30104,40875
+55177,36913,30103,40157
+52270,36914,30102,40627
+51555,36915,30101,40940
+56667,36916,30100,40280
+58425,36917,30099,40229
+56043,36918,30098,40018
+54435,36919,30097,40313
+51477,36920,30096,40088
+54502,36921,30095,40213
+56399,36922,30094,40238
+53658,36923,30093,40551
+55182,36924,30092,40819
+55180,36925,30091,40558
+54165,36926,30090,40317
+50640,36927,30089,40670
+57966,36928,30088,40593
+59759,36929,30087,40135
+58231,36930,30086,40499
+54606,36931,30085,40378
+51508,36932,30084,40086
+50767,36933,30083,40135
+50067,36934,30082,40128
+52160,36935,30081,40342
+52492,36936,30080,40517
+58050,36937,30079,40200
+52966,36938,30078,40327
+55236,36939,30077,40809
+58935,36940,30076,40503
+54477,36941,30075,40004
+59268,36942,30074,40362
+58025,36943,30073,40618
+50310,36944,30072,40851
+58523,36945,30071,40217
+59374,36946,30070,40657
+52736,36947,30069,40702
+52831,36948,30068,40933
+55380,36949,30067,40959
+53577,36950,30066,40458
+55022,36951,30065,40182
+57734,36952,30064,40361
+51030,36953,30063,40261
+50173,36954,30062,40447
+55879,36955,30061,40003
+58756,36956,30060,40694
+54112,36957,30059,40889
+58787,36958,30058,40893
+51284,36959,30057,40788
+50708,36960,30056,40988
+58961,36961,30055,40215
+59142,36962,30054,40905
+58526,36963,30053,40052
+53600,36964,30052,40360
+57101,36965,30051,40222
+51362,36966,30050,40931
+57312,36967,30049,40558
+54840,36968,30048,40677
+52248,36969,30047,40057
+59160,36970,30046,40508
+58024,36971,30045,40986
+53194,36972,30044,40563
+59272,36973,30043,40529
+53799,36974,30042,40665
+56133,36975,30041,40519
+59322,36976,30040,40538
+53126,36977,30039,40653
+50548,36978,30038,40439
+57802,36979,30037,40585
+54812,36980,30036,40228
+57725,36981,30035,40011
+52515,36982,30034,40390
+54905,36983,30033,40506
+56964,36984,30032,40120
+56121,36985,30031,40822
+53978,36986,30030,40945
+52473,36987,30029,40426
+59693,36988,30028,40931
+56153,36989,30027,40952
+51652,36990,30026,40330
+50657,36991,30025,40152
+52574,36992,30024,40462
+56526,36993,30023,40127
+52652,36994,30022,40239
+51161,36995,30021,40800
+57717,36996,30020,40213
+52425,36997,30019,40529
+52957,36998,30018,40434
+51471,36999,30017,40373
+50912,37000,30016,40628
+52816,37001,30015,40873
+54441,37002,30014,40410
+53792,37003,30013,40397
+57423,37004,30012,40849
+56456,37005,30011,40610
+52462,37006,30010,40860
+55882,37007,30009,40634
+50419,37008,30008,40519
+54124,37009,30007,40768
+55105,37010,30006,40814
+51392,37011,30005,40845
+51778,37012,30004,40943
+52500,37013,30003,40116
+54071,37014,30002,40704
+53461,37015,30001,40499
+56320,37016,30000,40678
+52990,37017,31000,40718
+52279,37018,30999,40259
+58863,37019,30998,40569
+50278,37020,30997,40035
+54793,37021,30996,40202
+58969,37022,30995,40683
+55925,37023,30994,40129
+50318,37024,30993,40743
+58228,37025,30992,40398
+52329,37026,30991,40991
+50385,37027,30990,40565
+55572,37028,30989,40525
+50883,37029,30988,40455
+55588,37030,30987,40869
+58975,37031,30986,40496
+52438,37032,30985,40809
+53338,37033,30984,40769
+59736,37034,30983,40112
+51259,37035,30982,40098
+52917,37036,30981,40435
+53151,37037,30980,40382
+57928,37038,30979,40372
+56695,37039,30978,40842
+51218,37040,30977,40171
+54555,37041,30976,40723
+56566,37042,30975,40654
+54658,37043,30974,40707
+50481,37044,30973,40139
+55737,37045,30972,40648
+50935,37046,30971,40694
+59057,37047,30970,40321
+50440,37048,30969,40706
+55798,37049,30968,40774
+56194,37050,30967,40918
+59721,37051,30966,40299
+56848,37052,30965,40333
+52735,37053,30964,40696
+59456,37054,30963,40913
+53218,37055,30962,40376
+55171,37056,30961,40150
+54517,37057,30960,40284
+54349,37058,30959,40767
+58597,37059,30958,40156
+52988,37060,30957,40655
+59515,37061,30956,40713
+56981,37062,30955,40805
+53807,37063,30954,40538
+52377,37064,30953,40317
+54854,37065,30952,40793
+58039,37066,30951,40863
+57979,37067,30950,40496
+53212,37068,30949,40255
+59340,37069,30948,40414
+58853,37070,30947,40128
+56475,37071,30946,40312
+54501,37072,30945,40309
+57573,37073,30944,40741
+51713,37074,30943,40566
+59024,37075,30942,40915
+50569,37076,30941,40139
+54349,37077,30940,40720
+52450,37078,30939,40376
+51740,37079,30938,40262
+57550,37080,30937,40469
+55807,37081,30936,40541
+54812,37082,30935,40961
+53159,37083,30934,40534
+50188,37084,30933,40459
+53275,37085,30932,40402
+51787,37086,30931,40668
+59799,37087,30930,40485
+54716,37088,30929,40636
+58992,37089,30928,40259
+55656,37090,30927,40575
+53829,37091,30926,40245
+54316,37092,30925,40396
+51954,37093,30924,40343
+55329,37094,30923,40146
+57742,37095,30922,40169
+50937,37096,30921,40879
+57477,37097,30920,40984
+56709,37098,30919,40943
+59168,37099,30918,40520
+56164,37100,30917,40109
+58351,37101,30916,40129
+54661,37102,30915,40977
+55592,37103,30914,40315
+58381,37104,30913,40158
+51193,37105,30912,40584
+55156,37106,30911,40327
+50986,37107,30910,40828
+57162,37108,30909,40157
+58926,37109,30908,40291
+52599,37110,30907,40599
+53192,37111,30906,40719
+59984,37112,30905,40895
+50347,37113,30904,40588
+59302,37114,30903,40845
+52313,37115,30902,40939
+55768,37116,30901,40810
+57181,37117,30900,40367
+53880,37118,30899,40147
+54452,37119,30898,40543
+53915,37120,30897,40680
+57797,37121,30896,40816
+56682,37122,30895,40184
+53522,37123,30894,40575
+50470,37124,30893,40835
+55734,37125,30892,40512
+58953,37126,30891,40412
+54814,37127,30890,40798
+58855,37128,30889,40096
+50213,37129,30888,40887
+55154,37130,30887,40423
+55560,37131,30886,40938
+56125,37132,30885,40574
+52840,37133,30884,40720
+59352,37134,30883,40831
+50013,37135,30882,40421
+56299,37136,30881,40878
+53143,37137,30880,40985
+50206,37138,30879,40704
+52160,37139,30878,40911
+54469,37140,30877,40802
+55566,37141,30876,40730
+56073,37142,30875,40771
+52052,37143,30874,40919
+59280,37144,30873,40279
+53496,37145,30872,40699
+54287,37146,30871,40313
+55007,37147,30870,40689
+56918,37148,30869,40453
+50485,37149,30868,40856
+55001,37150,30867,40767
+50665,37151,30866,40199
+54335,37152,30865,40747
+55332,37153,30864,40494
+52276,37154,30863,40690
+50154,37155,30862,40603
+57049,37156,30861,40245
+57817,37157,30860,40005
+52539,37158,30859,40970
+57631,37159,30858,40152
+51585,37160,30857,40888
+55352,37161,30856,40493
+58959,37162,30855,40208
+50420,37163,30854,40461
+57031,37164,30853,40066
+54391,37165,30852,40859
+56351,37166,30851,40017
+58058,37167,30850,40835
+57245,37168,30849,40708
+59328,37169,30848,40251
+51615,37170,30847,40306
+52259,37171,30846,40652
+51000,37172,30845,40241
+54140,37173,30844,40508
+52661,37174,30843,40795
+57585,37175,30842,40716
+54061,37176,30841,40534
+58632,37177,30840,40689
+57302,37178,30839,40239
+56218,37179,30838,40394
+53237,37180,30837,40570
+55891,37181,30836,40044
+59581,37182,30835,40440
+56006,37183,30834,40989
+51465,37184,30833,40117
+50877,37185,30832,40147
+58004,37186,30831,40476
+53198,37187,30830,40768
+58661,37188,30829,40122
+56426,37189,30828,40845
+54985,37190,30827,40732
+56367,37191,30826,40776
+51705,37192,30825,40614
+52813,37193,30824,40227
+59386,37194,30823,40562
+52869,37195,30822,40298
+57218,37196,30821,40984
+53084,37197,30820,40729
+59544,37198,30819,40840
+54416,37199,30818,40480
+57589,37200,30817,40025
+55906,37201,30816,40572
+51927,37202,30815,40989
+50145,37203,30814,40170
+54069,37204,30813,40171
+58295,37205,30812,40531
+57852,37206,30811,40088
+55885,37207,30810,40783
+56007,37208,30809,40420
+50122,37209,30808,40783
+51181,37210,30807,40669
+54152,37211,30806,40035
+50997,37212,30805,40722
+57799,37213,30804,40980
+51956,37214,30803,40525
+51723,37215,30802,40744
+55767,37216,30801,40109
+54369,37217,30800,40792
+50204,37218,30799,40775
+53871,37219,30798,40367
+53466,37220,30797,40433
+52041,37221,30796,40609
+52866,37222,30795,40212
+56911,37223,30794,40493
+54925,37224,30793,40291
+55412,37225,30792,40657
+54740,37226,30791,40732
+59417,37227,30790,40265
+55279,37228,30789,40601
+58920,37229,30788,40563
+52661,37230,30787,40347
+59661,37231,30786,40506
+54882,37232,30785,40197
+59582,37233,30784,40444
+58743,37234,30783,40176
+54860,37235,30782,40554
+53307,37236,30781,40160
+59201,37237,30780,40244
+58110,37238,30779,40930
+58073,37239,30778,40999
+51326,37240,30777,40560
+53102,37241,30776,40901
+50739,37242,30775,40787
+59003,37243,30774,40015
+55270,37244,30773,40532
+50130,37245,30772,40456
+57551,37246,30771,40231
+56649,37247,30770,40689
+51002,37248,30769,40444
+56606,37249,30768,40060
+54107,37250,30767,40803
+55260,37251,30766,40352
+56617,37252,30765,40003
+54710,37253,30764,40225
+59286,37254,30763,40560
+52624,37255,30762,40376
+56079,37256,30761,40591
+50573,37257,30760,40118
+59934,37258,30759,40246
+59122,37259,30758,40625
+50349,37260,30757,40973
+50505,37261,30756,40440
+50373,37262,30755,40235
+51506,37263,30754,40489
+53396,37264,30753,40279
+51509,37265,30752,40053
+50189,37266,30751,40796
+57683,37267,30750,40868
+56857,37268,30749,40439
+51018,37269,30748,40666
+54236,37270,30747,40304
+55622,37271,30746,40053
+57731,37272,30745,40907
+58756,37273,30744,40210
+56985,37274,30743,40842
+58146,37275,30742,40915
+50264,37276,30741,40769
+52227,37277,30740,40088
+59436,37278,30739,40981
+54974,37279,30738,40559
+54275,37280,30737,40994
+51446,37281,30736,40569
+58417,37282,30735,40031
+55253,37283,30734,40562
+50144,37284,30733,40528
+54977,37285,30732,40812
+52693,37286,30731,40520
+52416,37287,30730,40033
+50606,37288,30729,40881
+50936,37289,30728,40491
+55079,37290,30727,40714
+53295,37291,30726,40027
+51547,37292,30725,40707
+51953,37293,30724,40170
+51256,37294,30723,40032
+52496,37295,30722,40774
+51515,37296,30721,40344
+55750,37297,30720,40324
+52123,37298,30719,40721
+50389,37299,30718,40439
+54018,37300,30717,40369
+54576,37301,30716,40499
+51173,37302,30715,40427
+53075,37303,30714,40657
+58548,37304,30713,40229
+54156,37305,30712,40943
+51981,37306,30711,40522
+58605,37307,30710,40778
+53275,37308,30709,40504
+58505,37309,30708,40123
+59452,37310,30707,40752
+52924,37311,30706,40987
+55233,37312,30705,40854
+53240,37313,30704,40196
+52558,37314,30703,40964
+57545,37315,30702,40518
+53761,37316,30701,40079
+57610,37317,30700,40251
+59439,37318,30699,40165
+58929,37319,30698,40104
+56954,37320,30697,40346
+52485,37321,30696,40000
+59583,37322,30695,40646
+59776,37323,30694,40110
+51979,37324,30693,40756
+51503,37325,30692,40094
+53422,37326,30691,40121
+50374,37327,30690,40545
+55448,37328,30689,40535
+58656,37329,30688,40140
+50015,37330,30687,40582
+55493,37331,30686,40452
+51491,37332,30685,40870
+59268,37333,30684,40392
+52248,37334,30683,40115
+58847,37335,30682,40730
+58819,37336,30681,40045
+52070,37337,30680,40067
+54880,37338,30679,40466
+57596,37339,30678,40282
+51440,37340,30677,40427
+59830,37341,30676,40233
+51184,37342,30675,40264
+50114,37343,30674,40149
+54420,37344,30673,40440
+50852,37345,30672,40576
+54834,37346,30671,40474
+57339,37347,30670,40420
+52923,37348,30669,40353
+58002,37349,30668,40901
+59829,37350,30667,40615
+51138,37351,30666,40020
+58947,37352,30665,40313
+57924,37353,30664,40868
+50806,37354,30663,40435
+57173,37355,30662,40323
+57164,37356,30661,40952
+57539,37357,30660,40503
+53895,37358,30659,40159
+51252,37359,30658,40994
+52089,37360,30657,40445
+55509,37361,30656,40099
+55275,37362,30655,40684
+57264,37363,30654,40114
+55965,37364,30653,40245
+51255,37365,30652,40880
+59047,37366,30651,40487
+58828,37367,30650,40089
+52174,37368,30649,40995
+55331,37369,30648,40927
+58611,37370,30647,40467
+54111,37371,30646,40604
+55621,37372,30645,40119
+54041,37373,30644,40203
+54663,37374,30643,40140
+55344,37375,30642,40916
+52412,37376,30641,40274
+54218,37377,30640,40250
+53390,37378,30639,40104
+54168,37379,30638,40059
+57806,37380,30637,40942
+55831,37381,30636,40700
+56149,37382,30635,40950
+53411,37383,30634,40029
+54473,37384,30633,40735
+54742,37385,30632,40536
+51740,37386,30631,40240
+50014,37387,30630,40002
+52750,37388,30629,40326
+52685,37389,30628,40667
+56067,37390,30627,40274
+53562,37391,30626,40160
+59152,37392,30625,40882
+57124,37393,30624,40314
+56126,37394,30623,40938
+55559,37395,30622,40610
+58544,37396,30621,40495
+54471,37397,30620,40820
+52881,37398,30619,40104
+53593,37399,30618,40397
+50519,37400,30617,40601
+53058,37401,30616,40147
+56789,37402,30615,40397
+59526,37403,30614,40367
+59932,37404,30613,40199
+59154,37405,30612,40953
+59408,37406,30611,40165
+58838,37407,30610,40977
+55401,37408,30609,40898
+51946,37409,30608,40637
+50656,37410,30607,40990
+51185,37411,30606,40052
+52250,37412,30605,40813
+55179,37413,30604,40997
+50545,37414,30603,40457
+50884,37415,30602,40781
+56157,37416,30601,40100
+50410,37417,30600,40603
+54193,37418,30599,40958
+55756,37419,30598,40770
+54770,37420,30597,40767
+51645,37421,30596,40019
+55363,37422,30595,40953
+59400,37423,30594,40739
+53371,37424,30593,40365
+52910,37425,30592,40056
+51288,37426,30591,40417
+55962,37427,30590,40612
+50093,37428,30589,40315
+58774,37429,30588,40452
+59191,37430,30587,40253
+58294,37431,30586,40383
+56381,37432,30585,40101
+51314,37433,30584,40446
+56821,37434,30583,40047
+55474,37435,30582,40677
+57444,37436,30581,40769
+55714,37437,30580,40643
+56365,37438,30579,40435
+58100,37439,30578,40515
+53451,37440,30577,40249
+50644,37441,30576,40711
+59786,37442,30575,40174
+58055,37443,30574,40278
+54559,37444,30573,40702
+54477,37445,30572,40131
+53927,37446,30571,40960
+50341,37447,30570,40170
+59981,37448,30569,40383
+59567,37449,30568,40462
+56534,37450,30567,40172
+58109,37451,30566,40684
+57286,37452,30565,40922
+56179,37453,30564,40070
+56757,37454,30563,40661
+59486,37455,30562,40285
+54348,37456,30561,40183
+52001,37457,30560,40614
+55385,37458,30559,40001
+51786,37459,30558,40591
+57491,37460,30557,40234
+57363,37461,30556,40063
+59079,37462,30555,40151
+56292,37463,30554,40727
+59824,37464,30553,40095
+52101,37465,30552,40257
+54053,37466,30551,40770
+53569,37467,30550,40947
+52349,37468,30549,40086
+52759,37469,30548,40872
+56414,37470,30547,40733
+56442,37471,30546,40564
+56146,37472,30545,40694
+54344,37473,30544,40497
+59162,37474,30543,40806
+53816,37475,30542,40826
+56509,37476,30541,40123
+55740,37477,30540,40492
+50265,37478,30539,40949
+56470,37479,30538,40100
+54139,37480,30537,40137
+54695,37481,30536,40713
+57753,37482,30535,40947
+57635,37483,30534,40807
+55420,37484,30533,40328
+53824,37485,30532,40357
+52267,37486,30531,40070
+53501,37487,30530,40540
+50930,37488,30529,40008
+50846,37489,30528,40172
+53506,37490,30527,40105
+56038,37491,30526,40508
+52561,37492,30525,40140
+56750,37493,30524,40526
+52706,37494,30523,40814
+54513,37495,30522,40637
+51808,37496,30521,40153
+55834,37497,30520,40475
+53044,37498,30519,40549
+55109,37499,30518,40412
+57623,37500,30517,40485
+56466,37501,30516,40961
+50481,37502,30515,40748
+51609,37503,30514,40801
+53487,37504,30513,40930
+58359,37505,30512,40466
+51474,37506,30511,40638
+58902,37507,30510,40368
+53140,37508,30509,40282
+50938,37509,30508,40568
+59078,37510,30507,40134
+53624,37511,30506,40804
+50421,37512,30505,40266
+59585,37513,30504,40731
+56092,37514,30503,40715
+56076,37515,30502,40940
+58466,37516,30501,40377
+56578,37517,30500,40591
+58404,37518,30499,40739
+56867,37519,30498,40576
+55933,37520,30497,40719
+50758,37521,30496,40433
+55729,37522,30495,40858
+57949,37523,30494,40115
+53801,37524,30493,40888
+51415,37525,30492,40080
+51833,37526,30491,40081
+59727,37527,30490,40616
+57390,37528,30489,40032
+58513,37529,30488,40605
+50835,37530,30487,40465
+52621,37531,30486,40421
+53474,37532,30485,40519
+58884,37533,30484,40733
+55287,37534,30483,40735
+52479,37535,30482,40325
+50656,37536,30481,40084
+59584,37537,30480,40315
+54641,37538,30479,40888
+56176,37539,30478,40927
+58181,37540,30477,40803
+53863,37541,30476,40897
+57902,37542,30475,40699
+59434,37543,30474,40130
+59357,37544,30473,40585
+57374,37545,30472,40501
+55869,37546,30471,40428
+53242,37547,30470,40259
+56506,37548,30469,40436
+57803,37549,30468,40077
+53727,37550,30467,40820
+53858,37551,30466,40490
+58941,37552,30465,40957
+59080,37553,30464,40790
+52567,37554,30463,40652
+57713,37555,30462,40947
+55370,37556,30461,40007
+59246,37557,30460,40454
+50443,37558,30459,40961
+58556,37559,30458,40568
+57256,37560,30457,40063
+52379,37561,30456,40871
+53696,37562,30455,40534
+56793,37563,30454,40481
+53182,37564,30453,40467
+52044,37565,30452,40805
+50817,37566,30451,40559
+59518,37567,30450,40438
+50743,37568,30449,40549
+50418,37569,30448,40836
+55415,37570,30447,40826
+57478,37571,30446,40728
+53902,37572,30445,40287
+59265,37573,30444,40812
+51202,37574,30443,40574
+52859,37575,30442,40390
+55314,37576,30441,40887
+50300,37577,30440,40711
+54286,37578,30439,40325
+59516,37579,30438,40275
+59939,37580,30437,40882
+57194,37581,30436,40391
+58126,37582,30435,40267
+52552,37583,30434,40770
+57314,37584,30433,40361
+51268,37585,30432,40362
+58613,37586,30431,40902
+50241,37587,30430,40861
+54165,37588,30429,40721
+51667,37589,30428,40120
+58167,37590,30427,40829
+57338,37591,30426,40390
+50992,37592,30425,40836
+51254,37593,30424,40649
+59388,37594,30423,40107
+50793,37595,30422,40078
+57020,37596,30421,40243
+56634,37597,30420,40656
+59645,37598,30419,40772
+53301,37599,30418,40132
+51331,37600,30417,40317
+52655,37601,30416,40164
+52133,37602,30415,40948
+51080,37603,30414,40787
+57297,37604,30413,40150
+53849,37605,30412,40340
+54656,37606,30411,40372
+56925,37607,30410,40767
+59812,37608,30409,40507
+50508,37609,30408,40760
+59134,37610,30407,40374
+56450,37611,30406,40005
+54269,37612,30405,40450
+50447,37613,30404,40565
+50305,37614,30403,40414
+59767,37615,30402,40872
+52701,37616,30401,40653
+58474,37617,30400,40286
+56154,37618,30399,40609
+54429,37619,30398,40904
+58872,37620,30397,40645
+53607,37621,30396,40533
+54623,37622,30395,40598
+59368,37623,30394,40546
+57604,37624,30393,40872
+50323,37625,30392,40106
+58568,37626,30391,40833
+55373,37627,30390,40485
+53562,37628,30389,40724
+57118,37629,30388,40744
+55293,37630,30387,40885
+54900,37631,30386,40409
+52692,37632,30385,40116
+56870,37633,30384,40828
+56489,37634,30383,40447
+50870,37635,30382,40287
+58237,37636,30381,40952
+58992,37637,30380,40471
+50361,37638,30379,40088
+52239,37639,30378,40637
+53860,37640,30377,40622
+58927,37641,30376,40625
+51069,37642,30375,40697
+58269,37643,30374,40641
+59576,37644,30373,40389
+57002,37645,30372,40350
+59830,37646,30371,40661
+59598,37647,30370,40355
+51980,37648,30369,40454
+55098,37649,30368,40690
+53350,37650,30367,40348
+56982,37651,30366,40621
+57108,37652,30365,40580
+58152,37653,30364,40918
+55474,37654,30363,40796
+58470,37655,30362,40188
+59798,37656,30361,40026
+55059,37657,30360,40642
+57923,37658,30359,40528
+55013,37659,30358,40470
+57194,37660,30357,40897
+59400,37661,30356,40283
+54320,37662,30355,40627
+51771,37663,30354,40987
+53370,37664,30353,40033
+51783,37665,30352,40468
+54468,37666,30351,40907
+51168,37667,30350,40997
+51122,37668,30349,40524
+54501,37669,30348,40414
+53886,37670,30347,40152
+59665,37671,30346,40266
+53088,37672,30345,40696
+52356,37673,30344,40506
+53148,37674,30343,40939
+52523,37675,30342,40654
+52660,37676,30341,40579
+55995,37677,30340,40170
+59150,37678,30339,40178
+54766,37679,30338,40580
+52897,37680,30337,40383
+52660,37681,30336,40398
+56969,37682,30335,40416
+52554,37683,30334,40508
+58874,37684,30333,40973
+52936,37685,30332,40091
+53185,37686,30331,40855
+56180,37687,30330,40907
+55381,37688,30329,40035
+51061,37689,30328,40819
+56108,37690,30327,40015
+50258,37691,30326,40216
+59483,37692,30325,40304
+52360,37693,30324,40705
+57446,37694,30323,40629
+52666,37695,30322,40338
+56677,37696,30321,40532
+54531,37697,30320,40523
+50887,37698,30319,40247
+53143,37699,30318,40469
+54736,37700,30317,40011
+59511,37701,30316,40160
+58590,37702,30315,40061
+54851,37703,30314,40484
+52858,37704,30313,40544
+58762,37705,30312,40402
+58569,37706,30311,40527
+59774,37707,30310,40997
+56450,37708,30309,40027
+57714,37709,30308,40211
+52058,37710,30307,40106
+59685,37711,30306,40130
+55067,37712,30305,40158
+56472,37713,30304,40841
+55091,37714,30303,40885
+54811,37715,30302,40732
+57400,37716,30301,40945
+55670,37717,30300,40547
+57610,37718,30299,40310
+59027,37719,30298,40624
+55121,37720,30297,40803
+56569,37721,30296,40323
+57765,37722,30295,40471
+59330,37723,30294,40619
+53917,37724,30293,40658
+50990,37725,30292,40507
+56991,37726,30291,40325
+50753,37727,30290,40335
+57158,37728,30289,40086
+59144,37729,30288,40258
+50257,37730,30287,40300
+50505,37731,30286,40334
+58551,37732,30285,40580
+52536,37733,30284,40910
+52617,37734,30283,40502
+53542,37735,30282,40267
+50624,37736,30281,40663
+52547,37737,30280,40233
+57446,37738,30279,40803
+56068,37739,30278,40566
+54912,37740,30277,40350
+50432,37741,30276,40303
+59288,37742,30275,40504
+59988,37743,30274,40982
+50010,37744,30273,40459
+55927,37745,30272,40475
+57076,37746,30271,40274
+56631,37747,30270,40118
+52672,37748,30269,40576
+51410,37749,30268,40983
+58574,37750,30267,40377
+58353,37751,30266,40922
+56922,37752,30265,40361
+55071,37753,30264,40314
+55870,37754,30263,40439
+54183,37755,30262,40329
+55456,37756,30261,40822
+50571,37757,30260,40881
+55355,37758,30259,40344
+50353,37759,30258,40754
+54315,37760,30257,40535
+54056,37761,30256,40032
+54098,37762,30255,40466
+53288,37763,30254,40568
+59816,37764,30253,40288
+54311,37765,30252,40352
+56927,37766,30251,40508
+51576,37767,30250,40000
+59220,37768,30249,40709
+58014,37769,30248,40113
+56369,37770,30247,40216
+56207,37771,30246,40688
+50640,37772,30245,40444
+51795,37773,30244,40078
+57866,37774,30243,40962
+58364,37775,30242,40639
+53166,37776,30241,40919
+51103,37777,30240,40277
+59981,37778,30239,40432
+55696,37779,30238,40041
+53969,37780,30237,40753
+57212,37781,30236,40798
+55582,37782,30235,40274
+50368,37783,30234,40058
+52801,37784,30233,40038
+51399,37785,30232,40569
+57573,37786,30231,40353
+53808,37787,30230,40160
+51684,37788,30229,40005
+58110,37789,30228,40090
+56986,37790,30227,40945
+59176,37791,30226,40746
+57254,37792,30225,40135
+56049,37793,30224,40558
+59978,37794,30223,40632
+56179,37795,30222,40373
+54385,37796,30221,40207
+50484,37797,30220,40373
+59520,37798,30219,40814
+56619,37799,30218,40115
+59285,37800,30217,40465
+52812,37801,30216,40064
+57948,37802,30215,40877
+52638,37803,30214,40764
+56993,37804,30213,40900
+56669,37805,30212,40681
+54063,37806,30211,40369
+50363,37807,30210,40647
+56695,37808,30209,40802
+51234,37809,30208,40279
+56700,37810,30207,40439
+58277,37811,30206,40887
+53451,37812,30205,40637
+51220,37813,30204,40441
+56681,37814,30203,40558
+55086,37815,30202,40824
+57588,37816,30201,40537
+57105,37817,30200,40352
+56491,37818,30199,40666
+53997,37819,30198,40368
+52297,37820,30197,40750
+59158,37821,30196,40871
+51389,37822,30195,40286
+56888,37823,30194,40168
+58469,37824,30193,40333
+57829,37825,30192,40884
+50649,37826,30191,40191
+57770,37827,30190,40963
+53876,37828,30189,40576
+52640,37829,30188,40166
+57996,37830,30187,40430
+51742,37831,30186,40531
+58835,37832,30185,40036
+53227,37833,30184,40582
+51254,37834,30183,40979
+56897,37835,30182,40465
+52099,37836,30181,40039
+59405,37837,30180,40856
+56327,37838,30179,40468
+56659,37839,30178,40787
+56837,37840,30177,40116
+51375,37841,30176,40593
+55027,37842,30175,40902
+58534,37843,30174,40173
+51470,37844,30173,40331
+55499,37845,30172,40792
+51886,37846,30171,40336
+59965,37847,30170,40675
+59493,37848,30169,40043
+52029,37849,30168,40086
+52256,37850,30167,40965
+51534,37851,30166,40890
+59376,37852,30165,40419
+54725,37853,30164,40392
+50562,37854,30163,40976
+54576,37855,30162,40876
+52074,37856,30161,40514
+53667,37857,30160,40233
+53967,37858,30159,40191
+55309,37859,30158,40218
+59135,37860,30157,40052
+51049,37861,30156,40299
+52899,37862,30155,40873
+55328,37863,30154,40826
+54814,37864,30153,40341
+59452,37865,30152,40828
+55887,37866,30151,40122
+53032,37867,30150,40960
+51878,37868,30149,40909
+52563,37869,30148,40426
+56418,37870,30147,40332
+54323,37871,30146,40718
+53128,37872,30145,40524
+51750,37873,30144,40122
+53153,37874,30143,40390
+59539,37875,30142,40134
+55499,37876,30141,40173
+56943,37877,30140,40922
+59279,37878,30139,40126
+51612,37879,30138,40772
+57922,37880,30137,40716
+56908,37881,30136,40996
+56389,37882,30135,40216
+59427,37883,30134,40086
+51399,37884,30133,40650
+57210,37885,30132,40420
+52891,37886,30131,40174
+58784,37887,30130,40989
+53060,37888,30129,40781
+54331,37889,30128,40518
+54796,37890,30127,40823
+57532,37891,30126,40948
+53812,37892,30125,40274
+51985,37893,30124,40515
+51158,37894,30123,40922
+53779,37895,30122,40197
+51782,37896,30121,40541
+55408,37897,30120,40714
+57895,37898,30119,40257
+54515,37899,30118,40697
+59514,37900,30117,40141
+57359,37901,30116,40505
+57809,37902,30115,40505
+52667,37903,30114,40102
+51233,37904,30113,40578
+55303,37905,30112,40707
+53254,37906,30111,40254
+50748,37907,30110,40608
+56448,37908,30109,40632
+50994,37909,30108,40416
+57547,37910,30107,40889
+50299,37911,30106,40690
+54109,37912,30105,40620
+56764,37913,30104,40658
+56377,37914,30103,40643
+50220,37915,30102,40804
+57232,37916,30101,40907
+50901,37917,30100,40868
+53351,37918,30099,40721
+50104,37919,30098,40196
+57347,37920,30097,40231
+58205,37921,30096,40243
+59457,37922,30095,40751
+57592,37923,30094,40414
+53298,37924,30093,40755
+53238,37925,30092,40518
+59492,37926,30091,40615
+50535,37927,30090,40759
+53404,37928,30089,40596
+59478,37929,30088,40624
+52958,37930,30087,40545
+58547,37931,30086,40931
+53007,37932,30085,40128
+55153,37933,30084,40906
+54994,37934,30083,40298
+53643,37935,30082,40420
+50335,37936,30081,40160
+58866,37937,30080,40715
+54226,37938,30079,40831
+51794,37939,30078,40611
+58479,37940,30077,40319
+55314,37941,30076,40194
+59315,37942,30075,40664
+59850,37943,30074,40526
+51573,37944,30073,40098
+55343,37945,30072,40759
+56204,37946,30071,40919
+53866,37947,30070,40707
+55482,37948,30069,40991
+57975,37949,30068,40375
+55574,37950,30067,40952
+58088,37951,30066,40249
+51927,37952,30065,40968
+55209,37953,30064,40191
+58508,37954,30063,40425
+52303,37955,30062,40187
+58233,37956,30061,40215
+53258,37957,30060,40950
+54140,37958,30059,40585
+55066,37959,30058,40325
+54846,37960,30057,40273
+56157,37961,30056,40157
+55893,37962,30055,40418
+52086,37963,30054,40799
+59173,37964,30053,40645
+53370,37965,30052,40775
+57366,37966,30051,40352
+52211,37967,30050,40519
+53201,37968,30049,40532
+59831,37969,30048,40659
+58037,37970,30047,40553
+59891,37971,30046,40945
+59846,37972,30045,40112
+54385,37973,30044,40767
+59157,37974,30043,40180
+58569,37975,30042,40432
+59514,37976,30041,40174
+57620,37977,30040,40884
+58478,37978,30039,40860
+57451,37979,30038,40755
+51253,37980,30037,40590
+52574,37981,30036,40678
+52936,37982,30035,40708
+54369,37983,30034,40851
+54033,37984,30033,40439
+51632,37985,30032,40619
+57615,37986,30031,40931
+55955,37987,30030,40025
+57162,37988,30029,40589
+51196,37989,30028,40753
+53392,37990,30027,40831
+54669,37991,30026,40586
+54206,37992,30025,40829
+51319,37993,30024,40898
+56602,37994,30023,40315
+54334,37995,30022,40724
+51261,37996,30021,40965
+55976,37997,30020,40880
+59406,37998,30019,40570
+59508,37999,30018,40896
+59993,38000,30017,40902
+52791,38001,30016,40315
+54287,38002,30015,40155
+56401,38003,30014,40936
+54535,38004,30013,40451
+55006,38005,30012,40184
+59698,38006,30011,40613
+56361,38007,30010,40711
+51452,38008,30009,40809
+50523,38009,30008,40745
+57727,38010,30007,40386
+55841,38011,30006,40006
+51921,38012,30005,40911
+52283,38013,30004,40073
+58797,38014,30003,40085
+53041,38015,30002,40515
+53103,38016,30001,40093
+55872,38017,30000,40358
+53913,38018,31000,40837
+56786,38019,30999,40553
+53619,38020,30998,40598
+55164,38021,30997,40511
+52783,38022,30996,40578
+51633,38023,30995,40716
+50378,38024,30994,40192
+52035,38025,30993,40630
+50463,38026,30992,40869
+58233,38027,30991,40555
+56534,38028,30990,40896
+56765,38029,30989,40061
+59081,38030,30988,40393
+57740,38031,30987,40887
+59206,38032,30986,40767
+59792,38033,30985,40458
+57441,38034,30984,40078
+53355,38035,30983,40576
+58577,38036,30982,40722
+52713,38037,30981,40404
+53211,38038,30980,40370
+56536,38039,30979,40308
+56796,38040,30978,40715
+52804,38041,30977,40437
+59887,38042,30976,40096
+51339,38043,30975,40378
+53822,38044,30974,40875
+57679,38045,30973,40960
+50357,38046,30972,40803
+50202,38047,30971,40088
+58131,38048,30970,40426
+50507,38049,30969,40129
+56938,38050,30968,40057
+54510,38051,30967,40490
+53317,38052,30966,40282
+59514,38053,30965,40106
+52153,38054,30964,40226
+50500,38055,30963,40668
+57252,38056,30962,40169
+53665,38057,30961,40410
+59747,38058,30960,40815
+56537,38059,30959,40796
+59359,38060,30958,40069
+55303,38061,30957,40997
+51751,38062,30956,40613
+59645,38063,30955,40363
+53305,38064,30954,40887
+55156,38065,30953,40865
+53209,38066,30952,40794
+57249,38067,30951,40442
+56212,38068,30950,40883
+55459,38069,30949,40828
+55887,38070,30948,40732
+56648,38071,30947,40543
+58600,38072,30946,40492
+53935,38073,30945,40497
+57294,38074,30944,40055
+59801,38075,30943,40223
+57149,38076,30942,40570
+54681,38077,30941,40834
+57073,38078,30940,40133
+56970,38079,30939,40971
+56860,38080,30938,40813
+50800,38081,30937,40228
+54930,38082,30936,40381
+54022,38083,30935,40981
+54945,38084,30934,40632
+53975,38085,30933,40168
+52671,38086,30932,40030
+57064,38087,30931,40665
+55163,38088,30930,40105
+55154,38089,30929,40181
+59477,38090,30928,40410
+58781,38091,30927,40326
+59674,38092,30926,40917
+55560,38093,30925,40889
+52152,38094,30924,40526
+58038,38095,30923,40101
+57840,38096,30922,40694
+55794,38097,30921,40871
+57609,38098,30920,40382
+52803,38099,30919,40654
+54384,38100,30918,40030
+55498,38101,30917,40440
+52205,38102,30916,40995
+57528,38103,30915,40775
+55074,38104,30914,40675
+59325,38105,30913,40092
+50941,38106,30912,40190
+55771,38107,30911,40862
+59868,38108,30910,40155
+59577,38109,30909,40191
+59040,38110,30908,40203
+58147,38111,30907,40876
+55897,38112,30906,40438
+50351,38113,30905,40662
+54881,38114,30904,40113
+57450,38115,30903,40410
+50934,38116,30902,40009
+52881,38117,30901,40857
+54767,38118,30900,40231
+58268,38119,30899,40316
+56911,38120,30898,40559
+57940,38121,30897,40561
+52439,38122,30896,40758
+57413,38123,30895,40162
+55190,38124,30894,40263
+52863,38125,30893,40288
+52828,38126,30892,40811
+59799,38127,30891,40244
+52556,38128,30890,40667
+53258,38129,30889,40845
+58816,38130,30888,40683
+50465,38131,30887,40619
+51670,38132,30886,40740
+57599,38133,30885,40215
+51031,38134,30884,40454
+55000,38135,30883,40501
+53752,38136,30882,40937
+53868,38137,30881,40143
+51882,38138,30880,40254
+59473,38139,30879,40735
+59981,38140,30878,40551
+53199,38141,30877,40512
+59937,38142,30876,40695
+55300,38143,30875,40626
+55388,38144,30874,40655
+52376,38145,30873,40608
+54312,38146,30872,40318
+56986,38147,30871,40552
+52788,38148,30870,40652
+52361,38149,30869,40958
+55226,38150,30868,40148
+53152,38151,30867,40933
+51985,38152,30866,40844
+58489,38153,30865,40979
+54153,38154,30864,40402
+51058,38155,30863,40536
+56275,38156,30862,40292
+54348,38157,30861,40067
+52531,38158,30860,40620
+51712,38159,30859,40961
+52334,38160,30858,40751
+58588,38161,30857,40275
+53453,38162,30856,40344
+58347,38163,30855,40819
+50004,38164,30854,40213
+51811,38165,30853,40386
+59916,38166,30852,40613
+58040,38167,30851,40009
+58476,38168,30850,40605
+55342,38169,30849,40338
+53431,38170,30848,40398
+54715,38171,30847,40466
+55726,38172,30846,40925
+56827,38173,30845,40581
+54534,38174,30844,40615
+57117,38175,30843,40926
+59060,38176,30842,40678
+53531,38177,30841,40426
+51253,38178,30840,40500
+58759,38179,30839,40160
+51915,38180,30838,40867
+58688,38181,30837,40015
+51409,38182,30836,40302
+55396,38183,30835,40819
+59807,38184,30834,40584
+56319,38185,30833,40646
+57001,38186,30832,40535
+58818,38187,30831,40239
+59858,38188,30830,40668
+54911,38189,30829,40666
+58238,38190,30828,40170
+58161,38191,30827,40380
+54580,38192,30826,40905
+57128,38193,30825,40065
+51219,38194,30824,40262
+50610,38195,30823,40371
+51788,38196,30822,40793
+55381,38197,30821,40729
+51260,38198,30820,40642
+59608,38199,30819,40538
+57149,38200,30818,40856
+55186,38201,30817,40540
+54387,38202,30816,40608
+59466,38203,30815,40078
+52369,38204,30814,40038
+55485,38205,30813,40942
+53454,38206,30812,40875
+50798,38207,30811,40743
+58962,38208,30810,40068
+57016,38209,30809,40291
+53386,38210,30808,40210
+53839,38211,30807,40753
+58539,38212,30806,40115
+55525,38213,30805,40251
+58403,38214,30804,40573
+58792,38215,30803,40063
+59435,38216,30802,40610
+51064,38217,30801,40808
+59736,38218,30800,40060
+54551,38219,30799,40880
+54369,38220,30798,40863
+58665,38221,30797,40532
+56315,38222,30796,40821
+54202,38223,30795,40956
+54832,38224,30794,40402
+54255,38225,30793,40150
+59343,38226,30792,40637
+50276,38227,30791,40405
+50732,38228,30790,40935
+52757,38229,30789,40947
+53848,38230,30788,40219
+52040,38231,30787,40872
+56891,38232,30786,40298
+53719,38233,30785,40814
+57646,38234,30784,40801
+50172,38235,30783,40643
+57061,38236,30782,40074
+55670,38237,30781,40858
+55209,38238,30780,40540
+52344,38239,30779,40606
+53684,38240,30778,40768
+57279,38241,30777,40388
+57957,38242,30776,40201
+55700,38243,30775,40212
+57060,38244,30774,40567
+58568,38245,30773,40347
+54128,38246,30772,40792
+50796,38247,30771,40057
+50979,38248,30770,40459
+59619,38249,30769,40650
+57895,38250,30768,40627
+56511,38251,30767,40347
+53713,38252,30766,40452
+55768,38253,30765,40362
+59068,38254,30764,40733
+55083,38255,30763,40402
+51377,38256,30762,40849
+55663,38257,30761,40210
+50802,38258,30760,40644
+57683,38259,30759,40689
+53494,38260,30758,40495
+51091,38261,30757,40193
+51774,38262,30756,40353
+53353,38263,30755,40213
+52806,38264,30754,40930
+55455,38265,30753,40119
+57016,38266,30752,40796
+53219,38267,30751,40267
+57322,38268,30750,40400
+55876,38269,30749,40130
+55495,38270,30748,40282
+58825,38271,30747,40459
+59387,38272,30746,40243
+54945,38273,30745,40660
+50040,38274,30744,40321
+54933,38275,30743,40801
+52326,38276,30742,40125
+55561,38277,30741,40423
+54664,38278,30740,40136
+54289,38279,30739,40056
+57129,38280,30738,40442
+51376,38281,30737,40915
+51604,38282,30736,40569
+57100,38283,30735,40295
+53907,38284,30734,40435
+51118,38285,30733,40844
+56784,38286,30732,40417
+58437,38287,30731,40822
+50791,38288,30730,40016
+51274,38289,30729,40870
+52492,38290,30728,40020
+57821,38291,30727,40879
+59641,38292,30726,40579
+57599,38293,30725,40570
+50512,38294,30724,40107
+53513,38295,30723,40295
+51183,38296,30722,40369
+51674,38297,30721,40457
+54001,38298,30720,40119
+57422,38299,30719,40260
+55968,38300,30718,40169
+53449,38301,30717,40229
+55641,38302,30716,40101
+56035,38303,30715,40374
+52680,38304,30714,40100
+51581,38305,30713,40532
+50626,38306,30712,40081
+54881,38307,30711,40047
+53986,38308,30710,40205
+55464,38309,30709,40990
+51179,38310,30708,40541
+52006,38311,30707,40229
+59100,38312,30706,40561
+58959,38313,30705,40388
+59585,38314,30704,40065
+53517,38315,30703,40437
+50758,38316,30702,40245
+51241,38317,30701,40481
+51638,38318,30700,40330
+54679,38319,30699,40759
+56274,38320,30698,40972
+53694,38321,30697,40586
+57022,38322,30696,40557
+56020,38323,30695,40326
+54165,38324,30694,40056
+59593,38325,30693,40423
+55772,38326,30692,40242
+59866,38327,30691,40550
+51972,38328,30690,40926
+54726,38329,30689,40033
+51033,38330,30688,40611
+54605,38331,30687,40710
+57768,38332,30686,40069
+50579,38333,30685,40348
+52781,38334,30684,40900
+54340,38335,30683,40046
+54282,38336,30682,40052
+57266,38337,30681,40132
+59411,38338,30680,40730
+52650,38339,30679,40719
+59174,38340,30678,40424
+50293,38341,30677,40465
+57081,38342,30676,40270
+55658,38343,30675,40687
+57276,38344,30674,40509
+59873,38345,30673,40397
+57837,38346,30672,40897
+55554,38347,30671,40092
+54682,38348,30670,40538
+53867,38349,30669,40922
+58737,38350,30668,40487
+51024,38351,30667,40566
+55011,38352,30666,40448
+56653,38353,30665,40910
+53655,38354,30664,40510
+51703,38355,30663,40095
+54682,38356,30662,40833
+58129,38357,30661,40184
+54228,38358,30660,40117
+56171,38359,30659,40884
+51111,38360,30658,40250
+59732,38361,30657,40642
+59137,38362,30656,40007
+59328,38363,30655,40018
+58896,38364,30654,40283
+58388,38365,30653,40358
+51893,38366,30652,40087
+57214,38367,30651,40979
+54673,38368,30650,40011
+58130,38369,30649,40891
+58364,38370,30648,40348
+54799,38371,30647,40641
+55988,38372,30646,40958
+52465,38373,30645,40592
+50418,38374,30644,40748
+53951,38375,30643,40767
+51689,38376,30642,40657
+56356,38377,30641,40709
+59112,38378,30640,40138
+56315,38379,30639,40413
+56559,38380,30638,40786
+59347,38381,30637,40142
+52782,38382,30636,40289
+56418,38383,30635,40274
+56440,38384,30634,40146
+53526,38385,30633,40313
+57334,38386,30632,40920
+51303,38387,30631,40057
+54952,38388,30630,40188
+59048,38389,30629,40316
+55959,38390,30628,40565
+53372,38391,30627,40118
+58342,38392,30626,40693
+51653,38393,30625,40650
+52039,38394,30624,40364
+52231,38395,30623,40992
+59066,38396,30622,40476
+54747,38397,30621,40333
+56975,38398,30620,40032
+53504,38399,30619,40695
+55933,38400,30618,40281
+54796,38401,30617,40946
+59979,38402,30616,40390
+50177,38403,30615,40548
+58811,38404,30614,40463
+59402,38405,30613,40778
+54615,38406,30612,40062
+56228,38407,30611,40829
+57187,38408,30610,40339
+59674,38409,30609,40861
+50722,38410,30608,40091
+52776,38411,30607,40112
+53429,38412,30606,40540
+55697,38413,30605,40006
+58238,38414,30604,40311
+51964,38415,30603,40788
+59683,38416,30602,40275
+59353,38417,30601,40924
+55101,38418,30600,40138
+54786,38419,30599,40973
+52016,38420,30598,40922
+50204,38421,30597,40100
+59701,38422,30596,40154
+52349,38423,30595,40247
+57843,38424,30594,40718
+56919,38425,30593,40089
+56605,38426,30592,40874
+50642,38427,30591,40320
+55508,38428,30590,40927
+58424,38429,30589,40327
+50317,38430,30588,40725
+57786,38431,30587,40199
+59214,38432,30586,40717
+59984,38433,30585,40153
+51729,38434,30584,40608
+56494,38435,30583,40969
+51851,38436,30582,40906
+53522,38437,30581,40400
+59990,38438,30580,40621
+57849,38439,30579,40712
+57627,38440,30578,40196
+56385,38441,30577,40932
+52187,38442,30576,40093
+52793,38443,30575,40452
+57116,38444,30574,40849
+55493,38445,30573,40274
+59585,38446,30572,40899
+59438,38447,30571,40580
+55394,38448,30570,40986
+56598,38449,30569,40822
+53291,38450,30568,40733
+59449,38451,30567,40763
+51536,38452,30566,40874
+57039,38453,30565,40048
+57118,38454,30564,40347
+57454,38455,30563,40997
+59629,38456,30562,40390
+56924,38457,30561,40993
+58770,38458,30560,40000
+51042,38459,30559,40081
+54224,38460,30558,40285
+59548,38461,30557,40780
+57262,38462,30556,40470
+52572,38463,30555,40374
+57460,38464,30554,40855
+54191,38465,30553,40367
+58847,38466,30552,40828
+58631,38467,30551,40367
+52644,38468,30550,40200
+54200,38469,30549,40257
+53617,38470,30548,40073
+55534,38471,30547,40417
+52735,38472,30546,40403
+51755,38473,30545,40403
+52980,38474,30544,40039
+51135,38475,30543,40506
+58101,38476,30542,40992
+58144,38477,30541,40689
+50600,38478,30540,40835
+57193,38479,30539,40353
+58663,38480,30538,40728
+51822,38481,30537,40304
+59424,38482,30536,40897
+50973,38483,30535,40818
+56762,38484,30534,40541
+53520,38485,30533,40455
+57171,38486,30532,40538
+51903,38487,30531,40961
+53764,38488,30530,40069
+50531,38489,30529,40248
+54854,38490,30528,40231
+56901,38491,30527,40043
+57549,38492,30526,40638
+59221,38493,30525,40421
+55538,38494,30524,40020
+58727,38495,30523,40506
+55647,38496,30522,40192
+57815,38497,30521,40693
+55188,38498,30520,40280
+52445,38499,30519,40087
+53766,38500,30518,40968
+52284,38501,30517,40110
+55367,38502,30516,40841
+52804,38503,30515,40680
+59665,38504,30514,40481
+58663,38505,30513,40698
+50235,38506,30512,40954
+59257,38507,30511,40321
+57726,38508,30510,40426
+52256,38509,30509,40460
+56434,38510,30508,40111
+57787,38511,30507,40324
+59985,38512,30506,40269
+51375,38513,30505,40354
+54877,38514,30504,40653
+57276,38515,30503,40691
+52045,38516,30502,40715
+50351,38517,30501,40162
+58562,38518,30500,40732
+59489,38519,30499,40394
+56424,38520,30498,40666
+55740,38521,30497,40350
+52025,38522,30496,40742
+50269,38523,30495,40056
+59115,38524,30494,40329
+50053,38525,30493,40842
+51850,38526,30492,40143
+51257,38527,30491,40908
+58032,38528,30490,40784
+55962,38529,30489,40178
+54192,38530,30488,40612
+56783,38531,30487,40673
+57218,38532,30486,40763
+58350,38533,30485,40903
+55000,38534,30484,40926
+51737,38535,30483,40828
+51679,38536,30482,40997
+58373,38537,30481,40568
+55618,38538,30480,40638
+58566,38539,30479,40650
+56729,38540,30478,40399
+54327,38541,30477,40819
+57422,38542,30476,40368
+50181,38543,30475,40357
+56336,38544,30474,40520
+52603,38545,30473,40027
+55760,38546,30472,40610
+59512,38547,30471,40990
+57273,38548,30470,40035
+58684,38549,30469,40408
+57693,38550,30468,40763
+54626,38551,30467,40001
+56746,38552,30466,40462
+54967,38553,30465,40196
+53323,38554,30464,40510
+55081,38555,30463,40462
+51355,38556,30462,40559
+55706,38557,30461,40578
+54788,38558,30460,40526
+59796,38559,30459,40331
+55621,38560,30458,40407
+57798,38561,30457,40183
+59683,38562,30456,40994
+57229,38563,30455,40397
+52550,38564,30454,40840
+51870,38565,30453,40771
+53684,38566,30452,40264
+51055,38567,30451,40534
+59643,38568,30450,40609
+53973,38569,30449,40462
+55672,38570,30448,40523
+55672,38571,30447,40997
+59141,38572,30446,40662
+53747,38573,30445,40733
+59342,38574,30444,40246
+57432,38575,30443,40307
+52084,38576,30442,40330
+57712,38577,30441,40100
+50251,38578,30440,40936
+57744,38579,30439,40130
+55918,38580,30438,40398
+57447,38581,30437,40178
+58466,38582,30436,40743
+53060,38583,30435,40022
+52008,38584,30434,40945
+51236,38585,30433,40954
+50764,38586,30432,40911
+58681,38587,30431,40725
+55931,38588,30430,40432
+53254,38589,30429,40214
+56897,38590,30428,40578
+51622,38591,30427,40944
+54432,38592,30426,40684
+55069,38593,30425,40772
+56893,38594,30424,40704
+59943,38595,30423,40994
+59428,38596,30422,40257
+59404,38597,30421,40203
+51147,38598,30420,40238
+59701,38599,30419,40493
+57124,38600,30418,40178
+53784,38601,30417,40084
+57805,38602,30416,40088
+57470,38603,30415,40818
+50881,38604,30414,40264
+56966,38605,30413,40939
+59985,38606,30412,40296
+51651,38607,30411,40447
+50125,38608,30410,40538
+52336,38609,30409,40684
+53996,38610,30408,40934
+52004,38611,30407,40816
+51292,38612,30406,40684
+50641,38613,30405,40584
+50179,38614,30404,40560
+53734,38615,30403,40248
+56308,38616,30402,40098
+50692,38617,30401,40736
+50429,38618,30400,40712
+55197,38619,30399,40262
+53068,38620,30398,40971
+51848,38621,30397,40582
+50354,38622,30396,40803
+53025,38623,30395,40387
+51952,38624,30394,40960
+51390,38625,30393,40032
+57790,38626,30392,40006
+51821,38627,30391,40368
+52043,38628,30390,40172
+55592,38629,30389,40814
+57134,38630,30388,40329
+57691,38631,30387,40002
+50058,38632,30386,40051
+56748,38633,30385,40641
+57057,38634,30384,40837
+57272,38635,30383,40392
+55487,38636,30382,40436
+59855,38637,30381,40124
+52999,38638,30380,40807
+59750,38639,30379,40632
+58268,38640,30378,40889
+55690,38641,30377,40996
+54677,38642,30376,40253
+54260,38643,30375,40671
+52771,38644,30374,40692
+56488,38645,30373,40977
+55503,38646,30372,40693
+53207,38647,30371,40416
+53916,38648,30370,40362
+57909,38649,30369,40237
+56444,38650,30368,40682
+57941,38651,30367,40297
+53807,38652,30366,40401
+53248,38653,30365,40041
+56262,38654,30364,40016
+52388,38655,30363,40571
+54517,38656,30362,40714
+52585,38657,30361,40052
+50278,38658,30360,40565
+54271,38659,30359,40856
+55386,38660,30358,40473
+59523,38661,30357,40685
+55787,38662,30356,40504
+55168,38663,30355,40869
+56382,38664,30354,40642
+55779,38665,30353,40766
+52020,38666,30352,40300
+52663,38667,30351,40551
+59935,38668,30350,40916
+53566,38669,30349,40137
+59937,38670,30348,40891
+55761,38671,30347,40495
+53007,38672,30346,40967
+50014,38673,30345,40444
+51941,38674,30344,40468
+52241,38675,30343,40080
+57524,38676,30342,40639
+50206,38677,30341,40566
+51142,38678,30340,40647
+52874,38679,30339,40917
+56130,38680,30338,40800
+57529,38681,30337,40289
+50603,38682,30336,40221
+54609,38683,30335,40518
+57507,38684,30334,40929
+55022,38685,30333,40560
+58508,38686,30332,40066
+52431,38687,30331,40097
+50191,38688,30330,40907
+55102,38689,30329,40176
+58998,38690,30328,40253
+56141,38691,30327,40754
+57843,38692,30326,40201
+58492,38693,30325,40608
+52708,38694,30324,40172
+50487,38695,30323,40053
+57518,38696,30322,40888
+53083,38697,30321,40402
+54374,38698,30320,40178
+58857,38699,30319,40320
+53135,38700,30318,40597
+50611,38701,30317,40754
+50509,38702,30316,40835
+52996,38703,30315,40844
+53272,38704,30314,40931
+51740,38705,30313,40923
+52028,38706,30312,40692
+55929,38707,30311,40347
+58557,38708,30310,40972
+58625,38709,30309,40647
+50344,38710,30308,40885
+58028,38711,30307,40552
+59157,38712,30306,40589
+56798,38713,30305,40621
+52761,38714,30304,40088
+55154,38715,30303,40700
+58631,38716,30302,40556
+51954,38717,30301,40058
+52182,38718,30300,40057
+54179,38719,30299,40693
+58782,38720,30298,40163
+54525,38721,30297,40579
+58108,38722,30296,40925
+58376,38723,30295,40462
+51990,38724,30294,40940
+57954,38725,30293,40359
+57384,38726,30292,40621
+52845,38727,30291,40811
+59836,38728,30290,40308
+55462,38729,30289,40902
+59335,38730,30288,40418
+52975,38731,30287,40227
+59936,38732,30286,40738
+56191,38733,30285,40894
+53629,38734,30284,40664
+51711,38735,30283,40622
+57782,38736,30282,40926
+50869,38737,30281,40786
+59530,38738,30280,40813
+53213,38739,30279,40471
+52736,38740,30278,40396
+51139,38741,30277,40099
+55382,38742,30276,40792
+51374,38743,30275,40825
+54677,38744,30274,40952
+52678,38745,30273,40641
+57686,38746,30272,40156
+54214,38747,30271,40003
+50669,38748,30270,40090
+50253,38749,30269,40776
+51955,38750,30268,40652
+55614,38751,30267,40102
+54787,38752,30266,40049
+54871,38753,30265,40926
+59450,38754,30264,40462
+56062,38755,30263,40075
+52657,38756,30262,40233
+54407,38757,30261,40305
+52070,38758,30260,40330
+55971,38759,30259,40511
+50662,38760,30258,40687
+58627,38761,30257,40922
+56072,38762,30256,40586
+50042,38763,30255,40768
+54147,38764,30254,40841
+53264,38765,30253,40030
+55023,38766,30252,40313
+50387,38767,30251,40838
+51561,38768,30250,40344
+56663,38769,30249,40201
+51008,38770,30248,40926
+58206,38771,30247,40781
+59836,38772,30246,40807
+57907,38773,30245,40035
+54590,38774,30244,40080
+53920,38775,30243,40520
+54018,38776,30242,40475
+56208,38777,30241,40669
+50930,38778,30240,40072
+50815,38779,30239,40605
+55556,38780,30238,40666
+51597,38781,30237,40386
+52324,38782,30236,40772
+56529,38783,30235,40997
+55580,38784,30234,40326
+55274,38785,30233,40008
+57967,38786,30232,40424
+57617,38787,30231,40705
+57669,38788,30230,40745
+50810,38789,30229,40262
+57936,38790,30228,40940
+52458,38791,30227,40896
+58862,38792,30226,40904
+51618,38793,30225,40158
+51863,38794,30224,40115
+58265,38795,30223,40486
+57062,38796,30222,40162
+51240,38797,30221,40169
+55509,38798,30220,40776
+56158,38799,30219,40404
+50529,38800,30218,40704
+56148,38801,30217,40703
+51404,38802,30216,40272
+51419,38803,30215,40772
+51441,38804,30214,40140
+57217,38805,30213,40526
+53970,38806,30212,40572
+54899,38807,30211,40116
+53245,38808,30210,40972
+50492,38809,30209,40840
+52909,38810,30208,40134
+57212,38811,30207,40224
+51309,38812,30206,40349
+55524,38813,30205,40480
+54167,38814,30204,40072
+59005,38815,30203,40689
+54299,38816,30202,40810
+56648,38817,30201,40170
+52635,38818,30200,40858
+58656,38819,30199,40418
+59619,38820,30198,40036
+59404,38821,30197,40631
+58470,38822,30196,40795
+59264,38823,30195,40079
+59027,38824,30194,40145
+52947,38825,30193,40064
+52454,38826,30192,40156
+52268,38827,30191,40966
+56907,38828,30190,40295
+59221,38829,30189,40441
+53134,38830,30188,40986
+51570,38831,30187,40535
+57134,38832,30186,40447
+52571,38833,30185,40029
+53952,38834,30184,40915
+50852,38835,30183,40974
+57231,38836,30182,40188
+51869,38837,30181,40133
+51456,38838,30180,40493
+52598,38839,30179,40541
+54692,38840,30178,40923
+54187,38841,30177,40571
+58487,38842,30176,40715
+55821,38843,30175,40988
+55974,38844,30174,40827
+57184,38845,30173,40321
+50461,38846,30172,40465
+50433,38847,30171,40923
+56334,38848,30170,40740
+53712,38849,30169,40588
+51006,38850,30168,40781
+59355,38851,30167,40148
+52073,38852,30166,40121
+54491,38853,30165,40260
+55558,38854,30164,40974
+56711,38855,30163,40331
+52933,38856,30162,40199
+59542,38857,30161,40715
+58036,38858,30160,40350
+55603,38859,30159,40791
+58241,38860,30158,40789
+59225,38861,30157,40955
+55162,38862,30156,40498
+57645,38863,30155,40008
+55824,38864,30154,40988
+53835,38865,30153,40491
+53002,38866,30152,40318
+59401,38867,30151,40016
+50294,38868,30150,40919
+51064,38869,30149,40344
+57642,38870,30148,40074
+57048,38871,30147,40468
+59289,38872,30146,40057
+51307,38873,30145,40933
+54641,38874,30144,40378
+56242,38875,30143,40378
+57088,38876,30142,40075
+59667,38877,30141,40045
+57367,38878,30140,40775
+52172,38879,30139,40411
+57559,38880,30138,40173
+54418,38881,30137,40109
+50072,38882,30136,40217
+56498,38883,30135,40575
+53049,38884,30134,40609
+57896,38885,30133,40981
+56609,38886,30132,40126
+58382,38887,30131,40605
+59204,38888,30130,40909
+54716,38889,30129,40341
+56369,38890,30128,40153
+57694,38891,30127,40658
+55928,38892,30126,40159
+53955,38893,30125,40106
+59209,38894,30124,40950
+57057,38895,30123,40847
+57799,38896,30122,40280
+59142,38897,30121,40688
+54812,38898,30120,40906
+54122,38899,30119,40565
+56998,38900,30118,40116
+58608,38901,30117,40334
+51871,38902,30116,40317
+59035,38903,30115,40241
+58026,38904,30114,40126
+55597,38905,30113,40647
+58217,38906,30112,40121
+57659,38907,30111,40266
+59212,38908,30110,40342
+57228,38909,30109,40589
+50820,38910,30108,40729
+55344,38911,30107,40040
+52645,38912,30106,40820
+52046,38913,30105,40710
+57226,38914,30104,40149
+53605,38915,30103,40629
+59522,38916,30102,40379
+54516,38917,30101,40185
+57864,38918,30100,40210
+53829,38919,30099,40741
+50240,38920,30098,40233
+59875,38921,30097,40071
+52349,38922,30096,40484
+56285,38923,30095,40807
+57312,38924,30094,40083
+58195,38925,30093,40716
+51643,38926,30092,40510
+59021,38927,30091,40403
+58308,38928,30090,40099
+53835,38929,30089,40949
+54901,38930,30088,40998
+59111,38931,30087,40915
+52368,38932,30086,40228
+50279,38933,30085,40936
+50191,38934,30084,40715
+50958,38935,30083,40884
+58937,38936,30082,40257
+58514,38937,30081,40225
+52721,38938,30080,40368
+56744,38939,30079,40684
+59801,38940,30078,40283
+59435,38941,30077,40121
+51332,38942,30076,40848
+59420,38943,30075,40051
+54296,38944,30074,40914
+59972,38945,30073,40511
+52002,38946,30072,40681
+52916,38947,30071,40011
+55428,38948,30070,40768
+54189,38949,30069,40073
+52393,38950,30068,40859
+57596,38951,30067,40162
+56410,38952,30066,40515
+52429,38953,30065,40649
+50249,38954,30064,40120
+53504,38955,30063,40421
+57065,38956,30062,40178
+50297,38957,30061,40162
+54193,38958,30060,40523
+50641,38959,30059,40404
+58630,38960,30058,40780
+50571,38961,30057,40329
+58901,38962,30056,40955
+57537,38963,30055,40154
+59890,38964,30054,40182
+56556,38965,30053,40166
+55162,38966,30052,40311
+54018,38967,30051,40668
+52149,38968,30050,40851
+53657,38969,30049,40194
+56952,38970,30048,40288
+56474,38971,30047,40447
+58567,38972,30046,40363
+56225,38973,30045,40089
+54167,38974,30044,40852
+54433,38975,30043,40307
+56912,38976,30042,40983
+57476,38977,30041,40703
+55218,38978,30040,40223
+58288,38979,30039,40644
+58815,38980,30038,40496
+59893,38981,30037,40980
+54422,38982,30036,40705
+58211,38983,30035,40373
+51152,38984,30034,40197
+56617,38985,30033,40234
+52403,38986,30032,40629
+53784,38987,30031,40069
+51188,38988,30030,40139
+56214,38989,30029,40939
+50521,38990,30028,40421
+56334,38991,30027,40732
+52492,38992,30026,40924
+55044,38993,30025,40469
+59852,38994,30024,40937
+54454,38995,30023,40476
+53310,38996,30022,40651
+58158,38997,30021,40775
+50097,38998,30020,40183
+57253,38999,30019,40242
+57742,39000,30018,40812
+54690,39001,30017,40363
+58206,39002,30016,40657
+58559,39003,30015,40691
+54229,39004,30014,40278
+52924,39005,30013,40828
+52317,39006,30012,40780
+54093,39007,30011,40353
+52238,39008,30010,40130
+55390,39009,30009,40095
+57128,39010,30008,40765
+52648,39011,30007,40070
+57203,39012,30006,40139
+55083,39013,30005,40283
+59966,39014,30004,40859
+53627,39015,30003,40310
+56996,39016,30002,40029
+53239,39017,30001,40762
+54092,39018,30000,40882
+53508,39019,31000,40686
+50031,39020,30999,40927
+56456,39021,30998,40016
+59125,39022,30997,40604
+54642,39023,30996,40825
+56232,39024,30995,40715
+59927,39025,30994,40824
+58170,39026,30993,40575
+50664,39027,30992,40975
+53667,39028,30991,40099
+54255,39029,30990,40548
+57042,39030,30989,40323
+57496,39031,30988,40965
+56276,39032,30987,40267
+51597,39033,30986,40572
+54499,39034,30985,40763
+52201,39035,30984,40618
+55702,39036,30983,40123
+54430,39037,30982,40435
+55798,39038,30981,40225
+52529,39039,30980,40805
+54848,39040,30979,40340
+56435,39041,30978,40918
+56502,39042,30977,40595
+50827,39043,30976,40666
+52913,39044,30975,40935
+54655,39045,30974,40902
+59329,39046,30973,40362
+52034,39047,30972,40772
+52484,39048,30971,40075
+52311,39049,30970,40163
+59589,39050,30969,40112
+55347,39051,30968,40897
+54124,39052,30967,40734
+53841,39053,30966,40576
+53620,39054,30965,40471
+56783,39055,30964,40635
+50681,39056,30963,40096
+51477,39057,30962,40292
+53472,39058,30961,40150
+58294,39059,30960,40675
+52913,39060,30959,40629
+52527,39061,30958,40620
+55679,39062,30957,40309
+51996,39063,30956,40326
+53064,39064,30955,40681
+57722,39065,30954,40484
+55943,39066,30953,40277
+57527,39067,30952,40610
+57015,39068,30951,40587
+58405,39069,30950,40197
+59154,39070,30949,40218
+56526,39071,30948,40583
+52328,39072,30947,40876
+51874,39073,30946,40789
+54251,39074,30945,40107
+59787,39075,30944,40815
+59303,39076,30943,40327
+56817,39077,30942,40492
+55103,39078,30941,40486
+53594,39079,30940,40360
+53666,39080,30939,40160
+53673,39081,30938,40891
+51565,39082,30937,40006
+56087,39083,30936,40062
+52666,39084,30935,40399
+55791,39085,30934,40788
+52054,39086,30933,40423
+52451,39087,30932,40323
+58560,39088,30931,40916
+55293,39089,30930,40263
+50012,39090,30929,40197
+55536,39091,30928,40209
+51016,39092,30927,40338
+52844,39093,30926,40825
+56646,39094,30925,40995
+57487,39095,30924,40693
+55928,39096,30923,40286
+50553,39097,30922,40108
+56090,39098,30921,40447
+50633,39099,30920,40803
+57625,39100,30919,40406
+53987,39101,30918,40699
+52578,39102,30917,40400
+54083,39103,30916,40942
+55285,39104,30915,40523
+56250,39105,30914,40340
+51623,39106,30913,40571
+57863,39107,30912,40406
+54279,39108,30911,40641
+58578,39109,30910,40662
+57304,39110,30909,40118
+52257,39111,30908,40209
+54605,39112,30907,40524
+55117,39113,30906,40322
+57076,39114,30905,40533
+54543,39115,30904,40477
+57529,39116,30903,40147
+52148,39117,30902,40836
+51595,39118,30901,40531
+53024,39119,30900,40569
+59131,39120,30899,40630
+58399,39121,30898,40537
+53984,39122,30897,40102
+54054,39123,30896,40425
+58568,39124,30895,40153
+55511,39125,30894,40960
+58617,39126,30893,40717
+54643,39127,30892,40849
+59414,39128,30891,40129
+50586,39129,30890,40763
+58781,39130,30889,40443
+56143,39131,30888,40326
+55517,39132,30887,40731
+57751,39133,30886,40020
+53940,39134,30885,40323
+56591,39135,30884,40954
+56646,39136,30883,40049
+51405,39137,30882,40466
+58363,39138,30881,40883
+53983,39139,30880,40831
+54703,39140,30879,40158
+59440,39141,30878,40691
+53197,39142,30877,40539
+58377,39143,30876,40864
+57989,39144,30875,40911
+58468,39145,30874,40761
+54330,39146,30873,40861
+52155,39147,30872,40949
+58985,39148,30871,40882
+55647,39149,30870,40456
+51555,39150,30869,40252
+54597,39151,30868,40501
+50123,39152,30867,40568
+59531,39153,30866,40091
+56187,39154,30865,40230
+52840,39155,30864,40019
+56657,39156,30863,40308
+53121,39157,30862,40372
+52631,39158,30861,40812
+58023,39159,30860,40739
+54496,39160,30859,40330
+50286,39161,30858,40834
+56718,39162,30857,40311
+58665,39163,30856,40043
+54737,39164,30855,40947
+56001,39165,30854,40666
+56330,39166,30853,40498
+57460,39167,30852,40884
+50797,39168,30851,40622
+53034,39169,30850,40320
+55429,39170,30849,40164
+56035,39171,30848,40897
+53617,39172,30847,40918
+59661,39173,30846,40573
+53436,39174,30845,40868
+54113,39175,30844,40541
+58411,39176,30843,40400
+57644,39177,30842,40416
+52255,39178,30841,40765
+56380,39179,30840,40118
+58826,39180,30839,40395
+50608,39181,30838,40791
+58142,39182,30837,40274
+52066,39183,30836,40126
+56913,39184,30835,40159
+59393,39185,30834,40813
+57501,39186,30833,40408
+51910,39187,30832,40542
+52244,39188,30831,40944
+56968,39189,30830,40803
+51808,39190,30829,40288
+59643,39191,30828,40714
+51784,39192,30827,40955
+58419,39193,30826,40209
+53791,39194,30825,40259
+53399,39195,30824,40026
+50327,39196,30823,40621
+59857,39197,30822,40995
+56371,39198,30821,40878
+59762,39199,30820,40196
+55513,39200,30819,40913
+58289,39201,30818,40494
+54982,39202,30817,40566
+56500,39203,30816,40709
+50940,39204,30815,40960
+50231,39205,30814,40313
+59943,39206,30813,40779
+58809,39207,30812,40767
+52909,39208,30811,40855
+53047,39209,30810,40471
+52739,39210,30809,40022
+55808,39211,30808,40561
+55212,39212,30807,40542
+53479,39213,30806,40401
+56476,39214,30805,40164
+54935,39215,30804,40844
+57211,39216,30803,40382
+56417,39217,30802,40206
+55270,39218,30801,40114
+58928,39219,30800,40450
+57967,39220,30799,40936
+58030,39221,30798,40333
+54804,39222,30797,40224
+59807,39223,30796,40103
+56701,39224,30795,40386
+52603,39225,30794,40743
+51823,39226,30793,40089
+55898,39227,30792,40362
+59566,39228,30791,40202
+59367,39229,30790,40271
+51823,39230,30789,40023
+57495,39231,30788,40882
+51163,39232,30787,40466
+58755,39233,30786,40779
+59519,39234,30785,40512
+50752,39235,30784,40402
+55216,39236,30783,40660
+57552,39237,30782,40690
+52109,39238,30781,40765
+59874,39239,30780,40231
+59725,39240,30779,40046
+51316,39241,30778,40293
+58374,39242,30777,40175
+51736,39243,30776,40286
+59375,39244,30775,40869
+55109,39245,30774,40056
+53518,39246,30773,40227
+52276,39247,30772,40001
+51131,39248,30771,40459
+55195,39249,30770,40599
+54118,39250,30769,40245
+50263,39251,30768,40177
+53940,39252,30767,40442
+54258,39253,30766,40227
+57348,39254,30765,40213
+57092,39255,30764,40690
+55455,39256,30763,40431
+50053,39257,30762,40101
+58366,39258,30761,40590
+53552,39259,30760,40507
+53190,39260,30759,40034
+53124,39261,30758,40919
+50782,39262,30757,40052
+56948,39263,30756,40221
+54465,39264,30755,40091
+52574,39265,30754,40528
+58143,39266,30753,40627
+55465,39267,30752,40257
+51678,39268,30751,40092
+55239,39269,30750,40486
+57437,39270,30749,40629
+59034,39271,30748,40836
+50665,39272,30747,40267
+55187,39273,30746,40331
+52873,39274,30745,40626
+54165,39275,30744,40605
+53236,39276,30743,40662
+58520,39277,30742,40858
+54739,39278,30741,40438
+51306,39279,30740,40867
+50400,39280,30739,40182
+56768,39281,30738,40264
+53341,39282,30737,40080
+52010,39283,30736,40816
+58434,39284,30735,40958
+59732,39285,30734,40525
+57518,39286,30733,40639
+59112,39287,30732,40106
+56510,39288,30731,40172
+55601,39289,30730,40874
+54332,39290,30729,40453
+52587,39291,30728,40031
+52111,39292,30727,40636
+50550,39293,30726,40747
+50160,39294,30725,40454
+58841,39295,30724,40822
+54925,39296,30723,40753
+58308,39297,30722,40644
+51174,39298,30721,40039
+58733,39299,30720,40146
+53924,39300,30719,40238
+56028,39301,30718,40013
+56745,39302,30717,40977
+58690,39303,30716,40080
+55331,39304,30715,40878
+54260,39305,30714,40736
+54283,39306,30713,40516
+53430,39307,30712,40213
+54282,39308,30711,40809
+52194,39309,30710,40751
+59690,39310,30709,40478
+55927,39311,30708,40370
+52138,39312,30707,40736
+53470,39313,30706,40504
+58604,39314,30705,40516
+53816,39315,30704,40674
+50552,39316,30703,40870
+57428,39317,30702,40531
+51812,39318,30701,40349
+51629,39319,30700,40432
+57563,39320,30699,40182
+56031,39321,30698,40247
+50905,39322,30697,40300
+52314,39323,30696,40916
+50457,39324,30695,40545
+57935,39325,30694,40434
+55893,39326,30693,40748
+57863,39327,30692,40852
+57709,39328,30691,40391
+53475,39329,30690,40360
+59188,39330,30689,40299
+52062,39331,30688,40524
+50163,39332,30687,40178
+59999,39333,30686,40825
+52160,39334,30685,40233
+56347,39335,30684,40280
+51658,39336,30683,40456
+56551,39337,30682,40605
+57227,39338,30681,40888
+57083,39339,30680,40183
+50416,39340,30679,40568
+58721,39341,30678,40696
+59557,39342,30677,40675
+50781,39343,30676,40960
+57065,39344,30675,40841
+59159,39345,30674,40008
+59265,39346,30673,40912
+50915,39347,30672,40365
+58281,39348,30671,40800
+51138,39349,30670,40524
+57652,39350,30669,40296
+58433,39351,30668,40116
+59652,39352,30667,40672
+56271,39353,30666,40108
+58997,39354,30665,40308
+53949,39355,30664,40298
+50468,39356,30663,40361
+50654,39357,30662,40469
+59470,39358,30661,40119
+57581,39359,30660,40312
+51907,39360,30659,40387
+51997,39361,30658,40756
+58505,39362,30657,40752
+59611,39363,30656,40520
+54012,39364,30655,40130
+50064,39365,30654,40224
+59555,39366,30653,40150
+58196,39367,30652,40135
+54670,39368,30651,40209
+52028,39369,30650,40961
+56059,39370,30649,40500
+55108,39371,30648,40922
+57608,39372,30647,40001
+51671,39373,30646,40533
+51027,39374,30645,40084
+58907,39375,30644,40154
+53342,39376,30643,40809
+59530,39377,30642,40352
+51086,39378,30641,40798
+53290,39379,30640,40298
+57442,39380,30639,40692
+50878,39381,30638,40368
+50464,39382,30637,40736
+50390,39383,30636,40749
+56907,39384,30635,40768
+53794,39385,30634,40721
+58801,39386,30633,40772
+52100,39387,30632,40946
+51009,39388,30631,40115
+54674,39389,30630,40924
+52793,39390,30629,40367
+53563,39391,30628,40738
+50638,39392,30627,40798
+54561,39393,30626,40359
+52053,39394,30625,40982
+59300,39395,30624,40174
+59854,39396,30623,40099
+59312,39397,30622,40282
+52016,39398,30621,40512
+53613,39399,30620,40067
+51551,39400,30619,40953
+51524,39401,30618,40470
+50810,39402,30617,40104
+55105,39403,30616,40231
+56561,39404,30615,40516
+59715,39405,30614,40333
+59188,39406,30613,40794
+56673,39407,30612,40179
+59303,39408,30611,40159
+55111,39409,30610,40879
+59842,39410,30609,40595
+55885,39411,30608,40544
+55220,39412,30607,40747
+57577,39413,30606,40809
+53415,39414,30605,40156
+55387,39415,30604,40516
+58830,39416,30603,40479
+54742,39417,30602,40421
+54843,39418,30601,40957
+59055,39419,30600,40708
+53342,39420,30599,40552
+57901,39421,30598,40454
+51944,39422,30597,40899
+50113,39423,30596,40297
+58430,39424,30595,40359
+57071,39425,30594,40250
+59384,39426,30593,40708
+56586,39427,30592,40869
+56281,39428,30591,40456
+58317,39429,30590,40288
+50457,39430,30589,40063
+56918,39431,30588,40013
+50461,39432,30587,40943
+58317,39433,30586,40410
+55816,39434,30585,40094
+52850,39435,30584,40550
+51243,39436,30583,40890
+56261,39437,30582,40146
+53225,39438,30581,40181
+56911,39439,30580,40393
+59126,39440,30579,40443
+51518,39441,30578,40029
+54093,39442,30577,40802
+56077,39443,30576,40200
+50943,39444,30575,40325
+57601,39445,30574,40549
+54425,39446,30573,40893
+57312,39447,30572,40681
+51871,39448,30571,40564
+57453,39449,30570,40760
+50885,39450,30569,40319
+55390,39451,30568,40928
+55117,39452,30567,40615
+55263,39453,30566,40956
+59274,39454,30565,40916
+56159,39455,30564,40022
+50451,39456,30563,40591
+59063,39457,30562,40373
+56659,39458,30561,40650
+59147,39459,30560,40398
+51517,39460,30559,40111
+53369,39461,30558,40938
+50668,39462,30557,40978
+59103,39463,30556,40244
+51897,39464,30555,40787
+53465,39465,30554,40397
+57366,39466,30553,40356
+59303,39467,30552,40430
+57383,39468,30551,40923
+55405,39469,30550,40169
+59376,39470,30549,40796
+51077,39471,30548,40101
+53737,39472,30547,40095
+58863,39473,30546,40748
+55087,39474,30545,40833
+51928,39475,30544,40730
+55076,39476,30543,40650
+54680,39477,30542,40020
+53270,39478,30541,40474
+52421,39479,30540,40675
+59885,39480,30539,40264
+57702,39481,30538,40324
+54094,39482,30537,40753
+57842,39483,30536,40365
+55261,39484,30535,40201
+53766,39485,30534,40797
+57231,39486,30533,40578
+50165,39487,30532,40081
+53469,39488,30531,40278
+53977,39489,30530,40167
+57540,39490,30529,40362
+54376,39491,30528,40104
+55034,39492,30527,40209
+59251,39493,30526,40551
+58587,39494,30525,40729
+51416,39495,30524,40625
+53921,39496,30523,40047
+59207,39497,30522,40592
+50626,39498,30521,40986
+54881,39499,30520,40664
+50115,39500,30519,40582
+57852,39501,30518,40268
+51163,39502,30517,40688
+52451,39503,30516,40158
+54658,39504,30515,40481
+53834,39505,30514,40634
+59592,39506,30513,40137
+52432,39507,30512,40445
+54530,39508,30511,40889
+53549,39509,30510,40935
+55822,39510,30509,40932
+53787,39511,30508,40058
+56584,39512,30507,40699
+58512,39513,30506,40806
+55414,39514,30505,40828
+56329,39515,30504,40435
+54254,39516,30503,40174
+53875,39517,30502,40834
+58631,39518,30501,40012
+55955,39519,30500,40073
+53260,39520,30499,40395
+59526,39521,30498,40465
+52301,39522,30497,40562
+52242,39523,30496,40589
+53856,39524,30495,40837
+50235,39525,30494,40848
+53637,39526,30493,40382
+59857,39527,30492,40963
+52647,39528,30491,40673
+53415,39529,30490,40651
+50878,39530,30489,40321
+53695,39531,30488,40227
+59713,39532,30487,40948
+59632,39533,30486,40731
+59576,39534,30485,40872
+55529,39535,30484,40253
+56736,39536,30483,40409
+55392,39537,30482,40823
+51624,39538,30481,40753
+59475,39539,30480,40786
+58888,39540,30479,40386
+56400,39541,30478,40990
+55682,39542,30477,40803
+50304,39543,30476,40320
+59712,39544,30475,40420
+55469,39545,30474,40305
+54621,39546,30473,40911
+52709,39547,30472,40308
+59088,39548,30471,40237
+50039,39549,30470,40579
+57714,39550,30469,40306
+54042,39551,30468,40590
+59281,39552,30467,40359
+59660,39553,30466,40186
+53245,39554,30465,40738
+55416,39555,30464,40970
+54239,39556,30463,40188
+55505,39557,30462,40229
+58522,39558,30461,40871
+53291,39559,30460,40092
+55513,39560,30459,40489
+53881,39561,30458,40293
+54968,39562,30457,40937
+55056,39563,30456,40232
+54703,39564,30455,40717
+54443,39565,30454,40958
+58777,39566,30453,40422
+57400,39567,30452,40337
+56515,39568,30451,40825
+56911,39569,30450,40317
+54271,39570,30449,40010
+59779,39571,30448,40921
+55784,39572,30447,40919
+57409,39573,30446,40185
+54832,39574,30445,40670
+58502,39575,30444,40531
+59782,39576,30443,40284
+51053,39577,30442,40811
+57107,39578,30441,40931
+59621,39579,30440,40318
+51715,39580,30439,40239
+58480,39581,30438,40336
+58593,39582,30437,40342
+56568,39583,30436,40986
+52010,39584,30435,40946
+54061,39585,30434,40115
+55636,39586,30433,40716
+59498,39587,30432,40714
+50616,39588,30431,40798
+56565,39589,30430,40133
+52634,39590,30429,40180
+51524,39591,30428,40097
+57369,39592,30427,40344
+53645,39593,30426,40839
+59906,39594,30425,40338
+57782,39595,30424,40103
+56719,39596,30423,40728
+55003,39597,30422,40228
+51757,39598,30421,40463
+52606,39599,30420,40732
+58683,39600,30419,40486
+53705,39601,30418,40386
+59726,39602,30417,40259
+50575,39603,30416,40849
+52593,39604,30415,40805
+59248,39605,30414,40999
+52506,39606,30413,40046
+51483,39607,30412,40803
+59720,39608,30411,40586
+54478,39609,30410,40733
+55224,39610,30409,40920
+57417,39611,30408,40833
+54887,39612,30407,40460
+51312,39613,30406,40600
+53666,39614,30405,40662
+58575,39615,30404,40755
+55801,39616,30403,40603
+55576,39617,30402,40124
+51298,39618,30401,40077
+58615,39619,30400,40771
+53689,39620,30399,40882
+58135,39621,30398,40849
+59077,39622,30397,40206
+57134,39623,30396,40469
+54630,39624,30395,40526
+58051,39625,30394,40953
+59659,39626,30393,40891
+59872,39627,30392,40296
+57200,39628,30391,40350
+51810,39629,30390,40790
+54212,39630,30389,40094
+52701,39631,30388,40491
+57080,39632,30387,40675
+53341,39633,30386,40726
+57396,39634,30385,40990
+53367,39635,30384,40123
+58100,39636,30383,40823
+55790,39637,30382,40944
+54959,39638,30381,40978
+53505,39639,30380,40234
+59080,39640,30379,40999
+55153,39641,30378,40918
+51485,39642,30377,40357
+51185,39643,30376,40958
+53373,39644,30375,40035
+52502,39645,30374,40259
+54711,39646,30373,40075
+58414,39647,30372,40426
+58989,39648,30371,40139
+50589,39649,30370,40716
+57719,39650,30369,40560
+54642,39651,30368,40581
+51204,39652,30367,40345
+52792,39653,30366,40585
+53548,39654,30365,40941
+50776,39655,30364,40910
+54000,39656,30363,40683
+58252,39657,30362,40002
+56515,39658,30361,40748
+55454,39659,30360,40354
+57714,39660,30359,40663
+59023,39661,30358,40635
+54734,39662,30357,40609
+50228,39663,30356,40521
+59323,39664,30355,40566
+56747,39665,30354,40690
+50352,39666,30353,40144
+53195,39667,30352,40182
+52367,39668,30351,40307
+51257,39669,30350,40214
+57919,39670,30349,40219
+58039,39671,30348,40125
+52804,39672,30347,40587
+53399,39673,30346,40295
+58081,39674,30345,40187
+50445,39675,30344,40461
+56083,39676,30343,40477
+51106,39677,30342,40578
+59470,39678,30341,40404
+55429,39679,30340,40470
+51270,39680,30339,40673
+56524,39681,30338,40132
+50772,39682,30337,40299
+56504,39683,30336,40985
+50568,39684,30335,40050
+58278,39685,30334,40212
+58589,39686,30333,40625
+58482,39687,30332,40308
+56696,39688,30331,40822
+56438,39689,30330,40923
+53354,39690,30329,40438
+59120,39691,30328,40341
+58485,39692,30327,40538
+58690,39693,30326,40764
+54155,39694,30325,40480
+54563,39695,30324,40580
+57874,39696,30323,40704
+51299,39697,30322,40910
+50676,39698,30321,40060
+51515,39699,30320,40307
+53059,39700,30319,40242
+53939,39701,30318,40351
+59130,39702,30317,40545
+57693,39703,30316,40111
+55374,39704,30315,40622
+53022,39705,30314,40072
+58259,39706,30313,40338
+56109,39707,30312,40654
+51003,39708,30311,40273
+59060,39709,30310,40221
+56457,39710,30309,40562
+56660,39711,30308,40776
+58340,39712,30307,40446
+54853,39713,30306,40716
+57399,39714,30305,40399
+57578,39715,30304,40159
+57797,39716,30303,40894
+51139,39717,30302,40754
+58890,39718,30301,40697
+55148,39719,30300,40377
+53580,39720,30299,40351
+53875,39721,30298,40811
+50480,39722,30297,40721
+53751,39723,30296,40242
+55129,39724,30295,40592
+51163,39725,30294,40545
+54860,39726,30293,40765
+53258,39727,30292,40813
+59199,39728,30291,40440
+51045,39729,30290,40057
+56783,39730,30289,40552
+50936,39731,30288,40962
+54305,39732,30287,40427
+56218,39733,30286,40835
+59920,39734,30285,40860
+56931,39735,30284,40254
+51397,39736,30283,40015
+56568,39737,30282,40848
+56725,39738,30281,40561
+53890,39739,30280,40904
+52788,39740,30279,40880
+52588,39741,30278,40356
+55127,39742,30277,40828
+59995,39743,30276,40623
+51095,39744,30275,40242
+51368,39745,30274,40843
+58402,39746,30273,40707
+52233,39747,30272,40295
+51963,39748,30271,40957
+58686,39749,30270,40717
+53713,39750,30269,40533
+59062,39751,30268,40091
+52030,39752,30267,40848
+52546,39753,30266,40259
+55126,39754,30265,40061
+58433,39755,30264,40161
+52527,39756,30263,40976
+53851,39757,30262,40041
+59119,39758,30261,40717
+51466,39759,30260,40931
+53009,39760,30259,40695
+57025,39761,30258,40700
+59778,39762,30257,40381
+59920,39763,30256,40330
+58488,39764,30255,40447
+55520,39765,30254,40171
+54033,39766,30253,40635
+59225,39767,30252,40235
+56169,39768,30251,40014
+56068,39769,30250,40400
+51078,39770,30249,40073
+57546,39771,30248,40931
+50334,39772,30247,40498
+59931,39773,30246,40854
+52504,39774,30245,40748
+50938,39775,30244,40276
+50336,39776,30243,40316
+58309,39777,30242,40114
+52003,39778,30241,40152
+55987,39779,30240,40656
+55962,39780,30239,40806
+59863,39781,30238,40395
+59428,39782,30237,40111
+52753,39783,30236,40858
+52892,39784,30235,40030
+55827,39785,30234,40067
+52810,39786,30233,40249
+50000,39787,30232,40490
+55269,39788,30231,40027
+51696,39789,30230,40661
+53062,39790,30229,40211
+55105,39791,30228,40327
+52296,39792,30227,40868
+59320,39793,30226,40775
+57067,39794,30225,40922
+57461,39795,30224,40836
+54231,39796,30223,40366
+57108,39797,30222,40999
+55614,39798,30221,40782
+57952,39799,30220,40093
+57146,39800,30219,40756
+53762,39801,30218,40560
+59446,39802,30217,40977
+51415,39803,30216,40145
+53497,39804,30215,40238
+50111,39805,30214,40276
+55180,39806,30213,40231
+58402,39807,30212,40655
+51309,39808,30211,40109
+58221,39809,30210,40160
+57516,39810,30209,40398
+50234,39811,30208,40195
+59139,39812,30207,40658
+58094,39813,30206,40325
+58354,39814,30205,40621
+57451,39815,30204,40135
+57416,39816,30203,40652
+51142,39817,30202,40162
+50558,39818,30201,40702
+50883,39819,30200,40340
+57005,39820,30199,40121
+52220,39821,30198,40493
+58592,39822,30197,40525
+59408,39823,30196,40265
+58052,39824,30195,40425
+51897,39825,30194,40321
+56891,39826,30193,40696
+54198,39827,30192,40800
+59072,39828,30191,40167
+57798,39829,30190,40303
+59861,39830,30189,40438
+51718,39831,30188,40773
+52681,39832,30187,40897
+57378,39833,30186,40823
+56453,39834,30185,40378
+52776,39835,30184,40816
+50725,39836,30183,40317
+52490,39837,30182,40541
+54116,39838,30181,40928
+57889,39839,30180,40583
+58480,39840,30179,40940
+58847,39841,30178,40254
+58028,39842,30177,40708
+50573,39843,30176,40375
+50820,39844,30175,40321
+54915,39845,30174,40418
+51227,39846,30173,40071
+52118,39847,30172,40418
+54145,39848,30171,40648
+57867,39849,30170,40137
+54545,39850,30169,40963
+52474,39851,30168,40832
+53526,39852,30167,40145
+58765,39853,30166,40592
+51885,39854,30165,40348
+59320,39855,30164,40984
+52069,39856,30163,40276
+54723,39857,30162,40483
+58355,39858,30161,40862
+51619,39859,30160,40245
+56562,39860,30159,40923
+54103,39861,30158,40458
+51896,39862,30157,40139
+54381,39863,30156,40155
+51668,39864,30155,40285
+56324,39865,30154,40406
+53620,39866,30153,40276
+58909,39867,30152,40648
+55714,39868,30151,40567
+50875,39869,30150,40090
+58462,39870,30149,40261
+59113,39871,30148,40990
+59191,39872,30147,40546
+55162,39873,30146,40404
+59539,39874,30145,40371
+52412,39875,30144,40669
+55583,39876,30143,40826
+56406,39877,30142,40939
+56112,39878,30141,40263
+54320,39879,30140,40979
+54967,39880,30139,40499
+59922,39881,30138,40232
+52434,39882,30137,40769
+53045,39883,30136,40405
+59472,39884,30135,40532
+57365,39885,30134,40143
+59021,39886,30133,40095
+53512,39887,30132,40309
+51595,39888,30131,40893
+54904,39889,30130,40420
+56833,39890,30129,40686
+52300,39891,30128,40563
+59821,39892,30127,40170
+56005,39893,30126,40208
+55590,39894,30125,40189
+58548,39895,30124,40014
+54109,39896,30123,40951
+51497,39897,30122,40732
+56209,39898,30121,40855
+53990,39899,30120,40258
+55921,39900,30119,40635
+54081,39901,30118,40026
+58540,39902,30117,40862
+56409,39903,30116,40441
+51076,39904,30115,40599
+54208,39905,30114,40377
+58177,39906,30113,40090
+55719,39907,30112,40500
+56850,39908,30111,40142
+54183,39909,30110,40871
+55391,39910,30109,40618
+52843,39911,30108,40089
+57482,39912,30107,40612
+54684,39913,30106,40806
+53795,39914,30105,40136
+51044,39915,30104,40988
+58651,39916,30103,40257
+54515,39917,30102,40105
+53472,39918,30101,40919
+56608,39919,30100,40575
+54903,39920,30099,40221
+51559,39921,30098,40078
+54887,39922,30097,40445
+59985,39923,30096,40852
+56818,39924,30095,40787
+51521,39925,30094,40539
+51551,39926,30093,40771
+55531,39927,30092,40583
+57170,39928,30091,40480
+56943,39929,30090,40962
+52750,39930,30089,40554
+50675,39931,30088,40086
+53873,39932,30087,40026
+55518,39933,30086,40455
+55133,39934,30085,40733
+51825,39935,30084,40002
+53078,39936,30083,40771
+55424,39937,30082,40918
+50215,39938,30081,40235
+55150,39939,30080,40214
+50335,39940,30079,40037
+51767,39941,30078,40959
+55561,39942,30077,40283
+56742,39943,30076,40284
+58282,39944,30075,40804
+52807,39945,30074,40500
+55076,39946,30073,40191
+58487,39947,30072,40317
+56215,39948,30071,40697
+58567,39949,30070,40367
+58150,39950,30069,40316
+51376,39951,30068,40119
+57767,39952,30067,40260
+58784,39953,30066,40326
+54084,39954,30065,40782
+51650,39955,30064,40862
+56687,39956,30063,40551
+55820,39957,30062,40177
+57779,39958,30061,40270
+53506,39959,30060,40333
+54741,39960,30059,40823
+50552,39961,30058,40600
+55465,39962,30057,40533
+50708,39963,30056,40348
+51969,39964,30055,40531
+59090,39965,30054,40151
+53344,39966,30053,40720
+51009,39967,30052,40705
+56445,39968,30051,40485
+50826,39969,30050,40707
+55699,39970,30049,40755
+58385,39971,30048,40953
+58517,39972,30047,40966
+56453,39973,30046,40603
+59695,39974,30045,40832
+58961,39975,30044,40950
+50792,39976,30043,40099
+58965,39977,30042,40121
+57379,39978,30041,40439
+56078,39979,30040,40743
+56212,39980,30039,40452
+57569,39981,30038,40809
+52015,39982,30037,40309
+56424,39983,30036,40029
+56269,39984,30035,40745
+51411,39985,30034,40207
+50440,39986,30033,40727
+52112,39987,30032,40402
+59542,39988,30031,40301
+59493,39989,30030,40586
+58530,39990,30029,40373
+57401,39991,30028,40713
+59909,39992,30027,40029
+52130,39993,30026,40605
+59860,39994,30025,40569
+59146,39995,30024,40659
+57379,39996,30023,40687
+59787,39997,30022,40018
+51015,39998,30021,40983
+50825,39999,30020,40582
+51225,40000,30019,40637
+51759,40001,30018,40047
+52055,40002,30017,40261
+53617,40003,30016,40749
+50500,40004,30015,40353
+53040,40005,30014,40426
+59056,40006,30013,40602
+53437,40007,30012,40925
+50370,40008,30011,40256
+52549,40009,30010,40038
+51377,40010,30009,40176
+54778,40011,30008,40027
+53826,40012,30007,40025
+50594,40013,30006,40982
+52060,40014,30005,40123
+59272,40015,30004,40104
+58743,40016,30003,40021
+53441,40017,30002,40191
+50771,40018,30001,40873
+58857,40019,30000,40111
+50021,40020,31000,40910
+55852,40021,30999,40056
+53148,40022,30998,40865
+50640,40023,30997,40163
+50930,40024,30996,40586
+57679,40025,30995,40770
+56030,40026,30994,40694
+55027,40027,30993,40331
+52438,40028,30992,40494
+56826,40029,30991,40993
+57105,40030,30990,40299
+57065,40031,30989,40960
+55435,40032,30988,40341
+57245,40033,30987,40651
+50482,40034,30986,40407
+51468,40035,30985,40523
+50077,40036,30984,40064
+55642,40037,30983,40997
+51036,40038,30982,40408
+58776,40039,30981,40097
+57626,40040,30980,40228
+57527,40041,30979,40187
+50559,40042,30978,40761
+59097,40043,30977,40560
+53810,40044,30976,40825
+57400,40045,30975,40055
+55388,40046,30974,40653
+55873,40047,30973,40785
+57703,40048,30972,40192
+59272,40049,30971,40006
+53606,40050,30970,40339
+57535,40051,30969,40798
+54462,40052,30968,40218
+50081,40053,30967,40752
+50402,40054,30966,40850
+53336,40055,30965,40211
+55404,40056,30964,40789
+52942,40057,30963,40654
+59127,40058,30962,40212
+50129,40059,30961,40062
+51242,40060,30960,40583
+59712,40061,30959,40620
+58287,40062,30958,40220
+57538,40063,30957,40616
+53377,40064,30956,40410
+51622,40065,30955,40446
+57731,40066,30954,40849
+54308,40067,30953,40699
+54702,40068,30952,40857
+53234,40069,30951,40516
+58003,40070,30950,40755
+51779,40071,30949,40243
+50750,40072,30948,40863
+54971,40073,30947,40696
+58907,40074,30946,40971
+57789,40075,30945,40448
+56307,40076,30944,40020
+52533,40077,30943,40852
+51262,40078,30942,40204
+56449,40079,30941,40640
+52668,40080,30940,40705
+59370,40081,30939,40726
+56415,40082,30938,40358
+54961,40083,30937,40222
+57088,40084,30936,40936
+52004,40085,30935,40749
+56434,40086,30934,40423
+50319,40087,30933,40072
+54548,40088,30932,40151
+53151,40089,30931,40899
+57087,40090,30930,40570
+56867,40091,30929,40346
+51871,40092,30928,40763
+53195,40093,30927,40130
+59151,40094,30926,40034
+59714,40095,30925,40893
+52989,40096,30924,40639
+56600,40097,30923,40945
+57036,40098,30922,40405
+53640,40099,30921,40732
+54441,40100,30920,40924
+57881,40101,30919,40766
+50709,40102,30918,40546
+52815,40103,30917,40555
+57008,40104,30916,40296
+57441,40105,30915,40012
+57735,40106,30914,40797
+58021,40107,30913,40848
+58996,40108,30912,40361
+52216,40109,30911,40084
+52180,40110,30910,40896
+57658,40111,30909,40764
+53911,40112,30908,40145
+56965,40113,30907,40356
+56592,40114,30906,40065
+51022,40115,30905,40720
+59003,40116,30904,40761
+59765,40117,30903,40197
+56334,40118,30902,40721
+50215,40119,30901,40218
+53656,40120,30900,40887
+53811,40121,30899,40870
+58327,40122,30898,40496
+54409,40123,30897,40807
+53990,40124,30896,40122
+55981,40125,30895,40140
+51570,40126,30894,40748
+53883,40127,30893,40144
+53278,40128,30892,40106
+57893,40129,30891,40172
+59005,40130,30890,40880
+53000,40131,30889,40903
+50060,40132,30888,40283
+52925,40133,30887,40628
+56258,40134,30886,40951
+56699,40135,30885,40433
+55135,40136,30884,40424
+53337,40137,30883,40999
+55186,40138,30882,40568
+54241,40139,30881,40179
+54743,40140,30880,40865
+51216,40141,30879,40920
+55981,40142,30878,40816
+52138,40143,30877,40094
+57110,40144,30876,40935
+52843,40145,30875,40459
+52263,40146,30874,40614
+50680,40147,30873,40164
+55391,40148,30872,40693
+55364,40149,30871,40473
+53909,40150,30870,40063
+54479,40151,30869,40266
+58202,40152,30868,40262
+56856,40153,30867,40506
+58589,40154,30866,40400
+55571,40155,30865,40326
+53709,40156,30864,40747
+51285,40157,30863,40666
+51193,40158,30862,40338
+59916,40159,30861,40561
+51603,40160,30860,40572
+53692,40161,30859,40150
+52484,40162,30858,40765
+55412,40163,30857,40612
+51097,40164,30856,40723
+51279,40165,30855,40854
+51450,40166,30854,40231
+53543,40167,30853,40356
+57707,40168,30852,40766
+59459,40169,30851,40298
+59484,40170,30850,40276
+51473,40171,30849,40039
+53675,40172,30848,40516
+56975,40173,30847,40575
+55098,40174,30846,40253
+56147,40175,30845,40685
+57978,40176,30844,40660
+52923,40177,30843,40617
+57798,40178,30842,40786
+56879,40179,30841,40298
+59853,40180,30840,40675
+59218,40181,30839,40232
+57783,40182,30838,40531
+59111,40183,30837,40848
+54360,40184,30836,40492
+55967,40185,30835,40893
+50467,40186,30834,40328
+55697,40187,30833,40895
+55718,40188,30832,40839
+51827,40189,30831,40587
+59571,40190,30830,40206
+52117,40191,30829,40143
+51182,40192,30828,40020
+54061,40193,30827,40120
+59170,40194,30826,40826
+50812,40195,30825,40346
+52943,40196,30824,40495
+55440,40197,30823,40347
+57166,40198,30822,40013
+58627,40199,30821,40796
+57125,40200,30820,40819
+50832,40201,30819,40893
+54725,40202,30818,40816
+58709,40203,30817,40828
+59008,40204,30816,40361
+51940,40205,30815,40114
+59545,40206,30814,40783
+55396,40207,30813,40498
+53332,40208,30812,40184
+53925,40209,30811,40175
+54684,40210,30810,40640
+59777,40211,30809,40337
+50723,40212,30808,40148
+50221,40213,30807,40435
+57588,40214,30806,40282
+55519,40215,30805,40732
+54100,40216,30804,40672
+59961,40217,30803,40767
+56535,40218,30802,40544
+59619,40219,30801,40330
+51855,40220,30800,40365
+53975,40221,30799,40840
+53424,40222,30798,40838
+50646,40223,30797,40059
+59246,40224,30796,40523
+58408,40225,30795,40794
+51704,40226,30794,40328
+51681,40227,30793,40562
+55918,40228,30792,40965
+59358,40229,30791,40524
+59885,40230,30790,40471
+57903,40231,30789,40507
+58043,40232,30788,40447
+57791,40233,30787,40296
+56893,40234,30786,40253
+54030,40235,30785,40556
+50135,40236,30784,40097
+52224,40237,30783,40625
+55291,40238,30782,40213
+51805,40239,30781,40445
+56594,40240,30780,40850
+51017,40241,30779,40884
+50617,40242,30778,40590
+54191,40243,30777,40425
+58179,40244,30776,40399
+53079,40245,30775,40465
+58236,40246,30774,40607
+55943,40247,30773,40685
+53069,40248,30772,40803
+57221,40249,30771,40442
+59883,40250,30770,40610
+55882,40251,30769,40486
+51418,40252,30768,40462
+53745,40253,30767,40352
+55760,40254,30766,40794
+54543,40255,30765,40436
+59076,40256,30764,40961
+58589,40257,30763,40987
+53748,40258,30762,40806
+54993,40259,30761,40419
+53238,40260,30760,40182
+57579,40261,30759,40768
+50598,40262,30758,40662
+57803,40263,30757,40070
+51535,40264,30756,40593
+50335,40265,30755,40938
+55125,40266,30754,40122
+58874,40267,30753,40276
+51086,40268,30752,40747
+52705,40269,30751,40787
+51777,40270,30750,40256
+51167,40271,30749,40461
+59750,40272,30748,40187
+54360,40273,30747,40113
+59501,40274,30746,40877
+57946,40275,30745,40300
+57787,40276,30744,40053
+59247,40277,30743,40388
+58751,40278,30742,40416
+56959,40279,30741,40844
+52915,40280,30740,40485
+58211,40281,30739,40364
+51774,40282,30738,40187
+56468,40283,30737,40555
+55078,40284,30736,40986
+58178,40285,30735,40412
+58851,40286,30734,40955
+52144,40287,30733,40900
+54574,40288,30732,40709
+57261,40289,30731,40868
+56264,40290,30730,40757
+53326,40291,30729,40782
+53671,40292,30728,40361
+52582,40293,30727,40560
+51803,40294,30726,40761
+58919,40295,30725,40608
+51103,40296,30724,40370
+51634,40297,30723,40587
+54955,40298,30722,40577
+59579,40299,30721,40727
+55447,40300,30720,40256
+50216,40301,30719,40817
+55753,40302,30718,40648
+52664,40303,30717,40675
+55263,40304,30716,40819
+57632,40305,30715,40374
+58712,40306,30714,40532
+51136,40307,30713,40384
+56733,40308,30712,40950
+53381,40309,30711,40945
+57550,40310,30710,40469
+50376,40311,30709,40216
+53702,40312,30708,40229
+50787,40313,30707,40517
+51168,40314,30706,40950
+51091,40315,30705,40811
+57139,40316,30704,40256
+58568,40317,30703,40892
+53529,40318,30702,40217
+58762,40319,30701,40424
+55886,40320,30700,40536
+56454,40321,30699,40948
+58467,40322,30698,40293
+54982,40323,30697,40243
+56865,40324,30696,40975
+58447,40325,30695,40812
+57291,40326,30694,40753
+55876,40327,30693,40151
+54205,40328,30692,40130
+53255,40329,30691,40157
+58705,40330,30690,40215
+59824,40331,30689,40995
+51232,40332,30688,40752
+55878,40333,30687,40914
+53333,40334,30686,40180
+50122,40335,30685,40247
+52557,40336,30684,40345
+58291,40337,30683,40706
+52683,40338,30682,40260
+58067,40339,30681,40629
+53656,40340,30680,40289
+57865,40341,30679,40179
+55799,40342,30678,40992
+54096,40343,30677,40930
+58999,40344,30676,40276
+54533,40345,30675,40918
+54337,40346,30674,40153
+59558,40347,30673,40668
+56689,40348,30672,40841
+58261,40349,30671,40920
+50338,40350,30670,40027
+54331,40351,30669,40219
+50386,40352,30668,40762
+54106,40353,30667,40620
+50517,40354,30666,40301
+50571,40355,30665,40537
+52293,40356,30664,40416
+56374,40357,30663,40763
+55558,40358,30662,40846
+50000,40359,30661,40228
+57256,40360,30660,40432
+56847,40361,30659,40294
+53254,40362,30658,40295
+58457,40363,30657,40217
+53035,40364,30656,40314
+51994,40365,30655,40792
+55652,40366,30654,40516
+52403,40367,30653,40120
+54464,40368,30652,40486
+55088,40369,30651,40326
+51172,40370,30650,40340
+52930,40371,30649,40268
+53180,40372,30648,40207
+58379,40373,30647,40712
+56284,40374,30646,40831
+51065,40375,30645,40802
+52820,40376,30644,40026
+56953,40377,30643,40783
+54322,40378,30642,40069
+58829,40379,30641,40958
+56145,40380,30640,40957
+54056,40381,30639,40687
+59711,40382,30638,40499
+59269,40383,30637,40995
+50940,40384,30636,40682
+53512,40385,30635,40916
+55358,40386,30634,40574
+51308,40387,30633,40393
+50994,40388,30632,40147
+50172,40389,30631,40902
+59532,40390,30630,40391
+50205,40391,30629,40358
+57216,40392,30628,40645
+51192,40393,30627,40126
+55898,40394,30626,40495
+52006,40395,30625,40128
+57783,40396,30624,40711
+50675,40397,30623,40888
+52742,40398,30622,40938
+58373,40399,30621,40335
+52825,40400,30620,40785
+58390,40401,30619,40185
+50085,40402,30618,40278
+59917,40403,30617,40997
+58393,40404,30616,40335
+58492,40405,30615,40137
+59532,40406,30614,40490
+50194,40407,30613,40111
+52895,40408,30612,40734
+56927,40409,30611,40551
+55073,40410,30610,40145
+53560,40411,30609,40466
+54353,40412,30608,40131
+59473,40413,30607,40705
+58590,40414,30606,40118
+50295,40415,30605,40264
+54251,40416,30604,40905
+59920,40417,30603,40811
+53230,40418,30602,40551
+52825,40419,30601,40827
+59047,40420,30600,40009
+53599,40421,30599,40036
+52750,40422,30598,40421
+59242,40423,30597,40518
+52069,40424,30596,40624
+52067,40425,30595,40567
+53005,40426,30594,40655
+58919,40427,30593,40289
+56948,40428,30592,40342
+59859,40429,30591,40965
+57371,40430,30590,40332
+51823,40431,30589,40307
+52166,40432,30588,40227
+53175,40433,30587,40445
+56029,40434,30586,40879
+53428,40435,30585,40847
+57200,40436,30584,40147
+56922,40437,30583,40533
+58151,40438,30582,40762
+58074,40439,30581,40642
+59430,40440,30580,40768
+50024,40441,30579,40823
+50032,40442,30578,40206
+57687,40443,30577,40080
+54131,40444,30576,40892
+58800,40445,30575,40327
+56081,40446,30574,40038
+51761,40447,30573,40548
+57295,40448,30572,40894
+52626,40449,30571,40626
+54389,40450,30570,40514
+56242,40451,30569,40462
+53846,40452,30568,40081
+54033,40453,30567,40926
+57258,40454,30566,40743
+52759,40455,30565,40108
+58695,40456,30564,40658
+53825,40457,30563,40878
+52287,40458,30562,40905
+50124,40459,30561,40756
+54130,40460,30560,40152
+54697,40461,30559,40661
+17320,24209,30795,40000
+50017,40463,30557,40557
+54425,40464,30556,40764
+59282,40465,30555,40263
+51788,40466,30554,40730
+56360,40467,30553,40491
+54863,40468,30552,40856
+52199,40469,30551,40973
+53837,40470,30550,40399
+59843,40471,30549,40746
+53851,40472,30548,40186
+55539,40473,30547,40967
+58093,40474,30546,40380
+51386,40475,30545,40800
+53541,40476,30544,40791
+54069,40477,30543,40881
+54599,40478,30542,40963
+53299,40479,30541,40197
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_add_key_column.groovy b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_add_key_column.groovy
new file mode 100644
index 00000000000000..def763cd43c768
--- /dev/null
+++ b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_add_key_column.groovy
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.awaitility.Awaitility
+import static java.util.concurrent.TimeUnit.SECONDS
+
+suite("test_schema_change_add_key_column", "nonConcurrent") {
+    def tableName = "test_schema_change_add_key_column"
+
+    def getAlterTableState = {
+        waitForSchemaChangeDone {
+            sql """ SHOW ALTER TABLE COLUMN WHERE tablename='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+            time 100
+        }
+        return true
+    }
+
+    def getTabletStatus = { rowsetNum, lastRowsetSegmentNum ->
+        def tablets = sql_return_maparray """ show tablets from ${tableName}; """
+        logger.info("tablets: ${tablets}")
+        assertEquals(1, tablets.size())
+        String compactionUrl = ""
+        for (Map<String, String> tablet : tablets) {
+            compactionUrl = tablet["CompactionStatus"]
+        }
+        def (code, out, err) = curl("GET", compactionUrl)
+        logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+        assertEquals(code, 0)
+        def tabletJson = parseJson(out.trim())
+        assert tabletJson.rowsets instanceof List
+        assertTrue(tabletJson.rowsets.size() >= rowsetNum)
+        def rowset = tabletJson.rowsets.get(rowsetNum - 1)
+        logger.info("rowset: ${rowset}")
+        int start_index = rowset.indexOf("]")
+        int end_index = rowset.indexOf("DATA")
+        def segmentNumStr = rowset.substring(start_index + 1, end_index).trim()
+        logger.info("segmentNumStr: ${segmentNumStr}")
+        assertEquals(lastRowsetSegmentNum, Integer.parseInt(segmentNumStr))
+    }
+
+    // batch_size is 4164 in csv_reader.cpp
+    // _batch_size is 8192 in vtablet_writer.cpp
+    def backendId_to_params = get_be_param("doris_scanner_row_bytes")
+    onFinish {
+        GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush")
+        set_original_be_param("doris_scanner_row_bytes", backendId_to_params)
+    }
+    GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush")
+    set_be_param.call("doris_scanner_row_bytes", "1")
+
+    for (int i = 0; i < 2; i++) {
+        tableName = "test_schema_change_add_key_column_" + i
+        sql """ DROP TABLE IF EXISTS ${tableName} """
+        sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                `k1` int(11) NULL, 
+                `k2` int(11) NULL, 
+                `v3` int(11) NULL,
+                `v4` int(11) NULL
+            ) unique KEY(`k1`, `k2`) 
+            cluster by(`v3`, `v4`) 
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+            PROPERTIES (
+            """ + (i == 1 ? "\"function_column.sequence_col\"='v4', " : "") +
+            """
+                "replication_num" = "1",
+                "disable_auto_compaction" = "true"
+            );
+            """
+
+        streamLoad {
+            table "${tableName}"
+            set 'column_separator', ','
+            file 'test_schema_change_add_key_column.csv'
+            time 10000 // limit inflight 10s
+
+            check { result, exception, startTime, endTime ->
+                if (exception != null) {
+                    throw exception
+                }
+                def json = parseJson(result)
+                assertEquals("success", json.Status.toLowerCase())
+                assertEquals(8192, json.NumberTotalRows)
+                assertEquals(0, json.NumberFilteredRows)
+            }
+        }
+        // check generate 3 segments
+        sql """ select * from ${tableName} where `k1` = 12345; """
+        getTabletStatus(2, 3)
+
+        streamLoad {
+            table "${tableName}"
+            set 'column_separator', ','
+            file 'test_schema_change_add_key_column1.csv'
+            time 10000 // limit inflight 10s
+
+            check { result, exception, startTime, endTime ->
+                if (exception != null) {
+                    throw exception
+                }
+                def json = parseJson(result)
+                assertEquals("success", json.Status.toLowerCase())
+                assertEquals(20480, json.NumberTotalRows)
+                assertEquals(0, json.NumberFilteredRows)
+            }
+        }
+        // check generate 3 segments
+        sql """ select * from ${tableName} where `k1` = 12345; """
+        getTabletStatus(3, 6)
+
+        def rowCount1 = sql """ select count() from ${tableName}; """
+        logger.info("rowCount1: ${rowCount1}")
+
+        // do schema change
+        sql """ ALTER TABLE ${tableName} ADD COLUMN `k3` int(11) key """
+        getAlterTableState()
+        // check generate 1 segments
+        getTabletStatus(2, 1) // [2-3] or [2-2] [3-3]
+        // getTabletStatus(3, 1)
+
+        // check row count
+        def rowCount2 = sql """ select count() from ${tableName}; """
+        logger.info("rowCount2: ${rowCount2}")
+        assertEquals(rowCount1[0][0], rowCount2[0][0])
+        // check no duplicated key
+        def result = sql """ select `k1`, `k2`, count(*) a from ${tableName} group by `k1`, `k2` having a > 1; """
+        logger.info("result: ${result}")
+        assertEquals(0, result.size())
+        // check one row value
+        order_qt_select1 """ select * from ${tableName} where `k1` = 12345; """
+        order_qt_select2 """ select * from ${tableName} where `k1` = 17320; """
+        order_qt_select3 """ select * from ${tableName} where `k1` = 59832 and `k2` = 36673; """
+    }
+}

From 483c56ea1476a94586d7ff719f0e27416f4d2700 Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchi@selectdb.com>
Date: Thu, 12 Dec 2024 12:28:08 +0800
Subject: [PATCH 292/399] [fix](hive)Clear processed tasks (#45309)

### What problem does this PR solve?

Problem Summary:

Clear processed tasks, or it will be executed twice.
---
 .../doris/datasource/hive/HMSTransaction.java | 12 ++++++
 .../doris/datasource/hive/HmsCommitTest.java  | 40 ++++++++++++++++++-
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
index 02c99a695c8b5e..9b88f7a8dea83b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
@@ -534,6 +534,11 @@ public List<HivePartitionWithStatistics> getPartitions() {
             return partitions;
         }
 
+        public void clear() {
+            partitions.clear();
+            createdPartitionValues.clear();
+        }
+
         public void addPartition(HivePartitionWithStatistics partition) {
             partitions.add(partition);
         }
@@ -1143,6 +1148,7 @@ private void undoUpdateStatisticsTasks() {
             for (CompletableFuture<?> undoUpdateFuture : undoUpdateFutures.build()) {
                 MoreFutures.getFutureValue(undoUpdateFuture);
             }
+            updateStatisticsTasks.clear();
         }
 
         private void undoAddPartitionsTask() {
@@ -1157,6 +1163,7 @@ private void undoAddPartitionsTask() {
                 LOG.warn("Failed to rollback: add_partition for partition values {}.{}",
                         tableInfo, rollbackFailedPartitions);
             }
+            addPartitionsTask.clear();
         }
 
         private void waitForAsyncFileSystemTaskSuppressThrowable() {
@@ -1169,6 +1176,7 @@ private void waitForAsyncFileSystemTaskSuppressThrowable() {
                     // ignore
                 }
             }
+            asyncFileSystemTaskFutures.clear();
         }
 
         public void prepareInsertExistingTable(SimpleTableInfo tableInfo, TableAndMore tableAndMore) {
@@ -1319,6 +1327,7 @@ private void runDirectoryClearUpTasksForAbort() {
             for (DirectoryCleanUpTask cleanUpTask : directoryCleanUpTasksForAbort) {
                 recursiveDeleteItems(cleanUpTask.getPath(), cleanUpTask.isDeleteEmptyDir(), false);
             }
+            directoryCleanUpTasksForAbort.clear();
         }
 
         private void runRenameDirTasksForAbort() {
@@ -1334,6 +1343,7 @@ private void runRenameDirTasksForAbort() {
                     }
                 }
             }
+            renameDirectoryTasksForAbort.clear();
         }
 
         private void runClearPathsForFinish() {
@@ -1486,6 +1496,7 @@ private void abortMultiUploads() {
                             .build());
                 }, fileSystemExecutor));
             }
+            uncompletedMpuPendingUploads.clear();
         }
 
         public void doNothing() {
@@ -1520,6 +1531,7 @@ public void rollback() {
             for (CompletableFuture<?> future : asyncFileSystemTaskFutures) {
                 MoreFutures.getFutureValue(future, RuntimeException.class);
             }
+            asyncFileSystemTaskFutures.clear();
         }
 
         public void shutdownExecutorService() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
index 0e79f8ee2b56cd..61b373706d9f75 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java
@@ -675,5 +675,43 @@ public void testRollbackNewPartitionForPartitionedTableWithNewAppendPartition()
         Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
         assertNumRows(3, pa);
     }
-}
 
+    @Test
+    public void testCommitWithRollback() {
+        genQueryID();
+        List<THivePartitionUpdate> pus = new ArrayList<>();
+        try {
+            pus.add(createRandomAppend(null));
+            pus.add(createRandomAppend(null));
+            pus.add(createRandomAppend(null));
+        } catch (Throwable t) {
+            Assert.fail();
+        }
+
+        mockDoOther(() -> {
+            Table table = hmsClient.getTable(dbName, tbWithoutPartition);
+            assertNumRows(3, table);
+        });
+
+        HMSTransaction hmsTransaction = new HMSTransaction(hmsOps, fileSystemProvider, fileSystemExecutor);
+        try {
+            hmsTransaction.setHivePartitionUpdates(pus);
+            HiveInsertCommandContext ctx = new HiveInsertCommandContext();
+            String queryId = DebugUtil.printId(ConnectContext.get().queryId());
+            ctx.setQueryId(queryId);
+            ctx.setWritePath(getWritePath());
+            hmsTransaction.beginInsertTable(ctx);
+            hmsTransaction.finishInsertTable(new SimpleTableInfo(dbName, tbWithoutPartition));
+            hmsTransaction.commit();
+            Assert.fail();
+        } catch (Throwable t) {
+            Assert.assertTrue(t.getMessage().contains("failed to do nothing"));
+        }
+
+        try {
+            hmsTransaction.rollback();
+        } catch (Throwable t) {
+            Assert.fail();
+        }
+    }
+}

From 742e441dd70e494ea3f35a5779e7a21cd211850d Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchi@selectdb.com>
Date: Thu, 12 Dec 2024 12:28:24 +0800
Subject: [PATCH 293/399] [fix](iceberg)Fill in the detailed error information
 (#45285)

### What problem does this PR solve?

Problem Summary:

When dropping a database, fill in the detailed error information.
---
 .../iceberg/IcebergMetadataOps.java           |  3 ++-
 .../iceberg/CreateIcebergTableTest.java       | 26 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 970814b7acdc85..440a671afe58f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -160,7 +160,8 @@ public void dropDb(DropDbStmt stmt) throws DdlException {
                 return null;
             });
         } catch (Exception e) {
-            throw new DdlException("Failed to drop database: " + stmt.getDbName() + " ,error message is: ", e);
+            throw new DdlException(
+                "Failed to drop database: " + stmt.getDbName() + ", error message is: " + e.getMessage(), e);
         }
     }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
index 1c780d63c9cd8b..439f6f2aa7de21 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
@@ -21,6 +21,8 @@
 import org.apache.doris.analysis.CreateDbStmt;
 import org.apache.doris.analysis.CreateTableStmt;
 import org.apache.doris.analysis.DbName;
+import org.apache.doris.analysis.DropDbStmt;
+import org.apache.doris.common.DdlException;
 import org.apache.doris.common.UserException;
 import org.apache.doris.datasource.CatalogFactory;
 import org.apache.doris.nereids.parser.NereidsParser;
@@ -201,4 +203,28 @@ public String getTableName() {
         String s = "test_tb_" + UUID.randomUUID();
         return s.replaceAll("-", "");
     }
+
+    @Test
+    public void testDropDB() {
+        String dbName = "db_to_delete";
+        CreateDbStmt createDBStmt = new CreateDbStmt(false, new DbName("iceberg", dbName), new HashMap<>());
+        DropDbStmt dropDbStmt = new DropDbStmt(false, new DbName("iceberg", dbName), false);
+        DropDbStmt dropDbStmt2 = new DropDbStmt(false, new DbName("iceberg", "not_exists"), false);
+        try {
+            // create db success
+            ops.createDb(createDBStmt);
+            // drop db success
+            ops.dropDb(dropDbStmt);
+        } catch (Throwable t) {
+            Assert.fail();
+        }
+
+        try {
+            ops.dropDb(dropDbStmt2);
+            Assert.fail();
+        } catch (Throwable t) {
+            Assert.assertTrue(t instanceof DdlException);
+            Assert.assertTrue(t.getMessage().contains("database doesn't exist"));
+        }
+    }
 }

From 03407546ab53f98cb00f99990d8336aff4b342fe Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Thu, 12 Dec 2024 12:50:49 +0800
Subject: [PATCH 294/399] [Fix](job-case)Fix status typo (#45326)

### What problem does this PR solve?

Task cancellation status should be CANCELED
---
 regression-test/suites/job_p0/test_base_insert_job.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy
index fc97e367483b86..2bdf96cd5bdc3f 100644
--- a/regression-test/suites/job_p0/test_base_insert_job.groovy
+++ b/regression-test/suites/job_p0/test_base_insert_job.groovy
@@ -116,7 +116,7 @@ suite("test_base_insert_job") {
 
     def taskStatus = sql """select status from tasks("type"="insert") where JobName ='${jobName}'"""
     for (int i = 0; i < taskStatus.size(); i++) {
-        assert taskStatus.get(i).get(0) =="CANCELLED" || taskStatus.get(i).get(0) =="FINISHED"
+        assert taskStatus.get(i).get(0) =="CANCELED" || taskStatus.get(i).get(0) =="FINISHED"
     }
     sql """
        CREATE JOB ${jobMixedName}  ON SCHEDULE every 1 second  DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');

From 069a33a2faca6dbfa58800698c1257a42a6e6c25 Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Thu, 12 Dec 2024 14:52:56 +0800
Subject: [PATCH 295/399] [fix](regression) Fix some p0 case (#45333)

### What problem does this PR solve?


fix `test_partial_update_2pc_schema_change` and
`test_f_2pc_schema_change` case due to changes in
https://github.com/apache/doris/pull/45211
---
 .../test_partial_update_2pc_schema_change.groovy             | 5 +++--
 .../flexible/legacy/test_f_2pc_schema_change.groovy          | 5 +++--
 .../test_partial_update_2pc_schema_change.groovy             | 5 +++--
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_2pc_schema_change.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_2pc_schema_change.groovy
index e7205620ef319f..6b3c4572dab28b 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_2pc_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_2pc_schema_change.groovy
@@ -38,11 +38,12 @@ suite("test_partial_update_2pc_schema_change", "p0") {
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
-
+    def user = context.config.jdbcUser
+    def password = context.config.jdbcPassword
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
+        connect( user, password, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_partial_update_2pc_schema_change"
diff --git a/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_2pc_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_2pc_schema_change.groovy
index 23a9dce1001eb0..776fad1da782c7 100644
--- a/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_2pc_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_p0/flexible/legacy/test_f_2pc_schema_change.groovy
@@ -37,14 +37,15 @@ import org.apache.http.util.EntityUtils
 suite("test_f_2pc_schema_change", "p0") {
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
-
+    def user = context.config.jdbcUser
+    def password = context.config.jdbcPassword
     if (isCloudMode()) {
         return
     }
 
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
-        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
+        connect( user, password, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_f_2pc_schema_change"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy
index 9b323dec19c7a8..d506d0ee766e58 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy
@@ -38,11 +38,12 @@ suite("test_partial_update_2pc_schema_change", "p0") {
 
     String db = context.config.getDbNameByFile(context.file)
     sql "select 1;" // to create database
-
+    def user = context.config.jdbcUser
+    def password = context.config.jdbcPassword
     for (def use_row_store : [false, true]) {
         logger.info("current params: use_row_store: ${use_row_store}")
 
-        connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) {
+        connect( user, password, context.config.jdbcUrl) {
             sql "use ${db};"
 
             def tableName = "test_partial_update_2pc_schema_change"

From a139679ce0bec52ca4175af2929df96528b24f65 Mon Sep 17 00:00:00 2001
From: Dongyang Li <lidongyang@selectdb.com>
Date: Thu, 12 Dec 2024 15:05:17 +0800
Subject: [PATCH 296/399] [opt](ci) debug "Reach limit of connections" (#45173)

---
 regression-test/pipeline/cloud_p0/run.sh      |  6 ++-
 .../pipeline/common/doris-utils.sh            | 37 +++++++++++++++++++
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/regression-test/pipeline/cloud_p0/run.sh b/regression-test/pipeline/cloud_p0/run.sh
index 43847f86980c2a..c1e2570047696b 100644
--- a/regression-test/pipeline/cloud_p0/run.sh
+++ b/regression-test/pipeline/cloud_p0/run.sh
@@ -13,7 +13,7 @@ fi
 EOF
 ############################# run.sh content ########################################
 # shellcheck source=/dev/null
-# check_tpcds_table_rows, restart_doris, set_session_variable, check_tpcds_result
+# check_tpcds_table_rows, restart_doris, set_session_variable, check_tpcds_result, _monitor_connection
 source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
 # shellcheck source=/dev/null
 # create_an_issue_comment
@@ -47,6 +47,9 @@ export DORIS_HOME
 exit_flag=0
 need_collect_log=false
 
+# monitoring the log files in "${DORIS_HOME}"/regression-test/log/ for keyword 'Reach limit of connections'
+_monitor_regression_log &
+
 # shellcheck disable=SC2317
 run() {
     set -e
@@ -72,7 +75,6 @@ run() {
     export JAVA_HOME
     if "${teamcity_build_checkoutDir}"/run-regression-test.sh \
         --teamcity \
-        --clean \
         --run \
         --times "${repeat_times_from_trigger:-1}" \
         -parallel 18 \
diff --git a/regression-test/pipeline/common/doris-utils.sh b/regression-test/pipeline/common/doris-utils.sh
index 51bc0bece7af28..5f549db3c805f2 100644
--- a/regression-test/pipeline/common/doris-utils.sh
+++ b/regression-test/pipeline/common/doris-utils.sh
@@ -510,6 +510,43 @@ function set_doris_session_variables_from_file() {
     fi
 }
 
+_monitor_regression_log() {
+    if ! command -v inotifywait >/dev/null; then
+        apt install inotify-tools -y
+    fi
+
+    # Path to the log directory
+    local LOG_DIR="${DORIS_HOME}"/regression-test/log
+
+    # keyword to search for in the log files
+    local KEYWORD="Reach limit of connections"
+
+    local query_port
+    query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
+
+    echo "INFO: start monitoring the log files in ${LOG_DIR} for the keyword '${KEYWORD}'"
+
+    local start_row=1
+    # Monitor the log directory for new files and changes, only one file
+    # shellcheck disable=SC2034
+    inotifywait -m -e modify "${LOG_DIR}" | while read -r directory events filename; do
+        total_rows=$(wc -l "${directory}${filename}" | awk '{print $1}')
+        if [[ ${start_row} -ge ${total_rows} ]]; then
+            start_row=${total_rows}
+        fi
+        # shellcheck disable=SC2250
+        if sed -n "${start_row},\$p" "${directory}${filename}" | grep -a -q "${KEYWORD}"; then
+            matched=$(grep -a -n "${KEYWORD}" "${directory}${filename}")
+            start_row=$(echo "${matched}" | tail -n1 | cut -d: -f1)
+            echo "WARNING: find '${matched}' in ${directory}${filename}, run 'show processlist;' to check the connections" | tee -a "${DORIS_HOME}"/fe/log/monitor_regression_log.out
+            mysql -h127.0.0.1 -P"${query_port}" -uroot -e'show processlist;' | tee -a "${DORIS_HOME}"/fe/log/monitor_regression_log.out
+        fi
+        start_row=$((start_row + 1))
+        # echo "start_row ${start_row}" | tee -a "${DORIS_HOME}"/fe/log/monitor_regression_log.out
+    done
+
+}
+
 archive_doris_logs() {
     if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
     local archive_name="$1"

From 92015d941dd5e88fea1de7b08e2968374c4c4941 Mon Sep 17 00:00:00 2001
From: zhangstar333 <zhangsida@selectdb.com>
Date: Thu, 12 Dec 2024 15:30:14 +0800
Subject: [PATCH 297/399] [bug](function) fix first/last value return error
 with ignore null (#44996)

### What problem does this PR solve?

Problem Summary:
needs another prs of FE: https://github.com/apache/doris/pull/45065
https://github.com/apache/doris/pull/45264

1. the first value even if have set value, should not return directly,
need check it whether need arg_ignore_null, as maybe it's NULL

2. the last_value if need arg_ignore_null and not set value, should find
in while loop, if find could return directly, not find check iff has any
value before

### Release note

fix first_value/last_value return error with ignore null param is true
---
 .../exec/analytic_source_operator.cpp         |  14 +-
 .../aggregate_function_reader_first_last.h    |   2 +
 .../aggregate_function_window.cpp             |   2 +
 .../aggregate_function_window.h               |  49 +++--
 .../test_first_value_window.out               | 100 +++++++++
 .../test_first_value_window.groovy            | 189 ++++++++++++++++++
 6 files changed, 328 insertions(+), 28 deletions(-)

diff --git a/be/src/pipeline/exec/analytic_source_operator.cpp b/be/src/pipeline/exec/analytic_source_operator.cpp
index 3a9156f45b6758..fe0ab0b148e55a 100644
--- a/be/src/pipeline/exec/analytic_source_operator.cpp
+++ b/be/src/pipeline/exec/analytic_source_operator.cpp
@@ -352,17 +352,17 @@ Status AnalyticLocalState::_get_next_for_rows(size_t current_block_rows) {
         int64_t range_start, range_end;
         if (!_parent->cast<AnalyticSourceOperatorX>()._window.__isset.window_start &&
             _parent->cast<AnalyticSourceOperatorX>()._window.window_end.type ==
-                    TAnalyticWindowBoundaryType::
-                            CURRENT_ROW) { //[preceding, current_row],[current_row, following]
+                    TAnalyticWindowBoundaryType::CURRENT_ROW) {
+            // [preceding, current_row], [current_row, following] rewrite it's same
+            // as could reuse the previous calculate result, so don't call _reset_agg_status function
+            // going on calculate, add up data, no need to reset state
             range_start = _shared_state->current_row_position;
-            range_end = _shared_state->current_row_position +
-                        1; //going on calculate,add up data, no need to reset state
+            range_end = _shared_state->current_row_position + 1;
         } else {
             _reset_agg_status();
             range_end = _shared_state->current_row_position + _rows_end_offset + 1;
-            if (!_parent->cast<AnalyticSourceOperatorX>()
-                         ._window.__isset
-                         .window_start) { //[preceding, offset]        --unbound: [preceding, following]
+            //[preceding, offset]        --unbound: [preceding, following]
+            if (!_parent->cast<AnalyticSourceOperatorX>()._window.__isset.window_start) {
                 range_start = _partition_by_start.pos;
             } else {
                 range_start = _shared_state->current_row_position + _rows_start_offset;
diff --git a/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h b/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
index 2657feb9380acd..8efea2dc6fc8e4 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
@@ -142,6 +142,8 @@ struct ReaderFirstAndLastData {
 
     bool has_set_value() { return _has_value; }
 
+    bool is_null() { return _data_value.is_null(); }
+
 protected:
     StoreType _data_value;
     bool _has_value = false;
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.cpp b/be/src/vec/aggregate_functions/aggregate_function_window.cpp
index fcb0072abe8d6f..9df45611f0fc95 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_window.cpp
@@ -41,6 +41,8 @@ AggregateFunctionPtr create_function_lead_lag_first_last(const String& name,
     WhichDataType which(*type);
 
     bool arg_ignore_null_value = false;
+    // FE have rewrite case first_value(k1,false)--->first_value(k1)
+    // so size is 2, must will be arg_ignore_null_value
     if (argument_types.size() == 2) {
         DCHECK(name == "first_value" || name == "last_value") << "invalid function name: " << name;
         arg_ignore_null_value = true;
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.h b/be/src/vec/aggregate_functions/aggregate_function_window.h
index 0011ae3aba96b3..13fa8e74751df6 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_window.h
@@ -455,31 +455,28 @@ struct WindowFunctionLagImpl : Data {
     static const char* name() { return "lag"; }
 };
 
-// TODO: first_value && last_value in some corner case will be core,
-// if need to simply change it, should set them to always nullable insert into null value, and register in cpp maybe be change
-// But it's may be another better way to handle it
 template <typename Data, bool arg_ignore_null = false>
 struct WindowFunctionFirstImpl : Data {
     void add_range_single_place(int64_t partition_start, int64_t partition_end, int64_t frame_start,
                                 int64_t frame_end, const IColumn** columns) {
-        if (this->has_set_value()) {
+        // case 1: (has_set_value() = true && arg_ignore_null = false)
+        // case 2: (has_set_value() = true && arg_ignore_null = true && is_null() = false)
+        if ((this->has_set_value()) &&
+            (!arg_ignore_null || (arg_ignore_null && !this->is_null()))) {
             return;
         }
-        if (frame_start <= frame_end &&
-            frame_end <= partition_start) { //rewrite last_value when under partition
-            this->set_is_null();            //so no need more judge
+        DCHECK_LE(frame_start, frame_end);
+        if (frame_start >= partition_end || frame_end <= partition_start) {
+            this->set_is_null();
             return;
         }
         frame_start = std::max<int64_t>(frame_start, partition_start);
 
         if constexpr (arg_ignore_null) {
             frame_end = std::min<int64_t>(frame_end, partition_end);
-
-            auto& second_arg = assert_cast<const ColumnVector<UInt8>&>(*columns[1]);
-            auto ignore_null_value = second_arg.get_data()[0];
-
-            if (ignore_null_value && columns[0]->is_nullable()) {
-                auto& arg_nullable = assert_cast<const ColumnNullable&>(*columns[0]);
+            if (columns[0]->is_nullable()) {
+                const auto& arg_nullable = assert_cast<const ColumnNullable&>(*columns[0]);
+                // the valid range is: [frame_start, frame_end)
                 while (frame_start < frame_end - 1 && arg_nullable.is_null_at(frame_start)) {
                     frame_start++;
                 }
@@ -505,15 +502,25 @@ struct WindowFunctionLastImpl : Data {
 
         if constexpr (arg_ignore_null) {
             frame_start = std::max<int64_t>(frame_start, partition_start);
-
-            auto& second_arg = assert_cast<const ColumnVector<UInt8>&>(*columns[1]);
-            auto ignore_null_value = second_arg.get_data()[0];
-
-            if (ignore_null_value && columns[0]->is_nullable()) {
-                auto& arg_nullable = assert_cast<const ColumnNullable&>(*columns[0]);
-                while (frame_start < (frame_end - 1) && arg_nullable.is_null_at(frame_end - 1)) {
-                    frame_end--;
+            if (columns[0]->is_nullable()) {
+                const auto& arg_nullable = assert_cast<const ColumnNullable&>(*columns[0]);
+                // wants find a not null value in [frame_start, frame_end)
+                // iff has find: set_value and return directly
+                // iff not find: the while loop is finished
+                //     case 1: iff has_set_value, means the previous window have value, could reuse it, so return directly
+                //     case 2: iff not has_set_value, means there is none value, set it's to NULL
+                while (frame_start < frame_end) {
+                    if (arg_nullable.is_null_at(frame_end - 1)) {
+                        frame_end--;
+                    } else {
+                        this->set_value(columns, frame_end - 1);
+                        return;
+                    }
                 }
+                if (!this->has_set_value()) {
+                    this->set_is_null();
+                }
+                return;
             }
         }
 
diff --git a/regression-test/data/correctness_p0/test_first_value_window.out b/regression-test/data/correctness_p0/test_first_value_window.out
index 9951ad95c60bf4..73dbcf3ed34eda 100644
--- a/regression-test/data/correctness_p0/test_first_value_window.out
+++ b/regression-test/data/correctness_p0/test_first_value_window.out
@@ -41,3 +41,103 @@
 11	23	04-23-13	\N	10	10	10
 12	24	02-24-10-21	\N	\N	\N	\N
 
+-- !select_default4 --
+a	1	1	1	0
+a	\N	1	\N	1
+a	\N	1	\N	2
+a	\N	1	\N	3
+b	\N	\N	\N	4
+b	3	3	3	5
+b	\N	3	\N	6
+b	2	2	2	7
+
+-- !select_default5 --
+a	\N	\N	\N	0
+a	1	1	\N	1
+a	\N	1	\N	2
+a	\N	1	\N	3
+b	\N	\N	\N	4
+b	3	3	\N	5
+b	\N	3	\N	6
+b	2	3	\N	7
+
+-- !select_default_desc --
+a	2	3
+a	\N	2
+a	\N	1
+a	1	0
+b	2	7
+b	\N	6
+b	3	5
+b	\N	4
+
+-- !select_default_asc --
+a	1	0
+a	\N	1
+a	\N	2
+a	2	3
+b	\N	4
+b	3	5
+b	\N	6
+b	2	7
+
+-- !select_default_last_rewrite_first --
+a	1	1	0
+a	\N	1	1
+a	\N	1	2
+a	2	1	3
+b	\N	\N	4
+b	3	3	5
+b	\N	3	6
+b	2	3	7
+
+-- !select_default6 --
+a	\N	2	\N	0
+a	1	2	1	1
+a	2	2	2	2
+a	\N	2	2	3
+b	\N	2	\N	4
+b	3	2	3	5
+b	\N	2	3	6
+b	2	2	2	7
+
+-- !select_default_last_rewrite_first2 --
+a	1	1	0
+a	\N	1	1
+a	\N	1	2
+a	2	2	3
+b	\N	\N	4
+b	3	3	5
+b	\N	3	6
+b	2	2	7
+
+-- !select_default7 --
+a	1	1	1	1	1	0
+a	\N	1	1	1	1	1
+a	\N	1	1	1	1	2
+a	2	2	2	2	1	3
+b	\N	\N	\N	\N	\N	4
+b	3	3	3	3	3	5
+b	\N	3	3	3	3	6
+b	2	2	2	2	3	7
+
+-- !select_default8 --
+a	1	2	0
+a	\N	\N	1
+a	\N	\N	2
+a	2	\N	3
+b	\N	2	4
+b	3	\N	5
+b	\N	\N	6
+b	2	\N	7
+
+-- !select_default9 --
+a	1	2	0
+a	\N	\N	1
+a	\N	\N	2
+a	2	\N	3
+b	\N	2	4
+b	3	\N	5
+b	\N	\N	6
+b	2	\N	7
+
diff --git a/regression-test/suites/correctness_p0/test_first_value_window.groovy b/regression-test/suites/correctness_p0/test_first_value_window.groovy
index 8d0a3097056d18..7c1582e0e61b60 100644
--- a/regression-test/suites/correctness_p0/test_first_value_window.groovy
+++ b/regression-test/suites/correctness_p0/test_first_value_window.groovy
@@ -159,4 +159,193 @@ suite("test_first_value_window") {
             ,first_value(`state`, 1) over(partition by `myday` order by `time_col` rows between 1 preceding and 1 following) v3
         from ${tableName3} order by `id`, `myday`, `time_col`;
     """
+
+    qt_select_default4 """
+        SELECT uid
+            ,amt
+            ,LAST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) amt1
+            ,LAST_VALUE(amt, false) OVER(PARTITION BY uid ORDER BY time_s ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) amt2
+            ,time_s
+        FROM (
+            SELECT 'a' AS uid, 1    AS amt, 0 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 1 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 2 AS time_s UNION ALL
+            SELECT 'a' AS uid, null    AS amt, 3 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+            SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+            SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+            ) t
+        ORDER BY uid, time_s
+        ;
+    """
+
+    qt_select_default5 """
+        SELECT uid
+            ,amt
+            ,FIRST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) amt1
+            ,FIRST_VALUE(amt, false) OVER(PARTITION BY uid ORDER BY time_s ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) amt2
+            ,time_s
+        FROM (
+            SELECT 'a' AS uid, NULL    AS amt, 0 AS time_s UNION ALL
+            SELECT 'a' AS uid, 1 AS amt, 1 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 2 AS time_s UNION ALL
+            SELECT 'a' AS uid, null    AS amt, 3 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+            SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+            SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+            ) t
+        ORDER BY uid, time_s
+        ;
+    """
+    qt_select_default_desc """
+        SELECT uid
+            ,amt
+            ,time_s
+        FROM (
+            SELECT 'a' AS uid, 1    AS amt, 0 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 1 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 2 AS time_s UNION ALL
+            SELECT 'a' AS uid, 2    AS amt, 3 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+            SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+            SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+            ) t
+            order by uid,time_s desc;
+    """
+
+    qt_select_default_asc """
+        SELECT uid
+            ,amt
+            ,time_s
+        FROM (
+            SELECT 'a' AS uid, 1    AS amt, 0 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 1 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 2 AS time_s UNION ALL
+            SELECT 'a' AS uid, 2    AS amt, 3 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+            SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+            SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+            ) t
+            order by uid,time_s ASC;
+    """
+
+    // FIRST_VALUE: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+    qt_select_default_last_rewrite_first """ 
+            SELECT uid
+        ,amt
+        ,(LAST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s DESC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)) amt3
+        ,time_s
+    FROM (
+        SELECT 'a' AS uid, 1    AS amt, 0 AS time_s UNION ALL
+        SELECT 'a' AS uid, null AS amt, 1 AS time_s UNION ALL
+        SELECT 'a' AS uid, null AS amt, 2 AS time_s UNION ALL
+        SELECT 'a' AS uid, 2    AS amt, 3 AS time_s UNION ALL
+        SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+        SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+        SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+        SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+        ) t
+    ORDER BY uid, time_s;
+    """
+
+    qt_select_default6 """
+        SELECT uid
+        ,amt
+        ,LAST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED following) amt1
+        ,LAST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) amt2
+        ,time_s
+    FROM (
+        SELECT 'a' AS uid, null    AS amt, 0 AS time_s UNION ALL
+        SELECT 'a' AS uid, 1 AS amt, 1 AS time_s UNION ALL
+        SELECT 'a' AS uid, 2 AS amt, 2 AS time_s UNION ALL
+        SELECT 'a' AS uid, null    AS amt, 3 AS time_s UNION ALL
+        SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+        SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+        SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+        SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+        ) t
+    ORDER BY uid, time_s
+    ;
+    """
+
+    //last value: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+    qt_select_default_last_rewrite_first2 """
+            SELECT uid
+        ,amt
+        ,(FIRST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s DESC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)) amt3
+        ,time_s
+        FROM (
+            SELECT 'a' AS uid, 1    AS amt, 0 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 1 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 2 AS time_s UNION ALL
+            SELECT 'a' AS uid, 2    AS amt, 3 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+            SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+            SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+            ) t
+        ORDER BY uid, time_s;
+    """
+
+    qt_select_default7 """
+    SELECT uid
+        ,amt
+        ,COALESCE(LAST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)) amt1
+        ,COALESCE(LAST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s ASC ROWS BETWEEN 100 PRECEDING AND CURRENT ROW)) amt_not
+        ,COALESCE(FIRST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s DESC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)) amt2
+        ,COALESCE(LAST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s DESC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)) amt3
+        ,time_s
+    FROM (
+        SELECT 'a' AS uid, 1    AS amt, 0 AS time_s UNION ALL
+        SELECT 'a' AS uid, null AS amt, 1 AS time_s UNION ALL
+        SELECT 'a' AS uid, null AS amt, 2 AS time_s UNION ALL
+        SELECT 'a' AS uid, 2    AS amt, 3 AS time_s UNION ALL
+        SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+        SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+        SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+        SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+        ) t
+    ORDER BY uid, time_s
+    ;
+    """
+
+    qt_select_default8 """
+            SELECT uid
+        ,amt
+        ,(FIRST_VALUE(amt, true) OVER(PARTITION BY uid ORDER BY time_s ROWS between 3 following AND 6 FOLLOWING)) amt3
+        ,time_s
+        FROM (
+            SELECT 'a' AS uid, 1    AS amt, 0 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 1 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 2 AS time_s UNION ALL
+            SELECT 'a' AS uid, 2    AS amt, 3 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+            SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+            SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+            ) t
+        ORDER BY uid, time_s;
+    """
+
+    qt_select_default9 """
+            SELECT uid
+        ,amt
+        ,(FIRST_VALUE(amt) OVER(PARTITION BY uid ORDER BY time_s ROWS between 3 following AND 6 FOLLOWING)) amt3
+        ,time_s
+        FROM (
+            SELECT 'a' AS uid, 1    AS amt, 0 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 1 AS time_s UNION ALL
+            SELECT 'a' AS uid, null AS amt, 2 AS time_s UNION ALL
+            SELECT 'a' AS uid, 2    AS amt, 3 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 4 AS time_s UNION ALL
+            SELECT 'b' AS uid, 3    AS amt, 5 AS time_s UNION ALL
+            SELECT 'b' AS uid, null AS amt, 6 AS time_s UNION ALL
+            SELECT 'b' AS uid, 2    AS amt, 7 AS time_s 
+            ) t
+        ORDER BY uid, time_s;
+    """
 }

From 7e990d2812759ac84afdf555563bbc0450ac0531 Mon Sep 17 00:00:00 2001
From: zclllhhjj <zhaochangle@selectdb.com>
Date: Thu, 12 Dec 2024 15:48:05 +0800
Subject: [PATCH 298/399] [chore](test) Refactor fold constant test util in
 regression test (#45266)

make an abstract in `Suite.groovy`. no need to print result in `.out`
files.
---
 .../data/correctness/test_str_to_date.out     | 45 -------------
 .../math_functions/test_conv.out              | 54 ----------------
 .../nereids_syntax_p0/test_regexp_replace.out | 36 -----------
 .../conditional_functions/test_json_parse.out | 63 -------------------
 .../doris/regression/suite/Suite.groovy       |  9 +++
 .../correctness/test_str_to_date.groovy       | 27 ++------
 .../math_functions/test_conv.groovy           | 28 ++-------
 .../test_regexp_replace.groovy                | 24 ++-----
 .../test_json_parse.groovy                    | 30 +++------
 .../string_functions/test_translate.groovy    | 33 +++-------
 .../string_functions/test_trim_in.groovy      | 36 ++++-------
 .../test_template_three_args.groovy           | 20 ++----
 .../test_template_two_args.groovy             | 22 ++-----
 13 files changed, 61 insertions(+), 366 deletions(-)

diff --git a/regression-test/data/correctness/test_str_to_date.out b/regression-test/data/correctness/test_str_to_date.out
index dfbb90e234b35c..f145ff9e4bedc8 100644
--- a/regression-test/data/correctness/test_str_to_date.out
+++ b/regression-test/data/correctness/test_str_to_date.out
@@ -53,48 +53,3 @@
 6	\N
 7	\N
 
--- !check_fe --
-2019-12-01
-
--- !check_be --
-2019-12-01
-
--- !check_no_fold --
-2019-12-01
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
diff --git a/regression-test/data/nereids_p0/sql_functions/math_functions/test_conv.out b/regression-test/data/nereids_p0/sql_functions/math_functions/test_conv.out
index db787d1daf1c1e..b10406e6456122 100644
--- a/regression-test/data/nereids_p0/sql_functions/math_functions/test_conv.out
+++ b/regression-test/data/nereids_p0/sql_functions/math_functions/test_conv.out
@@ -23,57 +23,3 @@
 10	2.789
 11	3.14159
 
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-1111
-
--- !check_be --
-1111
-
--- !check_no_fold --
-1111
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-1111011
-
--- !check_be --
-1111011
-
--- !check_no_fold --
-1111011
-
diff --git a/regression-test/data/nereids_syntax_p0/test_regexp_replace.out b/regression-test/data/nereids_syntax_p0/test_regexp_replace.out
index f06114138f0dcc..578937e0ca1236 100644
--- a/regression-test/data/nereids_syntax_p0/test_regexp_replace.out
+++ b/regression-test/data/nereids_syntax_p0/test_regexp_replace.out
@@ -5,42 +5,6 @@ abcxyz
 -- !replace_chinese --
 汉123
 
--- !check_fe --
-abcxyz
-
--- !check_be --
-abcxyz
-
--- !check_no_fold --
-abcxyz
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
 -- !replace_in_table_chinese --
 1	abc123
 2	汉汉汉汉汉456
diff --git a/regression-test/data/query_p0/sql_functions/conditional_functions/test_json_parse.out b/regression-test/data/query_p0/sql_functions/conditional_functions/test_json_parse.out
index 05b650de68a545..a40cd6937d1560 100644
--- a/regression-test/data/query_p0/sql_functions/conditional_functions/test_json_parse.out
+++ b/regression-test/data/query_p0/sql_functions/conditional_functions/test_json_parse.out
@@ -28,66 +28,3 @@ null
 8	123
 9	[1,2,3]
 
--- !check_fe --
-{"key":"value"}
-
--- !check_be --
-{"key":"value"}
-
--- !check_no_fold --
-{"key":"value"}
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-\N
-
--- !check_be --
-\N
-
--- !check_no_fold --
-\N
-
--- !check_fe --
-null
-
--- !check_be --
-null
-
--- !check_no_fold --
-null
-
--- !check_fe --
-123
-
--- !check_be --
-123
-
--- !check_no_fold --
-123
-
--- !check_fe --
-[1,2,3]
-
--- !check_be --
-[1,2,3]
-
--- !check_no_fold --
-[1,2,3]
-
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index d622084c24bd82..59722d72d952fe 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -2714,4 +2714,13 @@ class Suite implements GroovyInterceptable {
             scpFiles("root", be_ip, udf_file_path, udf_file_path, false)
         }
     }
+
+    def check_fold_consistency = { test_sql ->
+        def re_fe = order_sql "select /*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
+        def re_be = order_sql "select /*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
+        def re_no_fold = order_sql "select /*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
+        logger.info("check sql: ${test_sql}")
+        assertEquals(re_fe, re_be)
+        assertEquals(re_fe, re_no_fold)
+    }
 }
diff --git a/regression-test/suites/correctness/test_str_to_date.groovy b/regression-test/suites/correctness/test_str_to_date.groovy
index a45be14902ba56..75234b1f3b3afd 100644
--- a/regression-test/suites/correctness/test_str_to_date.groovy
+++ b/regression-test/suites/correctness/test_str_to_date.groovy
@@ -72,26 +72,9 @@ suite("test_str_to_date") {
         SELECT id, STR_TO_DATE(s1, s2) as result from test_str_to_date_db order by id;
     """
 
-    def re_fe
-    def re_be
-    def re_no_fold
-
-    def check_three_ways = { test_sql ->
-        re_fe = order_sql "select /*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
-        re_be = order_sql "select /*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
-        re_no_fold = order_sql "select /*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
-        logger.info("check sql: ${test_sql}")
-        qt_check_fe "select /*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
-        qt_check_be "select /*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
-        qt_check_no_fold "select /*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
-        assertEquals(re_fe, re_be)
-        assertEquals(re_fe, re_no_fold)
-    }
-
-    check_three_ways "STR_TO_DATE('2019-12-01', 'yyyy-MM-dd')"
-    check_three_ways "STR_TO_DATE(null, 'yyyy-MM-dd')"
-    check_three_ways "STR_TO_DATE('2019-12-01', null)"
-    check_three_ways "STR_TO_DATE(null, null)"
-    check_three_ways "STR_TO_DATE('无效日期', 'yyyy-MM-dd')"
+    check_fold_consistency "STR_TO_DATE('2019-12-01', 'yyyy-MM-dd')"
+    check_fold_consistency "STR_TO_DATE(null, 'yyyy-MM-dd')"
+    check_fold_consistency "STR_TO_DATE('2019-12-01', null)"
+    check_fold_consistency "STR_TO_DATE(null, null)"
+    check_fold_consistency "STR_TO_DATE('无效日期', 'yyyy-MM-dd')"
 }
-
diff --git a/regression-test/suites/nereids_p0/sql_functions/math_functions/test_conv.groovy b/regression-test/suites/nereids_p0/sql_functions/math_functions/test_conv.groovy
index 631df018fbd726..1fa0bdcfc03ff9 100644
--- a/regression-test/suites/nereids_p0/sql_functions/math_functions/test_conv.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/math_functions/test_conv.groovy
@@ -36,26 +36,10 @@ suite("test_conv") {
 
     qt_select6 """ select conv(float_2,10,2), float_2 from test_tb_with_null; """
 
-    def re_fe
-    def re_be
-    def re_no_fold
-
-    def check_three_ways = { test_sql ->
-        re_fe = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
-        re_be = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
-        re_no_fold = order_sql "select/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
-        logger.info("check on sql: ${test_sql}")
-        qt_check_fe "select/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
-        qt_check_be "select/*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
-        qt_check_no_fold "select/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
-        assertEquals(re_fe, re_be)
-        assertEquals(re_fe, re_no_fold)
-    }
-
-    check_three_ways "conv(null, null, null)"
-    check_three_ways "conv(15, 10, 2)"
-    check_three_ways "conv(null, 10, 2)"
-    check_three_ways "conv(15, null, 2)"
-    check_three_ways "conv(15, 10, null)"
-    check_three_ways "conv('123', 10, 2)"
+    check_fold_consistency "conv(null, null, null)"
+    check_fold_consistency "conv(15, 10, 2)"
+    check_fold_consistency "conv(null, 10, 2)"
+    check_fold_consistency "conv(15, null, 2)"
+    check_fold_consistency "conv(15, 10, null)"
+    check_fold_consistency "conv('123', 10, 2)"
 }
\ No newline at end of file
diff --git a/regression-test/suites/nereids_syntax_p0/test_regexp_replace.groovy b/regression-test/suites/nereids_syntax_p0/test_regexp_replace.groovy
index 33c6c845491c22..ff94b640022d6d 100644
--- a/regression-test/suites/nereids_syntax_p0/test_regexp_replace.groovy
+++ b/regression-test/suites/nereids_syntax_p0/test_regexp_replace.groovy
@@ -20,26 +20,10 @@ suite("test_regexp_replace") {
 
     qt_replace_chinese "SELECT regexp_replace('这是一个测试字符串123', '\\\\p{Han}+', '汉');"
     
-    def re_fe
-    def re_be
-    def re_no_fold
-
-    def check_three_ways = { test_sql ->
-        re_fe = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
-        re_be = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
-        re_no_fold = order_sql "select/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
-        logger.info("check on sql: ${test_sql}")
-        qt_check_fe "select/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
-        qt_check_be "select/*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
-        qt_check_no_fold "select/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
-        assertEquals(re_fe, re_be)
-        assertEquals(re_fe, re_no_fold)
-    }
-
-    check_three_ways "regexp_replace('abc123', '123', 'xyz')"
-    check_three_ways "regexp_replace(null, 'abc', 'def')"
-    check_three_ways "regexp_replace('abc123', null, 'xyz')"
-    check_three_ways "regexp_replace('abc123', '123', null)"
+    check_fold_consistency "regexp_replace('abc123', '123', 'xyz')"
+    check_fold_consistency "regexp_replace(null, 'abc', 'def')"
+    check_fold_consistency "regexp_replace('abc123', null, 'xyz')"
+    check_fold_consistency "regexp_replace('abc123', '123', null)"
 
     sql """DROP TABLE IF EXISTS `test_table_for_regexp`;"""
     sql """CREATE TABLE test_table_for_regexp (id INT, name VARCHAR(100)) PROPERTIES ("replication_num"="1");"""
diff --git a/regression-test/suites/query_p0/sql_functions/conditional_functions/test_json_parse.groovy b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_json_parse.groovy
index 11674695037218..a51138774dbc4b 100644
--- a/regression-test/suites/query_p0/sql_functions/conditional_functions/test_json_parse.groovy
+++ b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_json_parse.groovy
@@ -42,28 +42,12 @@ suite("test_json_parse") {
 
     qt_parse_from_table "SELECT id, json_parse_error_to_null(json_str) FROM test_json_parse_table ORDER BY id;"
 
-    def re_fe
-    def re_be
-    def re_no_fold
-
-    def check_three_ways = { test_sql ->
-        re_fe = order_sql "SELECT /*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql};"
-        re_be = order_sql "SELECT /*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql};"
-        re_no_fold = order_sql "SELECT /*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql};"
-        logger.info("check on sql: ${test_sql}")
-        qt_check_fe "SELECT /*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql};"
-        qt_check_be "SELECT /*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql};"
-        qt_check_no_fold "SELECT /*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql};"
-        assertEquals(re_fe, re_be)
-        assertEquals(re_fe, re_no_fold)
-    }
-
-    check_three_ways "json_parse_error_to_null('{\"key\": \"value\"}')"
-    check_three_ways "json_parse_error_to_null('Invalid JSON')"
-    check_three_ways "json_parse_error_to_null(NULL)"
-    check_three_ways "json_parse_error_to_null('')"
-    check_three_ways "json_parse_error_to_null('null')"
-    check_three_ways "json_parse_error_to_null('123')"
-    check_three_ways "json_parse_error_to_null('[1, 2, 3]')"
+    check_fold_consistency "json_parse_error_to_null('{\"key\": \"value\"}')"
+    check_fold_consistency "json_parse_error_to_null('Invalid JSON')"
+    check_fold_consistency "json_parse_error_to_null(NULL)"
+    check_fold_consistency "json_parse_error_to_null('')"
+    check_fold_consistency "json_parse_error_to_null('null')"
+    check_fold_consistency "json_parse_error_to_null('123')"
+    check_fold_consistency "json_parse_error_to_null('[1, 2, 3]')"
 }
 
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_translate.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_translate.groovy
index e63f42ae5b4cf5..e6ba0ed44f390e 100644
--- a/regression-test/suites/query_p0/sql_functions/string_functions/test_translate.groovy
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_translate.groovy
@@ -88,30 +88,15 @@ suite("test_translate") {
     order_qt_const3 "select translate(b, a, 'abc') from test_translate"
 
     /// folding
-    def re_fe
-    def re_be
-    def re_no_fold
-    def check_three_ways = { test_sql ->
-        sql "set enable_fold_constant_by_be=false;"
-        re_fe = order_sql "select ${test_sql}"
-        sql "set enable_fold_constant_by_be=true;"
-        re_be = order_sql "select ${test_sql}"
-        sql "set debug_skip_fold_constant=true;"
-        re_no_fold = order_sql "select ${test_sql}"
-        logger.info("check on sql \${test_sql}")
-        assertEquals(re_fe, re_be)
-        assertEquals(re_fe, re_no_fold)
-    }
-
-    check_three_ways "translate('abcd', '', '');"
-    check_three_ways "translate('abcda', 'a', 'z');"
-    check_three_ways "translate('abcd', 'ac', 'z');"
-    check_three_ways "translate('abcd', 'aac', 'zq');"
-    check_three_ways "translate('abcd', 'aac', 'zqx');"
-    check_three_ways "translate('abcd', 'aac', '中文x');"
-    check_three_ways "translate('中文', '中', '文');"
-    check_three_ways "translate('中文', '中', 'a');"
-    check_three_ways "translate('\tt\tt\tt', '\t', 't');"
+    check_fold_consistency "translate('abcd', '', '');"
+    check_fold_consistency "translate('abcda', 'a', 'z');"
+    check_fold_consistency "translate('abcd', 'ac', 'z');"
+    check_fold_consistency "translate('abcd', 'aac', 'zq');"
+    check_fold_consistency "translate('abcd', 'aac', 'zqx');"
+    check_fold_consistency "translate('abcd', 'aac', '中文x');"
+    check_fold_consistency "translate('中文', '中', '文');"
+    check_fold_consistency "translate('中文', '中', 'a');"
+    check_fold_consistency "translate('\tt\tt\tt', '\t', 't');"
 
     order_qt_1 "select translate('abcd', '', '');"
     order_qt_2 "select translate('abcd', 'a', 'z')"
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_trim_in.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_trim_in.groovy
index ae6790fb0693e1..bf04ebb454a982 100644
--- a/regression-test/suites/query_p0/sql_functions/string_functions/test_trim_in.groovy
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_trim_in.groovy
@@ -82,30 +82,18 @@ suite("test_trim_in") {
 
 
     /// folding
-    def re_fe
-    def re_be
-    def re_no_fold
-    def check_three_ways = { test_sql ->
-        re_fe = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
-        re_be = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
-        re_no_fold = order_sql "select/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
-        logger.info("check on sql ${test_sql}")
-        assertEquals(re_fe, re_be)
-        assertEquals(re_fe, re_no_fold)
-    }
-
-    check_three_ways "trim_in(' hello world ', ' ld')"
-    check_three_ways "ltrim_in(' hello world ', ' ld')"
-    check_three_ways "rtrim_in(' hello world ', ' ld')"
-    check_three_ways "trim_in(' hello world ', ' ehlowrd')"
-    check_three_ways "ltrim_in(' hello world ', ' ehlowrd')"
-    check_three_ways "rtrim_in(' hello world ', ' ehlowrd')"
-    check_three_ways "trim_in(' hello world ', '')"
-    check_three_ways "ltrim_in(' hello world ', '')"
-    check_three_ways "rtrim_in(' hello world ', '')"
-    check_three_ways "trim_in(' hello world ', ' ')"
-    check_three_ways "ltrim_in(' hello world ', ' ')"
-    check_three_ways "rtrim_in(' hello world ', ' ')"
+    check_fold_consistency "trim_in(' hello world ', ' ld')"
+    check_fold_consistency "ltrim_in(' hello world ', ' ld')"
+    check_fold_consistency "rtrim_in(' hello world ', ' ld')"
+    check_fold_consistency "trim_in(' hello world ', ' ehlowrd')"
+    check_fold_consistency "ltrim_in(' hello world ', ' ehlowrd')"
+    check_fold_consistency "rtrim_in(' hello world ', ' ehlowrd')"
+    check_fold_consistency "trim_in(' hello world ', '')"
+    check_fold_consistency "ltrim_in(' hello world ', '')"
+    check_fold_consistency "rtrim_in(' hello world ', '')"
+    check_fold_consistency "trim_in(' hello world ', ' ')"
+    check_fold_consistency "ltrim_in(' hello world ', ' ')"
+    check_fold_consistency "rtrim_in(' hello world ', ' ')"
 
     order_qt_1 "SELECT ltrim_in('');"
     order_qt_2 "SELECT ltrim_in('   ');"
diff --git a/regression-test/suites/query_p0/sql_functions/test_template_three_args.groovy b/regression-test/suites/query_p0/sql_functions/test_template_three_args.groovy
index 13de80a0393024..64bb970c3f0cd6 100644
--- a/regression-test/suites/query_p0/sql_functions/test_template_three_args.groovy
+++ b/regression-test/suites/query_p0/sql_functions/test_template_three_args.groovy
@@ -95,20 +95,8 @@ suite("test_template_three_args") {
     order_qt_const3 "select concat(b, a, 'abc') from arg1_three_args"
 
     /// folding
-    def re_fe
-    def re_be
-    def re_no_fold
-    def check_three_ways = { test_sql ->
-        re_fe = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
-        re_be = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
-        re_no_fold = order_sql "select/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
-        logger.info("check on sql \${test_sql}")
-        assertEquals(re_fe, re_be)
-        assertEquals(re_fe, re_no_fold)
-    }
-
-    check_three_ways "concat('', '', '')"
-    check_three_ways "concat('\\t\\t', '\\t\\t', '\\t\\t')"
-    check_three_ways "concat('中文', '中文', '中文')"
-    check_three_ways "concat('abcde', 'abcde', 'abcde')"
+    check_fold_consistency "concat('', '', '')"
+    check_fold_consistency "concat('\\t\\t', '\\t\\t', '\\t\\t')"
+    check_fold_consistency "concat('中文', '中文', '中文')"
+    check_fold_consistency "concat('abcde', 'abcde', 'abcde')"
 }
\ No newline at end of file
diff --git a/regression-test/suites/query_p0/sql_functions/test_template_two_args.groovy b/regression-test/suites/query_p0/sql_functions/test_template_two_args.groovy
index 9d7e2643eca1c4..0794c79e817a40 100644
--- a/regression-test/suites/query_p0/sql_functions/test_template_two_args.groovy
+++ b/regression-test/suites/query_p0/sql_functions/test_template_two_args.groovy
@@ -88,21 +88,9 @@ suite("test_template_two_args") {
     order_qt_const_partial_nullable_no_null "select atan2(1e100, nullable(1e-10))"
 
     /// folding
-    def re_fe
-    def re_be
-    def re_no_fold
-    def check_three_ways = { test_sql ->
-        re_fe = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=false)*/ ${test_sql}"
-        re_be = order_sql "select/*+SET_VAR(enable_fold_constant_by_be=true)*/ ${test_sql}"
-        re_no_fold = order_sql "select/*+SET_VAR(debug_skip_fold_constant=true)*/ ${test_sql}"
-        logger.info("check on sql ${test_sql}")
-        assertEquals(re_fe, re_be)
-        assertEquals(re_fe, re_no_fold)
-    }
-
-    check_three_ways "atan2(-1, -2)"
-    check_three_ways "atan2(-1e100, 3.14)"
-    check_three_ways "atan2(0, 0)"
-    check_three_ways "atan2(1e100, 1e100)"
-    check_three_ways "atan2(-0.5, 0.5)"
+    check_fold_consistency "atan2(-1, -2)"
+    check_fold_consistency "atan2(-1e100, 3.14)"
+    check_fold_consistency "atan2(0, 0)"
+    check_fold_consistency "atan2(1e100, 1e100)"
+    check_fold_consistency "atan2(-0.5, 0.5)"
 }
\ No newline at end of file

From 7335ea8a2c0d2959c801d96282559235ca929364 Mon Sep 17 00:00:00 2001
From: Rijesh Kunhi Parambattu <147430310+rijeshkp@users.noreply.github.com>
Date: Thu, 12 Dec 2024 13:22:41 +0530
Subject: [PATCH 299/399] [Enhancement] (nereids)implement
 showQueryProfileCommand in nereids (#45271)

Issue Number: close #42772
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  8 +++
 .../doris/nereids/trees/plans/PlanType.java   |  3 +-
 .../commands/ShowQueryProfileCommand.java     | 55 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  6 ++
 .../show/test_show_commands_nereids.groovy    |  1 +
 6 files changed, 73 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryProfileCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index a385e1e5ddb3bb..274a354b63bc03 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -272,6 +272,7 @@ supportedShowStatement
     | SHOW TABLE CREATION ((FROM | IN) database=multipartIdentifier)?
         (LIKE STRING_LITERAL)?                                                      #showTableCreation
     | SHOW TABLET STORAGE FORMAT VERBOSE?                                           #showTabletStorageFormat
+    | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL                                 #showQueryProfile
     ;
 
 supportedLoadStatement
@@ -359,7 +360,6 @@ unsupportedShowStatement
         (FROM |IN) tableName=multipartIdentifier
         ((FROM | IN) database=multipartIdentifier)?                                 #showIndex
     | SHOW TRANSACTION ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showTransaction
-    | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL                                 #showQueryProfile
     | SHOW CACHE HOTSPOT tablePath=STRING_LITERAL                                   #showCacheHotSpot
     | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)?                     #showSyncJob
     | SHOW CATALOG RECYCLE BIN wildWhere?                                           #showCatalogRecycleBin
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 12ae62bac10930..56de50d620eb6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -266,6 +266,7 @@
 import org.apache.doris.nereids.DorisParser.ShowProcContext;
 import org.apache.doris.nereids.DorisParser.ShowProcedureStatusContext;
 import org.apache.doris.nereids.DorisParser.ShowProcessListContext;
+import org.apache.doris.nereids.DorisParser.ShowQueryProfileContext;
 import org.apache.doris.nereids.DorisParser.ShowReplicaDistributionContext;
 import org.apache.doris.nereids.DorisParser.ShowRepositoriesContext;
 import org.apache.doris.nereids.DorisParser.ShowRolesContext;
@@ -573,6 +574,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcessListCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowQueryProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowReplicaDistributionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand;
@@ -5043,5 +5045,11 @@ public LogicalPlan visitAdminShowTabletStorageFormat(AdminShowTabletStorageForma
     public LogicalPlan visitShowTabletStorageFormat(ShowTabletStorageFormatContext ctx) {
         return new ShowTabletStorageFormatCommand(ctx.VERBOSE() != null);
     }
+
+    @Override
+    public LogicalPlan visitShowQueryProfile(ShowQueryProfileContext ctx) {
+        String queryIdPath = stripQuotes(ctx.queryIdPath.getText());
+        return new ShowQueryProfileCommand(queryIdPath);
+    }
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 9bab39c946b45b..204d4b07111bec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -253,5 +253,6 @@ public enum PlanType {
     CREATE_WORKLOAD_GROUP_COMMAND,
     CREATE_FILE_COMMAND,
     CREATE_ROUTINE_LOAD_COMMAND,
-    SHOW_TABLE_CREATION_COMMAND
+    SHOW_TABLE_CREATION_COMMAND,
+    SHOW_QUERY_PROFILE_COMMAND
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryProfileCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryProfileCommand.java
new file mode 100644
index 00000000000000..1fe63619d0125e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryProfileCommand.java
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.StmtExecutor;
+
+/**
+ * Represents the command for SHOW COLLATION
+ */
+public class ShowQueryProfileCommand extends ShowCommand {
+    String queryIdPath;
+
+    public ShowQueryProfileCommand(String queryIdPath) {
+        super(PlanType.SHOW_QUERY_PROFILE_COMMAND);
+        this.queryIdPath = queryIdPath;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        String selfHost = Env.getCurrentEnv().getSelfNode().getHost();
+        int httpPort = Config.http_port;
+        String terminalMsg = String.format(
+                "try visit http://%s:%d/QueryProfile, show query/load profile syntax is a deprecated feature",
+                selfHost, httpPort);
+        throw new AnalysisException(terminalMsg);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowQueryProfileCommand(this, context);
+    }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 085b510b095e00..e2ac1c863e2f5a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -111,6 +111,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowProcessListCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowQueryProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowReplicaDistributionCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRepositoriesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand;
@@ -631,4 +632,9 @@ default R visitShowTabletStorageFormatCommand(ShowTabletStorageFormatCommand sho
                                                   C context) {
         return visitCommand(showTabletStorageFormatCommand, context);
     }
+
+    default R visitShowQueryProfileCommand(ShowQueryProfileCommand showQueryProfileCommand,
+                                           C context) {
+        return visitCommand(showQueryProfileCommand, context);
+    }
 }
diff --git a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
index ee243b42d3209a..92367db345a254 100644
--- a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
@@ -28,4 +28,5 @@ suite("test_show_commands_nereids") {
     checkNereidsExecute("""show triggers;""")
     checkNereidsExecute("""show events;""")
     checkNereidsExecute("""show load profile "/";""")
+    checkNereidsExecute("""show query profile "/";""")
 }

From 13add3c17164c1a5f33dcfd34b4b11932a16ee67 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Thu, 12 Dec 2024 16:42:19 +0800
Subject: [PATCH 300/399] [fix](job)Fix millisecond offset issue in time window
 scheduling trigger time calculation (#45176)

### Abstract:
In the current time window scheduling logic, the calculation of trigger
times was not strictly aligned to the second level, which could lead to
millisecond offsets. This offset caused issues such as consecutive
trigger times at 14:56:59 and 14:57:00, disrupting the correctness of
the scheduling.

This PR optimizes the calculation of trigger times to ensure that time
points are strictly aligned to the second level, preventing the
accumulation of millisecond errors.

### Issue Description:

Under a specified window (e.g., 14:50:00 to 14:59:00) and a fixed
interval (e.g., every minute), the scheduler generated erroneous trigger
times such as:

```
| 2024-12-04 14:56:59 |
| 2024-12-04 14:57:00 |
| 2024-12-04 14:57:59 |
| 2024-12-04 14:58:00 |
```
#### Cause:
The current firstTriggerTime and the loop calculation did not strictly
align trigger times to the second level, resulting in erroneous trigger
points due to floating-point or millisecond offset accumulation. The end
condition for the time window was not aligned to the second level, which
could lead to additional trigger times being included.

### Fix:
Modification 1: Strictly align the trigger time to the second level.
---
 .../org/apache/doris/common/util/TimeUtils.java     | 11 +++++++++++
 .../doris/job/base/JobExecutionConfiguration.java   |  2 +-
 .../org/apache/doris/job/base/TimerDefinition.java  |  7 ++++++-
 .../apache/doris/job/scheduler/JobScheduler.java    | 13 +++++++++----
 .../job/base/JobExecutionConfigurationTest.java     |  7 +++++++
 5 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
index e7066846c30919..d88971a6e72bde 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
@@ -257,6 +257,17 @@ public static long timeStringToLong(String timeStr) {
         return d.getTime();
     }
 
+    /**
+     * Converts a millisecond timestamp to a second-level timestamp.
+     *
+     * @param timestamp The millisecond timestamp to be converted.
+     * @return The timestamp rounded to the nearest second (in milliseconds).
+     */
+    public static long convertToSecondTimestamp(long timestamp) {
+        // Divide by 1000 to convert to seconds, then multiply by 1000 to return to milliseconds with no fractional part
+        return (timestamp / 1000) * 1000;
+    }
+
     public static long timeStringToLong(String timeStr, TimeZone timeZone) {
         DateTimeFormatter dateFormatTimeZone = getDatetimeFormatWithTimeZone();
         dateFormatTimeZone.withZone(timeZone.toZoneId());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecutionConfiguration.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecutionConfiguration.java
index 4c6ef4d2037f86..d564b1143122ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecutionConfiguration.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecutionConfiguration.java
@@ -155,7 +155,7 @@ private Long queryDelayTimeSecond(Long currentTimeMs, Long startTimeMs) {
             return 0L;
         }
 
-        return (startTimeMs - currentTimeMs) / 1000;
+        return (startTimeMs * 1000 / 1000 - currentTimeMs) / 1000;
     }
 
     // Returns a list of delay times in seconds for executing the job within the specified window
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/TimerDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/TimerDefinition.java
index 9068a18f693e12..96181877b9a568 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/base/TimerDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/TimerDefinition.java
@@ -17,6 +17,7 @@
 
 package org.apache.doris.job.base;
 
+import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.job.common.IntervalUnit;
 
 import com.google.gson.annotations.SerializedName;
@@ -40,11 +41,15 @@ public class TimerDefinition {
 
     public void checkParams() {
         if (null == startTimeMs) {
-            startTimeMs = System.currentTimeMillis() + intervalUnit.getIntervalMs(interval);
+            long currentTimeMs = TimeUtils.convertToSecondTimestamp(System.currentTimeMillis());
+            startTimeMs = currentTimeMs + intervalUnit.getIntervalMs(interval);
         }
         if (null != endTimeMs && endTimeMs < startTimeMs) {
             throw new IllegalArgumentException("endTimeMs must be greater than the start time");
         }
+        if (null != endTimeMs) {
+            endTimeMs = TimeUtils.convertToSecondTimestamp(endTimeMs);
+        }
 
         if (null != intervalUnit) {
             if (null == interval) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
index 921f333791cb44..2bd6fc04dace53 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
@@ -84,7 +84,8 @@ public void start() {
         taskDisruptorGroupManager = new TaskDisruptorGroupManager();
         taskDisruptorGroupManager.init();
         this.timerJobDisruptor = taskDisruptorGroupManager.getDispatchDisruptor();
-        latestBatchSchedulerTimerTaskTimeMs = System.currentTimeMillis();
+        long currentTimeMs = TimeUtils.convertToSecondTimestamp(System.currentTimeMillis());
+        latestBatchSchedulerTimerTaskTimeMs = currentTimeMs;
         batchSchedulerTimerJob();
         cycleSystemSchedulerTasks();
     }
@@ -94,7 +95,8 @@ public void start() {
      * Jobs will be re-registered after the task is completed
      */
     private void cycleSystemSchedulerTasks() {
-        log.info("re-register system scheduler timer tasks" + TimeUtils.longToTimeString(System.currentTimeMillis()));
+        log.info("re-register system scheduler timer tasks, time is " + TimeUtils
+                .longToTimeStringWithms(System.currentTimeMillis()));
         timerTaskScheduler.newTimeout(timeout -> {
             batchSchedulerTimerJob();
             cycleSystemSchedulerTasks();
@@ -144,7 +146,9 @@ public void close() throws IOException {
 
 
     private void cycleTimerJobScheduler(T job, long startTimeWindowMs) {
-        List<Long> delaySeconds = job.getJobConfig().getTriggerDelayTimes(System.currentTimeMillis(),
+        long currentTimeMs = TimeUtils.convertToSecondTimestamp(System.currentTimeMillis());
+        startTimeWindowMs = TimeUtils.convertToSecondTimestamp(startTimeWindowMs);
+        List<Long> delaySeconds = job.getJobConfig().getTriggerDelayTimes(currentTimeMs,
                 startTimeWindowMs, latestBatchSchedulerTimerTaskTimeMs);
         if (CollectionUtils.isEmpty(delaySeconds)) {
             log.info("skip job {} scheduler timer job, delay seconds is empty", job.getJobName());
@@ -190,7 +194,8 @@ private void executeTimerJobIdsWithinLastTenMinutesWindow() {
 
         long lastTimeWindowMs = latestBatchSchedulerTimerTaskTimeMs;
         if (latestBatchSchedulerTimerTaskTimeMs < System.currentTimeMillis()) {
-            this.latestBatchSchedulerTimerTaskTimeMs = System.currentTimeMillis();
+            long currentTimeMs = TimeUtils.convertToSecondTimestamp(System.currentTimeMillis());
+            this.latestBatchSchedulerTimerTaskTimeMs = currentTimeMs;
         }
         this.latestBatchSchedulerTimerTaskTimeMs += BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS;
         log.info("execute timer job ids within last ten minutes window, last time window is {}",
diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/base/JobExecutionConfigurationTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/base/JobExecutionConfigurationTest.java
index cce0a93c01daf8..163b2494189d90 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/job/base/JobExecutionConfigurationTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/job/base/JobExecutionConfigurationTest.java
@@ -75,7 +75,14 @@ public void testGetTriggerDelayTimesRecurring() {
         timerDefinition.setInterval(1L);
         Assertions.assertEquals(3, configuration.getTriggerDelayTimes(second * 5 + 10L, second * 3, second * 7).size());
         Assertions.assertEquals(3, configuration.getTriggerDelayTimes(second * 5, second * 5, second * 7).size());
+        timerDefinition.setStartTimeMs(1672531200000L);
+        timerDefinition.setIntervalUnit(IntervalUnit.MINUTE);
+        timerDefinition.setInterval(1L);
+        Assertions.assertArrayEquals(new Long[]{0L}, configuration.getTriggerDelayTimes(1672531800000L, 1672531200000L, 1672531800000L).toArray());
+
+        List<Long> expectDelayTimes = configuration.getTriggerDelayTimes(1672531200000L, 1672531200000L, 1672531850000L);
 
+        Assertions.assertArrayEquals(new Long[]{0L, 60L, 120L, 180L, 240L, 300L, 360L, 420L, 480L, 540L, 600L}, expectDelayTimes.toArray());
     }
 
     @Test

From 947100f9fa81154eae3122c4c8c867d79dfdfbb1 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Thu, 12 Dec 2024 16:43:18 +0800
Subject: [PATCH 301/399] [Chore](dependencies)Upgrade Fe dependencies (#44872)

### What problem does this PR solve?

upgrade guava to 32.2.1-jre
upgrade zookeeper to 3.9.3
upgrade azure to 1.2.29
exclusion ini4j and lucene
---
 fe/fe-core/pom.xml | 19 +++++++++++++++++++
 fe/pom.xml         | 30 +++++++++++++++++++++---------
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 70678c4262de33..1b40974d7e8b19 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -347,6 +347,7 @@ under the License.
             <groupId>com.alibaba.otter</groupId>
             <artifactId>canal.client</artifactId>
             <version>1.1.6</version>
+            <scope>provided</scope>
             <exclusions>
                 <exclusion>
                     <groupId>ch.qos.logback</groupId>
@@ -410,6 +411,7 @@ under the License.
             <groupId>com.huaweicloud</groupId>
             <artifactId>hadoop-huaweicloud</artifactId>
             <version>${huaweiobs.version}</version>
+            <scope>${obs.dependency.scope}</scope>
             <exclusions>
                 <exclusion>
                     <artifactId>jackson-databind</artifactId>
@@ -421,6 +423,7 @@ under the License.
             <groupId>com.qcloud.cos</groupId>
             <artifactId>hadoop-cos</artifactId>
             <version>${tencentcos.version}</version>
+            <scope>${cos.dependency.scope}</scope>
             <exclusions>
                 <exclusion>
                     <groupId>org.json</groupId>
@@ -758,6 +761,20 @@ under the License.
             <groupId>com.qcloud</groupId>
             <artifactId>cos_api</artifactId>
             <version>5.6.211</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.thoughtworks.xstream</groupId>
+                    <artifactId>xstream</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcprov-jdk15on</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>com.qcloud</groupId>
@@ -768,6 +785,7 @@ under the License.
             <groupId>com.huaweicloud</groupId>
             <artifactId>esdk-obs-java-bundle</artifactId>
             <version>[3.21.11,)</version>
+            <scope>${obs.dependency.scope}</scope>
         </dependency>
         <dependency>
             <groupId>com.baidubce</groupId>
@@ -791,6 +809,7 @@ under the License.
             <artifactId>gcs-connector</artifactId>
             <version>hadoop3-2.2.21</version>
             <classifier>shaded</classifier>
+            <scope>${gcs.dependency.scope}</scope>
             <exclusions>
                 <exclusion>
                     <groupId>*</groupId>
diff --git a/fe/pom.xml b/fe/pom.xml
index 0d2e3f70aa6e6a..c0f78b4423d0ba 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -222,13 +222,20 @@ under the License.
         <module>be-java-extensions</module>
     </modules>
     <properties>
+        <doris.hive.catalog.shade.version>2.1.1</doris.hive.catalog.shade.version>
+        <avro.version>1.11.4</avro.version>
+        <parquet.version>1.13.1</parquet.version>
+        <spark.version>3.4.3</spark.version>
+        <hudi.version>0.15.0</hudi.version>
+        <obs.dependency.scope>compile</obs.dependency.scope>
+        <cos.dependency.scope>compile</cos.dependency.scope>
+        <gcs.dependency.scope>compile</gcs.dependency.scope>
         <!--suppress UnresolvedMavenProperty -->
         <doris.thirdparty>${fe.dir}/../thirdparty</doris.thirdparty>
         <!--suppress UnresolvedMavenProperty -->
         <doris.home>${fe.dir}/../</doris.home>
         <revision>1.2-SNAPSHOT</revision>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <doris.hive.catalog.shade.version>2.1.1</doris.hive.catalog.shade.version>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
         <!--plugin parameters-->
@@ -248,7 +255,7 @@ under the License.
         <commons-text.version>1.10.0</commons-text.version>
         <commons-validator.version>1.7</commons-validator.version>
         <gson.version>2.10.1</gson.version>
-        <guava.version>32.1.2-jre</guava.version>
+        <guava.version>33.2.1-jre</guava.version>
         <jackson.version>2.16.0</jackson.version>
         <java-cup.version>0.11-a-czt02-cdh</java-cup.version>
         <javassist.version>3.18.2-GA</javassist.version>
@@ -301,7 +308,6 @@ under the License.
         <javax.activation.version>1.2.0</javax.activation.version>
         <jaxws-api.version>2.3.0</jaxws-api.version>
         <RoaringBitmap.version>0.8.13</RoaringBitmap.version>
-        <spark.version>3.4.3</spark.version>
         <hudi-spark.version>hudi-spark3.4.x</hudi-spark.version>
         <hive.version>3.1.3</hive.version>
         <hive.common.version>2.3.9</hive.common.version>
@@ -314,16 +320,13 @@ under the License.
          you can find avro version info in iceberg mvn repository -->
         <iceberg.version>1.4.3</iceberg.version>
         <maxcompute.version>0.49.0-public</maxcompute.version>
-        <avro.version>1.11.4</avro.version>
         <arrow.version>17.0.0</arrow.version>
-        <!-- hudi -->
-        <hudi.version>0.15.0</hudi.version>
         <presto.hadoop.version>2.7.4-11</presto.hadoop.version>
         <presto.hive.version>3.0.0-8</presto.hive.version>
         <!-- lakesoul -->
         <lakesoul.version>2.6.2</lakesoul.version>
 
-        <parquet.version>1.13.1</parquet.version>
+
         <commons-collections.version>3.2.2</commons-collections.version>
         <commons-compress.version>1.27.1</commons-compress.version>
         <scala.version>2.12.10</scala.version>
@@ -346,7 +349,7 @@ under the License.
         <spring.version>2.7.18</spring.version>
         <spring-framework.version>5.3.39</spring-framework.version>
         <orc.version>1.8.4</orc.version>
-        <zookeeper.version>3.9.2</zookeeper.version>
+        <zookeeper.version>3.9.3</zookeeper.version>
         <velocity-engine-core.version>2.4</velocity-engine-core.version>
         <ranger-plugins-common.version>2.4.0</ranger-plugins-common.version>
         <bcprov-jdk15on.version>1.70</bcprov-jdk15on.version>
@@ -372,7 +375,7 @@ under the License.
         <jakarta.annotation-api.version>2.1.1</jakarta.annotation-api.version>
         <asm.version>9.4</asm.version>
         <airlift.concurrent.version>202</airlift.concurrent.version>
-        <azure.sdk.version>1.2.27</azure.sdk.version>
+        <azure.sdk.version>1.2.29</azure.sdk.version>
         <azure.sdk.batch.version>12.22.0</azure.sdk.batch.version>
         <semver4j.version>5.3.0</semver4j.version>
         <aliyun-sdk-oss.version>3.15.0</aliyun-sdk-oss.version>
@@ -1319,6 +1322,10 @@ under the License.
                         <groupId>org.apache.arrow</groupId>
                         <artifactId>arrow-vector</artifactId>
                     </exclusion>
+                    <exclusion>
+                        <groupId>org.ini4j</groupId>
+                        <artifactId>ini4j</artifactId>
+                    </exclusion>
                 </exclusions>
             </dependency>
             <!-- For Iceberg, must be consistent with Iceberg version -->
@@ -1651,6 +1658,11 @@ under the License.
                 <artifactId>concurrent</artifactId>
                 <version>${airlift.concurrent.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.bouncycastle</groupId>
+                <artifactId>bcprov-jdk15on</artifactId>
+                <scope>test</scope>
+            </dependency>
             <dependency>
                 <groupId>com.azure</groupId>
                 <artifactId>azure-sdk-bom</artifactId>

From eab8d9f454f8da8668f75b7f5e790bea6c6d85b2 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Thu, 12 Dec 2024 17:22:40 +0800
Subject: [PATCH 302/399] [fix](backup) Fix read compressed backup job from old
 version (#45343)

---
 fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 1c897368624f11..de12670807f20e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -1126,7 +1126,7 @@ public void readFields(DataInput in) throws IOException {
             ByteArrayInputStream byteStream = new ByteArrayInputStream(text.getBytes());
             try (GZIPInputStream gzipStream = new GZIPInputStream(byteStream)) {
                 try (DataInputStream stream = new DataInputStream(gzipStream)) {
-                    readOthers(in);
+                    readOthers(stream);
                 }
             }
         } else {

From a288b825a8374ce548456b1496916e5349c59d77 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Thu, 12 Dec 2024 19:29:53 +0800
Subject: [PATCH 303/399] [chore](config) Enable batch download by default
 (#45324)

---
 be/src/common/config.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index c3d00e23c98e5e..08f1ca7dd58c5c 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -251,7 +251,7 @@ DEFINE_mInt32(download_low_speed_limit_kbps, "50");
 // download low speed time(seconds)
 DEFINE_mInt32(download_low_speed_time, "300");
 // whether to download small files in batch
-DEFINE_mBool(enable_batch_download, "false");
+DEFINE_mBool(enable_batch_download, "true");
 
 DEFINE_String(sys_log_dir, "");
 DEFINE_String(user_function_dir, "${DORIS_HOME}/lib/udf");

From 09619fc03da5d87053325dc2f4710a8eba3a5a59 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Thu, 12 Dec 2024 19:34:29 +0800
Subject: [PATCH 304/399] [fix](persist) Fix
 TableAddOrDropInvertedIndicesInfo.equals and remove useless logs (#45335)

---
 .../src/main/java/org/apache/doris/persist/EditLog.java       | 4 ----
 .../doris/persist/TableAddOrDropInvertedIndicesInfo.java      | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index 73a2c6338da5d7..65a161157c8967 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -2086,15 +2086,11 @@ public void logModifyTableAddOrDropColumns(TableAddOrDropColumnsInfo info) {
 
     public void logModifyTableAddOrDropInvertedIndices(TableAddOrDropInvertedIndicesInfo info) {
         long logId = logEdit(OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_INVERTED_INDICES, info);
-        LOG.info("walter log modify table add or drop inverted indices, infos: {}, json: {}",
-                info, info.toJson(), new RuntimeException("test"));
         Env.getCurrentEnv().getBinlogManager().addModifyTableAddOrDropInvertedIndices(info, logId);
     }
 
     public void logIndexChangeJob(IndexChangeJob indexChangeJob) {
         long logId = logEdit(OperationType.OP_INVERTED_INDEX_JOB, indexChangeJob);
-        LOG.info("walter log inverted index job, infos: {}, json: {}",
-                indexChangeJob, indexChangeJob.toJson(), new RuntimeException("test"));
         Env.getCurrentEnv().getBinlogManager().addIndexChangeJob(indexChangeJob, logId);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropInvertedIndicesInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropInvertedIndicesInfo.java
index 39a90046d2419e..effe33ce624dd1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropInvertedIndicesInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropInvertedIndicesInfo.java
@@ -120,7 +120,7 @@ public boolean equals(Object obj) {
 
         TableAddOrDropInvertedIndicesInfo info = (TableAddOrDropInvertedIndicesInfo) obj;
 
-        return (dbId == info.dbId && tableId == tableId
+        return (dbId == info.dbId && tableId == info.tableId
                 && indexSchemaMap.equals(info.indexSchemaMap)
                 && indexes.equals(info.indexes)
                 && alterInvertedIndexes.equals(info.alterInvertedIndexes)

From c0f4c89c97071e201e662130c5dcec7e4dac4d69 Mon Sep 17 00:00:00 2001
From: abmdocrt <lianyukang@selectdb.com>
Date: Thu, 12 Dec 2024 22:51:26 +0800
Subject: [PATCH 305/399] [Enhancement](Test) Add test config to recycler
 (#44761)

---
 cloud/src/common/config.h           |  4 +++
 cloud/src/recycler/meta_checker.cpp | 36 ++++++++++++++++++++++---
 cloud/src/recycler/recycler.cpp     | 41 ++++++++++++++++++++---------
 3 files changed, 65 insertions(+), 16 deletions(-)

diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index ac4403aeb24a3c..5befb0f8b2105e 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -87,6 +87,10 @@ CONF_mInt64(recycler_sleep_before_scheduling_seconds, "60");
 // log a warning if a recycle task takes longer than this duration
 CONF_mInt64(recycle_task_threshold_seconds, "10800"); // 3h
 
+// force recycler to recycle all useless object.
+// **just for TEST**
+CONF_Bool(force_immediate_recycle, "false");
+
 CONF_String(test_s3_ak, "");
 CONF_String(test_s3_sk, "");
 CONF_String(test_s3_endpoint, "");
diff --git a/cloud/src/recycler/meta_checker.cpp b/cloud/src/recycler/meta_checker.cpp
index 522015555de825..f1223068d4be6e 100644
--- a/cloud/src/recycler/meta_checker.cpp
+++ b/cloud/src/recycler/meta_checker.cpp
@@ -25,6 +25,7 @@
 #include <chrono>
 #include <set>
 
+#include "common/logging.h"
 #include "common/util.h"
 #include "meta-service/keys.h"
 #include "meta-service/txn_kv.h"
@@ -54,6 +55,7 @@ struct PartitionInfo {
     int64_t db_id;
     int64_t table_id;
     int64_t partition_id;
+    int64_t tablet_id;
     int64_t visible_version;
 };
 
@@ -173,6 +175,9 @@ bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) {
                     MYSQL_ROW row = mysql_fetch_row(result);
                     TabletInfo tablet_info = {0};
                     tablet_info.tablet_id = atoll(row[0]);
+                    VLOG_DEBUG << "get tablet info log"
+                               << ", db name" << elem.first << ", table name" << table
+                               << ",tablet id" << tablet_info.tablet_id;
                     tablet_info.schema_version = atoll(row[4]);
                     tablets.push_back(std::move(tablet_info));
                 }
@@ -201,6 +206,13 @@ bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) {
                 partition_info.db_id = atoll(row[4]);
                 partition_info.table_id = atoll(row[5]);
                 partition_info.partition_id = atoll(row[6]);
+                partition_info.tablet_id = tablet_info.tablet_id;
+                VLOG_DEBUG << "get partition info log"
+                           << ", db id" << partition_info.db_id << ", table id"
+                           << partition_info.table_id << ", partition id"
+                           << partition_info.partition_id << ", tablet id"
+                           << partition_info.tablet_id;
+
                 partitions.insert({partition_info.partition_id, std::move(partition_info)});
             }
         }
@@ -217,9 +229,16 @@ bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) {
                 int num_row = mysql_num_rows(result);
                 for (int i = 0; i < num_row; ++i) {
                     MYSQL_ROW row = mysql_fetch_row(result);
-                    int partition_id = atoll(row[0]);
-                    int visible_version = atoll(row[2]);
+                    int64_t partition_id = atoll(row[0]);
+                    int64_t visible_version = atoll(row[2]);
                     partitions[partition_id].visible_version = visible_version;
+                    VLOG_DEBUG << "get partition version log"
+                               << ", db name" << elem.first << ", table name" << table
+                               << ", raw partition id" << row[0] << ", first partition id"
+                               << partition_id << ", db id" << partitions[partition_id].db_id
+                               << ", table id" << partitions[partition_id].table_id
+                               << ", second partition id" << partitions[partition_id].partition_id
+                               << ", tablet id" << partitions[partition_id].tablet_id;
                 }
             }
             mysql_free_result(result);
@@ -354,14 +373,23 @@ bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) {
         int64_t db_id = elem.second.db_id;
         int64_t table_id = elem.second.table_id;
         int64_t partition_id = elem.second.partition_id;
+        int64_t tablet_id = elem.second.tablet_id;
         std::string ver_key = partition_version_key({instance_id_, db_id, table_id, partition_id});
         std::string ver_val;
         err = txn->get(ver_key, &ver_val);
         if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
-            LOG(WARNING) << "version key not found, partition id: " << partition_id;
+            LOG_WARNING("version key not found.")
+                    .tag("db id", db_id)
+                    .tag("table id", table_id)
+                    .tag("partition id", partition_id)
+                    .tag("tablet id", tablet_id);
             return false;
         } else if (err != TxnErrorCode::TXN_OK) {
-            LOG(WARNING) << "failed to get version: " << partition_id;
+            LOG_WARNING("failed to get version.")
+                    .tag("db id", db_id)
+                    .tag("table id", table_id)
+                    .tag("partition id", partition_id)
+                    .tag("tablet id", tablet_id);
             return false;
         }
 
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index f7000ea379225f..6877d7e433b253 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -24,6 +24,7 @@
 
 #include <atomic>
 #include <chrono>
+#include <cstdint>
 #include <deque>
 #include <string>
 #include <string_view>
@@ -747,7 +748,10 @@ int InstanceRecycler::recycle_indexes() {
                 .tag("num_recycled", num_recycled);
     });
 
-    auto calc_expiration = [](const RecycleIndexPB& index) {
+    auto calc_expiration = [](const RecycleIndexPB& index) -> int64_t {
+        if (config::force_immediate_recycle) {
+            return 0;
+        }
         int64_t expiration = index.expiration() > 0 ? index.expiration() : index.creation_time();
         int64_t retention_seconds = config::retention_seconds;
         if (index.state() == RecycleIndexPB::DROPPED) {
@@ -942,7 +946,10 @@ int InstanceRecycler::recycle_partitions() {
                 .tag("num_recycled", num_recycled);
     });
 
-    auto calc_expiration = [](const RecyclePartitionPB& partition) {
+    auto calc_expiration = [](const RecyclePartitionPB& partition) -> int64_t {
+        if (config::force_immediate_recycle) {
+            return 0;
+        }
         int64_t expiration =
                 partition.expiration() > 0 ? partition.expiration() : partition.creation_time();
         int64_t retention_seconds = config::retention_seconds;
@@ -1686,7 +1693,10 @@ int InstanceRecycler::recycle_rowsets() {
         return 0;
     };
 
-    auto calc_expiration = [](const RecycleRowsetPB& rs) {
+    auto calc_expiration = [](const RecycleRowsetPB& rs) -> int64_t {
+        if (config::force_immediate_recycle) {
+            return 0;
+        }
         // RecycleRowsetPB created by compacted or dropped rowset has no expiration time, and will be recycled when exceed retention time
         int64_t expiration = rs.expiration() > 0 ? rs.expiration() : rs.creation_time();
         int64_t retention_seconds = config::retention_seconds;
@@ -1923,8 +1933,9 @@ int InstanceRecycler::recycle_tmp_rowsets() {
         // ATTN: `txn_expiration` should > 0, however we use `creation_time` + a large `retention_time` (> 1 day in production environment)
         //  when `txn_expiration` <= 0 in some unexpected situation (usually when there are bugs). This is usually safe, coz loading
         //  duration or timeout always < `retention_time` in practice.
-        int64_t expiration =
-                rowset.txn_expiration() > 0 ? rowset.txn_expiration() : rowset.creation_time();
+        int64_t expiration = config::force_immediate_recycle ? 0
+                             : rowset.txn_expiration() > 0   ? rowset.txn_expiration()
+                                                             : rowset.creation_time();
         VLOG_DEBUG << "recycle tmp rowset scan, key=" << hex(k) << " num_scanned=" << num_scanned
                    << " num_expired=" << num_expired << " expiration=" << expiration
                    << " txn_expiration=" << rowset.txn_expiration()
@@ -2106,7 +2117,7 @@ int InstanceRecycler::abort_timeout_txn() {
                 LOG_WARNING("malformed txn_running_pb").tag("key", hex(k));
                 return -1;
             }
-            if (txn_running_pb.timeout_time() > current_time) {
+            if (!config::force_immediate_recycle && txn_running_pb.timeout_time() > current_time) {
                 return 0;
             }
             ++num_timeout;
@@ -2196,7 +2207,8 @@ int InstanceRecycler::recycle_expired_txn_label() {
             LOG_WARNING("malformed txn_running_pb").tag("key", hex(k));
             return -1;
         }
-        if ((recycle_txn_pb.has_immediate() && recycle_txn_pb.immediate()) ||
+        if ((config::force_immediate_recycle) ||
+            (recycle_txn_pb.has_immediate() && recycle_txn_pb.immediate()) ||
             (recycle_txn_pb.creation_time() + config::label_keep_max_second * 1000L <=
              current_time)) {
             LOG_INFO("found recycle txn").tag("key", hex(k));
@@ -2492,14 +2504,16 @@ int InstanceRecycler::recycle_copy_jobs() {
                 int64_t current_time =
                         duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
                 if (copy_job.finish_time_ms() > 0) {
-                    if (current_time <
-                        copy_job.finish_time_ms() + config::copy_job_max_retention_second * 1000) {
+                    if (!config::force_immediate_recycle &&
+                        current_time < copy_job.finish_time_ms() +
+                                               config::copy_job_max_retention_second * 1000) {
                         return 0;
                     }
                 } else {
                     // For compatibility, copy job does not contain finish time before 2.2.2, use start time
-                    if (current_time <
-                        copy_job.start_time_ms() + config::copy_job_max_retention_second * 1000) {
+                    if (!config::force_immediate_recycle &&
+                        current_time < copy_job.start_time_ms() +
+                                               config::copy_job_max_retention_second * 1000) {
                         return 0;
                     }
                 }
@@ -2508,7 +2522,7 @@ int InstanceRecycler::recycle_copy_jobs() {
             int64_t current_time =
                     duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
             // if copy job is timeout: delete all copy file kvs and copy job kv
-            if (current_time <= copy_job.timeout_time_ms()) {
+            if (!config::force_immediate_recycle && current_time <= copy_job.timeout_time_ms()) {
                 return 0;
             }
             ++num_expired;
@@ -2796,6 +2810,9 @@ int InstanceRecycler::recycle_expired_stage_objects() {
         int64_t expiration_time =
                 duration_cast<seconds>(system_clock::now().time_since_epoch()).count() -
                 config::internal_stage_objects_expire_time_second;
+        if (config::force_immediate_recycle) {
+            expiration_time = INT64_MAX;
+        }
         ret1 = accessor->delete_all(expiration_time);
         if (ret1 != 0) {
             LOG(WARNING) << "failed to recycle expired stage objects, instance_id=" << instance_id_

From 735db65d23e4f1042e70bd3a841308ef74b53b70 Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Fri, 13 Dec 2024 09:56:53 +0800
Subject: [PATCH 306/399] [regression](paimon) test Paimon's connectivity of
 cloud filesystems (#45268)

### What problem does this PR solve?

Problem Summary:
Add some cases to test Paimon's connectivity of oss, obs, cos and cosn
---
 .../paimon/paimon_base_filesystem.out         | 29 +++++----
 .../paimon/paimon_base_filesystem.groovy      | 65 +++++++++++++++----
 2 files changed, 68 insertions(+), 26 deletions(-)

diff --git a/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out b/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out
index d24ebd07ef6b16..6ce16ab2a38a2d 100644
--- a/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out
+++ b/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out
@@ -1,17 +1,22 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
--- !c1 --
-1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	2023-08-10T14:35:38.768
-1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	2023-08-10T15:58:54.364
+-- !oss --
+1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
 
--- !c2 --
-7	Hugo
-8	Stop
+-- !obs --
+1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
 
--- !c3 --
-1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	2023-08-10T14:35:38.768
-1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	2023-08-10T15:58:54.364
+-- !cos --
+1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
 
--- !c4 --
-7	Hugo
-8	Stop
+-- !cosn --
+1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
+
+-- !oss --
+1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
+
+-- !obs --
+1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
+
+-- !cos --
+1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
 
diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
index 1e4561c406556f..6ac76336986288 100644
--- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
+++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
@@ -17,29 +17,48 @@
 
 suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_docker_doris") {
     String enabled = context.config.otherConfigs.get("enablePaimonTest")
-    // if (enabled == null || !enabled.equalsIgnoreCase("true")) {
-    if (true) {
-        // temporary comment out, will add back when env is ready
+
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         return
     }
 
     try {
-        String catalog_cos = "paimon_base_filesystem_paimon_cos"
+        String catalog_obs = "paimon_base_filesystem_paimon_obs"
         String catalog_oss = "paimon_base_filesystem_paimon_oss"
+        String catalog_cos = "paimon_base_filesystem_paimon_cos"
+        String catalog_cosn = "paimon_base_filesystem_paimon_cosn"
         String aliYunAk = context.config.otherConfigs.get("aliYunAk")
         String aliYunSk = context.config.otherConfigs.get("aliYunSk")
+        String hwYunAk = context.config.otherConfigs.get("hwYunAk")
+        String hwYunSk = context.config.otherConfigs.get("hwYunSk")
         String txYunAk = context.config.otherConfigs.get("txYunAk")
         String txYunSk = context.config.otherConfigs.get("txYunSk")
 
-        def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cos}.zd.all_table order by c18"""
-        def oss = """select * from ${catalog_oss}.paimonossdb1.test_tableoss order by a"""
+        def obs = """select * from ${catalog_obs}.db1.all_table limit 1;"""
+        def oss = """select * from ${catalog_oss}.db1.all_table limit 1;"""
+        def cos = """select * from ${catalog_cos}.db1.all_table limit 1;"""
+        def cosn = """select * from ${catalog_cosn}.db1.all_table limit 1;"""
 
-        sql """drop catalog if exists ${catalog_cos};"""
+        sql """drop catalog if exists ${catalog_obs};"""
         sql """drop catalog if exists ${catalog_oss};"""
+        sql """drop catalog if exists ${catalog_cos};"""
+        sql """drop catalog if exists ${catalog_cosn};"""
+
         sql """
             create catalog if not exists ${catalog_cos} properties (
                 "type" = "paimon",
-                "warehouse" = "cosn://doris-build-1308700295/regression/paimoncos",
+                "paimon.catalog.type"="filesystem",
+                "warehouse" = "s3://doris-build-1308700295/regression/paimon1",
+                "s3.access_key" = "${txYunAk}",
+                "s3.secret_key" = "${txYunSk}",
+                "s3.endpoint" = "cos.ap-beijing.myqcloud.com"
+            );
+        """
+        sql """
+            create catalog if not exists ${catalog_cosn} properties (
+                "type" = "paimon",
+                "paimon.catalog.type"="filesystem",
+                "warehouse" = "cosn://doris-build-1308700295/regression/paimon1",
                 "cos.access_key" = "${txYunAk}",
                 "cos.secret_key" = "${txYunSk}",
                 "cos.endpoint" = "cos.ap-beijing.myqcloud.com"
@@ -48,22 +67,40 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock
         sql """
             create catalog if not exists ${catalog_oss} properties (
                 "type" = "paimon",
-                "warehouse" = "oss://doris-regression-bj/regression/paimonoss",
+                "paimon.catalog.type"="filesystem",
+                "warehouse" = "oss://doris-regression-bj/regression/paimon1",
                 "oss.access_key"="${aliYunAk}",
                 "oss.secret_key"="${aliYunSk}",
                 "oss.endpoint"="oss-cn-beijing.aliyuncs.com"
             );
         """
-        logger.info("catalog " + catalog_cos + " created")
+        sql """
+            create catalog if not exists ${catalog_obs} properties (
+                "type" = "paimon",
+                "paimon.catalog.type"="filesystem",
+                "warehouse" = "obs://doris-build/regression/paimon1",
+                "obs.access_key"="${hwYunAk}",
+                "obs.secret_key"="${hwYunSk}",
+                "obs.endpoint"="obs.cn-north-4.myhuaweicloud.com"
+            );
+        """
+        logger.info("catalog " + catalog_obs + " created")
         logger.info("catalog " + catalog_oss + " created")
+        logger.info("catalog " + catalog_cos + " created")
+        logger.info("catalog " + catalog_cosn + " created")
 
         sql """set force_jni_scanner=false"""
-        qt_c1 cos
-        qt_c2 oss
+        qt_oss oss
+        qt_obs obs
+        qt_cos cos
+        qt_cosn cosn
 
         sql """set force_jni_scanner=true"""
-        qt_c3 cos
-        qt_c4 oss
+        qt_oss oss
+        qt_obs obs
+        qt_cos cos
+        // java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.CosFileSystem not found
+        // qt_cosn cosn
 
     } finally {
         sql """set force_jni_scanner=false"""

From c7fefc367f992ee54ad08c8115d929f5ca4c9ba1 Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Fri, 13 Dec 2024 09:57:41 +0800
Subject: [PATCH 307/399] [fix](hive) fix block decompressor bug (#45289)

### What problem does this PR solve?

Problem Summary:
In the block decompressor, when it is found that the input data is less
than 4 bytes (the header size of the large block), should set
more_input_bytes instead of reporting an error.
---
 be/src/exec/decompressor.cpp | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/be/src/exec/decompressor.cpp b/be/src/exec/decompressor.cpp
index 9365bb00288db1..5da2e6acbb9bdf 100644
--- a/be/src/exec/decompressor.cpp
+++ b/be/src/exec/decompressor.cpp
@@ -492,15 +492,15 @@ Status Lz4BlockDecompressor::decompress(uint8_t* input, size_t input_len, size_t
     auto* output_ptr = output;
 
     while (input_len > 0) {
-        //if faild ,  fall back to large block begin
-        auto* large_block_input_ptr = input_ptr;
-        auto* large_block_output_ptr = output_ptr;
-
         if (input_len < sizeof(uint32_t)) {
-            return Status::InvalidArgument(strings::Substitute(
-                    "fail to do hadoop-lz4 decompress, input_len=$0", input_len));
+            *more_input_bytes = sizeof(uint32_t) - input_len;
+            break;
         }
 
+        //if faild, fall back to large block begin
+        auto* large_block_input_ptr = input_ptr;
+        auto* large_block_output_ptr = output_ptr;
+
         uint32_t remaining_decompressed_large_block_len = BigEndian::Load32(input_ptr);
 
         input_ptr += sizeof(uint32_t);
@@ -609,15 +609,15 @@ Status SnappyBlockDecompressor::decompress(uint8_t* input, size_t input_len,
     auto* output_ptr = output;
 
     while (input_len > 0) {
-        //if faild ,  fall back to large block begin
-        auto* large_block_input_ptr = input_ptr;
-        auto* large_block_output_ptr = output_ptr;
-
         if (input_len < sizeof(uint32_t)) {
-            return Status::InvalidArgument(strings::Substitute(
-                    "fail to do hadoop-snappy decompress, input_len=$0", input_len));
+            *more_input_bytes = sizeof(uint32_t) - input_len;
+            break;
         }
 
+        //if faild, fall back to large block begin
+        auto* large_block_input_ptr = input_ptr;
+        auto* large_block_output_ptr = output_ptr;
+
         uint32_t remaining_decompressed_large_block_len = BigEndian::Load32(input_ptr);
 
         input_ptr += sizeof(uint32_t);

From b9015f57fddd95c8b1ee57e303ce2dde0353fae7 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@selectdb.com>
Date: Fri, 13 Dec 2024 11:33:11 +0800
Subject: [PATCH 308/399] [Improment]add be avail cpu num metric (#45284)

---
 be/src/common/config.cpp                      |   2 +-
 be/src/common/config.h                        |   2 +-
 be/src/common/daemon.cpp                      |   2 +
 be/src/util/cgroup_util.cpp                   | 167 ++++++++++++++++++
 be/src/util/cgroup_util.h                     |  22 +++
 be/src/util/cpu_info.cpp                      |  55 +-----
 be/src/util/system_metrics.cpp                |  13 ++
 be/src/util/system_metrics.h                  |   2 +
 be/test/util/cgroup_util_test.cpp             |  90 ++++++++++
 .../util/test_data/cgroup_cpu_data/cpuset1    |   1 +
 .../util/test_data/cgroup_cpu_data/cpuset2    |   1 +
 .../util/test_data/cgroup_cpu_data/cpuset3    |   1 +
 .../cgroup_cpu_data/test11/child/cpu.max      |   1 +
 .../test_data/cgroup_cpu_data/test11/cpu.max  |   1 +
 .../cgroup_cpu_data/test12/child/cpu.max      |   1 +
 .../test_data/cgroup_cpu_data/test12/cpu.max  |   1 +
 .../cgroup_cpu_data/test13/child/cpu.max      |   1 +
 .../test_data/cgroup_cpu_data/test13/cpu.max  |   1 +
 .../cgroup_cpu_data/test14/child/cpu.max      |   1 +
 .../test_data/cgroup_cpu_data/test14/cpu.max  |   1 +
 .../test21/child/cpuset.cpus.effective        |   1 +
 .../test21/cpuset.cpus.effective              |   1 +
 .../test22/child/cpuset.cpus.effective        |   0
 .../test22/cpuset.cpus.effective              |   1 +
 .../test31/child/cpu.cfs_period_us            |   1 +
 .../test31/child/cpu.cfs_quota_us             |   1 +
 .../cgroup_cpu_data/test31/cpu.cfs_period_us  |   1 +
 .../cgroup_cpu_data/test31/cpu.cfs_quota_us   |   1 +
 .../test32/child/cpu.cfs_period_us            |   1 +
 .../test32/child/cpu.cfs_quota_us             |   1 +
 .../cgroup_cpu_data/test32/cpu.cfs_period_us  |   1 +
 .../cgroup_cpu_data/test32/cpu.cfs_quota_us   |   1 +
 .../test33/child/cpu.cfs_period_us            |   1 +
 .../test33/child/cpu.cfs_quota_us             |   1 +
 .../cgroup_cpu_data/test33/cpu.cfs_period_us  |   1 +
 .../cgroup_cpu_data/test33/cpu.cfs_quota_us   |   1 +
 .../cgroup_cpu_data/test41/cpuset.cpus        |   1 +
 37 files changed, 327 insertions(+), 55 deletions(-)
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/cpuset1
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/cpuset2
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/cpuset3
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test11/child/cpu.max
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test11/cpu.max
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test12/child/cpu.max
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test12/cpu.max
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test13/child/cpu.max
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test13/cpu.max
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test14/child/cpu.max
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test14/cpu.max
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test21/child/cpuset.cpus.effective
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test21/cpuset.cpus.effective
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test22/child/cpuset.cpus.effective
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test22/cpuset.cpus.effective
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test31/child/cpu.cfs_period_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test31/child/cpu.cfs_quota_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test31/cpu.cfs_period_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test31/cpu.cfs_quota_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test32/child/cpu.cfs_period_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test32/child/cpu.cfs_quota_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test32/cpu.cfs_period_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test32/cpu.cfs_quota_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test33/child/cpu.cfs_period_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test33/child/cpu.cfs_quota_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test33/cpu.cfs_period_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test33/cpu.cfs_quota_us
 create mode 100644 be/test/util/test_data/cgroup_cpu_data/test41/cpuset.cpus

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 08f1ca7dd58c5c..b70f492a29dfda 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1211,7 +1211,7 @@ DEFINE_Bool(exit_on_exception, "false");
 DEFINE_Bool(enable_flush_file_cache_async, "true");
 
 // cgroup
-DEFINE_mString(doris_cgroup_cpu_path, "");
+DEFINE_String(doris_cgroup_cpu_path, "");
 
 DEFINE_mBool(enable_be_proc_monitor, "false");
 DEFINE_mInt32(be_proc_monitor_interval_ms, "10000");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index c0b2e19b49a6be..984024cab7c77f 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1292,7 +1292,7 @@ DECLARE_mInt32(tablet_schema_cache_capacity);
 DECLARE_mBool(exit_on_exception);
 
 // cgroup
-DECLARE_mString(doris_cgroup_cpu_path);
+DECLARE_String(doris_cgroup_cpu_path);
 DECLARE_mBool(enable_be_proc_monitor);
 DECLARE_mInt32(be_proc_monitor_interval_ms);
 DECLARE_Int32(workload_group_metrics_interval_ms);
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index 73035ecf3957eb..12bf1749a5694d 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -437,6 +437,8 @@ void Daemon::calculate_metrics_thread() {
                 // update lst map
                 DorisMetrics::instance()->system_metrics()->get_network_traffic(
                         &lst_net_send_bytes, &lst_net_receive_bytes);
+
+                DorisMetrics::instance()->system_metrics()->update_be_avail_cpu_num();
             }
             update_rowsets_and_segments_num_metrics();
         }
diff --git a/be/src/util/cgroup_util.cpp b/be/src/util/cgroup_util.cpp
index 8f64fe699c6062..fc35be3dc35931 100644
--- a/be/src/util/cgroup_util.cpp
+++ b/be/src/util/cgroup_util.cpp
@@ -218,6 +218,10 @@ std::optional<std::string> CGroupUtil::get_cgroupsv2_path(const std::string& sub
 Status CGroupUtil::read_int_line_from_cgroup_file(const std::filesystem::path& file_path,
                                                   int64_t* val) {
     std::ifstream file_stream(file_path, std::ios::in);
+    if (!file_stream.is_open()) {
+        return Status::CgroupError("Error open {}", file_path.string());
+    }
+
     string line;
     getline(file_stream, line);
     if (file_stream.fail() || file_stream.bad()) {
@@ -264,4 +268,167 @@ void CGroupUtil::read_int_metric_from_cgroup_file(
     }
 }
 
+Status CGroupUtil::read_string_line_from_cgroup_file(const std::filesystem::path& file_path,
+                                                     std::string* line_ptr) {
+    std::ifstream file_stream(file_path, std::ios::in);
+    if (!file_stream.is_open()) {
+        return Status::CgroupError("Error open {}", file_path.string());
+    }
+    string line;
+    getline(file_stream, line);
+    if (file_stream.fail() || file_stream.bad()) {
+        return Status::CgroupError("Error reading {}: {}", file_path.string(), get_str_err_msg());
+    }
+    *line_ptr = line;
+    return Status::OK();
+}
+
+Status CGroupUtil::parse_cpuset_line(std::string cpuset_line, int* cpu_count_ptr) {
+    if (cpuset_line.empty()) {
+        return Status::CgroupError("cpuset line is empty");
+    }
+    std::vector<string> ranges;
+    boost::split(ranges, cpuset_line, boost::is_any_of(","));
+    int cpu_count = 0;
+
+    for (const std::string& range : ranges) {
+        std::vector<std::string> cpu_values;
+        boost::split(cpu_values, range, boost::is_any_of("-"));
+
+        if (cpu_values.size() == 2) {
+            int start = std::stoi(cpu_values[0]);
+            int end = std::stoi(cpu_values[1]);
+            cpu_count += (end - start) + 1;
+        } else {
+            cpu_count++;
+        }
+    }
+    *cpu_count_ptr = cpu_count;
+    return Status::OK();
+}
+
+int CGroupUtil::get_cgroup_limited_cpu_number(int physical_cores) {
+    if (physical_cores <= 0) {
+        return physical_cores;
+    }
+    int ret = physical_cores;
+#if defined(OS_LINUX)
+    // For cgroup v2
+    // Child cgroup's cpu.max may bigger than parent group's cpu.max,
+    //      so it should look up from current cgroup to top group.
+    // For cpuset, child cgroup's cpuset.cpus could not bigger thant parent's cpuset.cpus.
+    if (CGroupUtil::cgroupsv2_enable()) {
+        std::string cgroupv2_process_path = CGroupUtil::cgroupv2_of_process();
+        if (cgroupv2_process_path.empty()) {
+            return ret;
+        }
+        std::filesystem::path current_cgroup_path = (default_cgroups_mount / cgroupv2_process_path);
+        ret = get_cgroup_v2_cpu_quota_number(current_cgroup_path, default_cgroups_mount, ret);
+
+        current_cgroup_path = (default_cgroups_mount / cgroupv2_process_path);
+        ret = get_cgroup_v2_cpuset_number(current_cgroup_path, default_cgroups_mount, ret);
+    } else if (CGroupUtil::cgroupsv1_enable()) {
+        // cpu quota, should find first not empty config from current path to top.
+        // because if a process attach to current cgroup, its cpu quota may not be set.
+        std::string cpu_quota_path = "";
+        Status cpu_quota_ret = CGroupUtil::find_abs_cgroupv1_path("cpu", &cpu_quota_path);
+        if (cpu_quota_ret.ok() && !cpu_quota_path.empty()) {
+            std::filesystem::path current_cgroup_path = cpu_quota_path;
+            ret = get_cgroup_v1_cpu_quota_number(current_cgroup_path, default_cgroups_mount, ret);
+        }
+
+        //cpuset
+        // just lookup current process cgroup path is enough
+        // because if a process attach to current cgroup, its cpuset.cpus must be set.
+        std::string cpuset_path = "";
+        Status cpuset_ret = CGroupUtil::find_abs_cgroupv1_path("cpuset", &cpuset_path);
+        if (cpuset_ret.ok() && !cpuset_path.empty()) {
+            std::filesystem::path current_path = cpuset_path;
+            ret = get_cgroup_v1_cpuset_number(current_path, ret);
+        }
+    }
+#endif
+    return ret;
+}
+
+int CGroupUtil::get_cgroup_v2_cpu_quota_number(std::filesystem::path& current_path,
+                                               const std::filesystem::path& default_cg_mout_path,
+                                               int cpu_num) {
+    int ret = cpu_num;
+    while (current_path != default_cg_mout_path.parent_path()) {
+        std::ifstream cpu_max_file(current_path / "cpu.max");
+        if (cpu_max_file.is_open()) {
+            std::string cpu_limit_str;
+            double cpu_period;
+            cpu_max_file >> cpu_limit_str >> cpu_period;
+            if (cpu_limit_str != "max" && cpu_period != 0) {
+                double cpu_limit = std::stod(cpu_limit_str);
+                ret = std::min(static_cast<int>(std::ceil(cpu_limit / cpu_period)), ret);
+            }
+        }
+        current_path = current_path.parent_path();
+    }
+    return ret;
+}
+
+int CGroupUtil::get_cgroup_v2_cpuset_number(std::filesystem::path& current_path,
+                                            const std::filesystem::path& default_cg_mout_path,
+                                            int cpu_num) {
+    int ret = cpu_num;
+    while (current_path != default_cg_mout_path.parent_path()) {
+        std::ifstream cpuset_cpus_file(current_path / "cpuset.cpus.effective");
+        current_path = current_path.parent_path();
+        if (cpuset_cpus_file.is_open()) {
+            std::string cpuset_line;
+            cpuset_cpus_file >> cpuset_line;
+            if (cpuset_line.empty()) {
+                continue;
+            }
+            int cpus_count = 0;
+            static_cast<void>(CGroupUtil::parse_cpuset_line(cpuset_line, &cpus_count));
+            ret = std::min(cpus_count, ret);
+            break;
+        }
+    }
+    return ret;
+}
+
+int CGroupUtil::get_cgroup_v1_cpu_quota_number(std::filesystem::path& current_path,
+                                               const std::filesystem::path& default_cg_mout_path,
+                                               int cpu_num) {
+    int ret = cpu_num;
+    while (current_path != default_cg_mout_path.parent_path()) {
+        std::ifstream cpu_quota_file(current_path / "cpu.cfs_quota_us");
+        std::ifstream cpu_period_file(current_path / "cpu.cfs_period_us");
+        if (cpu_quota_file.is_open() && cpu_period_file.is_open()) {
+            double cpu_quota_value;
+            double cpu_period_value;
+            cpu_quota_file >> cpu_quota_value;
+            cpu_period_file >> cpu_period_value;
+            if (cpu_quota_value > 0 && cpu_period_value > 0) {
+                ret = std::min(ret,
+                               static_cast<int>(std::ceil(cpu_quota_value / cpu_period_value)));
+                break;
+            }
+        }
+        current_path = current_path.parent_path();
+    }
+    return ret;
+}
+
+int CGroupUtil::get_cgroup_v1_cpuset_number(std::filesystem::path& current_path, int cpu_num) {
+    int ret = cpu_num;
+    std::string cpuset_line = "";
+    Status cpuset_ret = CGroupUtil::read_string_line_from_cgroup_file(
+            (current_path / "cpuset.cpus"), &cpuset_line);
+    if (cpuset_ret.ok() && !cpuset_line.empty()) {
+        int cpuset_count = 0;
+        static_cast<void>(CGroupUtil::parse_cpuset_line(cpuset_line, &cpuset_count));
+        if (cpuset_count > 0) {
+            ret = std::min(ret, cpuset_count);
+        }
+    }
+    return ret;
+}
+
 } // namespace doris
diff --git a/be/src/util/cgroup_util.h b/be/src/util/cgroup_util.h
index bc1417453f41f6..54fc9494599f15 100644
--- a/be/src/util/cgroup_util.h
+++ b/be/src/util/cgroup_util.h
@@ -104,5 +104,27 @@ class CGroupUtil {
     static void read_int_metric_from_cgroup_file(
             const std::filesystem::path& file_path,
             std::unordered_map<std::string, int64_t>& metrics_map);
+
+    static Status read_string_line_from_cgroup_file(const std::filesystem::path& file_path,
+                                                    std::string* line_ptr);
+
+    // cpuset_line: 0-4,6,8-10
+    static Status parse_cpuset_line(std::string cpuset_line, int* cpu_count_ptr);
+
+    static int get_cgroup_limited_cpu_number(int physical_cores);
+
+    static int get_cgroup_v2_cpu_quota_number(std::filesystem::path& current_path,
+                                              const std::filesystem::path& default_cg_mout_path,
+                                              int cpu_num);
+
+    static int get_cgroup_v2_cpuset_number(std::filesystem::path& current_path,
+                                           const std::filesystem::path& default_cg_mout_path,
+                                           int cpu_num);
+
+    static int get_cgroup_v1_cpu_quota_number(std::filesystem::path& current_path,
+                                              const std::filesystem::path& default_cg_mout_path,
+                                              int cpu_num);
+
+    static int get_cgroup_v1_cpuset_number(std::filesystem::path& current_path, int cpu_num);
 };
 } // namespace doris
diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp
index 116dacb8da7ed4..b49985cdc06830 100644
--- a/be/src/util/cpu_info.cpp
+++ b/be/src/util/cpu_info.cpp
@@ -59,6 +59,7 @@
 #include "gflags/gflags.h"
 #include "gutil/stringprintf.h"
 #include "gutil/strings/substitute.h"
+#include "util/cgroup_util.h"
 #include "util/pretty_printer.h"
 
 using boost::algorithm::contains;
@@ -109,58 +110,6 @@ static struct {
         {"popcnt", CpuInfo::POPCNT}, {"avx", CpuInfo::AVX},       {"avx2", CpuInfo::AVX2},
 };
 
-int cgroup_bandwidth_quota(int physical_cores) {
-    namespace fs = std::filesystem;
-    fs::path cpu_max = "/sys/fs/cgroup/cpu.max";
-    fs::path cfs_quota = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us";
-    fs::path cfs_period = "/sys/fs/cgroup/cpu/cpu.cfs_period_us";
-
-    int64_t quota, period;
-    char byte_buffer[1000];
-    int64_t read_bytes;
-
-    if (fs::exists(cpu_max)) {
-        // cgroup v2
-        // https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
-        std::ifstream file(cpu_max);
-        file.read(byte_buffer, 999);
-        read_bytes = file.gcount();
-        byte_buffer[read_bytes] = '\0';
-        if (sscanf(byte_buffer, "%" SCNd64 " %" SCNd64 "", &quota, &period) != 2) {
-            return physical_cores;
-        }
-    } else if (fs::exists(cfs_quota) && fs::exists(cfs_period)) {
-        // cgroup v1
-        // https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management
-
-        // Read the quota, this indicates how many microseconds the CPU can be utilized by this cgroup per period
-        std::ifstream quota_file(cfs_quota);
-        quota_file.read(byte_buffer, 999);
-        read_bytes = quota_file.gcount();
-        byte_buffer[read_bytes] = '\0';
-        if (sscanf(byte_buffer, "%" SCNd64 "", &quota) != 1) {
-            return physical_cores;
-        }
-
-        // Read the time period, a cgroup can utilize the CPU up to quota microseconds every period
-        std::ifstream period_file(cfs_period);
-        period_file.read(byte_buffer, 999);
-        read_bytes = period_file.gcount();
-        byte_buffer[read_bytes] = '\0';
-        if (sscanf(byte_buffer, "%" SCNd64 "", &period) != 1) {
-            return physical_cores;
-        }
-    } else {
-        // No cgroup quota
-        return physical_cores;
-    }
-    if (quota > 0 && period > 0) {
-        return int64_t(ceil(double(quota) / double(period)));
-    } else {
-        return physical_cores;
-    }
-}
-
 // Helper function to parse for hardware flags.
 // values contains a list of space-separated flags.  check to see if the flags we
 // care about are present.
@@ -212,7 +161,7 @@ void CpuInfo::init() {
         }
     }
 
-    int num_cores = cgroup_bandwidth_quota(physical_num_cores);
+    int num_cores = CGroupUtil::get_cgroup_limited_cpu_number(physical_num_cores);
     if (max_mhz != 0) {
         cycles_per_ms_ = int64_t(max_mhz) * 1000;
     } else {
diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp
index 973f461d8defe7..fc2cdcc9262b31 100644
--- a/be/src/util/system_metrics.cpp
+++ b/be/src/util/system_metrics.cpp
@@ -33,18 +33,23 @@
 
 #include "gutil/strings/split.h" // for string split
 #include "gutil/strtoint.h"      //  for atoi64
+#include "util/cgroup_util.h"
 #include "util/mem_info.h"
 #include "util/perf_counters.h"
 
 namespace doris {
 
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(avail_cpu_num, MetricUnit::NOUNIT);
+
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(host_cpu_num, MetricUnit::NOUNIT);
 struct CpuNumberMetrics {
     CpuNumberMetrics(MetricEntity* ent) : entity(ent) {
         INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, host_cpu_num);
+        INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, avail_cpu_num);
     }
 
     IntAtomicCounter* host_cpu_num {nullptr};
+    IntAtomicCounter* avail_cpu_num {nullptr};
     MetricEntity* entity = nullptr;
 };
 
@@ -1004,6 +1009,14 @@ void SystemMetrics::_update_proc_metrics() {
     fclose(fp);
 }
 
+void SystemMetrics::update_be_avail_cpu_num() {
+    int64_t physical_cpu_num = _cpu_num_metrics->host_cpu_num->value();
+    if (physical_cpu_num > 0) {
+        physical_cpu_num = CGroupUtil::get_cgroup_limited_cpu_number(physical_cpu_num);
+        _cpu_num_metrics->avail_cpu_num->set_value(physical_cpu_num);
+    }
+}
+
 void SystemMetrics::get_metrics_from_proc_vmstat() {
 #ifdef BE_TEST
     FILE* fp = fopen(k_ut_vmstat_path, "r");
diff --git a/be/src/util/system_metrics.h b/be/src/util/system_metrics.h
index 29ce8c9c02b359..2c5446b81f4f71 100644
--- a/be/src/util/system_metrics.h
+++ b/be/src/util/system_metrics.h
@@ -66,6 +66,8 @@ class SystemMetrics {
     void update_max_network_receive_bytes_rate(int64_t max_receive_bytes_rate);
     void update_allocator_metrics();
 
+    void update_be_avail_cpu_num();
+
 private:
     void _install_cpu_metrics();
     // On Intel(R) Xeon(R) CPU E5-2450 0 @ 2.10GHz;
diff --git a/be/test/util/cgroup_util_test.cpp b/be/test/util/cgroup_util_test.cpp
index 92102120327840..4cc5c601b28ce1 100644
--- a/be/test/util/cgroup_util_test.cpp
+++ b/be/test/util/cgroup_util_test.cpp
@@ -87,4 +87,94 @@ TEST_F(CGroupUtilTest, memlimit) {
     }
 }
 
+TEST_F(CGroupUtilTest, readcpu) {
+    std::string dir_path = GetCurrentRunningDir();
+
+    std::string cpuset_1_path(dir_path + "/util/test_data/cgroup_cpu_data/cpuset1");
+    std::string cpuset_1_str = "";
+    Status ret1 = CGroupUtil::read_string_line_from_cgroup_file(cpuset_1_path, &cpuset_1_str);
+    EXPECT_TRUE(ret1.ok());
+    int cpu_count1 = 0;
+    static_cast<void>(CGroupUtil::parse_cpuset_line(cpuset_1_str, &cpu_count1));
+    EXPECT_TRUE(cpu_count1 == 3);
+
+    std::string cpuset_2_path(dir_path + "/util/test_data/cgroup_cpu_data/cpuset2");
+    std::string cpuset_2_str = "";
+    Status ret2 = CGroupUtil::read_string_line_from_cgroup_file(cpuset_2_path, &cpuset_2_str);
+    EXPECT_TRUE(ret2.ok());
+    int cpu_count2 = 0;
+    static_cast<void>(CGroupUtil::parse_cpuset_line(cpuset_2_str, &cpu_count2));
+    EXPECT_TRUE(cpu_count2 == 11);
+
+    std::string cpuset_3_path(dir_path + "/util/test_data/cgroup_cpu_data/cpuset3");
+    std::string cpuset_3_str = "";
+    Status ret3 = CGroupUtil::read_string_line_from_cgroup_file(cpuset_3_path, &cpuset_3_str);
+    EXPECT_TRUE(ret3.ok());
+    int cpu_count3 = 0;
+    static_cast<void>(CGroupUtil::parse_cpuset_line(cpuset_3_str, &cpu_count3));
+    EXPECT_TRUE(cpu_count3 == 10);
+
+    int ret = CGroupUtil::get_cgroup_limited_cpu_number(16);
+    EXPECT_TRUE(ret > 0);
+
+    // 1 read cgroup v2 quota
+    // 1.1 read default value
+    std::filesystem::path path11 = dir_path + "/util/test_data/cgroup_cpu_data/test11/child";
+    std::filesystem::path default_path_11 = dir_path + "/util/test_data/cgroup_cpu_data/test11";
+    int ret11 = CGroupUtil::get_cgroup_v2_cpu_quota_number(path11, default_path_11, 96);
+    EXPECT_TRUE(ret11 == 96);
+
+    // 1.2 read from child to parent
+    std::filesystem::path path12 = dir_path + "/util/test_data/cgroup_cpu_data/test12/child";
+    std::filesystem::path default_path_12 = dir_path + "/util/test_data/cgroup_cpu_data/test12";
+    int ret12 = CGroupUtil::get_cgroup_v2_cpu_quota_number(path12, default_path_12, 96);
+    EXPECT_TRUE(ret12 == 2);
+
+    // 1.3 read parent
+    std::filesystem::path path13 = dir_path + "/util/test_data/cgroup_cpu_data/test13/child";
+    std::filesystem::path default_path_13 = dir_path + "/util/test_data/cgroup_cpu_data/test13";
+    int ret13 = CGroupUtil::get_cgroup_v2_cpu_quota_number(path13, default_path_13, 96);
+    EXPECT_TRUE(ret13 == 2);
+
+    // 1.4 read child
+    std::filesystem::path path14 = dir_path + "/util/test_data/cgroup_cpu_data/test14/child";
+    std::filesystem::path default_path_14 = dir_path + "/util/test_data/cgroup_cpu_data/test14";
+    int ret14 = CGroupUtil::get_cgroup_v2_cpu_quota_number(path14, default_path_14, 96);
+    EXPECT_TRUE(ret14 == 3);
+
+    // 2 read cgroup v2 cpuset
+    // 2.1 read child
+    std::filesystem::path path21 = dir_path + "/util/test_data/cgroup_cpu_data/test21/child";
+    std::filesystem::path default_path_21 = dir_path + "/util/test_data/cgroup_cpu_data/test21";
+    int ret21 = CGroupUtil::get_cgroup_v2_cpuset_number(path21, default_path_21, 96);
+    EXPECT_TRUE(ret21 == 2);
+    // 2.2 read parent
+    std::filesystem::path path22 = dir_path + "/util/test_data/cgroup_cpu_data/test22/child";
+    std::filesystem::path default_path_22 = dir_path + "/util/test_data/cgroup_cpu_data/test22";
+    int ret22 = CGroupUtil::get_cgroup_v2_cpuset_number(path22, default_path_22, 96);
+    EXPECT_TRUE(ret22 == 7);
+
+    // 3 read cgroup v1 quota
+    // 3.1 read child
+    std::filesystem::path path31 = dir_path + "/util/test_data/cgroup_cpu_data/test31/child";
+    std::filesystem::path default_path_31 = dir_path + "/util/test_data/cgroup_cpu_data/test31";
+    int ret31 = CGroupUtil::get_cgroup_v1_cpu_quota_number(path31, default_path_31, 96);
+    EXPECT_TRUE(ret31 == 5);
+    // 3.2 read parent
+    std::filesystem::path path32 = dir_path + "/util/test_data/cgroup_cpu_data/test32/child";
+    std::filesystem::path default_path_32 = dir_path + "/util/test_data/cgroup_cpu_data/test32";
+    int ret32 = CGroupUtil::get_cgroup_v1_cpu_quota_number(path32, default_path_32, 96);
+    EXPECT_TRUE(ret32 == 6);
+    // 3.3 read default
+    std::filesystem::path path33 = dir_path + "/util/test_data/cgroup_cpu_data/test33/child";
+    std::filesystem::path default_path_33 = dir_path + "/util/test_data/cgroup_cpu_data/test33";
+    int ret33 = CGroupUtil::get_cgroup_v1_cpu_quota_number(path33, default_path_33, 96);
+    EXPECT_TRUE(ret33 == 96);
+
+    // 4 read cgroup v1 cpuset
+    std::filesystem::path path41 = dir_path + "/util/test_data/cgroup_cpu_data/test41";
+    int ret41 = CGroupUtil::get_cgroup_v1_cpuset_number(path41, 96);
+    EXPECT_TRUE(ret41 == 3);
+}
+
 } // namespace doris
diff --git a/be/test/util/test_data/cgroup_cpu_data/cpuset1 b/be/test/util/test_data/cgroup_cpu_data/cpuset1
new file mode 100644
index 00000000000000..f2fdae292d6460
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/cpuset1
@@ -0,0 +1 @@
+1,4,6
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/cpuset2 b/be/test/util/test_data/cgroup_cpu_data/cpuset2
new file mode 100644
index 00000000000000..9528de88c0a81c
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/cpuset2
@@ -0,0 +1 @@
+1-5,7-10,20-21
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/cpuset3 b/be/test/util/test_data/cgroup_cpu_data/cpuset3
new file mode 100644
index 00000000000000..02c93d74ecdcb9
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/cpuset3
@@ -0,0 +1 @@
+4,11-15,20,31-33
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test11/child/cpu.max b/be/test/util/test_data/cgroup_cpu_data/test11/child/cpu.max
new file mode 100644
index 00000000000000..b22d43b0084fe5
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test11/child/cpu.max
@@ -0,0 +1 @@
+max 100000 0
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test11/cpu.max b/be/test/util/test_data/cgroup_cpu_data/test11/cpu.max
new file mode 100644
index 00000000000000..b22d43b0084fe5
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test11/cpu.max
@@ -0,0 +1 @@
+max 100000 0
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test12/child/cpu.max b/be/test/util/test_data/cgroup_cpu_data/test12/child/cpu.max
new file mode 100644
index 00000000000000..434d117ae1c9d9
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test12/child/cpu.max
@@ -0,0 +1 @@
+300000 100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test12/cpu.max b/be/test/util/test_data/cgroup_cpu_data/test12/cpu.max
new file mode 100644
index 00000000000000..20b625d5126f34
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test12/cpu.max
@@ -0,0 +1 @@
+200000 100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test13/child/cpu.max b/be/test/util/test_data/cgroup_cpu_data/test13/child/cpu.max
new file mode 100644
index 00000000000000..b22d43b0084fe5
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test13/child/cpu.max
@@ -0,0 +1 @@
+max 100000 0
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test13/cpu.max b/be/test/util/test_data/cgroup_cpu_data/test13/cpu.max
new file mode 100644
index 00000000000000..20b625d5126f34
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test13/cpu.max
@@ -0,0 +1 @@
+200000 100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test14/child/cpu.max b/be/test/util/test_data/cgroup_cpu_data/test14/child/cpu.max
new file mode 100644
index 00000000000000..434d117ae1c9d9
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test14/child/cpu.max
@@ -0,0 +1 @@
+300000 100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test14/cpu.max b/be/test/util/test_data/cgroup_cpu_data/test14/cpu.max
new file mode 100644
index 00000000000000..4199eb01a27554
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test14/cpu.max
@@ -0,0 +1 @@
+400000 100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test21/child/cpuset.cpus.effective b/be/test/util/test_data/cgroup_cpu_data/test21/child/cpuset.cpus.effective
new file mode 100644
index 00000000000000..fb00853f18d5c2
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test21/child/cpuset.cpus.effective
@@ -0,0 +1 @@
+0-1
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test21/cpuset.cpus.effective b/be/test/util/test_data/cgroup_cpu_data/test21/cpuset.cpus.effective
new file mode 100644
index 00000000000000..745f3eb7203929
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test21/cpuset.cpus.effective
@@ -0,0 +1 @@
+0-6
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test22/child/cpuset.cpus.effective b/be/test/util/test_data/cgroup_cpu_data/test22/child/cpuset.cpus.effective
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/be/test/util/test_data/cgroup_cpu_data/test22/cpuset.cpus.effective b/be/test/util/test_data/cgroup_cpu_data/test22/cpuset.cpus.effective
new file mode 100644
index 00000000000000..745f3eb7203929
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test22/cpuset.cpus.effective
@@ -0,0 +1 @@
+0-6
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test31/child/cpu.cfs_period_us b/be/test/util/test_data/cgroup_cpu_data/test31/child/cpu.cfs_period_us
new file mode 100644
index 00000000000000..483fb82b6dd2b5
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test31/child/cpu.cfs_period_us
@@ -0,0 +1 @@
+100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test31/child/cpu.cfs_quota_us b/be/test/util/test_data/cgroup_cpu_data/test31/child/cpu.cfs_quota_us
new file mode 100644
index 00000000000000..516a58aff39342
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test31/child/cpu.cfs_quota_us
@@ -0,0 +1 @@
+500000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test31/cpu.cfs_period_us b/be/test/util/test_data/cgroup_cpu_data/test31/cpu.cfs_period_us
new file mode 100644
index 00000000000000..483fb82b6dd2b5
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test31/cpu.cfs_period_us
@@ -0,0 +1 @@
+100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test31/cpu.cfs_quota_us b/be/test/util/test_data/cgroup_cpu_data/test31/cpu.cfs_quota_us
new file mode 100644
index 00000000000000..212f56fce5dd11
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test31/cpu.cfs_quota_us
@@ -0,0 +1 @@
+600000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test32/child/cpu.cfs_period_us b/be/test/util/test_data/cgroup_cpu_data/test32/child/cpu.cfs_period_us
new file mode 100644
index 00000000000000..483fb82b6dd2b5
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test32/child/cpu.cfs_period_us
@@ -0,0 +1 @@
+100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test32/child/cpu.cfs_quota_us b/be/test/util/test_data/cgroup_cpu_data/test32/child/cpu.cfs_quota_us
new file mode 100644
index 00000000000000..d7d17fcbef95ca
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test32/child/cpu.cfs_quota_us
@@ -0,0 +1 @@
+-1
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test32/cpu.cfs_period_us b/be/test/util/test_data/cgroup_cpu_data/test32/cpu.cfs_period_us
new file mode 100644
index 00000000000000..483fb82b6dd2b5
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test32/cpu.cfs_period_us
@@ -0,0 +1 @@
+100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test32/cpu.cfs_quota_us b/be/test/util/test_data/cgroup_cpu_data/test32/cpu.cfs_quota_us
new file mode 100644
index 00000000000000..212f56fce5dd11
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test32/cpu.cfs_quota_us
@@ -0,0 +1 @@
+600000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test33/child/cpu.cfs_period_us b/be/test/util/test_data/cgroup_cpu_data/test33/child/cpu.cfs_period_us
new file mode 100644
index 00000000000000..483fb82b6dd2b5
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test33/child/cpu.cfs_period_us
@@ -0,0 +1 @@
+100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test33/child/cpu.cfs_quota_us b/be/test/util/test_data/cgroup_cpu_data/test33/child/cpu.cfs_quota_us
new file mode 100644
index 00000000000000..d7d17fcbef95ca
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test33/child/cpu.cfs_quota_us
@@ -0,0 +1 @@
+-1
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test33/cpu.cfs_period_us b/be/test/util/test_data/cgroup_cpu_data/test33/cpu.cfs_period_us
new file mode 100644
index 00000000000000..483fb82b6dd2b5
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test33/cpu.cfs_period_us
@@ -0,0 +1 @@
+100000
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test33/cpu.cfs_quota_us b/be/test/util/test_data/cgroup_cpu_data/test33/cpu.cfs_quota_us
new file mode 100644
index 00000000000000..d7d17fcbef95ca
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test33/cpu.cfs_quota_us
@@ -0,0 +1 @@
+-1
\ No newline at end of file
diff --git a/be/test/util/test_data/cgroup_cpu_data/test41/cpuset.cpus b/be/test/util/test_data/cgroup_cpu_data/test41/cpuset.cpus
new file mode 100644
index 00000000000000..b62d3ca93d6bd2
--- /dev/null
+++ b/be/test/util/test_data/cgroup_cpu_data/test41/cpuset.cpus
@@ -0,0 +1 @@
+1-3
\ No newline at end of file

From c5a23076378291b3885328c6c2298197a9738b71 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Fri, 13 Dec 2024 11:33:36 +0800
Subject: [PATCH 309/399] [fix](memory) Disable Jemalloc Hook and loading
 libjvm at runtime  (#45210)

### What problem does this PR solve?

fix:
```
*** Aborted at 1733734087 (unix time) try "date -d @1733734087" if you are using GNU date ***
*** Current BE git commitID: 43f06a5e26 ***
*** SIGSEGV address not mapped to object (@0x0) received by PID 1671420 (TID 1671420 OR 0x7f4f35f74ac0) from PID 0; stack trace: ***
 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /home/zcp/repo_center/doris_release/doris/be/src/common/signal_handler.h:421
 1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0] in /usr/local/jdk-17.0.2/lib/server/libjvm.so
 2# JVM_handle_linux_signal in /usr/local/jdk-17.0.2/lib/server/libjvm.so
 3# 0x00007F4F34340400 in /lib64/libc.so.6
 4# je_arena_dalloc_promoted at ../src/arena.c:1277
 5# je_free_default at ../src/jemalloc.c:3014
 6# __pthread_create_2_1 in /lib64/libpthread.so.0
 7# os::create_thread(Thread*, os::ThreadType, unsigned long) in /usr/local/jdk-17.0.2/lib/server/libjvm.so
 8# JVM_StartThread in /usr/local/jdk-17.0.2/lib/server/libjvm.so
 9# 0x00007F4F0A96C918
```
---
 be/src/http/default_path_handlers.cpp   |  2 +-
 be/src/runtime/CMakeLists.txt           |  4 +---
 be/src/runtime/memory/heap_profiler.cpp |  8 ++++----
 be/src/util/mem_info.cpp                |  2 +-
 be/src/util/mem_info.h                  | 10 +++++-----
 be/src/vec/common/allocator.h           |  2 +-
 build.sh                                | 14 --------------
 cloud/src/common/CMakeLists.txt         |  6 ------
 thirdparty/build-thirdparty.sh          |  4 +++-
 9 files changed, 16 insertions(+), 36 deletions(-)

diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp
index 04e1121cab63ba..b26ba67b54625c 100644
--- a/be/src/http/default_path_handlers.cpp
+++ b/be/src/http/default_path_handlers.cpp
@@ -146,7 +146,7 @@ void memory_info_handler(std::stringstream* output) {
         auto* _opaque = static_cast<std::string*>(opaque);
         _opaque->append(buf);
     };
-    jemalloc_stats_print(write_cb, &tmp, "a");
+    malloc_stats_print(write_cb, &tmp, "a");
     boost::replace_all(tmp, "\n", "<br>");
     (*output) << tmp;
 #else
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index a0b3b799a764cb..ab380f9711f1ac 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -25,9 +25,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/runtime")
 
 file(GLOB_RECURSE RUNTIME_FILES CONFIGURE_DEPENDS *.cpp *.cc)
 
-if (NOT USE_JEMALLOC OR NOT USE_MEM_TRACKER)
-    list(REMOVE_ITEM RUNTIME_FILES ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp)
-endif()
+list(REMOVE_ITEM RUNTIME_FILES ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp)
 
 add_library(Runtime STATIC
     ${RUNTIME_FILES}
diff --git a/be/src/runtime/memory/heap_profiler.cpp b/be/src/runtime/memory/heap_profiler.cpp
index 01ed82f76ef6d1..0b0448ce0eaaf6 100644
--- a/be/src/runtime/memory/heap_profiler.cpp
+++ b/be/src/runtime/memory/heap_profiler.cpp
@@ -30,8 +30,8 @@ void HeapProfiler::set_prof_active(bool prof) {
 #ifdef USE_JEMALLOC
     std::lock_guard guard(_mutex);
     try {
-        int err = jemallctl("prof.active", nullptr, nullptr, &prof, 1);
-        err |= jemallctl("prof.thread_active_init", nullptr, nullptr, &prof, 1);
+        int err = mallctl("prof.active", nullptr, nullptr, &prof, 1);
+        err |= mallctl("prof.thread_active_init", nullptr, nullptr, &prof, 1);
         if (err) {
             LOG(WARNING) << "jemalloc heap profiling start failed, " << err;
         } else {
@@ -48,7 +48,7 @@ bool HeapProfiler::get_prof_dump(const std::string& profile_file_name) {
     std::lock_guard guard(_mutex);
     const char* file_name_ptr = profile_file_name.c_str();
     try {
-        int err = jemallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*));
+        int err = mallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*));
         if (err) {
             LOG(WARNING) << "dump heap profile failed, " << err;
             return false;
@@ -93,7 +93,7 @@ bool HeapProfiler::check_heap_profiler() {
 #ifdef USE_JEMALLOC
     size_t value = 0;
     size_t sz = sizeof(value);
-    jemallctl("prof.active", &value, &sz, nullptr, 0);
+    mallctl("prof.active", &value, &sz, nullptr, 0);
     return value;
 #else
     return false;
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index fe9cf84b2aed54..97e529ac6c72e1 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -101,7 +101,7 @@ void MemInfo::refresh_allocator_mem() {
     // the current epoch number, which might be useful to log as a sanity check.
     uint64_t epoch = 0;
     size_t sz = sizeof(epoch);
-    jemallctl("epoch", &epoch, &sz, &epoch, sz);
+    mallctl("epoch", &epoch, &sz, &epoch, sz);
 
     // Number of extents of the given type in this arena in the bucket corresponding to page size index.
     // Large size class starts at 16384, the extents have three sizes before 16384: 4096, 8192, and 12288, so + 3
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 39ae9eb0b79cfb..0e14b64bd8f965 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -103,7 +103,7 @@ class MemInfo {
 #ifdef USE_JEMALLOC
         size_t value = 0;
         size_t sz = sizeof(value);
-        if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
+        if (mallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
             return value;
         }
 #endif
@@ -114,7 +114,7 @@ class MemInfo {
 #ifdef USE_JEMALLOC
         unsigned value = 0;
         size_t sz = sizeof(value);
-        if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
+        if (mallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
             return value;
         }
 #endif
@@ -146,8 +146,8 @@ class MemInfo {
         if (config::enable_je_purge_dirty_pages) {
             try {
                 // Purge all unused dirty pages for arena <i>, or for all arenas if <i> equals MALLCTL_ARENAS_ALL.
-                int err = jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(),
-                                    nullptr, nullptr, nullptr, 0);
+                int err = mallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(),
+                                  nullptr, nullptr, nullptr, 0);
                 if (err) {
                     LOG(WARNING) << "Jemalloc purge all unused dirty pages failed";
                 }
@@ -166,7 +166,7 @@ class MemInfo {
 #ifdef USE_JEMALLOC
         constexpr size_t TCACHE_LIMIT = (1ULL << 30); // 1G
         if (allocator_cache_mem() - je_dirty_pages_mem() > TCACHE_LIMIT) {
-            int err = jemallctl("thread.tcache.flush", nullptr, nullptr, nullptr, 0);
+            int err = mallctl("thread.tcache.flush", nullptr, nullptr, nullptr, 0);
             if (err) {
                 LOG(WARNING) << "Jemalloc thread.tcache.flush failed";
             }
diff --git a/be/src/vec/common/allocator.h b/be/src/vec/common/allocator.h
index b05128bc6933cc..a0ab9c8c0a8207 100644
--- a/be/src/vec/common/allocator.h
+++ b/be/src/vec/common/allocator.h
@@ -109,7 +109,7 @@ class DefaultMemoryAllocator {
 
     static void release_unused() {
 #if defined(USE_JEMALLOC)
-        jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), NULL, NULL, NULL, 0);
+        mallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), NULL, NULL, NULL, 0);
 #endif // defined(USE_JEMALLOC)
     }
 };
diff --git a/build.sh b/build.sh
index da4e701f42146a..89a22842b7d100 100755
--- a/build.sh
+++ b/build.sh
@@ -377,20 +377,6 @@ if [[ "${BUILD_TYPE_LOWWER}" == "asan" ]]; then
 elif [[ -z "${USE_JEMALLOC}" ]]; then
     USE_JEMALLOC='ON'
 fi
-if [[ -f "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" ]]; then
-    # compatible with old thirdparty
-    rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
-    rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a"
-    rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
-    rm -rf "${TP_INCLUDE_DIR}/rocksdb"
-    rm -rf "${TP_LIB_DIR}/librocksdb.a"
-
-    mv "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
-    mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a"
-    mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
-    mv "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" "${TP_LIB_DIR}/librocksdb.a"
-    mv -f "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" "${TP_INCLUDE_DIR}/rocksdb"
-fi
 if [[ -z "${USE_BTHREAD_SCANNER}" ]]; then
     USE_BTHREAD_SCANNER='OFF'
 fi
diff --git a/cloud/src/common/CMakeLists.txt b/cloud/src/common/CMakeLists.txt
index 429748909bebd9..008e8906044d51 100644
--- a/cloud/src/common/CMakeLists.txt
+++ b/cloud/src/common/CMakeLists.txt
@@ -15,12 +15,6 @@ set(COMMON_FILES
     network_util.cpp
 )
 
-if (USE_JEMALLOC)
-    set(COMMON_FILES ${COMMON_FILES}
-        jemalloc_hook.cpp
-        )
-endif()
-
 add_library(Common STATIC
     ${COMMON_FILES}
     )
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index f019f5f1e26bb2..a209977769c278 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -1529,8 +1529,10 @@ build_jemalloc_doris() {
         WITH_LG_PAGE=''
     fi
 
+    # CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris" "${WITH_LG_PAGE}" \
+    #     --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared
     CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris" "${WITH_LG_PAGE}" \
-        --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared
+        --enable-prof --disable-libdl --disable-shared
 
     make -j "${PARALLEL}"
     make install

From f85b40877c7ee20489d51fb287579f6ac92935f0 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Fri, 13 Dec 2024 11:43:01 +0800
Subject: [PATCH 310/399] [chore](arrow-flight-sql) Add Arrow Flight Sql demo
 for Java (#45306)

### What problem does this PR solve?

# How to use:

1. mvn clean install -U
2. mvn package
3. java
--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED
-cp java-0.1.jar doris.arrowflight.demo.Main "sql" "fe_ip"
"fe_arrow_flight_port" "fe_query_port"

# What can this demo do:

This is a java demo for doris arrow flight sql, you can use this to test
various connection
methods for sending queries to the doris arrow flight server, help you
understand how to use arrow flight sql
and test performance. You should install maven prior to run this demo.

# Performance test

Section 6.2 of https://github.com/apache/doris/issues/25514 is the
performance test results of the Doris Arrow Flight SQL using java.

# Output

```
WARNING: Unknown module: org.apache.arrow.memory.core specified to --add-opens
*************************************
|          FlightAdbcDriver         |
*************************************
FlightAdbcDriver > loadArrowBatch
SLF4J(W): No SLF4J providers were found.
SLF4J(W): Defaulting to no-operation (NOP) logger implementation
SLF4J(W): See https://www.slf4j.org/codes.html#noProviders for further details.
> Schema<l_shipdate: Date(DAY) not null>
> 1994-08-17,
> batchCount: 25, rowCount: 100000
> cost: 1704 ms.

FlightAdbcDriver > loadArrowBatchToString
> Schema<l_shipdate: Date(DAY) not null>
> 1992-01-02,
> batchCount: 25, rowCount: 100000
> cost: 1692 ms.

*************************************
|          FlightJdbcDriver         |
*************************************
FlightJdbcDriver > loadArrowBatch
> Schema<l_shipdate: Date(DAY) not null>
> 1997-01-30,
> batchCount: 98, rowCount: 100000
> cost: 1840 ms.

FlightJdbcDriver > loadArrowBatchToString
> Schema<l_shipdate: Date(DAY) not null>
> 1992-01-02,
> batchCount: 98, rowCount: 100000
> cost: 1712 ms.

*************************************
|          JdbcDriverManager        |
*************************************
JdbcDriverManager > jdbc:mysql > loadJdbcResult
> rowCount: 100000, columnCount: 1
> cost: 11431 ms.

JdbcDriverManager > jdbc:mysql > loadJdbcResultToString
> 1992-01-02,
> rowCount: 100000, columnCount: 1 resultSize: 100000
> cost: 5164 ms.

JdbcDriverManager > jdbc:arrow-flight-sql > loadJdbcResultToString
> rowCount: 100000, columnCount: 1
> cost: 1736 ms.

JdbcDriverManager > jdbc:arrow-flight-sql > loadJdbcResultToString
> 1997-01-29,
> rowCount: 100000, columnCount: 1 resultSize: 100000
> cost: 2442 ms.


*************************************
|           FlightSqlClient         |
*************************************
FlightSqlClient > getFlightInfoFromDorisFe
> Schema<l_shipdate: Date(DAY) not null>
> 1994-08-15,
> batchCount: 25, rowCount: 100000

FlightSqlClient > constructDummyFlightInfo, don't be afraid! expected to get error `INVALID_ARGUMENT: Malformed ticket`
org.apache.arrow.flight.FlightRuntimeException: INVALID_ARGUMENT: Malformed ticket, size: 1
	at org.apache.arrow.flight.CallStatus.toRuntimeException(CallStatus.java:121)
	at org.apache.arrow.flight.grpc.StatusUtils.fromGrpcRuntimeException(StatusUtils.java:161)
	at org.apache.arrow.flight.grpc.StatusUtils.fromThrowable(StatusUtils.java:182)
	at org.apache.arrow.flight.FlightStream$Observer.onError(FlightStream.java:489)
	at org.apache.arrow.flight.FlightClient$1.onError(FlightClient.java:371)
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:481)
	at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
	at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
	at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
	at org.apache.arrow.flight.grpc.ClientInterceptorAdapter$FlightClientCallListener.onClose(ClientInterceptorAdapter.java:118)
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:564)
	at io.grpc.internal.ClientCallImpl.access$100(ClientCallImpl.java:72)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:729)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:710)
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:842)

Process finished with exit code 0
```
---
 samples/arrow-flight-sql/java/README.md       |  35 ++++
 samples/arrow-flight-sql/java/pom.xml         | 186 ++++++++++++++++++
 .../arrowflight/demo/ArrowBatchReader.java    | 110 +++++++++++
 .../doris/arrowflight/demo/Configuration.java |  42 ++++
 .../arrowflight/demo/FlightAdbcDriver.java    |  90 +++++++++
 .../arrowflight/demo/FlightJdbcDriver.java    |  86 ++++++++
 .../arrowflight/demo/FlightSqlClient.java     | 144 ++++++++++++++
 .../arrowflight/demo/JdbcDriverManager.java   |  90 +++++++++
 .../arrowflight/demo/JdbcResultSetReader.java |  63 ++++++
 .../java/doris/arrowflight/demo/Main.java     |  30 +++
 .../arrowflight/demo/ConfigurationTest.java   |  35 ++++
 11 files changed, 911 insertions(+)
 create mode 100644 samples/arrow-flight-sql/java/README.md
 create mode 100644 samples/arrow-flight-sql/java/pom.xml
 create mode 100644 samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/ArrowBatchReader.java
 create mode 100644 samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Configuration.java
 create mode 100644 samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightAdbcDriver.java
 create mode 100644 samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightJdbcDriver.java
 create mode 100644 samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightSqlClient.java
 create mode 100644 samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcDriverManager.java
 create mode 100644 samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcResultSetReader.java
 create mode 100644 samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Main.java
 create mode 100644 samples/arrow-flight-sql/java/src/test/java/doris/arrowflight/demo/ConfigurationTest.java

diff --git a/samples/arrow-flight-sql/java/README.md b/samples/arrow-flight-sql/java/README.md
new file mode 100644
index 00000000000000..1eb574e2ab6626
--- /dev/null
+++ b/samples/arrow-flight-sql/java/README.md
@@ -0,0 +1,35 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# How to use:
+
+	1. mvn clean install -U
+    2. mvn package
+	3. java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -cp java-0.1.jar doris.arrowflight.demo.Main "sql" "fe_ip" "fe_arrow_flight_port" "fe_query_port"
+
+# What can this demo do:
+
+	This is a java demo for doris arrow flight sql, you can use this to test various connection
+    methods for  sending queries to the doris arrow flight server, help you understand how to use arrow flight sql
+    and test performance. You should install maven prior to run this demo.
+
+# Performance test
+
+    Section 6.2 of https://github.com/apache/doris/issues/25514 is the performance test
+    results of the doris arrow flight sql using java.
\ No newline at end of file
diff --git a/samples/arrow-flight-sql/java/pom.xml b/samples/arrow-flight-sql/java/pom.xml
new file mode 100644
index 00000000000000..d08e30d69aec0b
--- /dev/null
+++ b/samples/arrow-flight-sql/java/pom.xml
@@ -0,0 +1,186 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>doris.arrowflight.demo</groupId>
+    <artifactId>java</artifactId>
+    <version>0.1</version>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <!--
+        Notice !!! Arrow and ADBC versions cannot be combined arbitrarily.
+
+        1. For ADBC 0.15.0, please use Arrow 18.0.0 and after, not compatible with previous versions of Arrow.
+        2. Try not to use Arrow 17.0.0, may get the following error:
+            ```
+            Caused by: java.lang.ArrayIndexOutOfBoundsException: Index 2 out of bounds for length 2
+                at com.google.protobuf.GeneratedMessageV3$FieldAccessorTable.ensureFieldAccessorsInitialized(GeneratedMessageV3.java:2094)
+                at org.apache.arrow.flight.sql.impl.FlightSql$ActionCreatePreparedStatementRequest.internalGetFieldAccessorTable(FlightSql.java:16332)
+                at com.google.protobuf.GeneratedMessageV3.getDescriptorForType(GeneratedMessageV3.java:139)
+                at com.google.protobuf.Any.pack(Any.java:61)
+                at org.apache.arrow.flight.sql.FlightSqlClient.prepare(FlightSqlClient.java:767)
+                at org.apache.arrow.flight.sql.FlightSqlClient.prepare(FlightSqlClient.java:746)
+                at org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler.prepare(ArrowFlightSqlClientHandler.java:310)
+	        ```
+	        similar issue: https://github.com/protocolbuffers/protobuf/issues/15762
+	    3. A more stable version is Arrow 15.0.2 and ADBC 0.12.0, but we always hope to embrace the future with new versions!
+        -->
+        <arrow.version>18.1.0</arrow.version>
+        <adbc.version>0.15.0</adbc.version>
+        <log4j.version>2.17.1</log4j.version>
+    </properties>
+    <dependencies>
+        <!-- If Maven cannot find the Arrow or Adbc version of the dependency, update the Maven central repository.
+        Arrow and Adbc are updated frequently, and we always try to use the latest version as much as possible. -->
+        <dependency>
+            <groupId>org.apache.arrow.adbc</groupId>
+            <artifactId>adbc-driver-jdbc</artifactId>
+            <version>${adbc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow.adbc</groupId>
+            <artifactId>adbc-driver-flight-sql</artifactId>
+            <version>${adbc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>8.0.33</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory-core</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory-netty</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>flight-core</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>flight-sql</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.arrow.adbc</groupId>
+            <artifactId>adbc-core</artifactId>
+            <version>${adbc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow.adbc</groupId>
+            <artifactId>adbc-driver-manager</artifactId>
+            <version>${adbc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow.adbc</groupId>
+            <artifactId>adbc-sql</artifactId>
+            <version>${adbc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>flight-sql-jdbc-core</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.13.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>RELEASE</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!-- Java Compiler -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.10.1</version>
+                <configuration>
+                    <source>${maven.compiler.source}</source>
+                    <target>${maven.compiler.target}</target>
+                    <encoding>${project.build.sourceEncoding}</encoding>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.4.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                </transformer>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
\ No newline at end of file
diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/ArrowBatchReader.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/ArrowBatchReader.java
new file mode 100644
index 00000000000000..2522187db6d1b4
--- /dev/null
+++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/ArrowBatchReader.java
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package doris.arrowflight.demo;
+
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ipc.ArrowReader;
+import org.apache.arrow.vector.types.pojo.Field;
+
+import java.io.IOException;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * Iterate over each batch in ArrowReader.
+ * ArrowReader is the iterator returned by ADBC Client when executing a query.
+ */
+public class ArrowBatchReader {
+
+    @FunctionalInterface
+    public interface LoadArrowBatchFunc {
+        void load(ArrowReader reader) throws IOException;
+    }
+
+    /**
+     * Print one row in VectorSchemaRoot, if the output format is incorrect, may need to modify
+     * the output method of different types of ValueVector.
+     */
+    public static void printRow(VectorSchemaRoot root, int rowIndex) {
+        if (root == null || rowIndex < 0 || rowIndex >= root.getRowCount()) {
+            System.out.println("Invalid row index: " + rowIndex);
+            return;
+        }
+
+        System.out.print("> ");
+        for (Field field : root.getSchema().getFields()) {
+            ValueVector vector = root.getVector(field.getName());
+            if (vector != null) {
+                if (vector instanceof org.apache.arrow.vector.DateDayVector) {
+                    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+                    int dayOffset = ((org.apache.arrow.vector.DateDayVector) vector).get(rowIndex);
+                    LocalDate date = LocalDate.ofEpochDay(dayOffset);
+                    System.out.print(date.format(formatter));
+                } else if (vector instanceof org.apache.arrow.vector.BitVector) {
+                    System.out.print(((org.apache.arrow.vector.BitVector) vector).get(rowIndex) == 1);
+                } else {
+                    // other types field
+                    System.out.print(vector.getObject(rowIndex).toString());
+                }
+                System.out.print(", ");
+            }
+        }
+        System.out.println();
+    }
+
+    /**
+     * Iterate over each batch in ArrowReader with the least cost, only record the number of rows and batches,
+     * usually used to test performance.
+     */
+    public static LoadArrowBatchFunc loadArrowBatch = reader -> {
+        int rowCount = 0;
+        int batchCount = 0;
+        while (reader.loadNextBatch()) {
+            VectorSchemaRoot root = reader.getVectorSchemaRoot();
+            if (batchCount == 0) {
+                System.out.println("> " + root.getSchema().toString());
+                printRow(root, 1);  // only print first line
+            }
+            rowCount += root.getRowCount();
+            batchCount += 1;
+        }
+        System.out.println("> batchCount: " + batchCount + ", rowCount: " + rowCount);
+    };
+
+    /**
+     * Iterate over each batch in ArrowReader and convert the batch to String, this will take more time.
+     */
+    public static LoadArrowBatchFunc loadArrowBatchToString = reader -> {
+        int rowCount = 0;
+        List<String> result = new ArrayList<>();
+        while (reader.loadNextBatch()) {
+            VectorSchemaRoot root = reader.getVectorSchemaRoot();
+            if (result.size() == 0) {
+                System.out.println("> " + root.getSchema().toString());
+                printRow(root, 0);  // only print first line
+            }
+            result.add(root.contentToTSVString());
+            rowCount += root.getRowCount();
+        }
+        System.out.println("> batchCount: " + result.size() + ", rowCount: " + rowCount);
+    };
+}
diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Configuration.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Configuration.java
new file mode 100644
index 00000000000000..c40a46ca057f9b
--- /dev/null
+++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Configuration.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package doris.arrowflight.demo;
+
+public class Configuration {
+    public String sql = ""; // require
+    public String ip = "127.0.0.1"; // require
+    public String arrowFlightPort = "9090"; // require
+    public String mysqlPort = "9030";
+    public int retryTimes = 2; // The first execution is cold run
+    public String user = "root";
+    public String password = "";
+
+    Configuration(String[] args) {
+        for (int i = 0; i < args.length; i++) {
+            switch (i) {
+                case 0 -> sql = args[i];
+                case 1 -> ip = args[i];
+                case 2 -> arrowFlightPort = args[i];
+                case 3 -> mysqlPort = args[i];
+                case 4 -> retryTimes = Integer.parseInt(args[i]);
+                case 5 -> user = args[i];
+                case 6 -> password = args[i];
+            }
+        }
+    }
+}
diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightAdbcDriver.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightAdbcDriver.java
new file mode 100644
index 00000000000000..3c2202b2b07797
--- /dev/null
+++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightAdbcDriver.java
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package doris.arrowflight.demo;
+
+import doris.arrowflight.demo.ArrowBatchReader.LoadArrowBatchFunc;
+import org.apache.arrow.adbc.core.AdbcConnection;
+import org.apache.arrow.adbc.core.AdbcDatabase;
+import org.apache.arrow.adbc.core.AdbcDriver;
+import org.apache.arrow.adbc.core.AdbcStatement;
+import org.apache.arrow.adbc.core.AdbcStatement.QueryResult;
+import org.apache.arrow.adbc.driver.flightsql.FlightSqlDriver;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.ipc.ArrowReader;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Use the Arrow Flight ADBC driver to connect to the Doris Arrow Flight server and execute query.
+ */
+public class FlightAdbcDriver {
+    private static void connectAndExecute(Configuration configuration, LoadArrowBatchFunc loadArrowReader) {
+        final BufferAllocator allocator = new RootAllocator();
+        FlightSqlDriver driver = new FlightSqlDriver(allocator);
+        Map<String, Object> parameters = new HashMap<>();
+        AdbcDriver.PARAM_URI.set(parameters,
+                Location.forGrpcInsecure(configuration.ip, Integer.parseInt(configuration.arrowFlightPort)).getUri()
+                        .toString());
+        AdbcDriver.PARAM_USERNAME.set(parameters, configuration.user);
+        AdbcDriver.PARAM_PASSWORD.set(parameters, configuration.password);
+
+        try {
+            AdbcDatabase adbcDatabase = driver.open(parameters);
+            AdbcConnection connection = adbcDatabase.connect();
+            AdbcStatement stmt = connection.createStatement();
+            long start = System.currentTimeMillis();
+            stmt.setSqlQuery(configuration.sql);
+
+            // executeQuery, two steps:
+            // 1. Execute Query and get returned FlightInfo;
+            // 2. Create FlightInfoReader to sequentially traverse each Endpoint;
+            QueryResult queryResult = stmt.executeQuery();
+            ArrowReader reader = queryResult.getReader();
+            loadArrowReader.load(reader);
+            System.out.printf("> cost: %d ms.\n\n", (System.currentTimeMillis() - start));
+
+            reader.close();
+            queryResult.close();
+            stmt.close();
+            connection.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void run(Configuration configuration) {
+        System.out.println("*************************************");
+        System.out.println("|          FlightAdbcDriver         |");
+        System.out.println("*************************************");
+
+        System.out.println("FlightAdbcDriver > loadArrowBatch");
+        connectAndExecute(configuration, ArrowBatchReader.loadArrowBatch);
+        System.out.println("FlightAdbcDriver > loadArrowBatchToString");
+        connectAndExecute(configuration, ArrowBatchReader.loadArrowBatchToString);
+    }
+
+    public static void main(String[] args) throws Exception {
+        Configuration configuration = new Configuration(args);
+        for (int i = 0; i < configuration.retryTimes; i++) {
+            run(configuration);
+        }
+    }
+}
diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightJdbcDriver.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightJdbcDriver.java
new file mode 100644
index 00000000000000..1560e5172d7384
--- /dev/null
+++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightJdbcDriver.java
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package doris.arrowflight.demo;
+
+import doris.arrowflight.demo.ArrowBatchReader.LoadArrowBatchFunc;
+import org.apache.arrow.adbc.core.AdbcConnection;
+import org.apache.arrow.adbc.core.AdbcDatabase;
+import org.apache.arrow.adbc.core.AdbcDriver;
+import org.apache.arrow.adbc.core.AdbcStatement;
+import org.apache.arrow.adbc.driver.jdbc.JdbcDriver;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.ipc.ArrowReader;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Use the Arrow Flight JDBC driver to connect to the Doris Arrow Flight server and execute query.
+ * Unlike the Java JDBC DriverManager, this is a JDBC Driver provided by Arrow Flight, which may contain
+ * some optimizations (although no performance advantage was observed).
+ */
+public class FlightJdbcDriver {
+    private static void connectAndExecute(Configuration configuration, LoadArrowBatchFunc loadArrowReader) {
+        String DB_URL = "jdbc:arrow-flight-sql://" + configuration.ip + ":" + configuration.arrowFlightPort
+                + "?useServerPrepStmts=false" + "&cachePrepStmts=true&useSSL=false&useEncryption=false";
+        final Map<String, Object> parameters = new HashMap<>();
+        AdbcDriver.PARAM_URI.set(parameters, DB_URL);
+        AdbcDriver.PARAM_USERNAME.set(parameters, configuration.user);
+        AdbcDriver.PARAM_PASSWORD.set(parameters, configuration.password);
+
+        try {
+            BufferAllocator allocator = new RootAllocator();
+            AdbcDatabase db = new JdbcDriver(allocator).open(parameters);
+            AdbcConnection connection = db.connect();
+            AdbcStatement stmt = connection.createStatement();
+
+            long start = System.currentTimeMillis();
+            stmt.setSqlQuery(configuration.sql);
+            AdbcStatement.QueryResult queryResult = stmt.executeQuery();
+            ArrowReader reader = queryResult.getReader();
+            loadArrowReader.load(reader);
+            System.out.printf("> cost: %d ms.\n\n", (System.currentTimeMillis() - start));
+
+            reader.close();
+            queryResult.close();
+            stmt.close();
+            connection.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void run(Configuration configuration) {
+        System.out.println("*************************************");
+        System.out.println("|          FlightJdbcDriver         |");
+        System.out.println("*************************************");
+
+        System.out.println("FlightJdbcDriver > loadArrowBatch");
+        connectAndExecute(configuration, ArrowBatchReader.loadArrowBatch);
+        System.out.println("FlightJdbcDriver > loadArrowBatchToString");
+        connectAndExecute(configuration, ArrowBatchReader.loadArrowBatchToString);
+    }
+
+    public static void main(String[] args) {
+        Configuration configuration = new Configuration(args);
+        for (int i = 0; i < configuration.retryTimes; i++) {
+            run(configuration);
+        }
+    }
+}
diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightSqlClient.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightSqlClient.java
new file mode 100644
index 00000000000000..3a01ea16370d4b
--- /dev/null
+++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/FlightSqlClient.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package doris.arrowflight.demo;
+
+import com.google.protobuf.Any;
+import com.google.protobuf.ByteString;
+import org.apache.arrow.flight.FlightClient;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.Ticket;
+import org.apache.arrow.flight.auth2.BearerCredentialWriter;
+import org.apache.arrow.flight.grpc.CredentialCallOption;
+import org.apache.arrow.flight.sql.impl.FlightSql.TicketStatementQuery;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * Manually execute Arrow Flight SQL Rpc process, usually used for debug.
+ */
+public class FlightSqlClient {
+    public record FlightInfoResult<T, T2, T3>(T first, T2 second, T3 third) {
+    }
+
+    public record DummyFlightInfoResult<T, T2>(T first, T2 second) {
+    }
+
+    /**
+     * Connect to FE Arrow Flight Server to obtain Bearertoken and execute Query to get Ticket.
+     */
+    public static FlightInfoResult<FlightInfo, CredentialCallOption, org.apache.arrow.flight.sql.FlightSqlClient.PreparedStatement> getFlightInfoFromDorisFe(
+            Configuration configuration) throws URISyntaxException {
+        BufferAllocator allocatorFE = new RootAllocator(Integer.MAX_VALUE);
+        final Location clientLocationFE = new Location(
+                new URI("grpc", null, configuration.ip, Integer.parseInt(configuration.arrowFlightPort),
+                        null, null, null));
+        FlightClient clientFE = FlightClient.builder(allocatorFE, clientLocationFE).build();
+        org.apache.arrow.flight.sql.FlightSqlClient sqlClinetFE = new org.apache.arrow.flight.sql.FlightSqlClient(
+                clientFE);
+
+        // Use username and password authentication to obtain a Bearertoken for subsequent access to the Doris Arrow Flight Server.
+        CredentialCallOption credentialCallOption = clientFE.authenticateBasicToken(configuration.user,
+                configuration.password).get();
+        final org.apache.arrow.flight.sql.FlightSqlClient.PreparedStatement preparedStatement = sqlClinetFE.prepare(
+                configuration.sql,
+                credentialCallOption);
+        final FlightInfo info = preparedStatement.execute(credentialCallOption);
+        return new FlightInfoResult<>(info, credentialCallOption, preparedStatement);
+    }
+
+    /**
+     * Use the correct Bearertoken and the correct Ticket, and the expected return result is normal.
+     */
+    public static void getResultFromDorisBe(FlightInfo info, Ticket ticket, CredentialCallOption credentialCallOption)
+            throws Exception {
+        final Location locationBE = info.getEndpoints().get(0).getLocations().get(0);
+        // 连接 BE Arrow Flight Server
+        BufferAllocator allocatorBE = new RootAllocator(Integer.MAX_VALUE);
+        FlightClient clientBE = FlightClient.builder(allocatorBE, locationBE).build();
+        org.apache.arrow.flight.sql.FlightSqlClient sqlClinetBE = new org.apache.arrow.flight.sql.FlightSqlClient(
+                clientBE);
+
+        FlightStream stream = sqlClinetBE.getStream(ticket, credentialCallOption);
+        int rowCount = 0;
+        int batchCount = 0;
+        while (stream.next()) {
+            VectorSchemaRoot root = stream.getRoot();
+            if (batchCount == 0) {
+                System.out.println("> " + root.getSchema().toString());
+                ArrowBatchReader.printRow(root, 1);  // only print first line
+            }
+            rowCount += root.getRowCount();
+            batchCount += 1;
+        }
+        System.out.println("> batchCount: " + batchCount + ", rowCount: " + rowCount);
+        stream.close();
+    }
+
+    /**
+     * Construct a dummy Ticket and CredentialCallOption to simulate the BE Arrow Flight Server being hacked,
+     * to analyze data security.
+     *
+     * @return get error `INVALID_ARGUMENT: Malformed ticket`
+     */
+    public static DummyFlightInfoResult<Ticket, CredentialCallOption> constructDummyFlightInfo() {
+        String Bearertoken = "ojatddjr72k1ss20sqkatkhtd7";
+        String queryId = "18c64b4e15094922-af5fea3da80fb89f";
+        String query = "select * from clickbench.hits limit 10;";
+        CredentialCallOption dummyCredentialCallOption = new CredentialCallOption(
+                new BearerCredentialWriter(Bearertoken));
+        final ByteString handle = ByteString.copyFromUtf8(queryId + ":" + query);
+        TicketStatementQuery ticketStatement = TicketStatementQuery.newBuilder().setStatementHandle(handle).build();
+        final Ticket dummyTicket = new Ticket(Any.pack(ticketStatement).toByteArray());
+        return new DummyFlightInfoResult<>(dummyTicket, dummyCredentialCallOption);
+    }
+
+    public static void run(Configuration configuration) {
+        System.out.println("*************************************");
+        System.out.println("|           FlightSqlClient         |");
+        System.out.println("*************************************");
+
+        try {
+            System.out.println("FlightSqlClient > getFlightInfoFromDorisFe");
+            var flightInfo = getFlightInfoFromDorisFe(configuration);
+            getResultFromDorisBe(flightInfo.first(), flightInfo.first().getEndpoints().get(0).getTicket(),
+                    flightInfo.second());
+            System.out.println();
+
+            System.out.println(
+                    "FlightSqlClient > constructDummyFlightInfo, don't be afraid! expected to get error `INVALID_ARGUMENT: Malformed ticket`");
+            var dummyFlightInfo = constructDummyFlightInfo();
+            getResultFromDorisBe(flightInfo.first(), dummyFlightInfo.first(), dummyFlightInfo.second());
+            System.out.println();
+
+            flightInfo.third().close(flightInfo.second());
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void main(String[] args) {
+        Configuration configuration = new Configuration(args);
+        run(configuration);
+    }
+}
diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcDriverManager.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcDriverManager.java
new file mode 100644
index 00000000000000..3cc07c738470b3
--- /dev/null
+++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcDriverManager.java
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package doris.arrowflight.demo;
+
+import doris.arrowflight.demo.JdbcResultSetReader.LoadJdbcResultSetFunc;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Objects;
+
+/**
+ * Use the Java JDBC DriverManager to connect to the Doris Arrow Flight server and execute query.
+ * Usually, DriverManager is used to connect to the database using the Mysql protocol in Java. only need to replace
+ * `jdbc:mysql` in the URI with `jdbc:arrow-flight-sql` to connect to the database using the Arrow Flight SQL protocol
+ * (provided that the database implements the Arrow Flight server).
+ */
+public class JdbcDriverManager {
+    private static void connectAndExecute(Configuration configuration, String urlPrefix, String port,
+            LoadJdbcResultSetFunc loadJdbcResultSetFunc) {
+        String DB_URL = urlPrefix + "://" + configuration.ip + ":" + port + "?useServerPrepStmts=false"
+                + "&cachePrepStmts=true&useSSL=false&useEncryption=false";
+        try {
+            long start = System.currentTimeMillis();
+            Connection conn = DriverManager.getConnection(DB_URL, configuration.user, configuration.password);
+            Statement stmt = conn.createStatement();
+            stmt.execute(configuration.sql);
+
+            final ResultSet resultSet = stmt.getResultSet();
+            loadJdbcResultSetFunc.load(resultSet);
+            System.out.printf("> cost: %d ms.\n\n", (System.currentTimeMillis() - start));
+
+            stmt.close();
+            conn.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void run(Configuration configuration) {
+        System.out.println("*************************************");
+        System.out.println("|          JdbcDriverManager        |");
+        System.out.println("*************************************");
+
+        try {
+            if (!Objects.equals(configuration.mysqlPort, "")) {
+                Class.forName("com.mysql.cj.jdbc.Driver");
+                System.out.println("JdbcDriverManager > jdbc:mysql > loadJdbcResult");
+                connectAndExecute(configuration, "jdbc:mysql", configuration.mysqlPort,
+                        JdbcResultSetReader.loadJdbcResult);
+                System.out.println("JdbcDriverManager > jdbc:mysql > loadJdbcResultToString");
+                connectAndExecute(configuration, "jdbc:mysql", configuration.mysqlPort,
+                        JdbcResultSetReader.loadJdbcResultToString);
+            }
+
+            Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver");
+            System.out.println("JdbcDriverManager > jdbc:arrow-flight-sql > loadJdbcResultToString");
+            connectAndExecute(configuration, "jdbc:arrow-flight-sql", configuration.arrowFlightPort,
+                    JdbcResultSetReader.loadJdbcResult);
+            System.out.println("JdbcDriverManager > jdbc:arrow-flight-sql > loadJdbcResultToString");
+            connectAndExecute(configuration, "jdbc:arrow-flight-sql", configuration.arrowFlightPort,
+                    JdbcResultSetReader.loadJdbcResultToString);
+        } catch (ClassNotFoundException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void main(String[] args) throws ClassNotFoundException {
+        Configuration configuration = new Configuration(args);
+        for (int i = 0; i < configuration.retryTimes; i++) {
+            run(configuration);
+        }
+    }
+}
diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcResultSetReader.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcResultSetReader.java
new file mode 100644
index 00000000000000..ac9f24a5ca50d1
--- /dev/null
+++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/JdbcResultSetReader.java
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package doris.arrowflight.demo;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Iterate over each row in jdbc ResultSet.
+ */
+public class JdbcResultSetReader {
+
+    @FunctionalInterface
+    public interface LoadJdbcResultSetFunc {
+        void load(ResultSet resultSet) throws IOException, SQLException;
+    }
+
+    public static LoadJdbcResultSetFunc loadJdbcResult = resultSet -> {
+        int rowCount = 0;
+        final int columnCount = resultSet.getMetaData().getColumnCount();
+        while (resultSet.next()) {
+            rowCount += 1;
+        }
+        System.out.println("> rowCount: " + rowCount + ", columnCount: " + columnCount);
+    };
+
+    public static LoadJdbcResultSetFunc loadJdbcResultToString = resultSet -> {
+        int rowCount = 0;
+        final int columnCount = resultSet.getMetaData().getColumnCount();
+        List<String> result = new ArrayList<>();
+        while (resultSet.next()) {
+            StringBuilder line = new StringBuilder();
+            for (int i = 1; i <= columnCount; i++) {
+                line.append(resultSet.getString(i)).append(",");
+            }
+            if (rowCount == 0) { // only print first line
+                System.out.println("> " + line);
+            }
+            rowCount += 1;
+            result.add(line.toString());
+        }
+        System.out.println(
+                "> rowCount: " + rowCount + ", columnCount: " + columnCount + " resultSize: " + result.size());
+    };
+}
diff --git a/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Main.java b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Main.java
new file mode 100644
index 00000000000000..2f9a8d20a879d1
--- /dev/null
+++ b/samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Main.java
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package doris.arrowflight.demo;
+
+public class Main {
+    public static void main(String[] args) {
+        Configuration configuration = new Configuration(args);
+        for (int i = 0; i < configuration.retryTimes; i++) {
+            FlightAdbcDriver.run(configuration);
+            FlightJdbcDriver.run(configuration);
+            JdbcDriverManager.run(configuration);
+        }
+        FlightSqlClient.run(configuration);
+    }
+}
diff --git a/samples/arrow-flight-sql/java/src/test/java/doris/arrowflight/demo/ConfigurationTest.java b/samples/arrow-flight-sql/java/src/test/java/doris/arrowflight/demo/ConfigurationTest.java
new file mode 100644
index 00000000000000..4614730ede6f45
--- /dev/null
+++ b/samples/arrow-flight-sql/java/src/test/java/doris/arrowflight/demo/ConfigurationTest.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package doris.arrowflight.demo;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Unit test for simple App.
+ */
+public class ConfigurationTest {
+
+    /**
+     * Rigorous Test :-)
+     */
+    @Test
+    public void shouldAnswerWithTrue() {
+        assertTrue(true);
+    }
+}

From 6cead2fff51583bc0b567c93e813f33a3185976b Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Fri, 13 Dec 2024 11:48:01 +0800
Subject: [PATCH 311/399] [fix](ip)fix default value for ip (#45194)

before this pr: we do not support create table for IP type with default
value
like this:
```
mysql> CREATE TABLE table2
    -> (
    ->     col0 BIGINT  NOT NULL DEFAULT '1' ,
    ->     col24 IPV4  NULL DEFAULT '127.0.0.1'
    -> )
    -> DUPLICATE KEY(col0)
    -> DISTRIBUTED BY HASH(col0) BUCKETS 4
    -> PROPERTIES (
    ->     "replication_num" = "1"
    -> );
ERROR 1105 (HY000): errCode = 2, detailMessage = Unsupported type: ipv4
```
after this pr we deal with this problem
---
 .../org/apache/doris/analysis/ColumnDef.java  |  6 ++++
 .../apache/doris/analysis/IPv4Literal.java    |  2 +-
 .../apache/doris/analysis/IPv6Literal.java    |  2 +-
 .../data/datatype_p0/ip/test_ip_basic.out     | 31 +++++++++++++++++++
 .../datatype_p0/ip/test_ip_basic.groovy       | 26 ++++++++++++++++
 5 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index c2c55c4ce7c955..02e123397c69aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -652,6 +652,12 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV
             case BOOLEAN:
                 new BoolLiteral(defaultValue);
                 break;
+            case IPV4:
+                new IPv4Literal(defaultValue);
+                break;
+            case IPV6:
+                new IPv6Literal(defaultValue);
+                break;
             default:
                 throw new AnalysisException("Unsupported type: " + type);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java
index 6fa84d18b25f41..92063e6b9b30c3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java
@@ -115,7 +115,7 @@ public Expr clone() {
 
     @Override
     protected String toSqlImpl() {
-        return getStringValue();
+        return "\"" + getStringValue() + "\"";
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java
index a3ec7a3a349824..bb986e0ffe7a2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java
@@ -75,7 +75,7 @@ private void checkValueValid(String ipv6) throws AnalysisException {
 
     @Override
     protected String toSqlImpl() {
-        return getStringValue();
+        return "\"" + getStringValue() + "\"";
     }
 
     @Override
diff --git a/regression-test/data/datatype_p0/ip/test_ip_basic.out b/regression-test/data/datatype_p0/ip/test_ip_basic.out
index b69f9708a1b1fc..e1b85abe00c41b 100644
--- a/regression-test/data/datatype_p0/ip/test_ip_basic.out
+++ b/regression-test/data/datatype_p0/ip/test_ip_basic.out
@@ -373,3 +373,34 @@ ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff	4
 -- !sql --
 1	false	127.0.0.1	ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff
 
+-- !sql --
+1	5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b	0.0.0.1
+2	::	127.0.0.1
+
+-- !sql --
+table_ip_default	UNIQUE_KEYS	col0	bigint	bigint	No	true	\N		true		
+		col4	ipv6	ipv6	Yes	false	::	NONE	true		
+		col24	ipv4	ipv4	Yes	false	127.0.0.1	NONE	true		
+
+-- !sql --
+table_ip_default_like	UNIQUE_KEYS	col0	bigint	bigint	No	true	\N		true		
+		col4	ipv6	ipv6	Yes	false	::	NONE	true		
+		col24	ipv4	ipv4	Yes	false	127.0.0.1	NONE	true		
+
+-- !sql --
+2
+
+-- !sql --
+1	5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b	0.0.0.1
+2	::	127.0.0.1
+
+-- !sql --
+0
+
+-- !sql --
+0
+
+-- !sql --
+1	5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b	0.0.0.1	::	127.0.0.1
+2	::	127.0.0.1	::	127.0.0.1
+
diff --git a/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy b/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy
index 468b6f6f146e14..5a4ab2ca94aca9 100644
--- a/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy
+++ b/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy
@@ -160,4 +160,30 @@ suite("test_ip_basic") {
     qt_sql """ select * from table_ip where col0 = 1"""
     sql """ Update table_ip set col25 = 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff' where col0 = 1 """
     qt_sql """ select * from table_ip where col0 = 1"""
+
+    // test ip with default value
+    sql """ DROP TABLE IF EXISTS table_ip_default """
+    sql """ CREATE TABLE IF NOT EXISTS `table_ip_default` (`col0` bigint NOT NULL, `col4` ipv6 NULL DEFAULT "::",   `col24` ipv4 NULL DEFAULT "127.0.0.1") ENGINE=OLAP UNIQUE KEY(`col0`) DISTRIBUTED BY HASH(`col0`) BUCKETS 4 PROPERTIES ("replication_allocation" = "tag.location.default: 1") """
+    sql """ insert into table_ip_default values (1, "5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b", "0.0.0.1") """
+    sql """ insert into table_ip_default(col0) values (2); """
+    qt_sql """ select * from table_ip_default order by col0"""
+    // add cases for default value to make sure in all cases, the default value is not lost.
+    // show create table
+    // desc table
+    // create table like
+    // insert into table
+    // alter new ip column with default value
+    def result = sql """ show create table table_ip_default """
+    log.info("show result : ${result}")
+    assertTrue(result.toString().containsIgnoreCase("`col4` ipv6 NULL DEFAULT \"::\""))
+    assertTrue(result.toString().containsIgnoreCase("`col24` ipv4 NULL DEFAULT \"127.0.0.1\""))
+    qt_sql """ desc table_ip_default all"""
+    sql """ DROP TABLE IF EXISTS table_ip_default_like """
+    sql """ create table table_ip_default_like like table_ip_default """
+    qt_sql """ desc table_ip_default_like all"""
+    qt_sql """ insert into table_ip_default_like select * from table_ip_default """
+    qt_sql """ select * from table_ip_default_like order by col0 """
+    qt_sql """ alter table table_ip_default_like add column col25 ipv6 NULL DEFAULT "::" """
+    qt_sql """ alter table table_ip_default_like add column col26 ipv4 NULL DEFAULT "127.0.0.1" """
+    qt_sql """ select * from table_ip_default_like order by col0 """
 }

From 7e478ca5cee5d83420dea8fbbbc40ce086189a23 Mon Sep 17 00:00:00 2001
From: qiye <luen@selectdb.com>
Date: Fri, 13 Dec 2024 13:25:26 +0800
Subject: [PATCH 312/399] [test](index compaction)Add index compaction
 exception fault injection cases (#45127)

Problem Summary:

1. Add skip index compaction logic when index compaction encounters
exceptions
2. Optimize fault injection regression cases
---
 be/src/olap/compaction.cpp                    | 150 ++++++++++++++----
 be/src/olap/compaction.h                      |   4 +
 .../segment_v2/inverted_index_compaction.cpp  |  11 +-
 ...ompaction_exception_fault_injection.groovy |  29 ++--
 4 files changed, 151 insertions(+), 43 deletions(-)

diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 0d811d9348f1f5..aec38699e014a2 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -495,10 +495,35 @@ Status CompactionMixin::execute_compact_impl(int64_t permits) {
 Status Compaction::do_inverted_index_compaction() {
     const auto& ctx = _output_rs_writer->context();
     if (!config::inverted_index_compaction_enable || _input_row_num <= 0 ||
-        !_stats.rowid_conversion || ctx.columns_to_do_index_compaction.empty()) {
+        ctx.columns_to_do_index_compaction.empty()) {
         return Status::OK();
     }
 
+    auto error_handler = [this](int64_t index_id, int64_t column_uniq_id) {
+        LOG(WARNING) << "failed to do index compaction"
+                     << ". tablet=" << _tablet->tablet_id() << ". column uniq id=" << column_uniq_id
+                     << ". index_id=" << index_id;
+        for (auto& rowset : _input_rowsets) {
+            rowset->set_skip_index_compaction(column_uniq_id);
+            LOG(INFO) << "mark skipping inverted index compaction next time"
+                      << ". tablet=" << _tablet->tablet_id() << ", rowset=" << rowset->rowset_id()
+                      << ", column uniq id=" << column_uniq_id << ", index_id=" << index_id;
+        }
+    };
+
+    DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_rowid_conversion_null",
+                    { _stats.rowid_conversion = nullptr; })
+    if (!_stats.rowid_conversion) {
+        LOG(WARNING) << "failed to do index compaction, rowid conversion is null"
+                     << ". tablet=" << _tablet->tablet_id()
+                     << ", input row number=" << _input_row_num;
+        mark_skip_index_compaction(ctx, error_handler);
+
+        return Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(
+                "failed to do index compaction, rowid conversion is null. tablet={}",
+                _tablet->tablet_id());
+    }
+
     OlapStopWatch inverted_watch;
 
     // translation vec
@@ -521,8 +546,7 @@ Status Compaction::do_inverted_index_compaction() {
     auto src_segment_num = src_seg_to_id_map.size();
     auto dest_segment_num = dest_segment_num_rows.size();
 
-    DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_dest_segment_num_is_zero",
-                    { dest_segment_num = 0; })
+    // when all the input rowsets are deleted, the output rowset will be empty and dest_segment_num will be 0.
     if (dest_segment_num <= 0) {
         LOG(INFO) << "skip doing index compaction due to no output segments"
                   << ". tablet=" << _tablet->tablet_id() << ", input row number=" << _input_row_num
@@ -600,27 +624,62 @@ Status Compaction::do_inverted_index_compaction() {
         DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_find_rowset_error",
                         { find_it = rs_id_to_rowset_map.end(); })
         if (find_it == rs_id_to_rowset_map.end()) [[unlikely]] {
-            // DCHECK(false) << _tablet->tablet_id() << ' ' << rowset_id;
-            return Status::InternalError("cannot find rowset. tablet_id={} rowset_id={}",
-                                         _tablet->tablet_id(), rowset_id.to_string());
+            LOG(WARNING) << "failed to do index compaction, cannot find rowset. tablet_id="
+                         << _tablet->tablet_id() << " rowset_id=" << rowset_id.to_string();
+            mark_skip_index_compaction(ctx, error_handler);
+            return Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(
+                    "failed to do index compaction, cannot find rowset. tablet_id={} rowset_id={}",
+                    _tablet->tablet_id(), rowset_id.to_string());
         }
 
         auto* rowset = find_it->second;
         auto fs = rowset->rowset_meta()->fs();
         DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_get_fs_error", { fs = nullptr; })
         if (!fs) {
-            return Status::InternalError("get fs failed, resource_id={}",
-                                         rowset->rowset_meta()->resource_id());
+            LOG(WARNING) << "failed to do index compaction, get fs failed. resource_id="
+                         << rowset->rowset_meta()->resource_id();
+            mark_skip_index_compaction(ctx, error_handler);
+            return Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(
+                    "get fs failed, resource_id={}", rowset->rowset_meta()->resource_id());
         }
 
-        auto seg_path = DORIS_TRY(rowset->segment_path(seg_id));
+        auto seg_path = rowset->segment_path(seg_id);
+        DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_seg_path_nullptr", {
+            seg_path = ResultError(Status::Error<ErrorCode::INTERNAL_ERROR>(
+                    "do_inverted_index_compaction_seg_path_nullptr"));
+        })
+        if (!seg_path.has_value()) {
+            LOG(WARNING) << "failed to do index compaction, get segment path failed. tablet_id="
+                         << _tablet->tablet_id() << " rowset_id=" << rowset_id.to_string()
+                         << " seg_id=" << seg_id;
+            mark_skip_index_compaction(ctx, error_handler);
+            return Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(
+                    "get segment path failed. tablet_id={} rowset_id={} seg_id={}",
+                    _tablet->tablet_id(), rowset_id.to_string(), seg_id);
+        }
         auto inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
-                fs, std::string {InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)},
+                fs,
+                std::string {InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value())},
                 _cur_tablet_schema->get_inverted_index_storage_format(),
                 rowset->rowset_meta()->inverted_index_file_info(seg_id));
-        RETURN_NOT_OK_STATUS_WITH_WARN(
-                inverted_index_file_reader->init(config::inverted_index_read_buffer_size),
-                "inverted_index_file_reader init faiqled");
+        auto st = inverted_index_file_reader->init(config::inverted_index_read_buffer_size);
+        DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_init_inverted_index_file_reader",
+                        {
+                            st = Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+                                    "debug point: "
+                                    "Compaction::do_inverted_index_compaction_init_inverted_index_"
+                                    "file_reader error");
+                        })
+        if (!st.ok()) {
+            LOG(WARNING) << "failed to do index compaction, init inverted index file reader "
+                            "failed. tablet_id="
+                         << _tablet->tablet_id() << " rowset_id=" << rowset_id.to_string()
+                         << " seg_id=" << seg_id;
+            mark_skip_index_compaction(ctx, error_handler);
+            return Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(
+                    "init inverted index file reader failed. tablet_id={} rowset_id={} seg_id={}",
+                    _tablet->tablet_id(), rowset_id.to_string(), seg_id);
+        }
         inverted_index_file_readers[m.second] = std::move(inverted_index_file_reader);
     }
 
@@ -628,7 +687,20 @@ Status Compaction::do_inverted_index_compaction() {
     // format: rowsetId_segmentId
     auto& inverted_index_file_writers = dynamic_cast<BaseBetaRowsetWriter*>(_output_rs_writer.get())
                                                 ->inverted_index_file_writers();
-    DCHECK_EQ(inverted_index_file_writers.size(), dest_segment_num);
+    DBUG_EXECUTE_IF(
+            "Compaction::do_inverted_index_compaction_inverted_index_file_writers_size_error",
+            { inverted_index_file_writers.clear(); })
+    if (inverted_index_file_writers.size() != dest_segment_num) {
+        LOG(WARNING) << "failed to do index compaction, dest segment num not match. tablet_id="
+                     << _tablet->tablet_id() << " dest_segment_num=" << dest_segment_num
+                     << " inverted_index_file_writers.size()="
+                     << inverted_index_file_writers.size();
+        mark_skip_index_compaction(ctx, error_handler);
+        return Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(
+                "dest segment num not match. tablet_id={} dest_segment_num={} "
+                "inverted_index_file_writers.size()={}",
+                _tablet->tablet_id(), dest_segment_num, inverted_index_file_writers.size());
+    }
 
     // use tmp file dir to store index files
     auto tmp_file_dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir();
@@ -637,18 +709,6 @@ Status Compaction::do_inverted_index_compaction() {
               << ". tablet=" << _tablet->tablet_id() << ", source index size=" << src_segment_num
               << ", destination index size=" << dest_segment_num << ".";
 
-    auto error_handler = [this](int64_t index_id, int64_t column_uniq_id) {
-        LOG(WARNING) << "failed to do index compaction"
-                     << ". tablet=" << _tablet->tablet_id() << ". column uniq id=" << column_uniq_id
-                     << ". index_id=" << index_id;
-        for (auto& rowset : _input_rowsets) {
-            rowset->set_skip_index_compaction(column_uniq_id);
-            LOG(INFO) << "mark skipping inverted index compaction next time"
-                      << ". tablet=" << _tablet->tablet_id() << ", rowset=" << rowset->rowset_id()
-                      << ", column uniq id=" << column_uniq_id << ", index_id=" << index_id;
-        }
-    };
-
     Status status = Status::OK();
     for (auto&& column_uniq_id : ctx.columns_to_do_index_compaction) {
         auto col = _cur_tablet_schema->column_by_uid(column_uniq_id);
@@ -658,6 +718,10 @@ Status Compaction::do_inverted_index_compaction() {
         if (index_meta == nullptr) {
             status = Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(
                     fmt::format("Can not find index_meta for col {}", col.name()));
+            LOG(WARNING) << "failed to do index compaction, can not find index_meta for column"
+                         << ". tablet=" << _tablet->tablet_id()
+                         << ", column uniq id=" << column_uniq_id;
+            error_handler(-1, column_uniq_id);
             break;
         }
 
@@ -671,6 +735,11 @@ Status Compaction::do_inverted_index_compaction() {
                             "debug point: Compaction::open_index_file_reader error"));
                 })
                 if (!res.has_value()) {
+                    LOG(WARNING) << "failed to do index compaction, open inverted index file "
+                                    "reader failed"
+                                 << ". tablet=" << _tablet->tablet_id()
+                                 << ", column uniq id=" << column_uniq_id
+                                 << ", src_segment_id=" << src_segment_id;
                     throw Exception(ErrorCode::INVERTED_INDEX_COMPACTION_ERROR, res.error().msg());
                 }
                 src_idx_dirs[src_segment_id] = std::move(res.value());
@@ -682,6 +751,11 @@ Status Compaction::do_inverted_index_compaction() {
                             "debug point: Compaction::open_inverted_index_file_writer error"));
                 })
                 if (!res.has_value()) {
+                    LOG(WARNING) << "failed to do index compaction, open inverted index file "
+                                    "writer failed"
+                                 << ". tablet=" << _tablet->tablet_id()
+                                 << ", column uniq id=" << column_uniq_id
+                                 << ", dest_segment_id=" << dest_segment_id;
                     throw Exception(ErrorCode::INVERTED_INDEX_COMPACTION_ERROR, res.error().msg());
                 }
                 // Destination directories in dest_index_dirs do not need to be deconstructed,
@@ -714,6 +788,23 @@ Status Compaction::do_inverted_index_compaction() {
     return Status::OK();
 }
 
+void Compaction::mark_skip_index_compaction(
+        const RowsetWriterContext& context,
+        const std::function<void(int64_t, int64_t)>& error_handler) {
+    for (auto&& column_uniq_id : context.columns_to_do_index_compaction) {
+        auto col = _cur_tablet_schema->column_by_uid(column_uniq_id);
+        const auto* index_meta = _cur_tablet_schema->inverted_index(col);
+        if (index_meta == nullptr) {
+            LOG(WARNING) << "mark skip index compaction, can not find index_meta for column"
+                         << ". tablet=" << _tablet->tablet_id()
+                         << ", column uniq id=" << column_uniq_id;
+            error_handler(-1, column_uniq_id);
+            continue;
+        }
+        error_handler(index_meta->index_id(), column_uniq_id);
+    }
+}
+
 void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
     for (const auto& index : _cur_tablet_schema->inverted_indexes()) {
         auto col_unique_ids = index->col_unique_ids();
@@ -789,7 +880,8 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
                 // TODO: inverted_index_path
                 auto seg_path = rowset->segment_path(i);
                 DBUG_EXECUTE_IF("Compaction::construct_skip_inverted_index_seg_path_nullptr", {
-                    seg_path = ResultError(Status::Error<ErrorCode::INTERNAL_ERROR>("error"));
+                    seg_path = ResultError(Status::Error<ErrorCode::INTERNAL_ERROR>(
+                            "construct_skip_inverted_index_seg_path_nullptr"));
                 })
                 if (!seg_path) {
                     LOG(WARNING) << seg_path.error();
@@ -800,8 +892,8 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
                 try {
                     auto inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
                             fs,
-                            std::string {
-                                    InvertedIndexDescriptor::get_index_file_path_prefix(*seg_path)},
+                            std::string {InvertedIndexDescriptor::get_index_file_path_prefix(
+                                    seg_path.value())},
                             _cur_tablet_schema->get_inverted_index_storage_format(),
                             rowset->rowset_meta()->inverted_index_file_info(i));
                     auto st = inverted_index_file_reader->init(
diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h
index ccabf7dadb4733..057f4084b068b3 100644
--- a/be/src/olap/compaction.h
+++ b/be/src/olap/compaction.h
@@ -70,6 +70,10 @@ class Compaction {
     // merge inverted index files
     Status do_inverted_index_compaction();
 
+    // mark all columns in columns_to_do_index_compaction to skip index compaction next time.
+    void mark_skip_index_compaction(const RowsetWriterContext& context,
+                                    const std::function<void(int64_t, int64_t)>& error_handler);
+
     void construct_index_compaction_columns(RowsetWriterContext& ctx);
 
     virtual Status construct_output_rowset_writer(RowsetWriterContext& ctx) = 0;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
index f988c46c027c26..dcbdca921ab8e8 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
@@ -79,7 +79,16 @@ Status compact_column(int64_t index_id,
 
     // delete temporary segment_path, only when inverted_index_ram_dir_enable is false
     if (!config::inverted_index_ram_dir_enable) {
-        std::ignore = io::global_local_filesystem()->delete_directory(tmp_path.data());
+        auto st = io::global_local_filesystem()->delete_directory(tmp_path.data());
+        DBUG_EXECUTE_IF("compact_column_delete_tmp_path_error", {
+            st = Status::Error<ErrorCode::INVERTED_INDEX_COMPACTION_ERROR>(
+                    "debug point: compact_column_delete_tmp_path_error in index compaction");
+        })
+        if (!st.ok()) {
+            LOG(WARNING) << "compact column failed to delete tmp path: " << tmp_path
+                         << ", error: " << st.to_string();
+            return st;
+        }
     }
     return Status::OK();
 }
diff --git a/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy
index ac3cd8125a863c..b54f6374d833b8 100644
--- a/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy
@@ -181,30 +181,26 @@ suite("test_index_compaction_exception_fault_injection", "nonConcurrent") {
         "compact_column_create_index_writer_error",
         "compact_column_indexCompaction_error",
         "compact_column_index_writer_close_error",
-        "compact_column_src_index_dirs_close_error",
+        "compact_column_delete_tmp_path_error",
         "Compaction::do_inverted_index_compaction_find_rowset_error",
         "Compaction::do_inverted_index_compaction_get_fs_error",
-        "Compaction::do_inverted_index_compaction_index_file_reader_init_error",
-        // "Compaction::do_inverted_index_compaction_file_size_status_not_ok", // v2 do not do index compaction
         "Compaction::do_inverted_index_compaction_can_not_find_index_meta",
-        "Compaction::do_inverted_index_compaction_index_properties_different",
-        "Compaction::do_inverted_index_compaction_index_file_writer_close_not_ok",
-        "Compaction::construct_skip_inverted_index_index_reader_close_error"
+        "Compaction::do_inverted_index_compaction_rowid_conversion_null",
+        "Compaction::do_inverted_index_compaction_seg_path_nullptr",
+        "Compaction::do_inverted_index_compaction_init_inverted_index_file_reader",
+        "Compaction::do_inverted_index_compaction_inverted_index_file_writers_size_error"
     ]
 
     def debug_points_normal_compaction = [
-        "compact_column_local_tmp_dir_delete_error",
-        // "Compaction::do_inverted_index_compaction_dest_segment_num_is_zero", // query result not match without inverted index
-        "Compaction::do_inverted_index_compaction_index_file_reader_init_not_found",
         "Compaction::construct_skip_inverted_index_is_skip_index_compaction",
         "Compaction::construct_skip_inverted_index_get_fs_error",
         "Compaction::construct_skip_inverted_index_index_meta_nullptr",
         "Compaction::construct_skip_inverted_index_seg_path_nullptr",
+        "Compaction::do_inverted_index_compaction_index_properties_different",
+        "Compaction::construct_skip_inverted_index_index_files_count",
+        "Compaction::construct_skip_inverted_index_index_reader_close_error",
         "Compaction::construct_skip_inverted_index_index_file_reader_init_status_not_ok",
-        "Compaction::construct_skip_inverted_index_index_file_reader_exist_status_not_ok",
-        "Compaction::construct_skip_inverted_index_index_file_reader_exist_false",
-        "Compaction::construct_skip_inverted_index_index_file_reader_open_error",
-        "Compaction::construct_skip_inverted_index_index_files_count"
+        "Compaction::construct_skip_inverted_index_index_file_reader_open_error"
     ]
 
     def run_test = { tablets, debug_point, abnormal ->
@@ -221,6 +217,9 @@ suite("test_index_compaction_exception_fault_injection", "nonConcurrent") {
             }
         }
 
+        if (debug_point == "compact_column_delete_tmp_path_error") {
+            set_be_config.call("inverted_index_ram_dir_enable", "false")
+        }
         // before full compaction, there are 7 rowsets.
         int rowsetCount = get_rowset_count.call(tablets);
         assert (rowsetCount == 7 * replicaNum)
@@ -258,6 +257,10 @@ suite("test_index_compaction_exception_fault_injection", "nonConcurrent") {
         }
 
         run_sql.call()
+        
+        if (debug_point == "compact_column_delete_tmp_path_error") {
+            set_be_config.call("inverted_index_ram_dir_enable", "true")
+        }
     }
 
     def create_and_test_table = { table_name, key_type, debug_points, is_abnormal ->

From 044cb5a691918dd4008d3aca0d175a72e8433bd2 Mon Sep 17 00:00:00 2001
From: Uniqueyou <1520358997@qq.com>
Date: Fri, 13 Dec 2024 14:13:15 +0800
Subject: [PATCH 313/399] [fix](sql) Fix error result for column tosql (#44997)

---
 .../org/apache/doris/analysis/ColumnDef.java  | 30 ++++---
 .../analysis/AddColumnDefaultValueTest.java   | 89 +++++++++++++++++++
 2 files changed, 106 insertions(+), 13 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnDefaultValueTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index 02e123397c69aa..d74451a9331e8b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -30,6 +30,7 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.common.util.SqlUtils;
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.qe.SessionVariable;
 
@@ -180,17 +181,6 @@ public String getValue() {
             }
             return value;
         }
-
-        public String toSql() {
-            StringBuilder sb = new StringBuilder();
-            sb.append("DEFAULT ");
-            if (value != null) {
-                sb.append('"').append(value).append('"');
-            } else {
-                sb.append("NULL");
-            }
-            return sb.toString();
-        }
     }
 
     // parameter initialized in constructor
@@ -439,7 +429,7 @@ public void analyze(boolean isOlap) throws AnalysisException {
         }
 
         if (type.getPrimitiveType() == PrimitiveType.HLL) {
-            if (defaultValue.isSet) {
+            if (defaultValue != null && defaultValue.isSet) {
                 throw new AnalysisException("Hll type column can not set default value");
             }
             defaultValue = DefaultValue.HLL_EMPTY_DEFAULT_VALUE;
@@ -687,7 +677,21 @@ public String toSql() {
         }
 
         if (defaultValue.isSet) {
-            sb.append(defaultValue.toSql()).append(" ");
+            if (defaultValue.value != null) {
+                if (typeDef.getType().getPrimitiveType() != PrimitiveType.BITMAP
+                        && typeDef.getType().getPrimitiveType() != PrimitiveType.HLL) {
+                    if (defaultValue.defaultValueExprDef != null) {
+                        sb.append("DEFAULT ").append(defaultValue.value).append(" ");
+                    } else {
+                        sb.append("DEFAULT ").append("\"").append(SqlUtils.escapeQuota(defaultValue.value)).append("\"")
+                                .append(" ");
+                    }
+                } else if (typeDef.getType().getPrimitiveType() == PrimitiveType.BITMAP) {
+                    sb.append("DEFAULT ").append(defaultValue.defaultValueExprDef.getExprName()).append(" ");
+                }
+            } else {
+                sb.append("DEFAULT ").append("NULL").append(" ");
+            }
         }
         sb.append("COMMENT \"").append(comment).append("\"");
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnDefaultValueTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnDefaultValueTest.java
new file mode 100644
index 00000000000000..4f59ee068db01a
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnDefaultValueTest.java
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.analysis.ColumnDef.DefaultValue;
+import org.apache.doris.catalog.AggregateType;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+
+import com.google.common.collect.Lists;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.List;
+
+public class AddColumnDefaultValueTest {
+    private static Analyzer analyzer;
+
+    @BeforeClass
+    public static void setUp() {
+        analyzer = AccessTestUtil.fetchAdminAnalyzer(false);
+    }
+
+    @Test
+    public void testNormal() throws AnalysisException {
+        List<ColumnDef> columns = Lists.newArrayList();
+        ColumnDef definition = new ColumnDef("col1", new TypeDef(ScalarType.createType(PrimitiveType.DATETIME)),
+                true,
+                null, false,
+                DefaultValue.CURRENT_TIMESTAMP_DEFAULT_VALUE, "");
+        definition.setKeysType(KeysType.DUP_KEYS);
+        columns.add(definition);
+        definition = new ColumnDef("col2", new TypeDef(ScalarType.createType(PrimitiveType.INT)),
+                false,
+                null, false,
+                new DefaultValue(true, 1), "");
+        columns.add(definition);
+        definition = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.VARCHAR)),
+                false,
+                null, false,
+                new DefaultValue(true, "varchar"), "");
+        columns.add(definition);
+        definition = new ColumnDef("col4", new TypeDef(ScalarType.createType(PrimitiveType.BITMAP)),
+                false,
+                null, false,
+                DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE, "");
+        definition.setKeysType(KeysType.DUP_KEYS);
+        columns.add(definition);
+        definition = new ColumnDef("col5", new TypeDef(ScalarType.createType(PrimitiveType.VARCHAR)),
+                false,
+                null, false,
+                new DefaultValue(true, "xxx\"\"xxx"), "");
+        columns.add(definition);
+        definition = new ColumnDef("col6", new TypeDef(ScalarType.createType(PrimitiveType.HLL)),
+                false,
+                AggregateType.HLL_UNION, false,
+                null, "");
+        columns.add(definition);
+        AddColumnsClause clause = new AddColumnsClause(columns, null, null);
+        clause.analyze(analyzer);
+        System.out.println(clause.toString());
+        Assert.assertEquals(
+                "ADD COLUMN (`col1` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT \"\", "
+                        + "`col2` int NOT NULL DEFAULT \"1\" COMMENT \"\", "
+                        + "`col3` varchar(65533) NOT NULL DEFAULT \"varchar\" COMMENT \"\", "
+                        + "`col4` bitmap NOT NULL DEFAULT BITMAP_EMPTY COMMENT \"\", "
+                        + "`col5` varchar(65533) NOT NULL DEFAULT \"xxx\\\"\\\"xxx\" COMMENT \"\", "
+                        + "`col6` hll HLL_UNION NOT NULL COMMENT \"\")",
+                clause.toString());
+    }
+}

From ffa02e329b928150c610a01e7c7b6f45f39c3154 Mon Sep 17 00:00:00 2001
From: starocean999 <lichi@selectdb.com>
Date: Fri, 13 Dec 2024 14:33:31 +0800
Subject: [PATCH 314/399] [fix](nereids)EliminateGroupBy rule should keep
 output's datatype unchanged (#45359)

Related PR: #28615

Problem Summary:
`select id, count(age) from t group by id`
consider t is unique table and it's unique key is id. The sql will be
convert to:
`select id, if(age is null, 0, 1) from t;`
count(age) is replaced by if(age is null, 0, 1). And we should keep
if(age is null, 0, 1)'s datatype same as count(age) which is bigint.
---
 .../doris/nereids/rules/rewrite/EliminateGroupBy.java  | 10 +++-------
 .../nereids/rules/rewrite/EliminateGroupByTest.java    |  1 +
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java
index b50d3a41e8c3b5..9325607dd70769 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java
@@ -30,7 +30,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.PlanUtils;
@@ -84,12 +84,8 @@ public Rule build() {
                                         .castIfNotSameType(f.child(0), f.getDataType()), ne.getName()));
                             } else if (f instanceof Count) {
                                 newOutput.add((NamedExpression) ne.withChildren(
-                                        new If(
-                                            new IsNull(f.child(0)),
-                                            Literal.of(0),
-                                            Literal.of(1)
-                                        )
-                                ));
+                                        new If(new IsNull(f.child(0)), new BigIntLiteral(0),
+                                                new BigIntLiteral(1))));
                             } else {
                                 throw new IllegalStateException("Unexpected aggregate function: " + f);
                             }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByTest.java
index 0b6d2f199fc59f..70eb8e4a470746 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByTest.java
@@ -94,6 +94,7 @@ void eliminateCount() {
                         logicalEmptyRelation().when(p -> p.getProjects().get(0).toSql().equals("id")
                                 && p.getProjects().get(1).toSql()
                                 .equals("if(age IS NULL, 0, 1) AS `if(age IS NULL, 0, 1)`")
+                                && p.getProjects().get(1).getDataType().isBigIntType()
                         )
                 );
     }

From 1d64cb077cb60bfe1ddc2e6226dd9b77e40b3979 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Fri, 13 Dec 2024 15:37:03 +0800
Subject: [PATCH 315/399] [fix](coordinator) Fix wrong bucket assignments by
 coordinator (#45365)

Split scan ranges evenly into `parallelExecInstanceNum` instances.


 For a fragment contains co-located join,

      scan (id = 0) -> join build (id = 2)
                          |
      scan (id = 1) -> join probe (id = 2)

If both of `scan (id = 0)` and `scan (id = 1)` are serial operators, we
will plan local exchanger
 after them:

      scan (id = 0) -> local exchange -> join build (id = 2)
                                               |
      scan (id = 1) -> local exchange -> join probe (id = 2)


And there is another more complicated scenario, for example, `scan (id =
0)` has 10 partitions and
3 buckets which means 3 * 10 tablets and `scan (id = 1)` has 3 buckets
and no partition which means
3 tablets totally. If expected parallelism is 8, we will get a serial
scan (id = 0) and a
non-serial scan (id = 1). For this case, we will plan another plan with
local exchange:

      scan (id = 0)          ->         join build (id = 2)
                                               |
      scan (id = 1) -> local exchange -> join probe (id = 2)
---
 .../java/org/apache/doris/qe/Coordinator.java |  85 ++++++++++----
 ...t_colocate_join_with_different_tablets.out |  12 ++
 ...olocate_join_with_different_tablets.groovy | 109 ++++++++++++++++++
 3 files changed, 182 insertions(+), 24 deletions(-)
 create mode 100644 regression-test/data/correctness_p0/test_colocate_join_with_different_tablets.out
 create mode 100644 regression-test/suites/correctness_p0/test_colocate_join_with_different_tablets.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 42e2e0a3634870..1e327c469e7140 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -2731,7 +2731,8 @@ private void assignScanRanges(PlanFragmentId fragmentId, int parallelExecInstanc
          * 1. `parallelExecInstanceNum * numBackends` is larger than scan ranges.
          * 2. Use Nereids planner.
          */
-        boolean ignoreStorageDataDistribution = params.fragment != null && params.fragment.useSerialSource(context);
+        boolean ignoreStorageDataDistribution = scanNodes != null && !scanNodes.isEmpty()
+                && params.fragment != null && params.fragment.useSerialSource(context);
 
         FragmentScanRangeAssignment assignment = params.scanRangeAssignment;
         for (Map.Entry<TNetworkAddress, List<Pair<Integer, Map<Integer, List<TScanRangeParams>>>>> addressScanRange
@@ -2741,33 +2742,69 @@ private void assignScanRanges(PlanFragmentId fragmentId, int parallelExecInstanc
                     = findOrInsert(assignment, addressScanRange.getKey(), new HashMap<>());
 
             if (ignoreStorageDataDistribution) {
-                FInstanceExecParam instanceParam = new FInstanceExecParam(
-                        null, addressScanRange.getKey(), 0, params);
-
-                for (Pair<Integer, Map<Integer, List<TScanRangeParams>>> nodeScanRangeMap : scanRange) {
-                    for (Map.Entry<Integer, List<TScanRangeParams>> nodeScanRange
-                            : nodeScanRangeMap.second.entrySet()) {
-                        if (!instanceParam.perNodeScanRanges.containsKey(nodeScanRange.getKey())) {
-                            range.put(nodeScanRange.getKey(), Lists.newArrayList());
-                            instanceParam.perNodeScanRanges.put(nodeScanRange.getKey(), Lists.newArrayList());
+                List<List<Pair<Integer, Map<Integer, List<TScanRangeParams>>>>> perInstanceScanRanges
+                        = ListUtil.splitBySize(scanRange, parallelExecInstanceNum);
+                /**
+                 * Split scan ranges evenly into `parallelExecInstanceNum` instances.
+                 *
+                 *
+                 * For a fragment contains co-located join,
+                 *
+                 *      scan (id = 0) -> join build (id = 2)
+                 *                          |
+                 *      scan (id = 1) -> join probe (id = 2)
+                 *
+                 * If both of `scan (id = 0)` and `scan (id = 1)` are serial operators, we will plan local exchanger
+                 * after them:
+                 *
+                 *      scan (id = 0) -> local exchange -> join build (id = 2)
+                 *                                               |
+                 *      scan (id = 1) -> local exchange -> join probe (id = 2)
+                 *
+                 *
+                 * And there is another more complicated scenario, for example, `scan (id = 0)` has 10 partitions and
+                 * 3 buckets which means 3 * 10 tablets and `scan (id = 1)` has 3 buckets and no partition which means
+                 * 3 tablets totally. If expected parallelism is 8, we will get a serial scan (id = 0) and a
+                 * non-serial scan (id = 1). For this case, we will plan another plan with local exchange:
+                 *
+                 *      scan (id = 0) -> local exchange -> join build (id = 2)
+                 *                                               |
+                 *      scan (id = 1)          ->         join probe (id = 2)
+                 */
+                FInstanceExecParam firstInstanceParam = null;
+                for (List<Pair<Integer, Map<Integer, List<TScanRangeParams>>>> perInstanceScanRange
+                        : perInstanceScanRanges) {
+                    FInstanceExecParam instanceParam = new FInstanceExecParam(
+                            null, addressScanRange.getKey(), 0, params);
+
+                    if (firstInstanceParam == null) {
+                        firstInstanceParam = instanceParam;
+                    }
+                    for (Pair<Integer, Map<Integer, List<TScanRangeParams>>> nodeScanRangeMap : perInstanceScanRange) {
+                        instanceParam.addBucketSeq(nodeScanRangeMap.first);
+                        for (Map.Entry<Integer, List<TScanRangeParams>> nodeScanRange
+                                : nodeScanRangeMap.second.entrySet()) {
+                            int scanId = nodeScanRange.getKey();
+                            Optional<ScanNode> node = scanNodes.stream().filter(
+                                    scanNode -> scanNode.getId().asInt() == scanId).findFirst();
+                            Preconditions.checkArgument(node.isPresent());
+                            FInstanceExecParam instanceParamToScan = node.get().isSerialOperator()
+                                    ? firstInstanceParam : instanceParam;
+                            if (!instanceParamToScan.perNodeScanRanges.containsKey(nodeScanRange.getKey())) {
+                                range.put(nodeScanRange.getKey(), Lists.newArrayList());
+                                instanceParamToScan.perNodeScanRanges
+                                        .put(nodeScanRange.getKey(), Lists.newArrayList());
+                            }
+                            range.get(nodeScanRange.getKey()).addAll(nodeScanRange.getValue());
+                            instanceParamToScan.perNodeScanRanges.get(nodeScanRange.getKey())
+                                    .addAll(nodeScanRange.getValue());
                         }
-                        range.get(nodeScanRange.getKey()).addAll(nodeScanRange.getValue());
-                        instanceParam.perNodeScanRanges.get(nodeScanRange.getKey())
-                                .addAll(nodeScanRange.getValue());
                     }
+                    params.instanceExecParams.add(instanceParam);
                 }
-                List<FInstanceExecParam> instanceExecParams = new ArrayList<>();
-                instanceExecParams.add(instanceParam);
-                for (int i = 1; i < parallelExecInstanceNum; i++) {
-                    instanceExecParams.add(new FInstanceExecParam(
-                            null, addressScanRange.getKey(), 0, params));
-                }
-                int index = 0;
-                for (Pair<Integer, Map<Integer, List<TScanRangeParams>>> nodeScanRangeMap : scanRange) {
-                    instanceExecParams.get(index % instanceExecParams.size()).addBucketSeq(nodeScanRangeMap.first);
-                    index++;
+                for (int i = perInstanceScanRanges.size(); i < parallelExecInstanceNum; i++) {
+                    params.instanceExecParams.add(new FInstanceExecParam(null, addressScanRange.getKey(), 0, params));
                 }
-                params.instanceExecParams.addAll(instanceExecParams);
             } else {
                 int expectedInstanceNum = 1;
                 if (parallelExecInstanceNum > 1) {
diff --git a/regression-test/data/correctness_p0/test_colocate_join_with_different_tablets.out b/regression-test/data/correctness_p0/test_colocate_join_with_different_tablets.out
new file mode 100644
index 00000000000000..06c381356ea470
--- /dev/null
+++ b/regression-test/data/correctness_p0/test_colocate_join_with_different_tablets.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+10000007	10000007	48414	10000007
+10000007	10000007	48414	10000007
+10000007	10000007	48414	10000007
+10000007	10000007	60426	10000007
+10000007	10000007	60426	10000007
+10000007	10000007	60426	10000007
+10000007	10000007	94460	10000007
+10000007	10000007	94460	10000007
+10000007	10000007	94460	10000007
+
diff --git a/regression-test/suites/correctness_p0/test_colocate_join_with_different_tablets.groovy b/regression-test/suites/correctness_p0/test_colocate_join_with_different_tablets.groovy
new file mode 100644
index 00000000000000..661835916865c5
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_colocate_join_with_different_tablets.groovy
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_colocate_join_with_different_tablets") {
+    sql """
+    DROP TABLE IF EXISTS `USR_V_KHZHSJ_ES_POC1`;
+    DROP TABLE IF EXISTS `USR_TLBL_VAL_R1`;
+        CREATE TABLE `USR_V_KHZHSJ_ES_POC1` (
+          `khid` bigint NULL,
+          `khh` bigint NULL
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`khid`, `khh`)
+        DISTRIBUTED BY HASH(`khid`) BUCKETS 16
+        PROPERTIES (
+        "colocate_with" = "test_colocate_join_with_different_tabletsgroup1",
+        "replication_allocation" = "tag.location.default: 1"
+        );
+
+
+        CREATE TABLE `USR_TLBL_VAL_R1` (
+          `lbl_id` bigint NOT NULL COMMENT "标签ID",
+          `khh` bigint NULL COMMENT "客户号"
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`lbl_id`, `khh`)
+        COMMENT '标签结果日表'
+        PARTITION BY LIST (`lbl_id`)
+        (PARTITION p0 VALUES IN ("0"),
+        PARTITION p1 VALUES IN ("1"),
+        PARTITION p29 VALUES IN ("29"),
+        PARTITION p35 VALUES IN ("35"),
+        PARTITION p57 VALUES IN ("57"),
+        PARTITION p352 VALUES IN ("352"),
+        PARTITION p402 VALUES IN ("402"),
+        PARTITION p523 VALUES IN ("523"),
+        PARTITION p2347 VALUES IN ("2347"),
+        PARTITION p10376 VALUES IN ("10376"),
+        PARTITION p42408 VALUES IN ("42408"),
+        PARTITION p44410 VALUES IN ("44410"),
+        PARTITION p48414 VALUES IN ("48414"),
+        PARTITION p50416 VALUES IN ("50416"),
+        PARTITION p52418 VALUES IN ("52418"),
+        PARTITION p56422 VALUES IN ("56422"),
+        PARTITION p60426 VALUES IN ("60426"),
+        PARTITION p64430 VALUES IN ("64430"),
+        PARTITION p66432 VALUES IN ("66432"),
+        PARTITION p70436 VALUES IN ("70436"),
+        PARTITION p72438 VALUES IN ("72438"),
+        PARTITION p74440 VALUES IN ("74440"),
+        PARTITION p78444 VALUES IN ("78444"),
+        PARTITION p84450 VALUES IN ("84450"),
+        PARTITION p86452 VALUES IN ("86452"),
+        PARTITION p88454 VALUES IN ("88454"),
+        PARTITION p90456 VALUES IN ("90456"),
+        PARTITION p92458 VALUES IN ("92458"),
+        PARTITION p94460 VALUES IN ("94460"),
+        PARTITION p96462 VALUES IN ("96462"),
+        PARTITION p98464 VALUES IN ("98464"),
+        PARTITION p100466 VALUES IN ("100466"),
+        PARTITION p102468 VALUES IN ("102468"),
+        PARTITION p104470 VALUES IN ("104470"),
+        PARTITION p106472 VALUES IN ("106472"),
+        PARTITION p108474 VALUES IN ("108474"),
+        PARTITION p110476 VALUES IN ("110476"),
+        PARTITION p112478 VALUES IN ("112478"),
+        PARTITION p114480 VALUES IN ("114480"),
+        PARTITION p122488 VALUES IN ("122488"),
+        PARTITION p124490 VALUES IN ("124490"),
+        PARTITION p126492 VALUES IN ("126492"),
+        PARTITION p130496 VALUES IN ("130496"),
+        PARTITION p134500 VALUES IN ("134500"),
+        PARTITION p150516 VALUES IN ("150516"),
+        PARTITION p154520 VALUES IN ("154520"),
+        PARTITION p158524 VALUES IN ("158524"),
+        PARTITION p158525 VALUES IN ("158525"),
+        PARTITION p1848141 VALUES IN ("1848141"),
+        PARTITION p1848161 VALUES IN ("1848161"),
+        PARTITION p1848177 VALUES IN ("1848177"),
+        PARTITION p1848197 VALUES IN ("1848197"),
+        PARTITION p1848218 VALUES IN ("1848218"))
+        DISTRIBUTED BY HASH(`khh`) BUCKETS 16
+        PROPERTIES (
+        "colocate_with" = "test_colocate_join_with_different_tabletsgroup1",
+        "replication_allocation" = "tag.location.default: 1"
+        );
+
+        insert into USR_V_KHZHSJ_ES_POC1 values(10000007, 10000007);
+        insert into USR_V_KHZHSJ_ES_POC1 values(10000007, 10000007);
+        insert into USR_V_KHZHSJ_ES_POC1 values(10000007, 10000007);
+
+        insert into USR_TLBL_VAL_R1 values(48414, 10000007);
+        insert into USR_TLBL_VAL_R1 values(94460, 10000007);
+        insert into USR_TLBL_VAL_R1 values(60426, 10000007);
+    """
+    qt_sql """ select  *    from  USR_V_KHZHSJ_ES_POC1  A,USR_TLBL_VAL_R1  B  WHERE  A.khid  =  B.khh order by lbl_id; """
+}

From ad58cd134601c301b4da2e72cd3f7c51c7a6fe7d Mon Sep 17 00:00:00 2001
From: FreeOnePlus <54164178+FreeOnePlus@users.noreply.github.com>
Date: Fri, 13 Dec 2024 15:49:53 +0800
Subject: [PATCH 316/399] [feat](docker)Modify the init_be and start_be scripts
 to meet the requirements for rapid Docker startup. (#45269)

Related PR: #45267

Problem Summary:

To meet the needs of rapid Docker startup, I have made adjustments to two related scripts in the Docker startup process. First, I added a env `SKIP_CHECK_ULIMIT` to the `start_be.sh` script, which will skip the size checks for `swap`, `ulimit`, and `max_map_count`. At the same time, I used `--console` to start the process and print logs. The reason why I did not use the `--daemon` daemon command to execute is that starting with a foreground log printing method in a Docker container is the correct and reliable approach.

At the same time, I added a check logic for a `be.conf` configuration item in the `init_be.sh` script: if it is the first time starting, append the export `SKIP_CHECK_ULIMIT=true` to skip the `ulimit` value check in the BE process. In summary, these adjustments can meet the basic requirements for rapid Docker startup usage.
---
 bin/start_be.sh                       | 36 ++++++++++++++-------------
 docker/runtime/be/resource/init_be.sh |  1 +
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/bin/start_be.sh b/bin/start_be.sh
index 5b7df5ead589fa..35b453a407c688 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -182,29 +182,31 @@ if [[ "${RUN_VERSION}" -eq 1 ]]; then
     exit 0
 fi
 
-if [[ "$(uname -s)" != 'Darwin' ]]; then
-    MAX_MAP_COUNT="$(cat /proc/sys/vm/max_map_count)"
-    if [[ "${MAX_MAP_COUNT}" -lt 2000000 ]]; then
-        echo "Set kernel parameter 'vm.max_map_count' to a value greater than 2000000, example: 'sysctl -w vm.max_map_count=2000000'"
-        exit 1
+if [[ "${SKIP_CHECK_ULIMIT:- "false"}" != "true" ]]; then
+    if [[ "$(uname -s)" != 'Darwin' ]]; then
+        MAX_MAP_COUNT="$(cat /proc/sys/vm/max_map_count)"
+        if [[ "${MAX_MAP_COUNT}" -lt 2000000 ]]; then
+            echo "Set kernel parameter 'vm.max_map_count' to a value greater than 2000000, example: 'sysctl -w vm.max_map_count=2000000'"
+            exit 1
+        fi
+
+        if [[ "$(swapon -s | wc -l)" -gt 1 ]]; then
+            echo "Disable swap memory before starting be"
+            exit 1
+        fi
     fi
 
-    if [[ "$(swapon -s | wc -l)" -gt 1 ]]; then
-        echo "Disable swap memory before starting be"
+    MAX_FILE_COUNT="$(ulimit -n)"
+    if [[ "${MAX_FILE_COUNT}" -lt 60000 ]]; then
+        echo "Set max number of open file descriptors to a value greater than 60000."
+        echo "Ask your system manager to modify /etc/security/limits.conf and append content like"
+        echo "  * soft nofile 655350"
+        echo "  * hard nofile 655350"
+        echo "and then run 'ulimit -n 655350' to take effect on current session."
         exit 1
     fi
 fi
 
-MAX_FILE_COUNT="$(ulimit -n)"
-if [[ "${MAX_FILE_COUNT}" -lt 60000 ]]; then
-    echo "Set max number of open file descriptors to a value greater than 60000."
-    echo "Ask your system manager to modify /etc/security/limits.conf and append content like"
-    echo "  * soft nofile 655350"
-    echo "  * hard nofile 655350"
-    echo "and then run 'ulimit -n 655350' to take effect on current session."
-    exit 1
-fi
-
 # add java libs
 # Must add hadoop libs, because we should load specified jars
 # instead of jars in hadoop libs, such as avro
diff --git a/docker/runtime/be/resource/init_be.sh b/docker/runtime/be/resource/init_be.sh
index f9269f3ee7e7b8..4b77eb3fc0df08 100644
--- a/docker/runtime/be/resource/init_be.sh
+++ b/docker/runtime/be/resource/init_be.sh
@@ -197,6 +197,7 @@ _main() {
     fi
     check_be_status
     doris_note "Ready to start BE！"
+    export SKIP_CHECK_ULIMIT=true
     ${DORIS_HOME}/be/bin/start_be.sh --console &
     child_pid=$!
     wait $child_pid

From 429cda44664703c6bb9a10c7e2c76fda527f8ada Mon Sep 17 00:00:00 2001
From: FreeOnePlus <54164178+FreeOnePlus@users.noreply.github.com>
Date: Fri, 13 Dec 2024 15:52:05 +0800
Subject: [PATCH 317/399] [typo](docker) Adjust the indentation format of the
 init_be and entry_point scripts, as well as the duration of loop execution.
 (#45308)

Adjust the indentation format of the `init_be` and `entry_point` scripts, as well as the duration of loop execution.
Adjust the smallest unit of all indentations to a single tab character, and modify the loop duration when checking the BE startup status, changing both from 300 seconds to 30 seconds to speed up the overall Docker startup time.
---
 docker/runtime/be/resource/entry_point.sh | 214 +++++++++++-----------
 docker/runtime/be/resource/init_be.sh     | 164 ++++++++---------
 2 files changed, 189 insertions(+), 189 deletions(-)

diff --git a/docker/runtime/be/resource/entry_point.sh b/docker/runtime/be/resource/entry_point.sh
index 6e3dfaf3875d89..474fa984a4d86a 100755
--- a/docker/runtime/be/resource/entry_point.sh
+++ b/docker/runtime/be/resource/entry_point.sh
@@ -61,95 +61,95 @@ docker_setup_env() {
 
 # Check the variables required for startup
 docker_required_variables_env() {
-  declare -g RUN_TYPE
-  if [[ -n "$FE_SERVERS" && -n "$BE_ADDR" ]]; then
-      RUN_TYPE="ELECTION"
-      if [[ $FE_SERVERS =~ ^.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}(,.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4})*$ || $FE_SERVERS =~ ^.+:([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$ ]]; then
-          doris_warn "FE_SERVERS" $FE_SERVERS
-      else
-          doris_error "FE_SERVERS rule error！example: \$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT[,\$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT]..."
-      fi
-      if [[ $BE_ADDR =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}$ || $BE_ADDR =~ ^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:):[1-6]{0,1}[0-9]{1,4}$ ]]; then
-        doris_warn "BE_ADDR" $BE_ADDR
-      else
-        doris_error "BE_ADDR rule error！example: \$BE_IP:\$HEARTBEAT_SERVICE_PORT"
-      fi
-      export RUN_TYPE=${RUN_TYPE}
-      return
-  fi
+    declare -g RUN_TYPE
+    if [[ -n "$FE_SERVERS" && -n "$BE_ADDR" ]]; then
+        RUN_TYPE="ELECTION"
+        if [[ $FE_SERVERS =~ ^.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}(,.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4})*$ || $FE_SERVERS =~ ^.+:([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$ ]]; then
+            doris_warn "FE_SERVERS" $FE_SERVERS
+        else
+            doris_error "FE_SERVERS rule error！example: \$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT[,\$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT]..."
+        fi
+        if [[ $BE_ADDR =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}$ || $BE_ADDR =~ ^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:):[1-6]{0,1}[0-9]{1,4}$ ]]; then
+            doris_warn "BE_ADDR" $BE_ADDR
+        else
+            doris_error "BE_ADDR rule error！example: \$BE_IP:\$HEARTBEAT_SERVICE_PORT"
+        fi
+        export RUN_TYPE=${RUN_TYPE}
+        return
+    fi
 
-  if [[ -n "$FE_MASTER_IP"  && -n "$BE_IP" && -n "$BE_PORT" ]]; then
-      RUN_TYPE="ASSIGN"
-      if [[ $FE_MASTER_IP =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}$ || $FE_MASTER_IP =~ ^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$ ]]; then
-          doris_warn "FE_MASTER_IP" $FE_MASTER_IP
-      else
-          doris_error "FE_MASTER_IP rule error！example: \$FE_MASTER_IP"
-      fi
-      if [[ $BE_IP =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}$ || $BE_IP =~ ^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$ ]]; then
-          doris_warn "BE_IP" $BE_IP
-      else
-          doris_error "BE_IP rule error！example: \$BE_IP"
-      fi
-      if [[ $BE_PORT =~ ^[1-6]{0,1}[0-9]{1,4}$ ]]; then
-          doris_warn "BE_PORT" $BE_PORT
-      else
-          doris_error "BE_PORT rule error！example: \$BE_PORT."
-      fi
-      export RUN_TYPE=${RUN_TYPE}
-      return
-  fi
+    if [[ -n "$FE_MASTER_IP"  && -n "$BE_IP" && -n "$BE_PORT" ]]; then
+        RUN_TYPE="ASSIGN"
+        if [[ $FE_MASTER_IP =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}$ || $FE_MASTER_IP =~ ^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$ ]]; then
+            doris_warn "FE_MASTER_IP" $FE_MASTER_IP
+        else
+            doris_error "FE_MASTER_IP rule error！example: \$FE_MASTER_IP"
+        fi
+        if [[ $BE_IP =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}$ || $BE_IP =~ ^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$ ]]; then
+            doris_warn "BE_IP" $BE_IP
+        else
+            doris_error "BE_IP rule error！example: \$BE_IP"
+        fi
+        if [[ $BE_PORT =~ ^[1-6]{0,1}[0-9]{1,4}$ ]]; then
+            doris_warn "BE_PORT" $BE_PORT
+        else
+            doris_error "BE_PORT rule error！example: \$BE_PORT."
+        fi
+        export RUN_TYPE=${RUN_TYPE}
+        return
+    fi
 
-  doris_error EOF "
-               Note that you did not configure the required parameters!
-               plan 1:
-               FE_SERVERS & BE_ADDR
-               plan 2:
-               FE_MASTER_IP & FE_MASTER_PORT & BE_IP & BE_PORT"
-              EOF
+    doris_error EOF "
+                 Note that you did not configure the required parameters!
+                 plan 1:
+                 FE_SERVERS & BE_ADDR
+                 plan 2:
+                 FE_MASTER_IP & FE_MASTER_PORT & BE_IP & BE_PORT"
+                EOF
 }
 
 get_doris_args() {
-  declare -g MASTER_FE_IP CURRENT_BE_IP CURRENT_BE_PORT PRIORITY_NETWORKS
-  if [ $RUN_TYPE == "ELECTION" ]; then
-      local feServerArray=($(echo "${FE_SERVERS}" | awk '{gsub (/,/," "); print $0}'))
-      for i in "${feServerArray[@]}"; do
-        val=${i}
-        val=${val// /}
-        tmpFeName=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, ""); print$1}')
-        tmpFeIp=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$2}')
-        tmpFeEditLogPort=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$3}')
-        check_arg "TMP_FE_NAME" $tmpFeName
-        feIpArray[$tmpFeName]=${tmpFeIp}
-      done
+    declare -g MASTER_FE_IP CURRENT_BE_IP CURRENT_BE_PORT PRIORITY_NETWORKS
+    if [ $RUN_TYPE == "ELECTION" ]; then
+        local feServerArray=($(echo "${FE_SERVERS}" | awk '{gsub (/,/," "); print $0}'))
+        for i in "${feServerArray[@]}"; do
+            val=${i}
+            val=${val// /}
+            tmpFeName=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, ""); print$1}')
+            tmpFeIp=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$2}')
+            tmpFeEditLogPort=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$3}')
+            check_arg "TMP_FE_NAME" $tmpFeName
+            feIpArray[$tmpFeName]=${tmpFeIp}
+        done
 
-      FE_MASTER_IP=${feIpArray[1]}
-      check_arg "FE_MASTER_IP" $FE_MASTER_IP
-      BE_IP=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$1}')
-      check_arg "BE_IP" $BE_IP
-      BE_PORT=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$2}')
-      check_arg "BE_PORT" $BE_PORT
+        FE_MASTER_IP=${feIpArray[1]}
+        check_arg "FE_MASTER_IP" $FE_MASTER_IP
+        BE_IP=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$1}')
+        check_arg "BE_IP" $BE_IP
+        BE_PORT=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$2}')
+        check_arg "BE_PORT" $BE_PORT
 
-  elif [ $RUN_TYPE == "ASSIGN" ]; then
-      check_arg "FE_MASTER_IP" $FE_MASTER_IP
-      check_arg "BE_IP" $BE_IP
-      check_arg "BE_PORT" $BE_PORT
-  fi
+    elif [ $RUN_TYPE == "ASSIGN" ]; then
+        check_arg "FE_MASTER_IP" $FE_MASTER_IP
+        check_arg "BE_IP" $BE_IP
+        check_arg "BE_PORT" $BE_PORT
+    fi
 
-  PRIORITY_NETWORKS=$(echo "${BE_IP}" | awk -F '.' '{print$1"."$2"."$3".0/24"}')
-  check_arg "PRIORITY_NETWORKS" $PRIORITY_NETWORKS
+    PRIORITY_NETWORKS=$(echo "${BE_IP}" | awk -F '.' '{print$1"."$2"."$3".0/24"}')
+    check_arg "PRIORITY_NETWORKS" $PRIORITY_NETWORKS
 
-  # export be args
-  export MASTER_FE_IP=${FE_MASTER_IP}
-  export CURRENT_BE_IP=${BE_IP}
-  export CURRENT_BE_PORT=${BE_PORT}
-  export PRIORITY_NETWORKS=${PRIORITY_NETWORKS}
+    # export be args
+    export MASTER_FE_IP=${FE_MASTER_IP}
+    export CURRENT_BE_IP=${BE_IP}
+    export CURRENT_BE_PORT=${BE_PORT}
+    export PRIORITY_NETWORKS=${PRIORITY_NETWORKS}
 
-  doris_note "MASTER_FE_IP ${MASTER_FE_IP}"
-  doris_note "CURRENT_BE_IP ${CURRENT_BE_IP}"
-  doris_note "CURRENT_BE_PORT ${CURRENT_BE_PORT}"
-  doris_note "PRIORITY_NETWORKS ${PRIORITY_NETWORKS}"
+    doris_note "MASTER_FE_IP ${MASTER_FE_IP}"
+    doris_note "CURRENT_BE_IP ${CURRENT_BE_IP}"
+    doris_note "CURRENT_BE_PORT ${CURRENT_BE_PORT}"
+    doris_note "PRIORITY_NETWORKS ${PRIORITY_NETWORKS}"
 
-  check_be_status true
+    check_be_status true
 }
 
 # Execute sql script, passed via stdin
@@ -162,32 +162,32 @@ docker_process_sql() {
 }
 
 check_be_status() {
-  set +e
-  for i in {1..300}; do
-    if [[ $1 == true ]]; then
-      docker_process_sql <<<"show frontends" | grep "[[:space:]]${MASTER_FE_IP}[[:space:]]"
-    else
-      docker_process_sql <<<"show backends" | grep "[[:space:]]${CURRENT_BE_IP}[[:space:]]" | grep "[[:space:]]${CURRENT_BE_PORT}[[:space:]]"
-    fi
-    be_join_status=$?
-    if [[ "${be_join_status}" == 0 ]]; then
-      if [[ $1 == true ]]; then
-        doris_note "MASTER FE is started!"
-      else
-        doris_note "EntryPoint Check - Verify that BE is registered to FE successfully"
-        BE_ALREADY_EXISTS=true
-      fi
-      return
-    fi
-    if [[ $(( $i % 20 )) == 1 ]]; then
-      if [[ $1 == true ]]; then
-        doris_note "MASTER FE is not started. retry."
-      else
-        doris_note "BE is not register. retry."
-      fi
-    fi
-    sleep 1
-  done
+    set +e
+    for i in {1..30}; do
+        if [[ $1 == true ]]; then
+            docker_process_sql <<<"show frontends" | grep "[[:space:]]${MASTER_FE_IP}[[:space:]]"
+        else
+            docker_process_sql <<<"show backends" | grep "[[:space:]]${CURRENT_BE_IP}[[:space:]]" | grep "[[:space:]]${CURRENT_BE_PORT}[[:space:]]"
+        fi
+        be_join_status=$?
+        if [[ "${be_join_status}" == 0 ]]; then
+            if [[ $1 == true ]]; then
+                doris_note "MASTER FE is started!"
+            else
+                doris_note "EntryPoint Check - Verify that BE is registered to FE successfully"
+                BE_ALREADY_EXISTS=true
+            fi
+            return
+        fi
+        if [[ $(( $i % 15 )) == 1 ]]; then
+            if [[ $1 == true ]]; then
+                doris_note "MASTER FE is not started. retry."
+            else
+                doris_note "BE is not register. retry."
+            fi
+        fi
+        sleep 1
+    done
 }
 
 # usage: docker_process_init_files [file [file [...]]]
@@ -240,9 +240,9 @@ docker_process_init_files() {
 # Check whether the passed parameters are empty to avoid subsequent task execution failures. At the same time,
 # enumeration checks can be added, such as checking whether a certain parameter appears repeatedly, etc.
 check_arg() {
-  if [ -z $2 ]; then
-    doris_error "$1 is null!"
-  fi
+    if [ -z $2 ]; then
+        doris_error "$1 is null!"
+    fi
 }
 
 _main() {
diff --git a/docker/runtime/be/resource/init_be.sh b/docker/runtime/be/resource/init_be.sh
index 4b77eb3fc0df08..82b3007ba19154 100644
--- a/docker/runtime/be/resource/init_be.sh
+++ b/docker/runtime/be/resource/init_be.sh
@@ -28,50 +28,50 @@ DORIS_HOME="/opt/apache-doris"
 #    ie: doris_warn "task may be risky!"
 #   out: 2023-01-08T19:08:16+08:00 [Warn] [Entrypoint]: task may be risky!
 doris_log() {
-  local type="$1"
-  shift
-  # accept argument string or stdin
-  local text="$*"
-  if [ "$#" -eq 0 ]; then text="$(cat)"; fi
-  local dt="$(date -Iseconds)"
-  printf '%s [%s] [Entrypoint]: %s\n' "$dt" "$type" "$text"
+    local type="$1"
+    shift
+    # accept argument string or stdin
+    local text="$*"
+    if [ "$#" -eq 0 ]; then text="$(cat)"; fi
+    local dt="$(date -Iseconds)"
+    printf '%s [%s] [Entrypoint]: %s\n' "$dt" "$type" "$text"
 }
 doris_note() {
-  doris_log Note "$@"
+    doris_log Note "$@"
 }
 doris_warn() {
-  doris_log Warn "$@" >&2
+    doris_log Warn "$@" >&2
 }
 doris_error() {
-  doris_log ERROR "$@" >&2
-  exit 1
+    doris_log ERROR "$@" >&2
+    exit 1
 }
 
 # check to see if this file is being run or sourced from another script
 _is_sourced() {
-  [ "${#FUNCNAME[@]}" -ge 2 ] &&
+    [ "${#FUNCNAME[@]}" -ge 2 ] &&
     [ "${FUNCNAME[0]}" = '_is_sourced' ] &&
     [ "${FUNCNAME[1]}" = 'source' ]
 }
 
 docker_setup_env() {
-  declare -g DATABASE_ALREADY_EXISTS
-  if [ -d "${DORIS_HOME}/be/storage/data" ]; then
-    DATABASE_ALREADY_EXISTS='true'
-  fi
+    declare -g DATABASE_ALREADY_EXISTS
+    if [ -d "${DORIS_HOME}/be/storage/data" ]; then
+        DATABASE_ALREADY_EXISTS='true'
+    fi
 }
 
 add_priority_networks() {
-  doris_note "add priority_networks ${1} to ${DORIS_HOME}/be/conf/be.conf"
-  echo "priority_networks = ${1}" >>${DORIS_HOME}/be/conf/be.conf
+    doris_note "add priority_networks ${1} to ${DORIS_HOME}/be/conf/be.conf"
+    echo "priority_networks = ${1}" >>${DORIS_HOME}/be/conf/be.conf
 }
 
 show_be_args(){
-  doris_note "============= init args ================"
-  doris_note "MASTER_FE_IP " ${MASTER_FE_IP}
-  doris_note "CURRENT_BE_IP " ${CURRENT_BE_IP}
-  doris_note "CURRENT_BE_PORT " ${CURRENT_BE_PORT}
-  doris_note "RUN_TYPE " ${RUN_TYPE}
+    doris_note "============= init args ================"
+    doris_note "MASTER_FE_IP " ${MASTER_FE_IP}
+    doris_note "CURRENT_BE_IP " ${CURRENT_BE_IP}
+    doris_note "CURRENT_BE_PORT " ${CURRENT_BE_PORT}
+    doris_note "RUN_TYPE " ${RUN_TYPE}
 }
 
 # Execute sql script, passed via stdin
@@ -99,40 +99,40 @@ register_be_to_fe() {
     # check fe status
     local is_fe_start=false
     if [ -n "$DATABASE_ALREADY_EXISTS" ]; then
-      check_be_status
-      if [ -n "$BE_ALREADY_EXISTS" ]; then
-        doris_warn "Same backend already exists! No need to register again！"
-        return
-      fi
+        check_be_status
+        if [ -n "$BE_ALREADY_EXISTS" ]; then
+            doris_warn "Same backend already exists! No need to register again！"
+            return
+        fi
     fi
     for i in {1..300}; do
-      if [[ $RUN_TYPE == "ELECTION" || $RUN_TYPE == "ASSIGN" ]]; then
-          SQL="alter system add backend '${CURRENT_BE_IP}:${CURRENT_BE_PORT}';"
-          doris_note "Executing SQL: $SQL"
-          docker_process_sql <<<"$SQL"
-      elif [[ $RUN_TYPE == "FQDN" ]]; then
-          SQL="alter system add backend '${CURRENT_NODE_NAME}:${CURRENT_BE_PORT}';"
-          doris_note "Executing SQL: $SQL"
-          docker_process_sql <<<"$SQL"
-      fi
-      register_be_status=$?
-      if [[ $register_be_status == 0 ]]; then
-        doris_note "BE successfully registered to FE！"
-        is_fe_start=true
-        return
-      else
-        check_be_status
-        if [[ $IS_BE_JOIN_STATUS == "true" ]]; then
-          return
+        if [[ $RUN_TYPE == "ELECTION" || $RUN_TYPE == "ASSIGN" ]]; then
+            SQL="alter system add backend '${CURRENT_BE_IP}:${CURRENT_BE_PORT}';"
+            doris_note "Executing SQL: $SQL"
+            docker_process_sql <<<"$SQL"
+        elif [[ $RUN_TYPE == "FQDN" ]]; then
+            SQL="alter system add backend '${CURRENT_NODE_NAME}:${CURRENT_BE_PORT}';"
+            doris_note "Executing SQL: $SQL"
+            docker_process_sql <<<"$SQL"
+        fi
+        register_be_status=$?
+        if [[ $register_be_status == 0 ]]; then
+            doris_note "BE successfully registered to FE！"
+            is_fe_start=true
+            return
+        else
+            check_be_status
+            if [[ $IS_BE_JOIN_STATUS == "true" ]]; then
+                return
+            fi
+        fi
+        if [[ $(( $i % 20 )) == 1 ]]; then
+            doris_note "Register BE to FE is failed. retry."
         fi
-      fi
-      if [[ $(( $i % 20 )) == 1 ]]; then
-        doris_note "Register BE to FE is failed. retry."
-      fi
-      sleep 1
+        sleep 1
     done
     if ! [[ $is_fe_start ]]; then
-      doris_error "Failed to register BE to FE！Tried 30 times！Maybe FE Start Failed！"
+        doris_error "Failed to register BE to FE！Tried 30 times！Maybe FE Start Failed！"
     fi
 }
 
@@ -141,27 +141,27 @@ check_be_status() {
     declare -g IS_FE_START_STATUS IS_BE_JOIN_STATUS
     IS_FE_START_STATUS=false
     IS_BE_JOIN_STATUS=false
-    for i in {1..100}; do
-      if [[ $(($i % 20)) == 1 ]]; then
-        doris_warn "start check be register status~"
-      fi
-      if [[ $RUN_TYPE == "ELECTION" || $RUN_TYPE == "ASSIGN" ]]; then
-        docker_process_sql <<<"show backends" | grep "[[:space:]]${CURRENT_BE_IP}[[:space:]]" | grep "[[:space:]]${CURRENT_BE_PORT}[[:space:]]"
-      elif [[ $RUN_TYPE == "FQDN" ]]; then
-        docker_process_sql <<<"show backends" | grep "[[:space:]]${CURRENT_NODE_NAME}[[:space:]]" | grep "[[:space:]]${CURRENT_BE_PORT}[[:space:]]"
-      fi
-      be_join_status=$?
-      if [[ "${be_join_status}" == 0 ]]; then
-        doris_note "Verify that BE is registered to FE successfully"
-        IS_FE_START_STATUS=true
-        IS_BE_JOIN_STATUS=true
-        return
-      else
-      if [[ $(($i % 20)) == 1 ]]; then
-        doris_note "register is failed, wait next~"
-      fi
-      fi
-      sleep 1
+    for i in {1..30}; do
+        if [[ $(($i % 15)) == 1 ]]; then
+            doris_warn "start check be register status~"
+        fi
+        if [[ $RUN_TYPE == "ELECTION" || $RUN_TYPE == "ASSIGN" ]]; then
+            docker_process_sql <<<"show backends" | grep "[[:space:]]${CURRENT_BE_IP}[[:space:]]" | grep "[[:space:]]${CURRENT_BE_PORT}[[:space:]]"
+        elif [[ $RUN_TYPE == "FQDN" ]]; then
+            docker_process_sql <<<"show backends" | grep "[[:space:]]${CURRENT_NODE_NAME}[[:space:]]" | grep "[[:space:]]${CURRENT_BE_PORT}[[:space:]]"
+        fi
+        be_join_status=$?
+        if [[ "${be_join_status}" == 0 ]]; then
+            doris_note "Verify that BE is registered to FE successfully"
+            IS_FE_START_STATUS=true
+            IS_BE_JOIN_STATUS=true
+            return
+        else
+            if [[ $(($i % 15)) == 1 ]]; then
+                doris_note "register is failed, wait next~"
+            fi
+        fi
+        sleep 1
     done
     if [[ ! $IS_FE_START_STATUS ]]; then
         doris_error "Failed to register BE to FE！Tried 30 times！Maybe FE Start Failed！"
@@ -186,14 +186,14 @@ _main() {
     trap 'cleanup' SIGTERM SIGINT
     docker_setup_env
     if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
-      if [ $RUN_TYPE == "FQDN" ]; then
-        add_fqdn_conf
-      else
-        add_priority_networks $PRIORITY_NETWORKS
-      fi
-      node_role_conf
-      show_be_args
-      register_be_to_fe
+        if [ $RUN_TYPE == "FQDN" ]; then
+            add_fqdn_conf
+        else
+            add_priority_networks $PRIORITY_NETWORKS
+        fi
+        node_role_conf
+        show_be_args
+        register_be_to_fe
     fi
     check_be_status
     doris_note "Ready to start BE！"
@@ -205,5 +205,5 @@ _main() {
 }
 
 if ! _is_sourced; then
-  _main "$@"
+    _main "$@"
 fi

From 55f450cd00ead0728b592abc2b1020aee83ca009 Mon Sep 17 00:00:00 2001
From: abmdocrt <lianyukang@selectdb.com>
Date: Fri, 13 Dec 2024 16:50:44 +0800
Subject: [PATCH 318/399] [Enhancement](compaction) Do not set failure time
 when cumulative compaction dealing with delete rowset (#43466)

Before this PR, in cases where there is an alternating distribution of
data rowset -> delete rowset -> data rowset -> delete rowset, cumulative
compaction would only move the cumulative point forward to allow base
compaction to handle the delete rowset. Cumulative compaction itself
would not process the data and would return be marked as failure. This
would cause the compaction submission task process to pause for 5
seconds, impacting efficiency.

This PR modifies the return status to OK for such cases, which improves
the efficiency of the compaction submission task.
---
 be/src/cloud/cloud_cumulative_compaction.cpp  |   4 +
 be/src/cloud/cloud_storage_engine.cpp         |   3 +-
 be/src/common/config.cpp                      |   1 +
 be/src/common/config.h                        |   2 +
 be/src/olap/tablet.cpp                        |  13 +-
 .../test_cumu_compaction_with_delete.out      |   5 +
 .../test_cumu_compaction_with_delete.groovy   | 122 ++++++++++++++++++
 7 files changed, 148 insertions(+), 2 deletions(-)
 create mode 100644 regression-test/data/compaction/test_cumu_compaction_with_delete.out
 create mode 100644 regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy

diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp
index 6b4dbe360da651..1acf8efe32e62b 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -92,6 +92,10 @@ Status CloudCumulativeCompaction::prepare_compact() {
             // plus 1 to skip the delete version.
             // NOTICE: after that, the cumulative point may be larger than max version of this tablet, but it doesn't matter.
             update_cumulative_point();
+            if (!config::enable_sleep_between_delete_cumu_compaction) {
+                st = Status::Error<CUMULATIVE_NO_SUITABLE_VERSION>(
+                        "_last_delete_version.first not equal to -1");
+            }
         }
         return st;
     }
diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp
index b66a9cfbdb2245..650909a29157cd 100644
--- a/be/src/cloud/cloud_storage_engine.cpp
+++ b/be/src/cloud/cloud_storage_engine.cpp
@@ -677,7 +677,8 @@ Status CloudStorageEngine::_submit_cumulative_compaction_task(const CloudTabletS
     auto st = compaction->prepare_compact();
     if (!st.ok()) {
         long now = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
-        if (st.is<ErrorCode::CUMULATIVE_NO_SUITABLE_VERSION>()) {
+        if (st.is<ErrorCode::CUMULATIVE_NO_SUITABLE_VERSION>() &&
+            st.msg() != "_last_delete_version.first not equal to -1") {
             // Backoff strategy if no suitable version
             tablet->last_cumu_no_suitable_version_ms = now;
         }
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index b70f492a29dfda..95a3e61fb5517a 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1402,6 +1402,7 @@ DEFINE_mBool(enable_delete_bitmap_merge_on_compaction, "false");
 // Enable validation to check the correctness of table size.
 DEFINE_Bool(enable_table_size_correctness_check, "false");
 DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false");
+DEFINE_mBool(enable_sleep_between_delete_cumu_compaction, "false");
 
 // clang-format off
 #ifdef BE_TEST
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 984024cab7c77f..f8a9c3f7480b33 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1487,6 +1487,8 @@ DECLARE_Bool(force_regenerate_rowsetid_on_start_error);
 DECLARE_mBool(enable_delete_bitmap_merge_on_compaction);
 // Enable validation to check the correctness of table size.
 DECLARE_Bool(enable_table_size_correctness_check);
+// Enable sleep 5s between delete cumulative compaction.
+DECLARE_mBool(enable_sleep_between_delete_cumu_compaction);
 
 #ifdef BE_TEST
 // test s3
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index a1a56507ffc67a..705d0ad53097e8 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1750,8 +1750,13 @@ Status Tablet::prepare_compaction_and_calculate_permits(
         }
 
         if (!res.ok()) {
-            tablet->set_last_cumu_compaction_failure_time(UnixMillis());
             permits = 0;
+            // if we meet a delete version, should increase the cumulative point to let base compaction handle the delete version.
+            // no need to wait 5s.
+            if (!(res.msg() == "_last_delete_version.first not equal to -1") ||
+                config::enable_sleep_between_delete_cumu_compaction) {
+                tablet->set_last_cumu_compaction_failure_time(UnixMillis());
+            }
             if (!res.is<CUMULATIVE_NO_SUITABLE_VERSION>()) {
                 DorisMetrics::instance()->cumulative_compaction_request_failed->increment(1);
                 return Status::InternalError("prepare cumulative compaction with err: {}", res);
@@ -1759,6 +1764,12 @@ Status Tablet::prepare_compaction_and_calculate_permits(
             // return OK if OLAP_ERR_CUMULATIVE_NO_SUITABLE_VERSION, so that we don't need to
             // print too much useless logs.
             // And because we set permits to 0, so even if we return OK here, nothing will be done.
+            LOG_INFO(
+                    "cumulative compaction meet delete rowset, increase cumu point without other "
+                    "operation.")
+                    .tag("tablet id:", tablet->tablet_id())
+                    .tag("after cumulative compaction, cumu point:",
+                         tablet->cumulative_layer_point());
             return Status::OK();
         }
     } else if (compaction_type == CompactionType::BASE_COMPACTION) {
diff --git a/regression-test/data/compaction/test_cumu_compaction_with_delete.out b/regression-test/data/compaction/test_cumu_compaction_with_delete.out
new file mode 100644
index 00000000000000..642559699ac60c
--- /dev/null
+++ b/regression-test/data/compaction/test_cumu_compaction_with_delete.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select1 --
+
+-- !select2 --
+
diff --git a/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
new file mode 100644
index 00000000000000..7c6be0b177ce1e
--- /dev/null
+++ b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_cumu_compaction_with_delete") {
+    def tableName = "test_cumu_compaction_with_delete"
+    def check_cumu_point = { cumu_point ->
+        def tablets = sql_return_maparray """ show tablets from ${tableName}; """
+        int cumuPoint = 0
+        for (def tablet in tablets) {
+            String tablet_id = tablet.TabletId
+            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            cumuPoint = tabletJson["cumulative point"]
+        }
+        return cumuPoint > cumu_point
+    }
+
+    try {
+        sql """ DROP TABLE IF EXISTS ${tableName} """
+        sql """
+            CREATE TABLE ${tableName} (
+            `user_id` INT NOT NULL,
+            `value` INT NOT NULL)
+            UNIQUE KEY(`user_id`) 
+            DISTRIBUTED BY HASH(`user_id`) 
+            BUCKETS 1 
+            PROPERTIES ("replication_allocation" = "tag.location.default: 1",
+            "enable_mow_light_delete" = "true")"""
+
+        for(int i = 1; i <= 100; ++i){
+            sql """ INSERT INTO ${tableName} VALUES (1,1)"""
+            sql """ delete from ${tableName} where user_id = 1"""
+        }
+
+        now = System.currentTimeMillis()
+
+        while(true){
+            if(check_cumu_point(100)){
+                break;
+            }
+            Thread.sleep(1000)
+        }
+        time_diff = System.currentTimeMillis() - now
+        logger.info("time_diff:" + time_diff)
+        assertTrue(time_diff<200*1000)
+
+        qt_select1 """select * from ${tableName} order by user_id, value"""
+    } catch (Exception e){
+        logger.info(e.getMessage())
+        assertFalse(true)
+    } finally {
+        try_sql("DROP TABLE IF EXISTS ${tableName} FORCE")
+    }
+
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+    def set_be_config = { key, value ->
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+            logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+        }
+    }
+
+    try {
+        set_be_config.call("enable_sleep_between_delete_cumu_compaction", "true")
+        sql """ DROP TABLE IF EXISTS ${tableName} """
+        sql """
+            CREATE TABLE ${tableName} (
+            `user_id` INT NOT NULL,
+            `value` INT NOT NULL)
+            UNIQUE KEY(`user_id`) 
+            DISTRIBUTED BY HASH(`user_id`) 
+            BUCKETS 1 
+            PROPERTIES ("replication_allocation" = "tag.location.default: 1",
+            "enable_mow_light_delete" = "true")"""
+
+        for(int i = 1; i <= 100; ++i){
+            sql """ INSERT INTO ${tableName} VALUES (1,1)"""
+            sql """ delete from ${tableName} where user_id = 1"""
+        }
+
+        now = System.currentTimeMillis()
+
+        while(true){
+            if(check_cumu_point(100)){
+                break;
+            }
+            Thread.sleep(1000)
+        }
+        time_diff = System.currentTimeMillis() - now
+        logger.info("time_diff:" + time_diff)
+        assertTrue(time_diff>=200*1000)
+
+        qt_select2 """select * from ${tableName} order by user_id, value"""
+    } catch (Exception e){
+        logger.info(e.getMessage())
+        assertFalse(true)
+    } finally {
+        try_sql("DROP TABLE IF EXISTS ${tableName} FORCE")
+        set_be_config.call("enable_sleep_between_delete_cumu_compaction", "false")
+    }
+}
\ No newline at end of file

From 41e554dd7efca2762244f2cc27eed936b31e63f8 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Fri, 13 Dec 2024 16:57:10 +0800
Subject: [PATCH 319/399] [Fix](Insert-Job)Resolving Task Status Discrepancies
 in Query Execution Records (#45342)

### What problem does this PR solve?


When querying the execution records of tasks, the current system has two
sources that may describe task status:

Task status in the scheduling module: Represents the processing status
of tasks within the scheduling system.
Task status in LoadManager: Specifically refers to the status of
Insert-type tasks, containing more detailed execution information.

When the data in the LoadManager is deleted or expires, the completion
status will revert to the task status from the scheduling system, which
could cause confusion. Therefore, consistency needs to be maintained.

Since these two sources may have inconsistencies in their status
information, they can easily cause confusion for users and developers.
---
 .../org/apache/doris/job/extensions/insert/InsertTask.java    | 2 +-
 regression-test/suites/job_p0/test_base_insert_job.groovy     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java
index 23a367d5d6e1a1..a577250dc865ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java
@@ -244,7 +244,7 @@ public TRow getTvfInfo(String jobName) {
         trow.addToColumnValue(new TCell().setStringVal(String.valueOf(getJobId())));
         trow.addToColumnValue(new TCell().setStringVal(jobName));
         trow.addToColumnValue(new TCell().setStringVal(getJobId() + LABEL_SPLITTER + getTaskId()));
-        trow.addToColumnValue(new TCell().setStringVal(jobInfo.getState().name()));
+        trow.addToColumnValue(new TCell().setStringVal(getStatus().name()));
         trow.addToColumnValue(new TCell().setStringVal(getErrorMsg()));
         // create time
         trow.addToColumnValue(new TCell().setStringVal(TimeUtils.longToTimeString(getCreateTimeMs())));
diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy
index 2bdf96cd5bdc3f..33ae28443b290a 100644
--- a/regression-test/suites/job_p0/test_base_insert_job.groovy
+++ b/regression-test/suites/job_p0/test_base_insert_job.groovy
@@ -116,7 +116,7 @@ suite("test_base_insert_job") {
 
     def taskStatus = sql """select status from tasks("type"="insert") where JobName ='${jobName}'"""
     for (int i = 0; i < taskStatus.size(); i++) {
-        assert taskStatus.get(i).get(0) =="CANCELED" || taskStatus.get(i).get(0) =="FINISHED"
+        assert taskStatus.get(i).get(0) =="CANCELED" || taskStatus.get(i).get(0) =="SUCCESS"
     }
     sql """
        CREATE JOB ${jobMixedName}  ON SCHEDULE every 1 second  DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
@@ -168,7 +168,7 @@ suite("test_base_insert_job") {
     // table should have one record after job finished
     assert datas.size() == 1
     // one time job only has one task. when job finished, task status should be FINISHED
-    assert datas.get(0).get(0) == "FINISHED"
+    assert datas.get(0).get(0) == "SUCCESS"
     // check table data
     def dataCount1 = sql """select count(1) from ${tableName} where user_id=1001"""
     assert dataCount1.get(0).get(0) == 1

From ab863bb5923b383a5666ffdfe072258c2105c1a0 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Fri, 13 Dec 2024 17:27:44 +0800
Subject: [PATCH 320/399] [regression-test](framework) disable defining global
 variable in test groovy files (#44576)

---
 .../test_uniq_vals_schema_change.out          |  3 +++
 .../regression/suite/ScriptSource.groovy      |  9 ++++++-
 .../test_assistant_command_auth.groovy        |  2 +-
 .../test_backup_restore_db.groovy             |  2 +-
 .../test_backup_restore_exclude.groovy        |  2 +-
 .../test_backup_restore_multi_tables.groovy   |  2 +-
 ...ckup_restore_multi_tables_overwrite.groovy |  2 +-
 .../test_backup_restore_mv.groovy             | 10 ++++----
 ...st_restore_mix_exists_and_new_table.groovy |  2 +-
 .../test_drop_catalog_recycle_bin.groovy      | 24 +++++++++----------
 .../test_ingest_binlog.groovy                 |  2 +-
 .../inverted_index/test_ingest_binlog.groovy  |  2 +-
 .../ccr_syncer_p0/test_ingest_binlog.groovy   |  2 +-
 .../test_disable_revoke_admin_auth.groovy     |  1 +
 .../test_grant_revoke_cluster_to_user.groovy  |  5 ++--
 ..._grant_revoke_compute_group_to_user.groovy |  5 ++--
 .../compaction/test_full_compaction.groovy    |  2 +-
 .../compaction/test_full_compaction_ck.groovy |  2 +-
 .../test_trim_new_parameters.groovy           |  2 +-
 .../test_routine_load_generated_column.groovy |  2 +-
 .../test_create_table_properties.groovy       |  2 +-
 .../suites/export/test_array_export.groovy    |  6 ++---
 .../suites/export/test_map_export.groovy      |  4 ++--
 .../suites/export/test_struct_export.groovy   |  2 +-
 .../csv/test_outfile_empty_data.groovy        | 14 +++++------
 ...test_outfile_expr_generate_col_name.groovy | 10 ++++----
 .../suites/export_p0/test_export_basic.groovy |  8 +++----
 .../export_p0/test_outfile_file_suffix.groovy |  2 +-
 .../test_show_create_database.groovy          |  2 +-
 .../cache/test_file_cache_statistics.groovy   |  2 +-
 .../hive/test_autoinc_broker_load.groovy      | 22 ++++++++---------
 .../test_hive_parquet_alter_column.groovy     |  2 +-
 .../hive/test_hive_statistic.groovy           |  2 +-
 .../test_partial_update_broker_load.groovy    | 16 ++++++-------
 .../test_iceberg_optimize_count.groovy        | 10 ++++----
 .../test_iceberg_predicate_conversion.groovy  |  2 +-
 .../jdbc/test_mysql_jdbc_statistics.groovy    |  2 +-
 .../insert_group_commit_with_exception.groovy | 24 +++++++++----------
 ...t_group_commit_interval_ms_property.groovy |  2 +-
 ...est_add_drop_index_on_table_with_mv.groovy |  2 +-
 .../test_pk_uk_index_change.groovy            |  2 +-
 .../load/test_spark_load.groovy               |  6 ++---
 .../test_schema_change_storage_format.groovy  |  2 +-
 ..._array_contains_with_inverted_index.groovy |  4 ++--
 .../test_index_match_select.groovy            |  8 +++----
 .../test_primary_key_simple_case.groovy       |  2 +-
 .../http_stream/test_http_stream_2pc.groovy   |  8 +++----
 ...st_insert_random_distribution_table.groovy |  4 ++--
 .../load_p0/mysql_load/test_mysql_load.groovy |  2 +-
 .../test_routine_load_alter.groovy            |  4 ++--
 .../test_routine_load_with_udf.groovy         |  2 +-
 .../test_get_stream_load_state.groovy         |  6 ++---
 ...eam_load_with_nonexist_db_and_table.groovy |  6 ++---
 .../test_map_load_and_compaction.groovy       |  4 +++-
 .../stream_load/test_stream_load.groovy       | 22 ++++++++---------
 .../stream_load/test_stream_load_2pc.groovy   | 10 ++++----
 ..._stream_load_2pc_with_schema_change.groovy |  6 ++---
 .../test_stream_load_move_memtable.groovy     |  2 +-
 .../manager/test_manager_interface_2.groovy   |  4 ++--
 .../manager/test_manager_interface_4.groovy   |  2 +-
 .../suites/mtmv_p0/test_build_mtmv.groovy     |  2 +-
 .../suites/mtmv_p0/test_create_mv_mtmv.groovy |  2 +-
 .../mtmv_p0/test_create_rollup_mtmv.groovy    |  2 +-
 .../mtmv_p0/test_limit_partition_mtmv.groovy  |  2 +-
 .../suites/mtmv_p0/test_mtmv_outfile.groovy   |  2 +-
 .../mtmv_p0/test_rollup_partition_mtmv.groovy |  4 ++--
 .../test_create_mv/test_create_mv.groovy      |  2 +-
 .../nereids_arith_p0/topn/accept_null.groovy  |  4 ++--
 .../cache/parse_sql_from_sql_cache.groovy     |  2 +-
 .../fold_constant/fold_constant_by_fe.groovy  |  2 +-
 .../nereids_p0/outfile/test_outfile.groovy    | 12 ++++++----
 .../outfile/test_outfile_expr.groovy          |  2 +-
 .../outfile/test_outfile_parquet.groovy       |  8 ++++---
 ...te_or_datetime_computation_negative.groovy |  1 -
 .../mv/dimension/dimension_self_conn.groovy   |  2 +-
 .../suites/nereids_syntax_p0/group_bit.groovy |  2 +-
 .../nereids_syntax_p0/rollup/bitmap.groovy    |  2 +-
 .../test_auto_partition_behavior.groovy       |  2 +-
 .../test_auto_partition_load.groovy           |  4 ++--
 .../test_auto_range_partition.groovy          |  4 ++--
 .../test_dynamic_partition.groovy             |  2 +-
 ...amic_partition_mod_distribution_key.groovy |  2 +-
 .../test_dynamic_partition_with_alter.groovy  |  2 +-
 .../suites/query_p0/sort/topn_2pr_rule.groovy |  2 +-
 .../query_profile/s3_load_profile_test.groovy |  2 +-
 .../test_materialized_view_bitmap.groovy      |  2 +-
 .../rollup/test_materialized_view_hll.groovy  |  2 +-
 ...materialized_view_hll_with_light_sc.groovy |  2 +-
 .../test_agg_keys_schema_change.groovy        |  8 +++----
 .../test_agg_mv_schema_change.groovy          |  8 +++----
 .../test_agg_rollup_schema_change.groovy      | 10 ++++----
 .../test_agg_schema_key_change_modify1.groovy |  6 ++---
 .../test_agg_vals_schema_change.groovy        |  4 ++--
 .../test_alter_uniq_null.groovy               |  2 +-
 .../test_dup_keys_schema_change.groovy        |  8 +++----
 .../test_dup_mv_schema_change.groovy          |  4 ++--
 .../test_dup_rollup_schema_change.groovy      |  6 ++---
 .../test_dup_schema_key_change_modify1.groovy |  6 ++---
 .../test_dup_vals_schema_change.groovy        |  4 ++--
 .../test_enable_light_schema_change.groovy    |  2 +-
 .../test_schema_change_duplicate.groovy       |  2 +-
 .../test_uniq_keys_schema_change.groovy       |  4 ++--
 .../test_uniq_mv_schema_change.groovy         |  4 ++--
 .../test_uniq_rollup_schema_change.groovy     |  6 ++---
 .../test_uniq_vals_schema_change.groovy       |  8 +++----
 .../test_varchar_schema_change.groovy         |  4 ++--
 .../suites/show_p0/test_show_data.groovy      |  2 +-
 .../suites/statistics/analyze_stats.groovy    |  6 ++---
 ...al_update_delete_sign_with_conflict.groovy |  6 ++---
 .../test_partial_update_parallel.groovy       |  6 ++---
 .../flexible/upgrade/test.groovy              |  2 +-
 ...al_update_delete_sign_with_conflict.groovy |  6 ++---
 .../test_partial_update_parallel.groovy       |  6 ++---
 .../suites/update/test_update_configs.groovy  |  2 +-
 .../variant_p0/concurrent_insert.groovy       |  6 ++---
 .../suites/variant_p0/delete_update.groovy    |  6 ++---
 .../suites/variant_p0/nested.groovy           |  2 +-
 .../schema_change/schema_change.groovy        |  2 +-
 .../test_resource_tag.groovy                  | 14 +++++------
 119 files changed, 290 insertions(+), 276 deletions(-)

diff --git a/regression-test/data/schema_change_p0/test_uniq_vals_schema_change.out b/regression-test/data/schema_change_p0/test_uniq_vals_schema_change.out
index de0526b3e734cd..22d1644a86b381 100644
--- a/regression-test/data/schema_change_p0/test_uniq_vals_schema_change.out
+++ b/regression-test/data/schema_change_p0/test_uniq_vals_schema_change.out
@@ -11,6 +11,9 @@
 -- !sc --
 3
 
+-- !sc --
+3	2017-10-01	Beijing	10	1	2020-01-03T00:00	2020-01-03T00:00	1	32	20	2
+
 -- !sc --
 4	2017-10-01	Beijing	10	1	2020-01-03T00:00	2020-01-03T00:00	1	32	20	2
 
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy
index d73cf6afaf6b09..97d935c58ae05e 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy
@@ -34,7 +34,14 @@ class GroovyFileSource implements ScriptSource {
 
     @Override
     SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) {
-        SuiteScript suiteScript = shell.parse(file) as SuiteScript
+        def setPropertyFunction = '''
+\nvoid setProperty(String key, value) {
+    throw new IllegalArgumentException("defined global variables in script are not allowed: ${key}")
+}
+'''
+        def scriptContent = file.text
+        scriptContent = scriptContent + setPropertyFunction
+        SuiteScript suiteScript = shell.parse(scriptContent, file.getName()) as SuiteScript
         suiteScript.init(scriptContext)
         return suiteScript
     }
diff --git a/regression-test/suites/auth_call/test_assistant_command_auth.groovy b/regression-test/suites/auth_call/test_assistant_command_auth.groovy
index 49a149a28ce51f..1b47e566ff31e8 100644
--- a/regression-test/suites/auth_call/test_assistant_command_auth.groovy
+++ b/regression-test/suites/auth_call/test_assistant_command_auth.groovy
@@ -87,7 +87,7 @@ suite("test_assistant_command_auth","p0,auth_call") {
     }
 
     sql """grant select_PRIV on ${catalogName}.*.* to ${user}"""
-    connect(user, password="${pwd}", context.config.jdbcUrl) {
+    connect(user, "${pwd}", context.config.jdbcUrl) {
         sql """REFRESH CATALOG ${catalogName};"""
     }
 
diff --git a/regression-test/suites/backup_restore/test_backup_restore_db.groovy b/regression-test/suites/backup_restore/test_backup_restore_db.groovy
index 08b9f619d4fe03..2cca1356dc5d73 100644
--- a/regression-test/suites/backup_restore/test_backup_restore_db.groovy
+++ b/regression-test/suites/backup_restore/test_backup_restore_db.groovy
@@ -81,7 +81,7 @@ suite("test_backup_restore_db", "backup_restore") {
     syncer.waitAllRestoreFinish(dbName)
 
     for (def tableName in tables) {
-        result = sql "SELECT * FROM ${dbName}.${tableName}"
+        def result = sql "SELECT * FROM ${dbName}.${tableName}"
         assertEquals(result.size(), numRows);
         sql "DROP TABLE ${dbName}.${tableName} FORCE"
     }
diff --git a/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy b/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy
index e29ebfe3b6be1b..6993a0ca6fa22f 100644
--- a/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy
+++ b/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy
@@ -88,7 +88,7 @@ suite("test_backup_restore_exclude", "backup_restore") {
 
     qt_select "SELECT * FROM ${dbName}.${backupExcludeTable} ORDER BY id"
     for (def tableName in tables) {
-        result = sql "SELECT * FROM ${dbName}.${tableName}"
+        def result = sql "SELECT * FROM ${dbName}.${tableName}"
         assertEquals(result.size(), numRows);
         sql "DROP TABLE ${dbName}.${tableName} FORCE"
     }
diff --git a/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy b/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy
index 4fa274bbae305a..2be39d8eef14b5 100644
--- a/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy
+++ b/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy
@@ -84,7 +84,7 @@ suite("test_backup_restore_multi_tables", "backup_restore") {
     syncer.waitAllRestoreFinish(dbName)
 
     for (def tableName in tables) {
-        result = sql "SELECT * FROM ${dbName}.${tableName}"
+        def result = sql "SELECT * FROM ${dbName}.${tableName}"
         assertEquals(result.size(), numRows);
         sql "DROP TABLE ${dbName}.${tableName} FORCE"
     }
diff --git a/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy b/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy
index 30dc63dd195618..9ee5c848aef79e 100644
--- a/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy
+++ b/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy
@@ -86,7 +86,7 @@ suite("test_backup_restore_multi_tables_overwrite", "backup_restore") {
 
     qt_select "SELECT * FROM ${dbName}.${firstTableName} ORDER BY id"
     for (def tableName in tables) {
-        result = sql "SELECT * FROM ${dbName}.${tableName}"
+        def result = sql "SELECT * FROM ${dbName}.${tableName}"
         assertEquals(result.size(), numRows);
         sql "DROP TABLE ${dbName}.${tableName} FORCE"
     }
diff --git a/regression-test/suites/backup_restore/test_backup_restore_mv.groovy b/regression-test/suites/backup_restore/test_backup_restore_mv.groovy
index 0a54aa76e17b36..398b51cc995a90 100644
--- a/regression-test/suites/backup_restore/test_backup_restore_mv.groovy
+++ b/regression-test/suites/backup_restore/test_backup_restore_mv.groovy
@@ -57,13 +57,13 @@ suite("test_backup_restore_mv", "backup_restore") {
     """
 
     def alter_finished = false
-    for (i = 0; i < 60 && !alter_finished; i++) {
+    for (int i = 0; i < 60 && !alter_finished; i++) {
         result = sql_return_maparray "SHOW ALTER TABLE MATERIALIZED VIEW FROM ${dbName}"
         logger.info("result: ${result}")
-        for (int i = 0; i < result.size(); i++) {
-            if (result[i]['TableName'] == "${tableName}" &&
-                result[i]['RollupIndexName'] == "${mvName}" &&
-                result[i]['State'] == 'FINISHED') {
+        for (int j = 0; j < result.size(); j++) {
+            if (result[j]['TableName'] == "${tableName}" &&
+                result[j]['RollupIndexName'] == "${mvName}" &&
+                result[j]['State'] == 'FINISHED') {
                 alter_finished = true
                 break
             }
diff --git a/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy b/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy
index aea46af179de06..b1cc9136b25900 100644
--- a/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy
+++ b/regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy
@@ -84,7 +84,7 @@ suite("test_restore_mix_exists_and_new_table", "backup_restore") {
     syncer.waitAllRestoreFinish(dbName)
 
     for (def tableName in tables) {
-        result = sql "SELECT * FROM ${dbName}.${tableName}"
+        def result = sql "SELECT * FROM ${dbName}.${tableName}"
         assertEquals(result.size(), numRows);
         sql "DROP TABLE ${dbName}.${tableName} FORCE"
     }
diff --git a/regression-test/suites/catalog_recycle_bin_p0/test_drop_catalog_recycle_bin.groovy b/regression-test/suites/catalog_recycle_bin_p0/test_drop_catalog_recycle_bin.groovy
index 40b1175c856a58..ec2fdc891aaeba 100644
--- a/regression-test/suites/catalog_recycle_bin_p0/test_drop_catalog_recycle_bin.groovy
+++ b/regression-test/suites/catalog_recycle_bin_p0/test_drop_catalog_recycle_bin.groovy
@@ -60,28 +60,28 @@ suite("test_drop_catalog_recycle_bin") {
     """
 
     // test drop partition in catalog recycle bin
-    res = sql "SHOW CREATE TABLE `test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1`;"
+    def res = sql "SHOW CREATE TABLE `test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1`;"
     assertTrue(res.size() != 0)
 
     sql "use `test_drop_catalog_recycle_bin_db`"
     sql "ALTER TABLE `test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1` DROP PARTITION p1000;"
 
-    pre_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p1000" """
+    def pre_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p1000" """
     assertTrue(pre_res.size() > 0)
-    partition_id = pre_res[0][4]
+    def partition_id = pre_res[0][4]
     sql "DROP CATALOG RECYCLE BIN WHERE 'PartitionId' = ${partition_id};"
-    cur_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p1000" """
+    def cur_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p1000" """
     assertTrue(pre_res.size() - cur_res.size() == 1)
 
     // test drop table not in catalog recycle bin
     sql "use `test_drop_catalog_recycle_bin_db`"
     sql "ALTER TABLE `test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1` DROP PARTITION p111;"
 
-    pre_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
+    def pre_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
     assertTrue(pre_pt_res.size() > 0)
-    table_id = pre_res[0][3]
+    def table_id = pre_res[0][3]
     sql "DROP CATALOG RECYCLE BIN WHERE 'TableId' = ${table_id};"
-    cur_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
+    def cur_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
     assertTrue(pre_pt_res.size() - cur_pt_res.size() == 1)
 
     // test drop table in catalog recycle bin
@@ -89,13 +89,13 @@ suite("test_drop_catalog_recycle_bin") {
     sql "ALTER TABLE `test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1` DROP PARTITION p222;"
     sql "DROP TABLE `test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb1`;"
 
-    pre_tb_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_tb1" """
+    def pre_tb_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_tb1" """
     assertTrue(pre_tb_res.size() > 0)
     pre_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p222" """
     assertTrue(pre_pt_res.size() > 0)
     table_id = pre_res[0][3]
     sql "DROP CATALOG RECYCLE BIN WHERE 'TableId' = ${table_id};"
-    cur_tb_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_tb1" """
+    def cur_tb_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_tb1" """
     assertTrue(pre_tb_res.size() - cur_tb_res.size() == 1)
     cur_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
     assertTrue(pre_pt_res.size() - cur_pt_res.size() == 1)
@@ -103,13 +103,13 @@ suite("test_drop_catalog_recycle_bin") {
     // test drop db not in catalog recycle bin
     sql "ALTER TABLE `test_drop_catalog_recycle_bin_db`.`test_drop_catalog_recycle_bin_tb2` DROP PARTITION p111;"
 
-    pre_db_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_db" """
+    def pre_db_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_db" """
     assertTrue(pre_db_res.size() == 0)
     pre_tb_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_tb2" """
     assertTrue(pre_tb_res.size() == 0)
     pre_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p111" """
     assertTrue(pre_pt_res.size() > 0)
-    db_id = pre_res[0][2]
+    def db_id = pre_res[0][2]
     sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${db_id};"
     cur_pt_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "p222" """
     assertTrue(pre_pt_res.size() - cur_pt_res.size() == 1)
@@ -126,7 +126,7 @@ suite("test_drop_catalog_recycle_bin") {
     assertTrue(pre_pt_res.size() > 0)
     db_id = pre_res[0][2]
     sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${db_id};"
-    cur_db_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_db" """
+    def cur_db_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_db" """
     assertTrue(pre_db_res.size() - cur_db_res.size() == 1)
     cur_tb_res = sql """ SHOW CATALOG RECYCLE BIN WHERE NAME = "test_drop_catalog_recycle_bin_tb2" """
     assertTrue(pre_tb_res.size() - cur_tb_res.size() == 1)
diff --git a/regression-test/suites/ccr_mow_syncer_p0/test_ingest_binlog.groovy b/regression-test/suites/ccr_mow_syncer_p0/test_ingest_binlog.groovy
index e07529718ee467..16ce1d43c4fccb 100644
--- a/regression-test/suites/ccr_mow_syncer_p0/test_ingest_binlog.groovy
+++ b/regression-test/suites/ccr_mow_syncer_p0/test_ingest_binlog.groovy
@@ -78,7 +78,7 @@ suite("test_mow_ingest_binlog") {
     }
 
     target_sql " sync "
-    res = target_sql """SELECT * FROM ${tableName} WHERE test=${test_num}"""
+    def res = target_sql """SELECT * FROM ${tableName} WHERE test=${test_num}"""
     assertEquals(res.size(), insert_num)
 
 
diff --git a/regression-test/suites/ccr_syncer_p0/inverted_index/test_ingest_binlog.groovy b/regression-test/suites/ccr_syncer_p0/inverted_index/test_ingest_binlog.groovy
index 12ba49e084dae9..b46d256a52ff28 100644
--- a/regression-test/suites/ccr_syncer_p0/inverted_index/test_ingest_binlog.groovy
+++ b/regression-test/suites/ccr_syncer_p0/inverted_index/test_ingest_binlog.groovy
@@ -152,7 +152,7 @@ suite("test_ingest_binlog_index") {
         }
 
         target_sql " sync "
-        res = target_sql """SELECT * FROM ${tableName}"""
+        def res = target_sql """SELECT * FROM ${tableName}"""
         if (tableName.contains("mow")) {
             assertEquals(res.size(), insert_data(tableName).size() / 2 as Integer)
         } else {
diff --git a/regression-test/suites/ccr_syncer_p0/test_ingest_binlog.groovy b/regression-test/suites/ccr_syncer_p0/test_ingest_binlog.groovy
index 9f176cac9e09e4..414621b5086ed9 100644
--- a/regression-test/suites/ccr_syncer_p0/test_ingest_binlog.groovy
+++ b/regression-test/suites/ccr_syncer_p0/test_ingest_binlog.groovy
@@ -77,7 +77,7 @@ suite("test_ingest_binlog") {
     }
 
     target_sql " sync "
-    res = target_sql """SELECT * FROM ${tableName} WHERE test=${test_num}"""
+    def res = target_sql """SELECT * FROM ${tableName} WHERE test=${test_num}"""
     assertEquals(res.size(), insert_num)
 
 
diff --git a/regression-test/suites/cloud_p0/auth/test_disable_revoke_admin_auth.groovy b/regression-test/suites/cloud_p0/auth/test_disable_revoke_admin_auth.groovy
index 19add1d2c87503..bc1767a4ed4b76 100644
--- a/regression-test/suites/cloud_p0/auth/test_disable_revoke_admin_auth.groovy
+++ b/regression-test/suites/cloud_p0/auth/test_disable_revoke_admin_auth.groovy
@@ -22,6 +22,7 @@ suite("test_disable_revoke_admin_auth", "cloud_auth") {
     sql """create user ${user} identified by 'Cloud12345' default role 'admin'"""
 
     sql "sync"
+    def result
 
     try {
         result = sql """revoke 'admin' from 'admin'""";
diff --git a/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_to_user.groovy b/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_to_user.groovy
index a61b4f96954e9c..d2209d1e81ef52 100644
--- a/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_to_user.groovy
+++ b/regression-test/suites/cloud_p0/auth/test_grant_revoke_cluster_to_user.groovy
@@ -67,7 +67,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
     // 1. change user
     // ${user1} admin role
     sql """create user ${user1} identified by 'Cloud12345' default role 'admin'"""
-    result = sql_return_maparray """show grants for '${user1}'"""
+    def result = sql_return_maparray """show grants for '${user1}'"""
     commonAuth result, "'${user1}'@'%'" as String, "Yes", "admin", "Admin_priv"
     assertNull(result.CloudClusterPrivs[0])
 
@@ -113,7 +113,6 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
 
     // 2. grant cluster
     def cluster1 = "clusterA"
-    def result
 
     sql "sync"
 
@@ -195,7 +194,7 @@ suite("test_grant_revoke_cluster_to_user", "cloud_auth") {
 
     sql """GRANT USAGE_PRIV ON CLUSTER '${cluster1}' TO '${user2}'"""
     sql """GRANT USAGE_PRIV ON CLUSTER '${validCluster}' TO '${user2}'"""
-    show_cluster_2 = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
+    def show_cluster_2 = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
             getCluster(validCluster)
     }
 
diff --git a/regression-test/suites/cloud_p0/auth/test_grant_revoke_compute_group_to_user.groovy b/regression-test/suites/cloud_p0/auth/test_grant_revoke_compute_group_to_user.groovy
index 09205433a3988e..c74e239159acb0 100644
--- a/regression-test/suites/cloud_p0/auth/test_grant_revoke_compute_group_to_user.groovy
+++ b/regression-test/suites/cloud_p0/auth/test_grant_revoke_compute_group_to_user.groovy
@@ -73,7 +73,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
     // 1. change user
     // ${user1} admin role
     logAndExecuteSql """create user ${user1} identified by 'Cloud12345' default role 'admin'"""
-    result = sql_return_maparray """show grants for '${user1}'"""
+    def result = sql_return_maparray """show grants for '${user1}'"""
     commonAuth result, "'${user1}'@'%'" as String, "Yes", "admin", "Admin_priv"
     assertNull(result.ComputeGroupPrivs[0])
 
@@ -120,7 +120,6 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
 
     // 2. grant group
     def group1 = "groupA"
-    def result
 
     logAndExecuteSql "sync"
 
@@ -202,7 +201,7 @@ suite("test_grant_revoke_compute_group_to_user", "cloud_auth") {
 
     logAndExecuteSql """GRANT USAGE_PRIV ON COMPUTE GROUP '${group1}' TO '${user2}'"""
     logAndExecuteSql """GRANT USAGE_PRIV ON COMPUTE GROUP '${validCluster}' TO '${user2}'"""
-    show_group_2 = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
+    def show_group_2 = connect("${user2}", 'Cloud12345', context.config.jdbcUrl) {
             getCluster(validCluster)
     }
 
diff --git a/regression-test/suites/compaction/test_full_compaction.groovy b/regression-test/suites/compaction/test_full_compaction.groovy
index ed2d2ce5bc5e46..b54f92747e6c4e 100644
--- a/regression-test/suites/compaction/test_full_compaction.groovy
+++ b/regression-test/suites/compaction/test_full_compaction.groovy
@@ -120,7 +120,7 @@ suite("test_full_compaction") {
         for (def tablet in tablets) {
             String tablet_id = tablet.TabletId
             backend_id = tablet.BackendId
-            times = 1
+            def times = 1
 
             do{
                 (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
diff --git a/regression-test/suites/compaction/test_full_compaction_ck.groovy b/regression-test/suites/compaction/test_full_compaction_ck.groovy
index 8e2f86c596de65..ae6b467acafcb9 100644
--- a/regression-test/suites/compaction/test_full_compaction_ck.groovy
+++ b/regression-test/suites/compaction/test_full_compaction_ck.groovy
@@ -121,7 +121,7 @@ suite("test_full_compaction_ck") {
         for (def tablet in tablets) {
             String tablet_id = tablet.TabletId
             backend_id = tablet.BackendId
-            times = 1
+            def times = 1
 
             do{
                 (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
diff --git a/regression-test/suites/correctness/test_trim_new_parameters.groovy b/regression-test/suites/correctness/test_trim_new_parameters.groovy
index 17ac4a0c65eae5..a76ccb96356da9 100644
--- a/regression-test/suites/correctness/test_trim_new_parameters.groovy
+++ b/regression-test/suites/correctness/test_trim_new_parameters.groovy
@@ -68,6 +68,6 @@ suite("test_trim_new_parameters") {
     rtrim = sql "select rtrim('bcTTTabcabc','abc')"
     assertEquals(rtrim[0][0], 'bcTTT')   
 
-    trim_one = sql "select trim('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaabcTTTabcabcaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','a')"
+    def trim_one = sql "select trim('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaabcTTTabcabcaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','a')"
     assertEquals(trim_one[0][0], 'baaaaaaaaaaabcTTTabcabcaaaaaaaaaaaaaaaaaaaaaaaaaab')  
 }
diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy
index edf8ab2532345a..fa5858176284e4 100644
--- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy
+++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy
@@ -32,7 +32,7 @@ suite("test_routine_load_generated_column") {
         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
         def producer = new KafkaProducer<>(props)
-        filepath = getLoalFilePath "gen_col_data.csv"
+        def filepath = getLoalFilePath "gen_col_data.csv"
         def txt = new File("${filepath}").text
         def lines = txt.readLines();
         lines.each { line ->
diff --git a/regression-test/suites/ddl_p0/test_create_table_properties.groovy b/regression-test/suites/ddl_p0/test_create_table_properties.groovy
index c6908d80b07cda..32fd0cabcaf8cf 100644
--- a/regression-test/suites/ddl_p0/test_create_table_properties.groovy
+++ b/regression-test/suites/ddl_p0/test_create_table_properties.groovy
@@ -336,7 +336,7 @@ suite("test_create_table_properties") {
         )
     """
     sql """ insert into ${bool_tab} values (1, '2020-12-12 12:12:12', '2000-01-01 12:12:12.123456'), (0, '20201212 121212', '2000-01-01'), (1, '20201212121212', '2000-01-01'), (0, 'AaA', '2000-01-01') """
-    result = sql "show partitions from ${bool_tab}"
+    def result = sql "show partitions from ${bool_tab}"
     logger.info("${result}")
     assertEquals(result.size(), 2)
     
diff --git a/regression-test/suites/export/test_array_export.groovy b/regression-test/suites/export/test_array_export.groovy
index 4186ee21d9f01a..4ce9786e9972bb 100644
--- a/regression-test/suites/export/test_array_export.groovy
+++ b/regression-test/suites/export/test_array_export.groovy
@@ -136,7 +136,7 @@ suite("test_array_export", "export") {
     def check_export_result = {checklabel->
         max_try_milli_secs = 15000
         while(max_try_milli_secs) {
-            result = sql "show export where label='${checklabel}'"
+            def result = sql "show export where label='${checklabel}'"
             if(result[0][2] == "FINISHED") {
                 break
             } else {
@@ -171,7 +171,7 @@ suite("test_array_export", "export") {
         } else {
             throw new IllegalStateException("""${outFilePath} already exists! """)
         }
-        result = sql """
+        def result = sql """
             SELECT * FROM ${tableName} t ORDER BY k1 INTO OUTFILE "file://${outFile}/";
         """
         def url = result[0][3]
@@ -203,7 +203,7 @@ suite("test_array_export", "export") {
             path.delete();
         }
         if (csvFiles != "") {
-            cmd = "rm -rf ${csvFiles}"
+            def cmd = "rm -rf ${csvFiles}"
             sshExec("root", urlHost, cmd)
         }
     }
diff --git a/regression-test/suites/export/test_map_export.groovy b/regression-test/suites/export/test_map_export.groovy
index fdb207f8628bfd..101ab07ebbe4fd 100644
--- a/regression-test/suites/export/test_map_export.groovy
+++ b/regression-test/suites/export/test_map_export.groovy
@@ -98,7 +98,7 @@ suite("test_map_export", "export") {
         def result = sql """
                     SELECT * FROM ${testTable} ORDER BY id INTO OUTFILE "file://${outFile}/";
         """
-        url = result[0][3]
+        def url = result[0][3]
         urlHost = url.substring(8, url.indexOf("${outFile}"))
         if (backends.size() > 1) {
             // custer will scp files
@@ -146,7 +146,7 @@ suite("test_map_export", "export") {
             path.delete();
         }
         if (csvFiles != "") {
-            cmd = "rm -rf ${csvFiles}"
+            def cmd = "rm -rf ${csvFiles}"
             sshExec("root", urlHost, cmd)
         }
     }
diff --git a/regression-test/suites/export/test_struct_export.groovy b/regression-test/suites/export/test_struct_export.groovy
index 8df3a40906e67c..bc3912af39ebaa 100644
--- a/regression-test/suites/export/test_struct_export.groovy
+++ b/regression-test/suites/export/test_struct_export.groovy
@@ -151,7 +151,7 @@ suite("test_struct_export", "export") {
             path.delete();
         }
         if (csvFiles != "") {
-            cmd = "rm -rf ${csvFiles}"
+            def cmd = "rm -rf ${csvFiles}"
             sshExec("root", urlHost, cmd)
         }
     }
diff --git a/regression-test/suites/export_p0/outfile/csv/test_outfile_empty_data.groovy b/regression-test/suites/export_p0/outfile/csv/test_outfile_empty_data.groovy
index d8da398ea813bf..d14dc119ddb805 100644
--- a/regression-test/suites/export_p0/outfile/csv/test_outfile_empty_data.groovy
+++ b/regression-test/suites/export_p0/outfile/csv/test_outfile_empty_data.groovy
@@ -45,7 +45,7 @@ suite("test_outfile_empty_data", "external,hive,tvf,external_docker") {
     String ak = getS3AK()
     String sk = getS3SK()
     String s3_endpoint = getS3Endpoint()
-    String region = region = getS3Region()
+    String region = getS3Region()
     String bucket = context.config.otherConfigs.get("s3BucketName");
 
     // broker
@@ -67,8 +67,8 @@ suite("test_outfile_empty_data", "external,hive,tvf,external_docker") {
         // select ... into outfile ...
         def uuid = UUID.randomUUID().toString()
 
-        hdfs_outfile_path = "/user/doris/tmp_data/${uuid}"
-        uri = "${defaultFS}" + "${hdfs_outfile_path}/exp_"
+        def hdfs_outfile_path = "/user/doris/tmp_data/${uuid}"
+        def uri = "${defaultFS}" + "${hdfs_outfile_path}/exp_"
 
         def res = sql """
             SELECT * FROM ${export_table_name} t ORDER BY user_id
@@ -87,8 +87,8 @@ suite("test_outfile_empty_data", "external,hive,tvf,external_docker") {
         // select ... into outfile ...
         def uuid = UUID.randomUUID().toString()
 
-        hdfs_outfile_path = "/user/doris/tmp_data/${uuid}"
-        uri = "${defaultFS}" + "${hdfs_outfile_path}/exp_"
+        def hdfs_outfile_path = "/user/doris/tmp_data/${uuid}"
+        def uri = "${defaultFS}" + "${hdfs_outfile_path}/exp_"
 
         def res = sql """
             SELECT * FROM ${export_table_name} t ORDER BY user_id
@@ -106,8 +106,8 @@ suite("test_outfile_empty_data", "external,hive,tvf,external_docker") {
 
     def outfile_to_S3_directly = {
         // select ... into outfile ...
-        s3_outfile_path = "${bucket}/outfile/csv/test-outfile-empty/"
-        uri = "s3://${s3_outfile_path}/exp_"
+        def s3_outfile_path = "${bucket}/outfile/csv/test-outfile-empty/"
+        def uri = "s3://${s3_outfile_path}/exp_"
 
         def res = sql """
             SELECT * FROM ${export_table_name} t ORDER BY user_id
diff --git a/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expr_generate_col_name.groovy b/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expr_generate_col_name.groovy
index e5a9ec7ea490b3..1d31c64ba7585f 100644
--- a/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expr_generate_col_name.groovy
+++ b/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expr_generate_col_name.groovy
@@ -129,7 +129,7 @@ suite("test_outfile_expr_generate_col_name", "p0") {
                 "s3.access_key" = "${ak}"
             );
         """
-        outfile_url = res[0][3]
+        def outfile_url = res[0][3]
         
         check_outfile_data(outfile_url, outfile_format)
         check_outfile_column_name(outfile_url, outfile_format)
@@ -150,7 +150,7 @@ suite("test_outfile_expr_generate_col_name", "p0") {
                 "s3.access_key" = "${ak}"
             );
         """
-        outfile_url = res[0][3]
+        def outfile_url = res[0][3]
         
         check_outfile_data(outfile_url, outfile_format)
         check_outfile_column_name(outfile_url, outfile_format)
@@ -171,7 +171,7 @@ suite("test_outfile_expr_generate_col_name", "p0") {
                 "s3.access_key" = "${ak}"
             );
         """
-        outfile_url = res[0][3]
+        def outfile_url = res[0][3]
         
         check_outfile_data(outfile_url, outfile_format)
         check_outfile_column_name(outfile_url, outfile_format)
@@ -211,7 +211,7 @@ suite("test_outfile_expr_generate_col_name", "p0") {
                 "s3.access_key" = "${ak}"
             );
         """
-        outfile_url = res[0][3]
+        def outfile_url = res[0][3]
         
         check_outfile_data(outfile_url, outfile_format)
         check_outfile_column_name(outfile_url, outfile_format)
@@ -235,7 +235,7 @@ suite("test_outfile_expr_generate_col_name", "p0") {
                 "s3.access_key" = "${ak}"
             );
         """
-        outfile_url = res[0][3]
+        def outfile_url = res[0][3]
         
         check_outfile_data(outfile_url, outfile_format)
         check_outfile_column_name(outfile_url, outfile_format)
diff --git a/regression-test/suites/export_p0/test_export_basic.groovy b/regression-test/suites/export_p0/test_export_basic.groovy
index ca838232e3fdec..152f1ab4e6e9e1 100644
--- a/regression-test/suites/export_p0/test_export_basic.groovy
+++ b/regression-test/suites/export_p0/test_export_basic.groovy
@@ -414,11 +414,11 @@ suite("test_export_basic", "p0") {
     }
 
     // 5. test order by and limit clause
-    uuid1 = UUID.randomUUID().toString()
+    def uuid1 = UUID.randomUUID().toString()
     outFilePath = """${outfile_path_prefix}_${uuid1}"""
-    label1 = "label_${uuid1}"
-    uuid2 = UUID.randomUUID().toString()
-    label2 = "label_${uuid2}"
+    def label1 = "label_${uuid1}"
+    def uuid2 = UUID.randomUUID().toString()
+    def label2 = "label_${uuid2}"
     try {
         // check export path
         check_path_exists.call("${outFilePath}")
diff --git a/regression-test/suites/export_p0/test_outfile_file_suffix.groovy b/regression-test/suites/export_p0/test_outfile_file_suffix.groovy
index d7a583824311be..4a453bce28d7ee 100644
--- a/regression-test/suites/export_p0/test_outfile_file_suffix.groovy
+++ b/regression-test/suites/export_p0/test_outfile_file_suffix.groovy
@@ -42,7 +42,7 @@ suite("test_outfile_file_suffix", "p0") {
 
     def outFilePath = """s3://${bucket}/outfile_"""
     def csv_suffix_result = { file_suffix, file_format ->
-        result = sql """
+        def result = sql """
                 select * from ${table_name}
                 into outfile "${outFilePath}"
                 FORMAT AS ${file_format}
diff --git a/regression-test/suites/export_p0/test_show_create_database.groovy b/regression-test/suites/export_p0/test_show_create_database.groovy
index 03f0e8b291cdb5..44e54c46028f6d 100644
--- a/regression-test/suites/export_p0/test_show_create_database.groovy
+++ b/regression-test/suites/export_p0/test_show_create_database.groovy
@@ -18,7 +18,7 @@
 suite("test_show_create_database", "p0,external,hive,external_docker,external_docker_hive") {
 
     sql """create database if not exists db_test"""
-    result = sql """show create database db_test"""
+    def result = sql """show create database db_test"""
     assertEquals(result.size(), 1)
     assertEquals(result[0][1], "CREATE DATABASE `db_test`")
 
diff --git a/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy b/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy
index dfceba90cec906..a4796c73d4ed74 100644
--- a/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy
+++ b/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy
@@ -45,7 +45,7 @@ suite("test_file_cache_statistics", "external_docker,hive,external_docker_hive,p
     order_qt_2 """select * from ${catalog_name}.${ex_db_name}.parquet_partition_table order by l_orderkey limit 1;"""
     // brpc metrics will be updated at most 20 seconds
     Awaitility.await().atMost(30, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until{
-        result = sql """select METRIC_VALUE from information_schema.file_cache_statistics where METRIC_NAME like "%hits_ratio%" order by METRIC_VALUE limit 1;"""
+        def result = sql """select METRIC_VALUE from information_schema.file_cache_statistics where METRIC_NAME like "%hits_ratio%" order by METRIC_VALUE limit 1;"""
         logger.info("result " + result)
         if (result.size() == 0) {
             return false;
diff --git a/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy b/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy
index 3171c72b710822..376da31c8a7987 100644
--- a/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy
@@ -20,15 +20,15 @@ suite("test_autoinc_broker_load", "p0,external,hive,external_docker,external_doc
 
     String enabled = context.config.otherConfigs.get("enableHiveTest")
     if (enabled != null && enabled.equalsIgnoreCase("true")) {
-        brokerName = getBrokerName()
-        hdfsUser = getHdfsUser()
-        hdfsPasswd = getHdfsPasswd()
-        hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
-        externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+        def brokerName = getBrokerName()
+        def hdfsUser = getHdfsUser()
+        def hdfsPasswd = getHdfsPasswd()
+        def hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
+        def externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
 
         def test_dir = "user/doris/preinstalled_data/data_case/autoinc"
 
-        def load_from_hdfs = {columns, testTable, label, testFile, format, brokerName, hdfsUser, hdfsPasswd ->
+        def load_from_hdfs = {columns, testTable, label, testFile, format ->
             def result1= sql """ LOAD LABEL ${label} (
                                 DATA INFILE("hdfs://${externalEnvIp}:${hdfs_port}/${test_dir}/${testFile}")
                                 INTO TABLE ${testTable}
@@ -46,9 +46,9 @@ suite("test_autoinc_broker_load", "p0,external,hive,external_docker,external_doc
         }
 
         def wait_for_load_result = {checklabel, testTable ->
-            max_try_milli_secs = 10000
+            def max_try_milli_secs = 10000
             while(max_try_milli_secs) {
-                result = sql "show load where label = '${checklabel}'"
+                def result = sql "show load where label = '${checklabel}'"
                 if(result[0][2] == "FINISHED") {
                     break
                 } else {
@@ -61,7 +61,7 @@ suite("test_autoinc_broker_load", "p0,external,hive,external_docker,external_doc
             }
         }
 
-        table = "test_autoinc_broker_load"
+        def table = "test_autoinc_broker_load"
         sql "drop table if exists ${table}"
         sql """ CREATE TABLE IF NOT EXISTS `${table}` (
             `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT "用户 ID",
@@ -78,7 +78,7 @@ suite("test_autoinc_broker_load", "p0,external,hive,external_docker,external_doc
             "enable_unique_key_merge_on_write" = "true") """
         
         def test_load_label = UUID.randomUUID().toString().replaceAll("-", "")
-        load_from_hdfs("name, value", table, test_load_label, "auto_inc_basic.csv", "csv", brokerName, hdfsUser, hdfsPasswd)
+        load_from_hdfs("name, value", table, test_load_label, "auto_inc_basic.csv", "csv")
         wait_for_load_result(test_load_label, table)
         qt_sql "select * from ${table};"
         sql """ insert into ${table} values(0, "Bob", 123), (2, "Tom", 323), (4, "Carter", 523);"""
@@ -102,7 +102,7 @@ suite("test_autoinc_broker_load", "p0,external,hive,external_docker,external_doc
             "storage_format" = "V2",
             "enable_unique_key_merge_on_write" = "true");"""
         test_load_label = UUID.randomUUID().toString().replaceAll("-", "")
-        load_from_hdfs("id, name, value", table, test_load_label, "auto_inc_with_null.csv", "csv", brokerName, hdfsUser, hdfsPasswd)
+        load_from_hdfs("id, name, value", table, test_load_label, "auto_inc_with_null.csv", "csv")
         wait_for_load_result(test_load_label, table)
         sql "sync"
         qt_sql "select * from ${table};"
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_parquet_alter_column.groovy b/regression-test/suites/external_table_p0/hive/test_hive_parquet_alter_column.groovy
index 9020b99c183740..5e3afa8d7fbfdd 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_parquet_alter_column.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_parquet_alter_column.groovy
@@ -43,7 +43,7 @@ suite("test_hive_parquet_alter_column", "p0,external,hive,external_docker,extern
 
 
 
-        types = ["int","smallint","tinyint","bigint","float","double","boolean","string","char","varchar","date","timestamp","decimal"]
+        def types = ["int","smallint","tinyint","bigint","float","double","boolean","string","char","varchar","date","timestamp","decimal"]
 
         for( String type1 in types) {
             qt_desc """ desc parquet_alter_column_to_${type1} ; """
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_statistic.groovy b/regression-test/suites/external_table_p0/hive/test_hive_statistic.groovy
index 5da056ff00ebbc..971e6bc0253ec8 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_statistic.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_statistic.groovy
@@ -259,7 +259,7 @@ suite("test_hive_statistic", "p0,external,hive,external_docker,external_docker_h
         def ctlId
         def dbId
         def tblId
-        result = sql """show catalogs"""
+        def result = sql """show catalogs"""
 
         for (int i = 0; i < result.size(); i++) {
             if (result[i][1] == catalog_name) {
diff --git a/regression-test/suites/external_table_p0/hive/test_partial_update_broker_load.groovy b/regression-test/suites/external_table_p0/hive/test_partial_update_broker_load.groovy
index e444bbc24239e5..4b5f6a02b3b111 100644
--- a/regression-test/suites/external_table_p0/hive/test_partial_update_broker_load.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_partial_update_broker_load.groovy
@@ -20,12 +20,12 @@ suite("test_primary_key_partial_update_broker_load", "p0,external,hive,external_
 
     String enabled = context.config.otherConfigs.get("enableHiveTest")
     if (enabled != null && enabled.equalsIgnoreCase("true")) {
-        brokerName = getBrokerName()
-        hdfsUser = getHdfsUser()
-        hdfsPasswd = getHdfsPasswd()
-        hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
-        externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
-        def load_from_hdfs = {testTable, label, hdfsFilePath, format, brokerName, hdfsUser, hdfsPasswd ->
+        def brokerName = getBrokerName()
+        def hdfsUser = getHdfsUser()
+        def hdfsPasswd = getHdfsPasswd()
+        def hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
+        def externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+        def load_from_hdfs = {testTable, label, hdfsFilePath, format ->
             def result1= sql """
                             LOAD LABEL ${label} (
                                 DATA INFILE("${hdfsFilePath}")
@@ -77,13 +77,13 @@ suite("test_primary_key_partial_update_broker_load", "p0,external,hive,external_
         sql """insert into ${tableName} values(2, "bob", 2000, 223, 2),(1, "alice", 1000, 123, 1),(3, "tom", 3000, 323, 3);"""
         qt_sql """ select * from ${tableName} order by id; """
         def test_load_label = UUID.randomUUID().toString().replaceAll("-", "")
-        load_from_hdfs(tableName, test_load_label, "hdfs://${externalEnvIp}:${hdfs_port}/user/doris/preinstalled_data/data_case/partial_update/update.csv", "csv", brokerName, hdfsUser, hdfsPasswd)
+        load_from_hdfs(tableName, test_load_label, "hdfs://${externalEnvIp}:${hdfs_port}/user/doris/preinstalled_data/data_case/partial_update/update.csv", "csv")
         wait_for_load_result(test_load_label, tableName)
         qt_sql """select * from ${tableName} order by id;"""
 
         sql "sync;"
         def test_load_label2 = UUID.randomUUID().toString().replaceAll("-", "")
-        load_from_hdfs(tableName, test_load_label2, "hdfs://${externalEnvIp}:${hdfs_port}/user/doris/preinstalled_data/data_case/partial_update/update2.csv", "csv", brokerName, hdfsUser, hdfsPasswd)
+        load_from_hdfs(tableName, test_load_label2, "hdfs://${externalEnvIp}:${hdfs_port}/user/doris/preinstalled_data/data_case/partial_update/update2.csv", "csv")
         wait_for_load_result(test_load_label2, tableName)
         qt_sql """select * from ${tableName} order by id;"""
         sql "drop table if exists ${tableName};"
diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy
index 7a9e90a61fe2cb..556314225c5a81 100644
--- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy
+++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy
@@ -42,10 +42,10 @@ suite("test_iceberg_optimize_count", "p0,external,doris,external_docker,external
         sql """ switch ${catalog_name} """
         sql """ use format_v2 """
 
-        sqlstr1 = """ select count(*) from sample_cow_orc; """
-        sqlstr2 = """ select count(*) from sample_cow_parquet; """
-        sqlstr3 = """ select count(*) from sample_mor_orc; """
-        sqlstr4 = """ select count(*) from sample_mor_parquet; """
+        def sqlstr1 = """ select count(*) from sample_cow_orc; """
+        def sqlstr2 = """ select count(*) from sample_cow_parquet; """
+        def sqlstr3 = """ select count(*) from sample_mor_orc; """
+        def sqlstr4 = """ select count(*) from sample_mor_parquet; """
 
         // use push down count
         sql """ set enable_count_push_down_for_external_table=true; """
@@ -99,7 +99,7 @@ suite("test_iceberg_optimize_count", "p0,external,doris,external_docker,external
 
         // There has `dangling delete` after rewrite
         sql """ set enable_count_push_down_for_external_table=true; """
-        sqlstr5 = """ select count(*) from ${catalog_name}.test_db.dangling_delete_after_write; """
+        def sqlstr5 = """ select count(*) from ${catalog_name}.test_db.dangling_delete_after_write; """
 
         qt_q09 """${sqlstr5}""" 
 
diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_predicate_conversion.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_predicate_conversion.groovy
index bbca6d8f02352f..c1e2a792e9f208 100644
--- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_predicate_conversion.groovy
+++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_predicate_conversion.groovy
@@ -70,7 +70,7 @@ suite("test_iceberg_predicate_conversion", "p0,external,doris,external_docker,ex
         }
 
         sqlstr = """select l_shipdate, l_shipmode from tb_predict where l_shipdate in ("1997-05-18", "1996-05-06") or NOT(l_shipmode = "MAIL") order by l_shipdate, l_shipmode limit 10"""
-        plan = """(ref(name="l_shipdate") in ("1997-05-18", "1996-05-06") or not(ref(name="l_shipmode") == "MAIL"))"""
+        def plan = """(ref(name="l_shipdate") in ("1997-05-18", "1996-05-06") or not(ref(name="l_shipmode") == "MAIL"))"""
         order_qt_q04 """${sqlstr}""" 
         explain {
             sql("""${sqlstr}""")
diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_statistics.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_statistics.groovy
index 617644cda99e8c..6a5fb0222355f4 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_statistics.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_statistics.groovy
@@ -38,7 +38,7 @@ suite("test_mysql_jdbc_statistics", "p0,external,mysql,external_docker,external_
         sql """use ${catalog_name}.doris_test"""
 
         sql """analyze table ex_tb0 with sync"""
-        result = sql """show column stats ex_tb0 (name)"""
+        def result = sql """show column stats ex_tb0 (name)"""
         assertEquals(result.size(), 1)
         assertEquals(result[0][0], "name")
         assertEquals(result[0][2], "5.0")
diff --git a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
index e207e71f2cc964..054add11d9f3a3 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
@@ -78,7 +78,7 @@ suite("insert_group_commit_with_exception") {
             assertTrue(e.getMessage().contains("Column count doesn't match value count"))
         }
 
-        result = sql """ insert into ${table} values(3, 'c', 30)  """
+        def result = sql """ insert into ${table} values(3, 'c', 30)  """
         logger.info("insert result: " + result)
 
         // insert into with column
@@ -125,7 +125,7 @@ suite("insert_group_commit_with_exception") {
                 ps.setObject(3, 70);
                 ps.setObject(4, "a");
                 ps.addBatch();
-                int[] result = ps.executeBatch();
+                result = ps.executeBatch();
                 assertTrue(false)
             } catch (Exception e) {
                 assertTrue(e.getMessage().contains("Column count doesn't match value count"))
@@ -135,7 +135,7 @@ suite("insert_group_commit_with_exception") {
                 ps.setObject(1, 9);
                 ps.setObject(2, "f");
                 ps.addBatch();
-                int[] result = ps.executeBatch();
+                result = ps.executeBatch();
                 assertTrue(false)
             } catch (Exception e) {
                 assertTrue(e.getMessage().contains("Column count doesn't match value count"))
@@ -146,7 +146,7 @@ suite("insert_group_commit_with_exception") {
                 ps.setObject(2, "f");
                 ps.setObject(3, 90);
                 ps.addBatch();
-                int[] result = ps.executeBatch();
+                result = ps.executeBatch();
                 logger.info("prepare insert result: " + result)
             }
 
@@ -155,7 +155,7 @@ suite("insert_group_commit_with_exception") {
                 ps.setObject(1, 11);
                 ps.setObject(2, "f");
                 ps.addBatch();
-                int[] result = ps.executeBatch();
+                result = ps.executeBatch();
                 logger.info("prepare insert result: " + result)
             }
 
@@ -164,7 +164,7 @@ suite("insert_group_commit_with_exception") {
                 ps.setObject(2, "f");
                 ps.setObject(3, "f");
                 ps.addBatch();
-                int[] result = ps.executeBatch();
+                result = ps.executeBatch();
                 assertTrue(false)
             } catch (Exception e) {
                 assertTrue(e.getMessage().contains("Column count doesn't match value count"))
@@ -173,7 +173,7 @@ suite("insert_group_commit_with_exception") {
             try (PreparedStatement ps = connection.prepareStatement("insert into ${table}(id, name) values(?)")) {
                 ps.setObject(1, 13);
                 ps.addBatch();
-                int[] result = ps.executeBatch();
+                result = ps.executeBatch();
                 assertTrue(false)
             } catch (Exception e) {
                 assertTrue(e.getMessage().contains("Column count doesn't match value count"))
@@ -183,7 +183,7 @@ suite("insert_group_commit_with_exception") {
                 ps.setObject(1, 12);
                 ps.setObject(2, "f");
                 ps.addBatch();
-                int[] result = ps.executeBatch();
+                result = ps.executeBatch();
                 assertTrue(false)
             } catch (Exception e) {
                 assertTrue(e.getMessage().contains("Unknown column 'names'"))
@@ -198,7 +198,7 @@ suite("insert_group_commit_with_exception") {
                     ps.setObject(2, "f");
                     ps.setObject(3, 90);
                     ps.addBatch();
-                    int[] result = ps.executeBatch();
+                    result = ps.executeBatch();
                     logger.info("prepare insert result: " + result)
                 }
             }
@@ -214,7 +214,7 @@ suite("insert_group_commit_with_exception") {
                     ps.setObject(5, "f");
                     ps.setObject(6, 90);
                     ps.addBatch();
-                    int[] result = ps.executeBatch();
+                    result = ps.executeBatch();
                     logger.info("prepare insert result: " + result)
                 }
             }
@@ -226,7 +226,7 @@ suite("insert_group_commit_with_exception") {
                 ps.setObject(2, "f")
                 ps.setObject(3, 90)
                 ps.addBatch()
-                int[] result = ps.executeBatch()
+                result = ps.executeBatch()
                 logger.info("prepare insert result: " + result)
 
                 sql """ alter table ${table} ADD column age int after name; """
@@ -252,7 +252,7 @@ suite("insert_group_commit_with_exception") {
                 ps.setObject(2, "f")
                 ps.setObject(3, 90)
                 ps.addBatch()
-                int[] result = ps.executeBatch()
+                result = ps.executeBatch()
                 logger.info("prepare insert result: " + result)
 
                 sql """ alter table ${table} DROP column age; """
diff --git a/regression-test/suites/insert_p0/test_group_commit_interval_ms_property.groovy b/regression-test/suites/insert_p0/test_group_commit_interval_ms_property.groovy
index e05acca703aeb7..c42d251ee03540 100644
--- a/regression-test/suites/insert_p0/test_group_commit_interval_ms_property.groovy
+++ b/regression-test/suites/insert_p0/test_group_commit_interval_ms_property.groovy
@@ -41,7 +41,7 @@ suite("test_group_commit_interval_ms_property") {
 
 
     try {
-        test_table = table
+        def test_table = table
         sql """ drop table if exists ${test_table} force; """
         sql """
             CREATE table ${test_table} (
diff --git a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_on_table_with_mv.groovy b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_on_table_with_mv.groovy
index 98729dbc91b1ea..d43b0f825f7d13 100644
--- a/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_on_table_with_mv.groovy
+++ b/regression-test/suites/inverted_index_p0/index_change/test_add_drop_index_on_table_with_mv.groovy
@@ -104,7 +104,7 @@ suite("test_add_drop_index_on_table_with_mv") {
 
     sql """ SHOW ALTER TABLE MATERIALIZED VIEW """
 
-    max_try_secs = 60
+    def max_try_secs = 60
     while (max_try_secs--) {
         String res = getJobState(tableName)
         if (res == "FINISHED" || res == "CANCELLED") {
diff --git a/regression-test/suites/inverted_index_p0/index_change/test_pk_uk_index_change.groovy b/regression-test/suites/inverted_index_p0/index_change/test_pk_uk_index_change.groovy
index 5fc593d0b4a93a..4e80266ee50a51 100644
--- a/regression-test/suites/inverted_index_p0/index_change/test_pk_uk_index_change.groovy
+++ b/regression-test/suites/inverted_index_p0/index_change/test_pk_uk_index_change.groovy
@@ -302,7 +302,7 @@ suite("test_pk_uk_index_change", "inverted_index") {
                         """  
         assertTrue(result0.size()==result1.size())
         for (int i = 0; i < result0.size(); ++i) {
-            for (j = 0; j < result0[0].size(); j++) {
+            for (int j = 0; j < result0[0].size(); j++) {
                 logger.info("result: " + result0[i][j] + "|" + result1[i][j])
                 assertTrue(result0[i][j]==result1[i][j])
             }
diff --git a/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy b/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy
index 0fd0ca35627408..488b7c6b35bde7 100644
--- a/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy
+++ b/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy
@@ -36,9 +36,9 @@ suite("test_spark_load_with_index_p0", "p0") {
         def yarnAddress = "master:8032"
         def hdfsAddress = "hdfs://master:9000"
         def hdfsWorkingDir = "hdfs://master:9000/doris"
-        brokerName =getBrokerName()
-        hdfsUser = getHdfsUser()
-        hdfsPasswd = getHdfsPasswd()
+        def brokerName =getBrokerName()
+        def hdfsUser = getHdfsUser()
+        def hdfsPasswd = getHdfsPasswd()
         
         def create_test_table = {testTablex ->
             def result1 = sql """
diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_schema_change_storage_format.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_schema_change_storage_format.groovy
index fbccf0f8a62dea..28e9b03dce30cd 100644
--- a/regression-test/suites/inverted_index_p0/storage_format/test_schema_change_storage_format.groovy
+++ b/regression-test/suites/inverted_index_p0/storage_format/test_schema_change_storage_format.groovy
@@ -104,7 +104,7 @@ suite("test_local_schema_change_storge_format", "p0") {
     def backendId_to_backendHttpPort = [:]
     getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
 
-    tablets = sql_return_maparray """ show tablets from ${table_name}; """
+    def tablets = sql_return_maparray """ show tablets from ${table_name}; """
     String tablet_id = tablets[0].TabletId
     String backend_id = tablets[0].BackendId
     String ip = backendId_to_backendIP.get(backend_id)
diff --git a/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy
index 7b4075fe8a5337..6239f644f9854f 100644
--- a/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy
+++ b/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy
@@ -67,7 +67,7 @@ suite("test_array_contains_with_inverted_index"){
 
     qt_sql """ select count() from ${indexTblName}"""
     def param_contains = ["\'s\'", "\'\'", null]
-    for (i = 0 ; i < param_contains.size(); ++i) {
+    for (int i = 0 ; i < param_contains.size(); ++i) {
         def p = param_contains[i]
         log.info("param: ${p}")
         order_qt_sql """ select * from tai where array_contains(inventors, ${p}) order by id; """
@@ -84,7 +84,7 @@ suite("test_array_contains_with_inverted_index"){
     // test arrays_overlap with inverted index
     // now if we use inverted index we will not eval exprs
     def param = [["\'s\'", "\'t\'"], [], null, ["\'s\'", "\'\'", "\'t\'"], ["\'s\'", null, "\'t\'"], [null, "\'\'"], ["\'s\'", null, "\'t\'", "\'\'"]] // null for arrays_overlap will return null which in predicate will lead to return empty set
-    for (i = 0 ; i < param.size(); ++i) {
+    for (int i = 0 ; i < param.size(); ++i) {
         def p = param[i]
         log.info("param: ${p}")
         order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where arrays_overlap(inventors, ${p}) order by id; """
diff --git a/regression-test/suites/inverted_index_p0/test_index_match_select.groovy b/regression-test/suites/inverted_index_p0/test_index_match_select.groovy
index c5cfa69fefe8c5..387893e470f1e8 100644
--- a/regression-test/suites/inverted_index_p0/test_index_match_select.groovy
+++ b/regression-test/suites/inverted_index_p0/test_index_match_select.groovy
@@ -207,7 +207,7 @@ suite("test_index_match_select", "inverted_index_select"){
         }
 
         // cas2.2 test varchar standard match same term with different way and repeate 5 times
-        for (test_times = 0; test_times < 5; test_times++) {
+        for (int test_times = 0; test_times < 5; test_times++) {
             qt_sql """ select * from ${indexTbName1} where ${varchar_colume3} match_any 'zhang yi' order by name """
             qt_sql """ select * from ${indexTbName1} where ${varchar_colume3} match_all "zhang yi" order by name """
             qt_sql """ select * from ${indexTbName1} where ${varchar_colume3} match_any '"zhang yi"' order by name """
@@ -217,7 +217,7 @@ suite("test_index_match_select", "inverted_index_select"){
         }
 
         // case3: test char standard match same term with different way and repeate 5 times
-        for (test_times = 0; test_times < 5; test_times++) {
+        for (int test_times = 0; test_times < 5; test_times++) {
             qt_sql """ select * from ${indexTbName1} where ${char_colume1} match_any 'tall:100cm, weight: 30kg, hobbies:' order by name """
             qt_sql """ select * from ${indexTbName1} where ${char_colume1} match_all "tall:100cm, weight: 30kg, hobbies:" order by name """
             qt_sql """ select * from ${indexTbName1} where ${char_colume1} match_any '"tall:100cm, weight: 30kg, hobbies:"' order by name """
@@ -227,7 +227,7 @@ suite("test_index_match_select", "inverted_index_select"){
         }
 
         // case4: test string simple match same term with different way and repeate 5 times
-        for (test_times = 0; test_times < 5; test_times++) {
+        for (int test_times = 0; test_times < 5; test_times++) {
             qt_sql """ select * from ${indexTbName1} where ${string_colume1} match_all 'A naughty boy' order by name """
             qt_sql """ select * from ${indexTbName1} where ${string_colume1} match_any "A naughty boy" order by name """
             qt_sql """ select * from ${indexTbName1} where ${string_colume1} match_any '"A naughty boy"' order by name """
@@ -236,7 +236,7 @@ suite("test_index_match_select", "inverted_index_select"){
         }
 
         // case5: test text standard match same term with different way and repeate 5 times
-        for (test_times = 0; test_times < 5; test_times++) {
+        for (int test_times = 0; test_times < 5; test_times++) {
             qt_sql """ select * from ${indexTbName1} where ${text_colume1} match_all 'i just want go outside' order by name """
             qt_sql """ select * from ${indexTbName1} where ${text_colume1} match_any "i just want go outside" order by name """
             qt_sql """ select * from ${indexTbName1} where ${text_colume1} match_all '"i just want go outside"' order by name """
diff --git a/regression-test/suites/inverted_index_p0/unique_with_mow/test_primary_key_simple_case.groovy b/regression-test/suites/inverted_index_p0/unique_with_mow/test_primary_key_simple_case.groovy
index 8884ff837a641a..0c437e4fe1b50f 100644
--- a/regression-test/suites/inverted_index_p0/unique_with_mow/test_primary_key_simple_case.groovy
+++ b/regression-test/suites/inverted_index_p0/unique_with_mow/test_primary_key_simple_case.groovy
@@ -68,7 +68,7 @@ suite("test_primary_key_simple_case", "inverted_index") {
         """
     sql """ set enable_common_expr_pushdown = true """
 
-    result = sql """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+    def result = sql """ SELECT * FROM ${tableName} t ORDER BY user_id; """
     assertTrue(result.size() == 5)
     assertTrue(result[0].size() == 11)
 
diff --git a/regression-test/suites/load_p0/http_stream/test_http_stream_2pc.groovy b/regression-test/suites/load_p0/http_stream/test_http_stream_2pc.groovy
index b05ce6c42b81c9..bc07d65eccefca 100644
--- a/regression-test/suites/load_p0/http_stream/test_http_stream_2pc.groovy
+++ b/regression-test/suites/load_p0/http_stream/test_http_stream_2pc.groovy
@@ -65,16 +65,16 @@ suite("test_http_stream_2pc", "p0") {
         log.info("http_stream execute 2pc: ${command}")
 
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
-        json2pc = parseJson(out)
+        def code = process.waitFor()
+        def out = process.text
+        def json2pc = parseJson(out)
         log.info("http_stream 2pc result: ${out}".toString())
         assertEquals(code, 0)
         assertEquals("success", json2pc.status.toLowerCase())
 
         def count = 0
         while (true) {
-            res = sql "select count(*) from ${tableName1}"
+            def res = sql "select count(*) from ${tableName1}"
             if (res[0][0] > 0) {
                 break
             }
diff --git a/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy b/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy
index f9e456b5d3c801..9510a4f7980864 100644
--- a/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy
+++ b/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy
@@ -151,7 +151,7 @@ suite("test_insert_random_distribution_table", "p0") {
         res = sql "show tablets from ${tableName} partition ${partitions[p]}"
         partitionTablets[p] = getTablets.call(res)
         partitionRowCounts[p] = []
-        numTablets = partitionTablets[p].size()
+        def numTablets = partitionTablets[p].size()
         for (int i = numTablets - 1; i >= 0; i--) {
             def countResult = sql "select count() from ${tableName} tablet(${partitionTablets[p][i]})"
             partitionRowCounts[p][i] = countResult[0][0]
@@ -191,7 +191,7 @@ suite("test_insert_random_distribution_table", "p0") {
 
 
     for (int p = 0; p < 3; p++) {
-        numTablets = partitionTablets[p].size()
+        def numTablets = partitionTablets[p].size()
         for (int i = numTablets - 1; i >= 0; i--) {
             def countResult = sql "select count() from ${tableName} tablet(${partitionTablets[p][i]})"
             partitionRowCounts[p][i] = countResult[0][0]
diff --git a/regression-test/suites/load_p0/mysql_load/test_mysql_load.groovy b/regression-test/suites/load_p0/mysql_load/test_mysql_load.groovy
index 9eb948bf55a098..2b5a9537b2ff33 100644
--- a/regression-test/suites/load_p0/mysql_load/test_mysql_load.groovy
+++ b/regression-test/suites/load_p0/mysql_load/test_mysql_load.groovy
@@ -116,7 +116,7 @@ suite("test_mysql_load", "p0") {
     """
 
     sql "sync"
-    rowCount = sql "select count(1) from ${tableName}"
+    def rowCount = sql "select count(1) from ${tableName}"
     assertEquals(3, rowCount[0][0])
 
 
diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy
index e1cfa4ae9aaefc..0089aff61e56b4 100644
--- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy
+++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy
@@ -89,7 +89,7 @@ suite("test_routine_load_alter","p0") {
 
             def count = 0
             while (true) {
-                res = sql "select count(*) from ${tableName}"
+                def res = sql "select count(*) from ${tableName}"
                 def state = sql "show routine load for ${jobName}"
                 log.info("routine load state: ${state[0][8].toString()}".toString())
                 log.info("routine load statistic: ${state[0][14].toString()}".toString())
@@ -131,7 +131,7 @@ suite("test_routine_load_alter","p0") {
 
             count = 0
             while (true) {
-                res = sql "select count(*) from ${tableName}"
+                def res = sql "select count(*) from ${tableName}"
                 log.info("count: ${res[0][0]}".toString())
                 def state = sql "show routine load for ${jobName}"
                 log.info("routine load state: ${state[0][8].toString()}".toString())
diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy
index 11a562ba9d4660..9f4ae866e992e5 100644
--- a/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy
+++ b/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy
@@ -102,7 +102,7 @@ suite("test_routine_load_with_udf","p0") {
 
             def count = 0
             while (true) {
-                res = sql "select count(*) from ${tableName}"
+                def res = sql "select count(*) from ${tableName}"
                 def state = sql "show routine load for test_udf_load"
                 log.info("routine load state: ${state[0][8].toString()}".toString())
                 log.info("routine load statistic: ${state[0][14].toString()}".toString())
diff --git a/regression-test/suites/load_p0/stream_load/test_get_stream_load_state.groovy b/regression-test/suites/load_p0/stream_load/test_get_stream_load_state.groovy
index 783aa8b492cda5..7bba1394bf38aa 100644
--- a/regression-test/suites/load_p0/stream_load/test_get_stream_load_state.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_get_stream_load_state.groovy
@@ -60,9 +60,9 @@ suite("test_get_stream_load_state", "p0") {
     def command = "curl --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword} http://${context.config.feHttpAddress}/api/${db}/get_load_state?label=${label}"
     log.info("test_get_stream_load_state: ${command}")
     def process = command.execute()
-    code = process.waitFor()
-    out = process.text
-    json = parseJson(out)
+    def code = process.waitFor()
+    def out = process.text
+    def json = parseJson(out)
     log.info("test_get_stream_load_state: ${out}".toString())
     assertEquals("success", json.msg.toLowerCase())
     assertEquals("VISIBLE", json.data)
diff --git a/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy b/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy
index 57054b2a42088a..8fd638147152c5 100644
--- a/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy
@@ -28,13 +28,13 @@ suite("test_group_commit_stream_load_with_nonexist_db_and_table") {
         log.info("stream load command: ${command}")
 
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
+        def code = process.waitFor()
+        def out = process.text
         log.info("stream lad result: ${out}".toString())
         assertTrue(out.toString().contains("OlapTable not found"))
     } catch (Exception e) {
         logger.info("failed: " + e.getMessage())
-        assertTrue(false)
+        throw e
     } finally {
 
     }
diff --git a/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy b/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy
index 703f5dde345587..fd22c12afaf545 100644
--- a/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy
@@ -117,6 +117,8 @@ suite("test_map_load_and_compaction", "p0") {
         def compactJson = parseJson(out.trim())
         assertEquals("success", compactJson.status.toLowerCase())
 
+        def running = false
+
         // wait compactions done
         do {
             Thread.sleep(1000)
@@ -131,7 +133,7 @@ suite("test_map_load_and_compaction", "p0") {
         checkCompactionStatus.call(compactionStatus, 1)
 
         // finally check backend alive
-        backends = sql """ show backends; """
+        def backends = sql """ show backends; """
         assertTrue(backends.size() > 0)
         for (String[] b : backends) {
             assertEquals("true", b[9])
diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load.groovy
index 9783f178c3a2ad..54731a949584be 100644
--- a/regression-test/suites/load_p0/stream_load/test_stream_load.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_stream_load.groovy
@@ -178,7 +178,7 @@ suite("test_stream_load", "p0") {
     }
 
     sql "sync"
-    rowCount = sql "select count(1) from ${tableName}"
+    def rowCount = sql "select count(1) from ${tableName}"
     assertEquals(3, rowCount[0][0])
 
     // test load_nullable_to_not_nullable
@@ -1344,9 +1344,9 @@ suite("test_stream_load", "p0") {
         def command = "curl --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword} -H column_separator:| -H Transfer-Encoding:chunked -H columns:k1,k2,v1,v2,v3  -T ${context.dataPath}/test_chunked_transfer.csv http://${context.config.feHttpAddress}/api/${db}/${tableName16}/_stream_load"
         log.info("test chunked transfer command: ${command}")
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
-        json2pc = parseJson(out)
+        def code = process.waitFor()
+        def out = process.text
+        def json2pc = parseJson(out)
         log.info("test chunked transfer result: ${out}".toString())
         sql "sync"
         qt_sql_chunked_transfer_csv "select * from ${tableName16} order by k1"
@@ -1372,9 +1372,9 @@ suite("test_stream_load", "p0") {
         def command = "curl --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword} -H Transfer-Encoding:chunked -H format:json -H read_json_by_line:true -T ${context.dataPath}/test_chunked_transfer.json http://${context.config.feHttpAddress}/api/${db}/${tableName16}/_stream_load"
         log.info("test chunked transfer command: ${command}")
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
-        json2pc = parseJson(out)
+        def code = process.waitFor()
+        def out = process.text
+        def json2pc = parseJson(out)
         log.info("test chunked transfer result: ${out}".toString())
         sql "sync"
         qt_sql_chunked_transfer_json "select * from ${tableName16} order by k1"
@@ -1652,8 +1652,8 @@ suite("test_stream_load", "p0") {
        def command = "curl --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword} -H column_separator:| -H ${db}:${tableName16} -H Content-Length:0  -H Transfer-Encoding:chunked -H columns:k1,k2,v1,v2,v3 -T ${context.dataPath}/test_chunked_transfer.csv http://${beHost}:${beHttpPort}/api/${db}/${tableName16}/_stream_load"
        log.info("test chunked transfer command: ${command}")
        def process = command.execute()
-       code = process.waitFor()
-       out = process.text
+       def code = process.waitFor()
+       def out = process.text
        log.info("test chunked transfer result: ${out}".toString())
        def json = parseJson(out)
        assertEquals("fail", json.Status.toLowerCase())
@@ -1682,8 +1682,8 @@ suite("test_stream_load", "p0") {
         def command = "curl --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword} -H column_separator:| -H ${db}:${tableName16} -H Content-Length:  -H Transfer-Encoding: -T ${context.dataPath}/test_chunked_transfer.csv http://${beHost}:${beHttpPort}/api/${db}/${tableName16}/_stream_load"
         log.info("test chunked transfer command: ${command}")
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
+        def code = process.waitFor()
+        def out = process.text
         log.info("test chunked transfer result: ${out}".toString())
         def json = parseJson(out)
         assertEquals("fail", json.Status.toLowerCase())
diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_2pc.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_2pc.groovy
index a193a3f503db6f..ec51c76a62a986 100644
--- a/regression-test/suites/load_p0/stream_load/test_stream_load_2pc.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_stream_load_2pc.groovy
@@ -346,8 +346,8 @@ suite("test_stream_load_2pc", "p0") {
         log.info("http_stream execute 2pc: ${command}")
 
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
+        def code = process.waitFor()
+        def out = process.text
         log.info("http_stream 2pc result: ${out}".toString())
         def json2pc = parseJson(out)
         return json2pc
@@ -361,8 +361,8 @@ suite("test_stream_load_2pc", "p0") {
         log.info("http_stream execute 2pc: ${command}")
 
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
+        def code = process.waitFor()
+        def out = process.text
         log.info("http_stream 2pc result: ${out}".toString())
         def json2pc = parseJson(out)
         return json2pc
@@ -522,7 +522,7 @@ suite("test_stream_load_2pc", "p0") {
 
             def count = 0
             while (true) {
-                res = sql "select count(*) from ${tbl}"
+                def res = sql "select count(*) from ${tbl}"
                 if (res[0][0] > 0) {
                     break
                 }
diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_2pc_with_schema_change.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_2pc_with_schema_change.groovy
index 8695101be62ea1..17104133f7d9f8 100644
--- a/regression-test/suites/load_p0/stream_load/test_stream_load_2pc_with_schema_change.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_stream_load_2pc_with_schema_change.groovy
@@ -30,8 +30,8 @@ suite("test_stream_load_2pc_with_schema_change", "p0") {
         log.info("http_stream execute 2pc: ${command}")
 
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
+        def code = process.waitFor()
+        def out = process.text
         log.info("http_stream 2pc result: ${out}".toString())
         def json2pc = parseJson(out)
         return json2pc
@@ -84,7 +84,7 @@ suite("test_stream_load_2pc_with_schema_change", "p0") {
 
     Thread.sleep(5000) // 5s
 
-    json2pc = do_streamload_2pc_commit_by_txn_id.call(txnId)
+    def json2pc = do_streamload_2pc_commit_by_txn_id.call(txnId)
     assertEquals("success", json2pc.status.toLowerCase())
 
     waitForSchemaChangeDone {
diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy
index 7244cd24f4ac47..bbd532a76259cc 100644
--- a/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy
@@ -185,7 +185,7 @@ suite("test_stream_load_move_memtable", "p0") {
     }
 
     sql "sync"
-    rowCount = sql "select count(1) from ${tableName}"
+    def rowCount = sql "select count(1) from ${tableName}"
     assertEquals(3, rowCount[0][0])
 
     // test load_nullable_to_not_nullable
diff --git a/regression-test/suites/manager/test_manager_interface_2.groovy b/regression-test/suites/manager/test_manager_interface_2.groovy
index 9c53941c157392..1270d77be5572a 100644
--- a/regression-test/suites/manager/test_manager_interface_2.groovy
+++ b/regression-test/suites/manager/test_manager_interface_2.groovy
@@ -44,7 +44,7 @@ suite('test_manager_interface_2',"p0") {
         );"""
 
     
-        result =sql  """ show data """ 
+        def result =sql  """ show data """ 
         for(int i = 0 ; i < result.size();i++) {
             assert(result[i][0].toLowerCase() != "null") //TableName 
             assert(result[i][1].toLowerCase() != "null") //Size 
@@ -183,7 +183,7 @@ suite('test_manager_interface_2',"p0") {
     
         sql """ALTER SYSTEM ADD BACKEND "${address}:${notExistPort}";"""
 
-        result = sql """SHOW BACKENDS;"""
+        def result = sql """SHOW BACKENDS;"""
         logger.info("result = ${result}" )
         def x = 0 
         for(int i  =0 ;i<result.size();i++) {
diff --git a/regression-test/suites/manager/test_manager_interface_4.groovy b/regression-test/suites/manager/test_manager_interface_4.groovy
index 355d96e1e0310a..78d8d451b7b796 100644
--- a/regression-test/suites/manager/test_manager_interface_4.groovy
+++ b/regression-test/suites/manager/test_manager_interface_4.groovy
@@ -122,7 +122,7 @@ suite('test_manager_interface_4',"p0") {
         def healthy_num = 0;
         def total_tablet_num  = 0;
         def total_healthy_num = 0;
-        result = sql """  SHOW PROC '/cluster_health/tablet_health' """ 
+        def result = sql """  SHOW PROC '/cluster_health/tablet_health' """ 
         logger.info("result = ${result}" )
 
         for( int i =0 ;i < result.size();i++ ){
diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
index 17b4d470485793..a360664707ded0 100644
--- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
@@ -202,7 +202,7 @@ suite("test_build_mtmv") {
         AS
         SELECT ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id;
     """
-    jobName = getJobName("regression_test_mtmv_p0", mvName);
+    def jobName = getJobName("regression_test_mtmv_p0", mvName);
     println jobName
     waitingMTMVTaskFinished(jobName)
     order_qt_select "SELECT * FROM ${mvName}"
diff --git a/regression-test/suites/mtmv_p0/test_create_mv_mtmv.groovy b/regression-test/suites/mtmv_p0/test_create_mv_mtmv.groovy
index c6b6b4386d5075..189517ba8ea244 100644
--- a/regression-test/suites/mtmv_p0/test_create_mv_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_create_mv_mtmv.groovy
@@ -50,7 +50,7 @@ suite("test_create_mv_mtmv","mtmv") {
         CREATE MATERIALIZED VIEW mv_mtmv1  as select k2 from ${mvName};
         """
 
-    max_try_secs = 60
+    def max_try_secs = 60
     while (max_try_secs--) {
         def jobStateResult = sql """  SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${mvName}' ORDER BY CreateTime DESC LIMIT 1; """
         String res = jobStateResult[0][8]
diff --git a/regression-test/suites/mtmv_p0/test_create_rollup_mtmv.groovy b/regression-test/suites/mtmv_p0/test_create_rollup_mtmv.groovy
index df4040d9a9e605..4fd926fa3ac800 100644
--- a/regression-test/suites/mtmv_p0/test_create_rollup_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_create_rollup_mtmv.groovy
@@ -51,7 +51,7 @@ suite("test_create_rollup_mtmv","mtmv") {
         alter table ${mvName} ADD ROLLUP rollup1(k3);
         """
 
-    max_try_secs = 60
+    def max_try_secs = 60
     while (max_try_secs--) {
         def jobStateResult = sql """SHOW ALTER TABLE ROLLUP WHERE TableName='${mvName}' ORDER BY CreateTime DESC LIMIT 1; """
         String res = jobStateResult[0][8]
diff --git a/regression-test/suites/mtmv_p0/test_limit_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_limit_partition_mtmv.groovy
index 19aa229743d6a5..6e4426313032f9 100644
--- a/regression-test/suites/mtmv_p0/test_limit_partition_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_limit_partition_mtmv.groovy
@@ -57,7 +57,7 @@ suite("test_limit_partition_mtmv") {
             AS
             SELECT * FROM ${tableName};
     """
-    showPartitionsResult = sql """show partitions from ${mvName}"""
+    def showPartitionsResult = sql """show partitions from ${mvName}"""
     logger.info("showPartitionsResult: " + showPartitionsResult.toString())
     assertEquals(1, showPartitionsResult.size())
     assertTrue(showPartitionsResult.toString().contains("p_20380101"))
diff --git a/regression-test/suites/mtmv_p0/test_mtmv_outfile.groovy b/regression-test/suites/mtmv_p0/test_mtmv_outfile.groovy
index 693df86e014844..28c76a5bdd820c 100644
--- a/regression-test/suites/mtmv_p0/test_mtmv_outfile.groovy
+++ b/regression-test/suites/mtmv_p0/test_mtmv_outfile.groovy
@@ -64,7 +64,7 @@ suite("test_mtmv_outfile","mtmv") {
     String ak = getS3AK()
     String sk = getS3SK()
     String s3_endpoint = getS3Endpoint()
-    String region = region = getS3Region()
+    String region = getS3Region()
     String bucket = context.config.otherConfigs.get("s3BucketName");
 
     def outfile_to_S3_directly = {
diff --git a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
index a9eeacc1c898bf..6ab3fa57c47236 100644
--- a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
@@ -96,7 +96,7 @@ suite("test_rollup_partition_mtmv") {
             AS
             SELECT * FROM ${tableName};
     """
-    showPartitionsResult = sql """show partitions from ${mvName}"""
+    def showPartitionsResult = sql """show partitions from ${mvName}"""
     logger.info("showPartitionsResult: " + showPartitionsResult.toString())
     assertEquals(2, showPartitionsResult.size())
     assertTrue(showPartitionsResult.toString().contains("2020-01-01"))
@@ -199,7 +199,7 @@ suite("test_rollup_partition_mtmv") {
     sql """
             REFRESH MATERIALIZED VIEW ${mvName} AUTO
         """
-    jobName = getJobName(dbName, mvName);
+    def jobName = getJobName(dbName, mvName);
     log.info(jobName)
     waitingMTMVTaskFinished(jobName)
     order_qt_date_range_month "SELECT * FROM ${mvName} order by k1,k2"
diff --git a/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy
index 8ee24cb5ba2bd9..49817cee2783e5 100644
--- a/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy
+++ b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy
@@ -67,7 +67,7 @@ suite("test_create_mv") {
 
     sql """ SHOW ALTER TABLE MATERIALIZED VIEW """
 
-    max_try_secs = 60
+    def max_try_secs = 60
     while (max_try_secs--) {
         String res = getJobState(tableName)
         if (res == "FINISHED" || res == "CANCELLED") {
diff --git a/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy b/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy
index 09713c76172268..b12aef3a656f9a 100644
--- a/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy
+++ b/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy
@@ -66,7 +66,7 @@ suite ("accept_null") {
    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
    boolean disableAutoCompaction = true
    for(int i=0;i<backendId_to_backendIP.keySet().size();i++){
-      backend_id = backendId_to_backendIP.keySet()[i]
+      def backend_id = backendId_to_backendIP.keySet()[i]
       def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
       logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
       assertEquals(code, 0)
@@ -86,7 +86,7 @@ suite ("accept_null") {
       def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """
       logger.info("tablet"+tablet_info)
       def table_id = tablet_info[0].TableId
-      backend_id = tablet.BackendId
+      def backend_id = tablet.BackendId
       def times = 1
       def code, out, err
       do{
diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
index 8c9e77bba7876e..9e9e6507fa6ad7 100644
--- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
+++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
@@ -151,7 +151,7 @@ suite("parse_sql_from_sql_cache") {
             assertNoCache "select * from test_use_plan_cache5"
         }),
         extraThread("testUpdate",{
-            createTestTable "test_use_plan_cache6", uniqueTable=true
+            createTestTable("test_use_plan_cache6", true)
 
             // after partition changed 10s, the sql cache can be used
             sleep(10000)
diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_fe.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_fe.groovy
index 23deb27450160b..67fda161bba02e 100644
--- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_fe.groovy
+++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_fe.groovy
@@ -66,7 +66,7 @@ suite("test_fold_constant_by_fe") {
         qt_sql "select to_monday('${date}'), last_day('${date}'), to_date('${date}'), to_days('${date}'), date('${date}'), datev2('${date}')"
     }
 
-    test_year = [2001, 2013, 123, 1969, 2023]
+    def test_year = [2001, 2013, 123, 1969, 2023]
     for (year in test_year) {
         for (integer in test_int) {
             qt_sql "select /*+SET_VAR(time_zone=\"Asia/Shanghai\")*/ makedate(${year}, ${integer}), from_days(${year * integer}), from_unixtime(${year / 10 * year * integer})"
diff --git a/regression-test/suites/nereids_p0/outfile/test_outfile.groovy b/regression-test/suites/nereids_p0/outfile/test_outfile.groovy
index f256df9180944c..6a7be107510958 100644
--- a/regression-test/suites/nereids_p0/outfile/test_outfile.groovy
+++ b/regression-test/suites/nereids_p0/outfile/test_outfile.groovy
@@ -109,7 +109,7 @@ suite("test_outfile") {
             SELECT * FROM ${tableName} t ORDER BY user_id INTO OUTFILE "file://${outFile}/";
         """
 
-        url = result[0][3]
+        def url = result[0][3]
         urlHost = url.substring(8, url.indexOf("${outFile}"))
         def filePrifix = url.split("${outFile}")[1]
         csvFiles = "${outFile}${filePrifix}*.csv"
@@ -146,7 +146,7 @@ suite("test_outfile") {
             path.delete();
         }
 
-        cmd = "rm -rf ${csvFiles}"
+        def cmd = "rm -rf ${csvFiles}"
         sshExec ("root", urlHost, cmd)
     }
 
@@ -184,7 +184,7 @@ suite("test_outfile") {
             SELECT * FROM ${tableName} t ORDER BY k1, v2 INTO OUTFILE "file://${outFile}/"
         """
 
-        url = result[0][3]
+        def url = result[0][3]
         urlHost = url.substring(8, url.indexOf("${outFile}"))
         def filePrifix = url.split("${outFile}")[1]
         csvFiles = "${outFile}${filePrifix}*.csv"
@@ -209,7 +209,7 @@ suite("test_outfile") {
             path.delete();
         }
 
-        cmd = "rm -rf ${csvFiles}"
+        def cmd = "rm -rf ${csvFiles}"
         sshExec ("root", urlHost, cmd)
     }
 
@@ -239,6 +239,8 @@ suite("test_outfile") {
 
         sql "set enable_parallel_outfile = true;"
         sql """select * from select_into_file into outfile "file://${outFilePath}/" properties("success_file_name" = "SUCCESS");"""
+    } catch (Exception e) {
+        logger.info("export exception: ${e}")
     } finally {
         try_sql("DROP TABLE IF EXISTS select_into_file")
         File path = new File(outFilePath)
@@ -249,7 +251,7 @@ suite("test_outfile") {
             path.delete();
         }
 
-        cmd = "rm -rf ${csvFiles}"
+        def cmd = "rm -rf ${csvFiles}"
         sshExec ("root", urlHost, cmd)
     }
 }
diff --git a/regression-test/suites/nereids_p0/outfile/test_outfile_expr.groovy b/regression-test/suites/nereids_p0/outfile/test_outfile_expr.groovy
index 7ac4b1b43b25a9..9429906ec3c896 100644
--- a/regression-test/suites/nereids_p0/outfile/test_outfile_expr.groovy
+++ b/regression-test/suites/nereids_p0/outfile/test_outfile_expr.groovy
@@ -142,7 +142,7 @@ suite("test_outfile_expr") {
             path.delete();
         }
 
-        cmd = "rm -rf ${csvFiles}"
+        def cmd = "rm -rf ${csvFiles}"
         sshExec ("root", urlHost, cmd)
     }
 
diff --git a/regression-test/suites/nereids_p0/outfile/test_outfile_parquet.groovy b/regression-test/suites/nereids_p0/outfile/test_outfile_parquet.groovy
index e6cdb541dbfb77..cdc210b8aea061 100644
--- a/regression-test/suites/nereids_p0/outfile/test_outfile_parquet.groovy
+++ b/regression-test/suites/nereids_p0/outfile/test_outfile_parquet.groovy
@@ -113,7 +113,7 @@ suite("test_outfile_parquet") {
             SELECT * FROM ${tableName} t ORDER BY user_id INTO OUTFILE "file://${outFile}/" FORMAT AS PARQUET;
         """
 
-        url = result[0][3]
+        def url = result[0][3]
         urlHost = url.substring(8, url.indexOf("${outFile}"))
         def filePrifix = url.split("${outFile}")[1]
         parquetFiles = "${outFile}${filePrifix}*.parquet"
@@ -158,7 +158,9 @@ suite("test_outfile_parquet") {
         logger.info("Run command: command=" + command + ",code=" + code + ", out=" + out + ", err=" + err)
         assertEquals(code, 0)
         qt_select_default """ SELECT * FROM ${tableName2} t ORDER BY user_id; """
-    } finally {
+    } catch (Exception e) {
+        logger.info("export exception: ${e}")
+    }finally {
         try_sql("DROP TABLE IF EXISTS ${tableName}")
         try_sql("DROP TABLE IF EXISTS ${tableName2}")
         File path = new File(outFilePath)
@@ -169,7 +171,7 @@ suite("test_outfile_parquet") {
             path.delete();
         }
 
-        cmd = "rm -rf ${parquetFiles}"
+        def cmd = "rm -rf ${parquetFiles}"
         sshExec ("root", urlHost, cmd)
     }
 }
diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
index 1a5d3ca7a0acfb..282a28a903e4a0 100644
--- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
@@ -14,7 +14,6 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-test_date_or_datetime_computation_negative
 suite("test_date_or_datetime_computation_negative") {
     sql """ CREATE TABLE IF NOT EXISTS test_date_or_datetime_computation_negative (
                 `row_id` LARGEINT NOT NULL,
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
index 776f1b996f1826..178307f15f8afb 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
@@ -305,7 +305,7 @@ suite("partition_mv_rewrite_dimension_self_conn") {
 
     // agg
     // agg + without group by + with agg function
-    agg_mv_stmt = """
+    def agg_mv_stmt = """
         select t2.o_orderkey, 
         sum(t1.O_TOTALPRICE) as sum_total,
         max(t1.o_totalprice) as max_total, 
diff --git a/regression-test/suites/nereids_syntax_p0/group_bit.groovy b/regression-test/suites/nereids_syntax_p0/group_bit.groovy
index b0a96f80ddf518..2244a5af4e07c8 100644
--- a/regression-test/suites/nereids_syntax_p0/group_bit.groovy
+++ b/regression-test/suites/nereids_syntax_p0/group_bit.groovy
@@ -18,7 +18,7 @@
 suite("group_bit") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
-    table = "group_bit_and_or_xor"
+    def table = "group_bit_and_or_xor"
     sql """ CREATE TABLE if not exists ${table} (
         `k` int(11) NULL
         ) ENGINE=OLAP
diff --git a/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy
index 2407c70d8a451c..77344c4fc86188 100644
--- a/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy
+++ b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy
@@ -37,7 +37,7 @@ suite("bitmap", "rollup") {
     sql """alter table test_materialized_view_bitmap1 modify column k1 set stats ('row_count'='2');"""
 
     sql "CREATE MATERIALIZED VIEW test_neg as select k1,bitmap_union(to_bitmap(k2)), bitmap_union(to_bitmap(k3)) FROM ${tbName1} GROUP BY k1;"
-    max_try_secs = 60
+    def max_try_secs = 60
     while (max_try_secs--) {
         String res = getJobState(tbName1)
         if (res == "FINISHED" || res == "CANCELLED") {
diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
index 808bb6a8798f40..0334bb684749e3 100644
--- a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
+++ b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
@@ -284,7 +284,7 @@ suite("test_auto_partition_behavior") {
     logger.info("get table replica num: " + replicaNum)
 
     sql """ insert into test_change values ("20201212"); """
-    part_result = sql " show tablets from test_change "
+    def part_result = sql " show tablets from test_change "
     assertEquals(part_result.size, 2 * replicaNum)
     sql """ ALTER TABLE test_change MODIFY DISTRIBUTION DISTRIBUTED BY HASH(k0) BUCKETS 50; """
     sql """ insert into test_change values ("20001212"); """
diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_load.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_load.groovy
index d728ba9a1457de..0e0440dc89ef8b 100644
--- a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_load.groovy
+++ b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_load.groovy
@@ -45,7 +45,7 @@ suite("test_auto_partition_load") {
     sql """ insert into load_table1 values (14, '2002-12-12 12:12:12.123', '2001-11-14 12:12:12.123456') """
 
     qt_select1 "select * from load_table1 order by k1"
-    result1 = sql "show partitions from load_table1"
+    def result1 = sql "show partitions from load_table1"
     logger.info("${result1}")
     assertEquals(result1.size(), 8)
 
@@ -79,7 +79,7 @@ suite("test_auto_partition_load") {
     sql """ insert into load_table2 values (14, '12', '2123-11-14 12:12:12.123456') """
 
     qt_select2 "select * from load_table2 order by k1"
-    result2 = sql "show partitions from load_table2"
+    def result2 = sql "show partitions from load_table2"
     logger.info("${result2}")
     assertEquals(result2.size(), 11)
 }
diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy
index 845bc2320feb81..557560dfefc477 100644
--- a/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy
+++ b/regression-test/suites/partition_p0/auto_partition/test_auto_range_partition.groovy
@@ -98,7 +98,7 @@ suite("test_auto_range_partition") {
     sql " insert into right_bound values ('9999-12-31 23:59:59'); "
     sql " insert into right_bound values ('9999-12-31 23:59:59.999999'); "
     qt_right_bound " select * from right_bound order by k0; "
-    result2 = sql "show partitions from right_bound"
+    def result2 = sql "show partitions from right_bound"
     logger.info("${result2}")
     assertEquals(result2.size(), 2)
 
@@ -117,7 +117,7 @@ suite("test_auto_range_partition") {
         );
     """
     sql " insert into week_range values (20240408), (20240409); "
-    def result2 = sql "show partitions from week_range"
+    result2 = sql "show partitions from week_range"
     logger.info("${result2}")
     assertEquals(result2.size(), 1)
 
diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
index cbd58007e6cf15..f10cf3086ae16d 100644
--- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
+++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 suite("test_dynamic_partition") {
-    config_row = sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'force_olap_table_replication_allocation'; """
+    def config_row = sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'force_olap_table_replication_allocation'; """
     String old_conf_value = config_row[0][1]
     sql """ ADMIN SET FRONTEND CONFIG ("force_olap_table_replication_allocation" = ""); """
     // todo: test dynamic partition
diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_mod_distribution_key.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_mod_distribution_key.groovy
index a2c1e3035a680e..1079ca04f810d7 100644
--- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_mod_distribution_key.groovy
+++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_mod_distribution_key.groovy
@@ -53,7 +53,7 @@ suite("test_dynamic_partition_mod_distribution_key") {
 
         sql """ ADMIN SET FRONTEND CONFIG ('dynamic_partition_check_interval_seconds' = '1') """
         sql """ alter table ${tableName} set('dynamic_partition.end'='5') """
-        result = sql "show partitions from ${tableName}"
+        def result = sql "show partitions from ${tableName}"
         for (def retry = 0; retry < 10; retry++) { // at most wait 120s
             if (result.size() == 9) {
                 break;
diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_alter.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_alter.groovy
index fe27fdbd77aabb..450c079b644960 100644
--- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_alter.groovy
+++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_with_alter.groovy
@@ -35,7 +35,7 @@ suite("test_dynamic_partition_with_alter") {
             "dynamic_partition.replication_allocation" = "tag.location.default: 1")
         """
 
-    result = sql "show partitions from ${tbl}"
+    def result = sql "show partitions from ${tbl}"
     assertEquals(7, result.size())
 
     try {
diff --git a/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy b/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy
index f5e20a97dd7b88..4b74cdad8538e4 100644
--- a/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy
+++ b/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy
@@ -21,7 +21,7 @@ suite("topn_2pr_rule") {
 
     def create_table = { table_name, key_type="DUPLICATE" ->
         sql "DROP TABLE IF EXISTS ${table_name}"
-        value_type = "v string"
+        def value_type = "v string"
         if ("${key_type}" == "AGGREGATE") {
             value_type = "v string REPLACE_IF_NOT_NULL NULL" 
         }
diff --git a/regression-test/suites/query_profile/s3_load_profile_test.groovy b/regression-test/suites/query_profile/s3_load_profile_test.groovy
index b0b1c357beb487..be4f38cfdbce7a 100644
--- a/regression-test/suites/query_profile/s3_load_profile_test.groovy
+++ b/regression-test/suites/query_profile/s3_load_profile_test.groovy
@@ -171,7 +171,7 @@ PROPERTIES (
     qt_select """ select count(*) from $loadAttribute.dataDesc.tableName """
 
     def profileString = getProfile(jobId)
-    profileJson = new JsonSlurper().parseText(profileString)
+    def profileJson = new JsonSlurper().parseText(profileString)
     assertEquals(0, profileJson.code)
     profileDataString = profileJson.data
     logger.info("profileDataString:" + profileDataString)
diff --git a/regression-test/suites/rollup/test_materialized_view_bitmap.groovy b/regression-test/suites/rollup/test_materialized_view_bitmap.groovy
index 25403a6db1cae4..fd2e7fe824fdc6 100644
--- a/regression-test/suites/rollup/test_materialized_view_bitmap.groovy
+++ b/regression-test/suites/rollup/test_materialized_view_bitmap.groovy
@@ -37,7 +37,7 @@ suite("test_materialized_view_bitmap", "rollup") {
         """
 
     sql "CREATE MATERIALIZED VIEW test_neg as select k1,bitmap_union(to_bitmap(k2)), bitmap_union(to_bitmap(k3)) FROM ${tbName1} GROUP BY k1;"
-    max_try_secs = 60
+    def max_try_secs = 60
     Awaitility.await().atMost(max_try_secs, SECONDS).pollInterval(2, SECONDS).until{
         String res = getJobState(tbName1)
         if (res == "FINISHED" || res == "CANCELLED") {
diff --git a/regression-test/suites/rollup/test_materialized_view_hll.groovy b/regression-test/suites/rollup/test_materialized_view_hll.groovy
index 1d186cbbd5f4ae..59dd2c82fbfd19 100644
--- a/regression-test/suites/rollup/test_materialized_view_hll.groovy
+++ b/regression-test/suites/rollup/test_materialized_view_hll.groovy
@@ -39,7 +39,7 @@ suite("test_materialized_view_hll", "rollup") {
         """
 
     sql "CREATE materialized VIEW amt_count AS SELECT store_id, hll_union(hll_hash(sale_amt)) FROM ${tbName1} GROUP BY store_id;"
-    max_try_secs = 60
+    def max_try_secs = 60
     Awaitility.await().atMost(max_try_secs, SECONDS).pollInterval(2, SECONDS).until{
         String res = getJobState(tbName1)
         if (res == "FINISHED" || res == "CANCELLED") {
diff --git a/regression-test/suites/rollup/test_materialized_view_hll_with_light_sc.groovy b/regression-test/suites/rollup/test_materialized_view_hll_with_light_sc.groovy
index 2d14df30d6db90..2e72408ecb3c08 100644
--- a/regression-test/suites/rollup/test_materialized_view_hll_with_light_sc.groovy
+++ b/regression-test/suites/rollup/test_materialized_view_hll_with_light_sc.groovy
@@ -39,7 +39,7 @@ suite("test_materialized_view_hll_with_light_sc", "rollup") {
         """
 
     sql "CREATE materialized VIEW amt_count1 AS SELECT store_id, hll_union(hll_hash(sale_amt)) FROM ${tbName1} GROUP BY store_id;"
-    max_try_secs = 60
+    def max_try_secs = 60
     Awaitility.await().atMost(max_try_secs, SECONDS).pollInterval(2, SECONDS).until{
         String res = getJobState(tbName1)
         if (res == "FINISHED" || res == "CANCELLED") {
diff --git a/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy
index 2672db7c751f88..13063ad34dc9db 100644
--- a/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy
@@ -176,9 +176,9 @@ suite ("test_agg_keys_schema_change") {
         String[][] tablets = sql """ show tablets from ${tableName}; """
         for (String[] tablet in tablets) {
                 String tablet_id = tablet[0]
-                backend_id = tablet[2]
+                def backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
         }
 
@@ -188,8 +188,8 @@ suite ("test_agg_keys_schema_change") {
                 do {
                     Thread.sleep(100)
                     String tablet_id = tablet[0]
-                    backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def backend_id = tablet[2]
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy
index 2e68bd7608162e..ed6e31e5f6a88f 100644
--- a/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy
@@ -151,9 +151,9 @@ suite ("test_agg_mv_schema_change") {
         String[][] tablets = sql """ show tablets from ${tableName}; """
         for (String[] tablet in tablets) {
                 String tablet_id = tablet[0]
-                backend_id = tablet[2]
+                def backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
                 //assertEquals(code, 0)
         }
@@ -164,8 +164,8 @@ suite ("test_agg_mv_schema_change") {
                 do {
                     Thread.sleep(100)
                     String tablet_id = tablet[0]
-                    backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def backend_id = tablet[2]
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy
index 9302133295cf90..719082e79bb054 100644
--- a/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy
@@ -111,7 +111,7 @@ suite ("test_agg_rollup_schema_change") {
             ALTER TABLE ${tableName} DROP COLUMN cost
             """
 
-        max_try_time = 3000
+        def max_try_time = 3000
         while (max_try_time--){
             String result = getJobState(tableName)
             if (result == "FINISHED") {
@@ -159,9 +159,9 @@ suite ("test_agg_rollup_schema_change") {
         String[][] tablets = sql """ show tablets from ${tableName}; """
         for (String[] tablet in tablets) {
                 String tablet_id = tablet[0]
-                backend_id = tablet[2]
+                def backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
                 //assertEquals(code, 0)
         }
@@ -172,8 +172,8 @@ suite ("test_agg_rollup_schema_change") {
                 do {
                     Thread.sleep(100)
                     String tablet_id = tablet[0]
-                    backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def backend_id = tablet[2]
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_agg_schema_key_change_modify1.groovy b/regression-test/suites/schema_change_p0/test_agg_schema_key_change_modify1.groovy
index 78d203c4f738d6..f6d1fdb4222382 100644
--- a/regression-test/suites/schema_change_p0/test_agg_schema_key_change_modify1.groovy
+++ b/regression-test/suites/schema_change_p0/test_agg_schema_key_change_modify1.groovy
@@ -84,7 +84,7 @@ suite("test_agg_schema_key_change_modify1","p0") {
              "               (789012345, 'Grace', 2123483141, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-07-07 22:00:00');"
 
      //TODO Test the agg model by modify a key type from LARGEINT  to BOOLEAN
-     errorMessage = "errCode = 2, detailMessage = Can not change LARGEINT to BOOLEAN"
+     def errorMessage = "errCode = 2, detailMessage = Can not change LARGEINT to BOOLEAN"
      expectException({
           sql initTable
           sql initTableData
@@ -219,7 +219,7 @@ suite("test_agg_schema_key_change_modify1","p0") {
      }, insertSql, false, "${tbName1}")
 
      sql """ DROP TABLE IF EXISTS ${tbName2} """
-     initTable2 = " CREATE TABLE IF NOT EXISTS ${tbName2}\n" +
+     def initTable2 = " CREATE TABLE IF NOT EXISTS ${tbName2}\n" +
              "          (\n" +
              "              `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" +
              "              `username` VARCHAR(50) NOT NULL COMMENT \"用户昵称\",\n" +
@@ -237,7 +237,7 @@ suite("test_agg_schema_key_change_modify1","p0") {
              "          \"replication_allocation\" = \"tag.location.default: 1\"\n" +
              "          );"
 
-     initTableData2 = "insert into ${tbName2} values(123456789, 'Alice', 2147483641, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," +
+     def initTableData2 = "insert into ${tbName2} values(123456789, 'Alice', 2147483641, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," +
              "               (234567890, 'Bob', 214748364, 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-02-02 12:00:00')," +
              "               (345678901, 'Carol', 2147483441, 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou', '2022-03-03 14:00:00')," +
              "               (456789012, 'Dave', 2147483141, 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-04-04 16:00:00')," +
diff --git a/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy
index 477ea202e4c38e..211bdaefe3c766 100644
--- a/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy
@@ -139,7 +139,7 @@ suite ("test_agg_vals_schema_change") {
         String[][] tablets = sql """ show tablets from ${tableName}; """
         for (String[] tablet in tablets) {
                 String tablet_id = tablet[0]
-                backend_id = tablet[2]
+                def backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
                 def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
@@ -152,7 +152,7 @@ suite ("test_agg_vals_schema_change") {
                 do {
                     Thread.sleep(100)
                     String tablet_id = tablet[0]
-                    backend_id = tablet[2]
+                    def backend_id = tablet[2]
                     def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
diff --git a/regression-test/suites/schema_change_p0/test_alter_uniq_null.groovy b/regression-test/suites/schema_change_p0/test_alter_uniq_null.groovy
index fcedf4c2fa46de..6ef5039b00e4a0 100644
--- a/regression-test/suites/schema_change_p0/test_alter_uniq_null.groovy
+++ b/regression-test/suites/schema_change_p0/test_alter_uniq_null.groovy
@@ -52,7 +52,7 @@ suite("test_alter_uniq_null") {
 
     sql """alter table ${tableName} modify column `v2` INT NULL"""
     sleep(10)
-    max_try_num = 1000
+    def max_try_num = 1000
     while (max_try_num--) {
         String res = getJobState(tableName)
         if (res == "FINISHED" || res == "CANCELLED") {
diff --git a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy
index 19b1c96ce4b1f2..044eaf62258be7 100644
--- a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy
@@ -142,9 +142,9 @@ suite ("test_dup_keys_schema_change") {
         String[][] tablets = sql """ show tablets from ${tableName}; """
         for (String[] tablet in tablets) {
                 String tablet_id = tablet[0]
-                backend_id = tablet[2]
+                def backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
                 //assertEquals(code, 0)
         }
@@ -153,8 +153,8 @@ suite ("test_dup_keys_schema_change") {
         for (String[] tablet in tablets) {
                 Awaitility.await().untilAsserted(() -> {
                     String tablet_id = tablet[0]
-                    backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def backend_id = tablet[2]
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
index 2896470e2a9d83..7c3ea7703272bb 100644
--- a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
@@ -154,7 +154,7 @@ suite ("test_dup_mv_schema_change") {
                 String tablet_id = tablet[0]
                 backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
                 //assertEquals(code, 0)
         }
@@ -164,7 +164,7 @@ suite ("test_dup_mv_schema_change") {
                 Awaitility.await().untilAsserted(() -> {
                     String tablet_id = tablet[0]
                     backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy
index b97f8c78b6fcae..7f55b7fcd16e2c 100644
--- a/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy
@@ -125,7 +125,7 @@ suite ("test_dup_rollup_schema_change") {
         sql """
             ALTER TABLE ${tableName} DROP COLUMN sex
             """
-        max_try_time = 3000
+        def max_try_time = 3000
         while (max_try_time--){
             String result = getJobState(tableName)
             if (result == "FINISHED") {
@@ -172,7 +172,7 @@ suite ("test_dup_rollup_schema_change") {
                 String tablet_id = tablet[0]
                 backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
                 //assertEquals(code, 0)
         }
@@ -182,7 +182,7 @@ suite ("test_dup_rollup_schema_change") {
            Awaitility.await().untilAsserted(() -> {
                     String tablet_id = tablet[0]
                     backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_dup_schema_key_change_modify1.groovy b/regression-test/suites/schema_change_p0/test_dup_schema_key_change_modify1.groovy
index 74a5c1f86d8bbf..4683b735faa3a2 100644
--- a/regression-test/suites/schema_change_p0/test_dup_schema_key_change_modify1.groovy
+++ b/regression-test/suites/schema_change_p0/test_dup_schema_key_change_modify1.groovy
@@ -83,7 +83,7 @@ suite("test_dup_schema_key_change_modify1","p0") {
              "               (789012345, 'Grace', 2123483141, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-07-07 22:00:00');"
 
      //TODO Test the dup model by modify a key type from LARGEINT  to BOOLEAN
-     errorMessage = "errCode = 2, detailMessage = Can not change LARGEINT to BOOLEAN"
+     def errorMessage = "errCode = 2, detailMessage = Can not change LARGEINT to BOOLEAN"
      expectException({
           sql initTable
           sql initTableData
@@ -218,7 +218,7 @@ suite("test_dup_schema_key_change_modify1","p0") {
      }, insertSql, false, "${tbName1}")
 
      sql """ DROP TABLE IF EXISTS ${tbName2} """
-     initTable2 = " CREATE TABLE IF NOT EXISTS ${tbName2}\n" +
+     def initTable2 = " CREATE TABLE IF NOT EXISTS ${tbName2}\n" +
              "          (\n" +
              "              `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" +
              "              `username` VARCHAR(50) NOT NULL COMMENT \"用户昵称\",\n" +
@@ -236,7 +236,7 @@ suite("test_dup_schema_key_change_modify1","p0") {
              "          \"replication_allocation\" = \"tag.location.default: 1\"\n" +
              "          );"
 
-     initTableData2 = "insert into ${tbName2} values(123456789, 'Alice', 2147483641, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," +
+     def initTableData2 = "insert into ${tbName2} values(123456789, 'Alice', 2147483641, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," +
              "               (234567890, 'Bob', 214748364, 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-02-02 12:00:00')," +
              "               (345678901, 'Carol', 2147483441, 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou', '2022-03-03 14:00:00')," +
              "               (456789012, 'Dave', 2147483141, 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-04-04 16:00:00')," +
diff --git a/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy
index db5c4bbd8697ca..e1914bb6c8f249 100644
--- a/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy
@@ -132,7 +132,7 @@ suite ("test_dup_vals_schema_change") {
                 String tablet_id = tablet[0]
                 backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
                 //assertEquals(code, 0)
         }
@@ -142,7 +142,7 @@ suite ("test_dup_vals_schema_change") {
             Awaitility.await().untilAsserted(() -> {
                     String tablet_id = tablet[0]
                     backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_enable_light_schema_change.groovy b/regression-test/suites/schema_change_p0/test_enable_light_schema_change.groovy
index 88d2c8179e0322..96e76a4168d75f 100644
--- a/regression-test/suites/schema_change_p0/test_enable_light_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_enable_light_schema_change.groovy
@@ -76,7 +76,7 @@ suite("test_enable_light_schema_change", "p0") {
 
     sql """ alter table ${tableName1} order by (k1, k2, k4, k3) """
 
-    max_try_num = 60
+    def max_try_num = 60
     while (max_try_num--) {
         String res = getJobState(tableName1)
         if (res == "FINISHED" || res == "CANCELLED") {
diff --git a/regression-test/suites/schema_change_p0/test_schema_change_duplicate.groovy b/regression-test/suites/schema_change_p0/test_schema_change_duplicate.groovy
index 6c6660d10551c4..81e4271965ade3 100644
--- a/regression-test/suites/schema_change_p0/test_schema_change_duplicate.groovy
+++ b/regression-test/suites/schema_change_p0/test_schema_change_duplicate.groovy
@@ -148,7 +148,7 @@ suite("test_schema_change_duplicate", "p0") {
         time 600
     }
 
-    int val = 100000 + max_try_num
+    int val = 100000 + 500
     sql """ insert into ${tableName3} values (${val}, 2, 3, 4, 5, 6.6, 1.7, 8.8,
 'a', 'b', 'c', '2021-10-30', '2021-10-30 00:00:00', 9527) """
 
diff --git a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy
index 856f49acd2bb57..b8dcf97e791582 100644
--- a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy
@@ -127,7 +127,7 @@ suite ("test_uniq_keys_schema_change") {
                 String tablet_id = tablet[0]
                 backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
                 //assertEquals(code, 0)
         }
@@ -137,7 +137,7 @@ suite ("test_uniq_keys_schema_change") {
                 Awaitility.await().untilAsserted(() -> {
                     String tablet_id = tablet[0]
                     backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
index 54a790248b38f5..f2c961b5aa4c16 100644
--- a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
@@ -170,7 +170,7 @@ suite ("test_uniq_mv_schema_change") {
             String tablet_id = tablet[0]
             backend_id = tablet[2]
             logger.info("run compaction:" + tablet_id)
-            (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+            def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
             logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
             //assertEquals(code, 0)
     }
@@ -180,7 +180,7 @@ suite ("test_uniq_mv_schema_change") {
             Awaitility.await().untilAsserted(() -> {
                 String tablet_id = tablet[0]
                 backend_id = tablet[2]
-                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                 assertEquals(code, 0)
                 def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy
index 1b82e913e05c15..6fb74ceda4bb53 100644
--- a/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy
@@ -131,7 +131,7 @@ suite ("test_uniq_rollup_schema_change") {
           ALTER TABLE ${tableName} DROP COLUMN cost
           """
 
-    max_try_time = 300
+    def max_try_time = 300
     Awaitility.await().atMost(max_try_time, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> {
         String result = getJobState(tableName)
         if (result == "FINISHED") {
@@ -173,7 +173,7 @@ suite ("test_uniq_rollup_schema_change") {
             String tablet_id = tablet[0]
             backend_id = tablet[2]
             logger.info("run compaction:" + tablet_id)
-            (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+            def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
             logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
             //assertEquals(code, 0)
     }
@@ -183,7 +183,7 @@ suite ("test_uniq_rollup_schema_change") {
             Awaitility.await().untilAsserted(() -> {
                 String tablet_id = tablet[0]
                 backend_id = tablet[2]
-                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                 assertEquals(code, 0)
                 def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy
index 7dc530cecb8605..9ca8111d0ff737 100644
--- a/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy
@@ -65,7 +65,7 @@ suite ("test_uniq_vals_schema_change") {
                 (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20)
             """
 
-        qt_sc"""
+        qt_sc """
                         select count(*) from ${tableName}
                         """
 
@@ -101,7 +101,7 @@ suite ("test_uniq_vals_schema_change") {
         sql """
             ALTER TABLE ${tableName} DROP COLUMN last_visit_date
             """
-        qt_sc = sql """ select * from ${tableName} where user_id = 3 """
+        qt_sc """ select * from ${tableName} where user_id = 3 """
 
         sql """ INSERT INTO ${tableName} VALUES
                 (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20, 2)
@@ -135,7 +135,7 @@ suite ("test_uniq_vals_schema_change") {
                 String tablet_id = tablet[0]
                 backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
                 //assertEquals(code, 0)
         }
@@ -145,7 +145,7 @@ suite ("test_uniq_vals_schema_change") {
             Awaitility.await().untilAsserted(() -> {
                     String tablet_id = tablet[0]
                     backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy b/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy
index e4006e6e62b27f..2bb5e823e8bd17 100644
--- a/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy
@@ -113,7 +113,7 @@ suite ("test_varchar_schema_change") {
                 String tablet_id = tablet[0]
                 backend_id = tablet[2]
                 logger.info("run compaction:" + tablet_id)
-                (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
         }
 
@@ -122,7 +122,7 @@ suite ("test_varchar_schema_change") {
             Awaitility.await().untilAsserted(() -> {
                     String tablet_id = tablet[0]
                     backend_id = tablet[2]
-                    (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                    def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                     logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
                     assertEquals(code, 0)
                     def compactionStatus = parseJson(out.trim())
diff --git a/regression-test/suites/show_p0/test_show_data.groovy b/regression-test/suites/show_p0/test_show_data.groovy
index 10714a542d49f8..20adc299d55d83 100644
--- a/regression-test/suites/show_p0/test_show_data.groovy
+++ b/regression-test/suites/show_p0/test_show_data.groovy
@@ -22,7 +22,7 @@ suite("test_show_data") {
 
     def jdbcUrlWithoutDbStr = (context.config.jdbcUrl).split(context.config.defaultDb)
     logger.info("jdbcUrlWithoutDbStr:${jdbcUrlWithoutDbStr}");
-    def result2 = connect(context.config.jdbcUser, password = context.config.jdbcPassword, url = jdbcUrlWithoutDbStr[0]) {
+    def result2 = connect(context.config.jdbcUser, context.config.jdbcPassword, jdbcUrlWithoutDbStr[0]) {
         sql """show data;"""
     }
 
diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy
index 69360da6911ba5..3916dd116ad1da 100644
--- a/regression-test/suites/statistics/analyze_stats.groovy
+++ b/regression-test/suites/statistics/analyze_stats.groovy
@@ -357,7 +357,7 @@ suite("test_analyze") {
         ANALYZE TABLE analyze_partitioned_tbl_test WITH SYNC
     """
 
-    part_tbl_analyze_result = sql """
+    def part_tbl_analyze_result = sql """
         SHOW COLUMN CACHED STATS analyze_partitioned_tbl_test(col1)
     """
 
@@ -1006,7 +1006,7 @@ PARTITION `p599` VALUES IN (599)
     sql """ANALYZE TABLE test_600_partition_table_analyze WITH SYNC"""
 
     //  0:column_name | 1:index_name | 2:count | 3:ndv  | 4:num_null | 5:data_size | 6:avg_size_byte | 7:min  | 8:max  | 9:method | 10:type | 11:trigger | 12:query_times | 13:updated_time
-    id_col_stats = sql """
+    def id_col_stats = sql """
         SHOW COLUMN CACHED STATS test_600_partition_table_analyze(id);
     """
 
@@ -1230,7 +1230,7 @@ PARTITION `p599` VALUES IN (599)
 
     def check_column = { r, expected ->
         expected_result = convert_col_list_str_to_java_collection(expected)
-        actual_result = convert_col_list_str_to_java_collection(r[0][4])
+        def actual_result = convert_col_list_str_to_java_collection(r[0][4])
         System.out.println(expected_result)
         System.out.println(actual_result)
         return expected_result.containsAll(actual_result) && actual_result.containsAll(expected_result)
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete_sign_with_conflict.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete_sign_with_conflict.groovy
index 0f67d5cb7c60e1..9438541227ed92 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete_sign_with_conflict.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_delete_sign_with_conflict.groovy
@@ -69,9 +69,9 @@ suite("test_partial_update_delete_sign_with_conflict") {
         log.info("http_stream execute 2pc: ${command}")
 
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
-        json2pc = parseJson(out)
+        def code = process.waitFor()
+        def out = process.text
+        def json2pc = parseJson(out)
         log.info("http_stream 2pc result: ${out}".toString())
         assertEquals(code, 0)
         assertEquals("success", json2pc.status.toLowerCase())
diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_parallel.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_parallel.groovy
index c8523d235b4a67..e76b5d9f7109d0 100644
--- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_parallel.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_parallel.groovy
@@ -40,7 +40,7 @@ suite("test_primary_key_partial_update_parallel", "p0") {
         (4, "doris4", 4000, 423, 4),
         (3, "doris3", 3000, 323, 3);"""
 
-    t1 = Thread.startDaemon {
+    def t1 = Thread.startDaemon {
         streamLoad {
             table "${tableName}"
 
@@ -65,7 +65,7 @@ suite("test_primary_key_partial_update_parallel", "p0") {
         }
     }
 
-    t2 = Thread.startDaemon {
+    def t2 = Thread.startDaemon {
         streamLoad {
             table "${tableName}"
 
@@ -90,7 +90,7 @@ suite("test_primary_key_partial_update_parallel", "p0") {
         }
     }
 
-    t3 = Thread.startDaemon {
+    def t3 = Thread.startDaemon {
         streamLoad {
             table "${tableName}"
 
diff --git a/regression-test/suites/unique_with_mow_p0/flexible/upgrade/test.groovy b/regression-test/suites/unique_with_mow_p0/flexible/upgrade/test.groovy
index 91fa2267032ea1..5fabb4b49be201 100644
--- a/regression-test/suites/unique_with_mow_p0/flexible/upgrade/test.groovy
+++ b/regression-test/suites/unique_with_mow_p0/flexible/upgrade/test.groovy
@@ -21,7 +21,7 @@ suite('test_flexible_partial_update_upgrade', 'p0,restart_fe') {
         logger.info("current params: use_row_store: ${use_row_store}")
         def tableName = "test_f_upgrade_${use_row_store}"
         sql """alter table ${tableName} enable feature "UPDATE_FLEXIBLE_COLUMNS"; """
-        show_res = sql "show create table ${tableName}"
+        def show_res = sql "show create table ${tableName}"
         assertTrue(show_res.toString().contains('"enable_unique_key_skip_bitmap_column" = "true"'))
         qt_sql "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;"
 
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign_with_conflict.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign_with_conflict.groovy
index 7e2cd9cdfe308a..ec03ac336e7dae 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign_with_conflict.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign_with_conflict.groovy
@@ -65,9 +65,9 @@ suite("test_partial_update_delete_sign_with_conflict") {
         log.info("http_stream execute 2pc: ${command}")
 
         def process = command.execute()
-        code = process.waitFor()
-        out = process.text
-        json2pc = parseJson(out)
+        def code = process.waitFor()
+        def out = process.text
+        def json2pc = parseJson(out)
         log.info("http_stream 2pc result: ${out}".toString())
         assertEquals(code, 0)
         assertEquals("success", json2pc.status.toLowerCase())
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_parallel.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_parallel.groovy
index e2fa3a7903a0c0..611d4d1da432c9 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_parallel.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_parallel.groovy
@@ -38,7 +38,7 @@ suite("test_primary_key_partial_update_parallel", "p0") {
         (4, "doris4", 4000, 423, 4),
         (3, "doris3", 3000, 323, 3);"""
 
-    t1 = Thread.startDaemon {
+    def t1 = Thread.startDaemon {
         streamLoad {
             table "${tableName}"
 
@@ -52,7 +52,7 @@ suite("test_primary_key_partial_update_parallel", "p0") {
         }
     }
 
-    t2 = Thread.startDaemon {
+    def t2 = Thread.startDaemon {
         streamLoad {
             table "${tableName}"
 
@@ -66,7 +66,7 @@ suite("test_primary_key_partial_update_parallel", "p0") {
         }
     }
 
-    t3 = Thread.startDaemon {
+    def t3 = Thread.startDaemon {
         streamLoad {
             table "${tableName}"
 
diff --git a/regression-test/suites/update/test_update_configs.groovy b/regression-test/suites/update/test_update_configs.groovy
index cbce43293c1c9a..6032682103c673 100644
--- a/regression-test/suites/update/test_update_configs.groovy
+++ b/regression-test/suites/update/test_update_configs.groovy
@@ -49,7 +49,7 @@ suite("test_update_configs", "p0") {
     (code, out, err) = show_be_config(beIp, bePort)
     logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
     assertEquals(code, 0)
-    configList2 = parseJson(out.trim())
+    def configList2 = parseJson(out.trim())
     assert configList instanceof List
     for (Object ele in (List) configList2) {
         assert ele instanceof List<String>
diff --git a/regression-test/suites/variant_p0/concurrent_insert.groovy b/regression-test/suites/variant_p0/concurrent_insert.groovy
index 0bbbfce789a3f3..4daa7750045cc1 100644
--- a/regression-test/suites/variant_p0/concurrent_insert.groovy
+++ b/regression-test/suites/variant_p0/concurrent_insert.groovy
@@ -28,19 +28,19 @@ suite("regression_test_variant_concurrent_schema_update", ""){
         DISTRIBUTED BY HASH(k) BUCKETS 3
         properties("replication_num" = "1");
     """
-    t1 = Thread.startDaemon {
+    def t1 = Thread.startDaemon {
         for (int k = 1; k <= 60; k++) {
             int x = k % 10;
             sql """insert into ${table_name} values(${x}, '{"k${x}" : ${x}, "x${k}" : 123}', '{"k${x}" : ${x}, "x${k}" : 123}')"""
         } 
     }
-    t2 = Thread.startDaemon {
+    def t2 = Thread.startDaemon {
         for (int k = 61; k <= 120; k++) {
             int x = k % 10;
             sql """insert into ${table_name} values(${x}, '{"k${x}" : ${x}, "x${k}" : 123}', '{"k${x}" : ${x}, "x${k}" : 123}')"""
         }
     }
-    t3 = Thread.startDaemon {
+    def t3 = Thread.startDaemon {
         for (int k = 121; k <= 180; k++) {
             int x = k % 10;
             sql """insert into ${table_name} values(${x}, '{"k${x}" : ${x}, "x${k}" : 123}', '{"k${x}" : ${x}, "x${k}" : 123}')"""
diff --git a/regression-test/suites/variant_p0/delete_update.groovy b/regression-test/suites/variant_p0/delete_update.groovy
index 3d905d07e05c6d..7317b2c3145cea 100644
--- a/regression-test/suites/variant_p0/delete_update.groovy
+++ b/regression-test/suites/variant_p0/delete_update.groovy
@@ -75,19 +75,19 @@ suite("regression_test_variant_delete_and_update", "variant_type"){
     // delete & insert concurrently
     sql "set enable_unique_key_partial_update=true;"
     sql "sync"
-    t1 = Thread.startDaemon {
+    def t1 = Thread.startDaemon {
         for (int k = 1; k <= 60; k++) {
             int x = new Random().nextInt(61) % 10;
             sql """insert into ${table_name}(k,vs) values(${x}, '{"k${x}" : ${x}}'),(${x+1}, '{"k${x+1}" : ${x+1}}'),(${x+2}, '{"k${x+2}" : ${x+2}}'),(${x+3}, '{"k${x+3}" : ${x+3}}')"""
         } 
     }
-    t2 = Thread.startDaemon {
+    def t2 = Thread.startDaemon {
         for (int k = 1; k <= 60; k++) {
             int x = new Random().nextInt(61) % 10;
             sql """insert into ${table_name}(k,v) values(${x}, '{"k${x}" : ${x}}'),(${x+1}, '{"k${x+1}" : ${x+1}}'),(${x+2}, '{"k${x+2}" : ${x+2}}'),(${x+3}, '{"k${x+3}" : ${x+3}}')"""
         } 
     }
-    t3 = Thread.startDaemon {
+    def t3 = Thread.startDaemon {
         for (int k = 1; k <= 60; k++) {
             int x = new Random().nextInt(61) % 10;
             sql """insert into ${table_name}(k,v) values(${x}, '{"k${x}" : ${x}}'),(${x+1}, '{"k${x+1}" : ${x+1}}'),(${x+2}, '{"k${x+2}" : ${x+2}}'),(${x+3}, '{"k${x+3}" : ${x+3}}')"""
diff --git a/regression-test/suites/variant_p0/nested.groovy b/regression-test/suites/variant_p0/nested.groovy
index 25bd682d43aca4..7df361c5731644 100644
--- a/regression-test/suites/variant_p0/nested.groovy
+++ b/regression-test/suites/variant_p0/nested.groovy
@@ -73,7 +73,7 @@ suite("regression_test_variant_nested", "p0"){
         qt_sql """
             select * from var_nested order by k limit 101
         """ 
-        for (i = 101; i < 121; ++i) {
+        for (int i = 101; i < 121; ++i) {
             sql """insert into var_nested values (${i}, '{"nested${i}" : {"nested": [{"yyyxxxx" : "11111"},{"ax1111" : "1111"},{"axxxb": 100, "xxxy111": 111}, {"ddsss":1024, "aaa" : "11"}, {"xx" : 10}]}, "not nested" : 1024, "not nested2" : {"llll" : 123}}');"""
         }
         def trigger_full_compaction_on_tablets = { tablets ->
diff --git a/regression-test/suites/variant_p0/schema_change/schema_change.groovy b/regression-test/suites/variant_p0/schema_change/schema_change.groovy
index 42cef32c8e5641..877874b7e03040 100644
--- a/regression-test/suites/variant_p0/schema_change/schema_change.groovy
+++ b/regression-test/suites/variant_p0/schema_change/schema_change.groovy
@@ -32,7 +32,7 @@ suite("regression_test_variant_schema_change", "variant_type"){
     def useTime = 0
     def wait_for_latest_op_on_table_finish = { tableName, OpTimeout ->
         for(int t = delta_time; t <= OpTimeout; t += delta_time){
-            alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;"""
+            def alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;"""
             alter_res = alter_res.toString()
             if(alter_res.contains("FINISHED")) {
                 sleep(3000) // wait change table state to normal
diff --git a/regression-test/suites/workload_manager_p0/test_resource_tag.groovy b/regression-test/suites/workload_manager_p0/test_resource_tag.groovy
index d6557645eb8759..fa7ba680143248 100644
--- a/regression-test/suites/workload_manager_p0/test_resource_tag.groovy
+++ b/regression-test/suites/workload_manager_p0/test_resource_tag.groovy
@@ -29,7 +29,7 @@ suite("test_resource_tag") {
         }
 
         // test query
-        connect(user = 'test_rg', password = '', url = context.config.jdbcUrl) {
+        connect('test_rg', '', context.config.jdbcUrl) {
                 sql "drop table if exists test_skip_rg_bad_replica_tab;"
                 sql """
             CREATE TABLE test_skip_rg_bad_replica_tab
@@ -55,7 +55,7 @@ suite("test_resource_tag") {
         }
         sql "set property for test_rg 'allow_resource_tag_downgrade' = 'true';"
 
-        connect(user = 'test_rg', password = '', url = context.config.jdbcUrl) {
+        connect('test_rg', '', context.config.jdbcUrl) {
                 sql "select count(1) as t2 from test_skip_rg_bad_replica_tab;"
                 sql "drop table test_skip_rg_bad_replica_tab";
         }
@@ -76,8 +76,8 @@ suite("test_resource_tag") {
         def test_failed_command = "curl --location-trusted -u test_rg: -H column_separator:| -H Transfer-Encoding:chunked -H columns:k1,k2  -T ${context.dataPath}/skip_rg_test_table.csv http://${context.config.feHttpAddress}/api/${context.config.defaultDb}/skip_rg_test_table/_stream_load"
         log.info("stream load skip_rg_test_table failed test cmd: ${test_failed_command}")
         def process = test_failed_command.execute()
-        code1 = process.waitFor()
-        out1 = process.text
+        def code1 = process.waitFor()
+        def out1 = process.text
         log.info("stream load skip_rg_test_table failed test result, ${out1}".toString())
         assertTrue("${out1}".toString().contains("No backend load available") || "${out1}".toString().contains("No available backends"))
 
@@ -85,9 +85,9 @@ suite("test_resource_tag") {
 
         def test_succ_command = "curl --location-trusted -u test_rg: -H column_separator:| -H Transfer-Encoding:chunked -H columns:k1,k2  -T ${context.dataPath}/skip_rg_test_table.csv http://${context.config.feHttpAddress}/api/${context.config.defaultDb}/skip_rg_test_table/_stream_load"
         def process2 = test_succ_command.execute()
-        code2 = process2.waitFor()
-        out2 = process2.text
-        jsonRet = parseJson(out2)
+        def code2 = process2.waitFor()
+        def out2 = process2.text
+        def jsonRet = parseJson(out2)
         log.info("stream load skip_rg_test_table succ test result, ${out2}".toString())
         assertFalse("${out2}".toString().contains("No backend load available"))
         assertTrue(jsonRet['Status'] == 'Success')

From 2ca753feb84c6931aedb760c1d2b3e9be3ead310 Mon Sep 17 00:00:00 2001
From: LiBinfeng <libinfeng@selectdb.com>
Date: Fri, 13 Dec 2024 17:40:52 +0800
Subject: [PATCH 321/399] [fix](Nereids) fix cases unstable when using profile
 to check is nereids execution (#45384)

Problem Summary:
when commands running in doris using profile to check whether it's plan
is generated by nereids, it would randomly failed because profile is not
always generated on time.
solved by remove this check and check whether command was running by
nereids by reviewer
---
 .../doris/regression/suite/Suite.groovy       | 35 ++-----------------
 .../test_nereids_admin_check_tablet.groovy    |  4 +++
 .../ddl/show_trash/test_nereids_trash.groovy  |  4 +++
 ..._nereids_show_tablet_storage_format.groovy |  4 +++
 4 files changed, 15 insertions(+), 32 deletions(-)

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 59722d72d952fe..f5d811514b375d 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -713,41 +713,12 @@ class Suite implements GroovyInterceptable {
     }
 
     void checkNereidsExecute(String sqlString) {
-        String tag = UUID.randomUUID().toString();
-        log.info("start check" + tag)
-        String finalSqlString = "--" + tag + "\n" + sqlString
-        ProfileAction profileAction = new ProfileAction(context, tag)
-        profileAction.run {
-            log.info("start profile run" + tag)
-            sql (finalSqlString)
-        }
-        profileAction.check {
-            profileString, exception ->
-                log.info("start profile check" + tag)
-                log.info(profileString)
-                Assertions.assertTrue(profileString.contains("-  Is  Nereids:  Yes"))
-        }
-        profileAction.run()
+        sql (sqlString)
     }
 
     String checkNereidsExecuteWithResult(String sqlString) {
-        String tag = UUID.randomUUID().toString();
-        String result = null;
-        log.info("start check" + tag)
-        String finalSqlString = "--" + tag + "\n" + sqlString
-        ProfileAction profileAction = new ProfileAction(context, tag)
-        profileAction.run {
-            log.info("start profile run" + tag)
-            result = sql (finalSqlString)
-        }
-        profileAction.check {
-            profileString, exception ->
-                log.info("start profile check" + tag)
-                log.info(profileString)
-                Assertions.assertTrue(profileString.contains("-  Is  Nereids:  Yes"))
-        }
-        profileAction.run()
-        return result;
+        String result = sql (sqlString);
+        return result
     }
 
     void createMV(String sql) {
diff --git a/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy b/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy
index 8812e09ea9f6f3..f96d8329f02a08 100644
--- a/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy
+++ b/regression-test/suites/nereids_p0/admin/test_nereids_admin_check_tablet.groovy
@@ -17,6 +17,10 @@
 
 
 suite("test_nereids_admin_check_tablet") {
+    //cloud-mode
+    if (isCloudMode()) {
+        return
+    }
     def table = "test_nereids_admin_check_tablet"
     // create table and insert data
     sql """ drop table if exists ${table} force"""
diff --git a/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy b/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
index 26a57c7e82d06f..f9b486a97fd014 100644
--- a/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
+++ b/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
@@ -20,6 +20,10 @@ suite("show_trash_nereids") {
     checkNereidsExecute("""show trash;""")
     checkNereidsExecute("""show trash on "127.0.0.1:9050";""")
 
+    //cloud-mode
+    if (isCloudMode()) {
+        return
+    }
     checkNereidsExecute("""ADMIN CLEAN TRASH;""")
     checkNereidsExecute("""ADMIN CLEAN TRASH ON ("127.0.0.1:9050");""")
     checkNereidsExecute("""ADMIN CLEAN TRASH ON ("192.168.0.1:9050", "192.168.0.2:9050", "192.168.0.3:9050");""")
diff --git a/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy b/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy
index b4b78bd1c28729..85b087fc19b291 100644
--- a/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy
+++ b/regression-test/suites/nereids_p0/show/test_nereids_show_tablet_storage_format.groovy
@@ -17,6 +17,10 @@
 
 
 suite("test_nereids_show_tablet_storage_format") {
+    //cloud-mode
+    if (isCloudMode()) {
+        return
+    }
     checkNereidsExecute("""show tablet storage format;""")
     checkNereidsExecute("""show tablet storage format verbose;""")
     checkNereidsExecute("""admin show tablet storage format;""")

From 475571b15204280b04e499e7b97c337b372a4e46 Mon Sep 17 00:00:00 2001
From: yujun <yujun@selectdb.com>
Date: Fri, 13 Dec 2024 18:53:54 +0800
Subject: [PATCH 322/399] [feat](nereids) simplify comparison predicate rule
 add check data type limit (#44732)

simplify comparison, check data type's limit, like:

suppose a is tinyint, so its range should be [-128, 127], then we can
simplify:

a  = -129    => false
cast (a as small int) =  small int(-129) => false
a  <= -129  => false
a  > -129    =>  true
a <= -128   =>  a = -128

currently data type check only support tinyint, small int, int, big int,
decimalv3.

if data type is float like, compare them with literal may lost
precision, but maybe suport it later.

This PR need more test.
---
 .../rules/SimplifyComparisonPredicate.java    | 184 ++++++++----
 .../doris/nereids/util/ExpressionUtils.java   |  16 ++
 .../doris/nereids/util/TypeCoercionUtils.java |  44 +++
 .../SimplifyComparisonPredicateTest.java      | 263 +++++++++++++++++-
 .../pull_up_predicate_literal.out             | 110 +-------
 .../predicate_infer/infer_predicate.out       |  10 +-
 .../predicate_infer/infer_predicate.groovy    |   7 +-
 7 files changed, 462 insertions(+), 172 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java
index cb61795865239b..5f14d9625e0849 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java
@@ -17,18 +17,17 @@
 
 package org.apache.doris.nereids.rules.expression.rules;
 
+import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.rules.expression.AbstractExpressionRewriteRule;
 import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
 import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
 import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
-import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.Cast;
 import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.GreaterThan;
 import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
-import org.apache.doris.nereids.trees.expressions.IsNull;
 import org.apache.doris.nereids.trees.expressions.LessThan;
 import org.apache.doris.nereids.trees.expressions.LessThanEqual;
 import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
@@ -44,16 +43,16 @@
 import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
-import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NumericLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
-import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.types.DateTimeType;
 import org.apache.doris.nereids.types.DateTimeV2Type;
 import org.apache.doris.nereids.types.DateType;
 import org.apache.doris.nereids.types.DateV2Type;
 import org.apache.doris.nereids.types.DecimalV3Type;
 import org.apache.doris.nereids.types.coercion.DateLikeType;
+import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.TypeCoercionUtils;
 
 import com.google.common.base.Preconditions;
@@ -62,9 +61,10 @@
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.util.List;
+import java.util.Optional;
 
 /**
- * simplify comparison
+ * simplify comparison, not support large int.
  * such as: cast(c1 as DateV2) >= DateV2Literal --> c1 >= DateLiteral
  *          cast(c1 AS double) > 2.0 --> c1 >= 2 (c1 is integer like type)
  */
@@ -98,22 +98,25 @@ public static Expression simplify(ComparisonPredicate cp) {
         Expression left = cp.left();
         Expression right = cp.right();
 
-        // float like type: float, double
-        if (left.getDataType().isFloatLikeType() && right.getDataType().isFloatLikeType()) {
-            return processFloatLikeTypeCoercion(cp, left, right);
-        }
+        Expression result;
 
-        // decimalv3 type
-        if (left.getDataType() instanceof DecimalV3Type && right.getDataType() instanceof DecimalV3Type) {
-            return processDecimalV3TypeCoercion(cp, left, right);
+        // process type coercion
+        if (left.getDataType().isFloatLikeType() && right.getDataType().isFloatLikeType()) {
+            result = processFloatLikeTypeCoercion(cp, left, right);
+        } else if (left.getDataType() instanceof DecimalV3Type && right.getDataType() instanceof DecimalV3Type) {
+            result = processDecimalV3TypeCoercion(cp, left, right);
+        } else if (left.getDataType() instanceof DateLikeType && right.getDataType() instanceof DateLikeType) {
+            result = processDateLikeTypeCoercion(cp, left, right);
+        } else {
+            result = cp;
         }
 
-        // date like type
-        if (left.getDataType() instanceof DateLikeType && right.getDataType() instanceof DateLikeType) {
-            return processDateLikeTypeCoercion(cp, left, right);
+        if (result instanceof ComparisonPredicate && ((ComparisonPredicate) result).right() instanceof NumericLiteral) {
+            ComparisonPredicate cmp = (ComparisonPredicate) result;
+            result = processTypeRangeLimitComparison(cmp, cmp.left(), (NumericLiteral) cmp.right());
         }
 
-        return cp;
+        return result;
     }
 
     private static Expression processComparisonPredicateDateTimeV2Literal(
@@ -128,17 +131,13 @@ private static Expression processComparisonPredicateDateTimeV2Literal(
                 if (right.getMicroSecond() == originValue) {
                     return comparisonPredicate.withChildren(left, right);
                 } else {
-                    if (left.nullable()) {
-                        // TODO: the ideal way is to return an If expr like:
-                        // return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
-                        // BooleanLiteral.of(false));
-                        // but current fold constant rule can't handle such complex expr with null literal
-                        // before supporting complex conjuncts with null literal folding rules,
-                        // we use a trick way like this:
-                        return new And(new IsNull(left), new NullLiteral(BooleanType.INSTANCE));
-                    } else {
-                        return BooleanLiteral.of(false);
-                    }
+                    // TODO: the ideal way is to return an If expr like:
+                    // return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
+                    // BooleanLiteral.of(false));
+                    // but current fold constant rule can't handle such complex expr with null literal
+                    // before supporting complex conjuncts with null literal folding rules,
+                    // we use a trick way like this:
+                    return ExpressionUtils.falseOrNull(left);
                 }
             } else if (comparisonPredicate instanceof NullSafeEqual) {
                 long originValue = right.getMicroSecond();
@@ -239,18 +238,13 @@ private static Expression processDecimalV3TypeCoercion(ComparisonPredicate compa
                                     comparisonPredicate.withChildren(left, new DecimalV3Literal(
                                             literal.getValue().setScale(toScale, RoundingMode.UNNECESSARY))));
                         } catch (ArithmeticException e) {
-                            if (left.nullable()) {
-                                // TODO: the ideal way is to return an If expr like:
-                                // return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
-                                // BooleanLiteral.of(false));
-                                // but current fold constant rule can't handle such complex expr with null literal
-                                // before supporting complex conjuncts with null literal folding rules,
-                                // we use a trick way like this:
-                                return new And(new IsNull(left),
-                                        new NullLiteral(BooleanType.INSTANCE));
-                            } else {
-                                return BooleanLiteral.of(false);
-                            }
+                            // TODO: the ideal way is to return an If expr like:
+                            // return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
+                            // BooleanLiteral.of(false));
+                            // but current fold constant rule can't handle such complex expr with null literal
+                            // before supporting complex conjuncts with null literal folding rules,
+                            // we use a trick way like this:
+                            return ExpressionUtils.falseOrNull(left);
                         }
                     } else if (comparisonPredicate instanceof NullSafeEqual) {
                         try {
@@ -281,21 +275,18 @@ private static Expression processDecimalV3TypeCoercion(ComparisonPredicate compa
     private static Expression processIntegerDecimalLiteralComparison(
             ComparisonPredicate comparisonPredicate, Expression left, BigDecimal literal) {
         // we only process isIntegerLikeType, which are tinyint, smallint, int, bigint
-        if (literal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0) {
+        if (literal.compareTo(new BigDecimal(Long.MIN_VALUE)) >= 0
+                && literal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0) {
             literal = literal.stripTrailingZeros();
             if (literal.scale() > 0) {
                 if (comparisonPredicate instanceof EqualTo) {
-                    if (left.nullable()) {
-                        // TODO: the ideal way is to return an If expr like:
-                        // return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
-                        // BooleanLiteral.of(false));
-                        // but current fold constant rule can't handle such complex expr with null literal
-                        // before supporting complex conjuncts with null literal folding rules,
-                        // we use a trick way like this:
-                        return new And(new IsNull(left), new NullLiteral(BooleanType.INSTANCE));
-                    } else {
-                        return BooleanLiteral.of(false);
-                    }
+                    // TODO: the ideal way is to return an If expr like:
+                    // return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
+                    // BooleanLiteral.of(false));
+                    // but current fold constant rule can't handle such complex expr with null literal
+                    // before supporting complex conjuncts with null literal folding rules,
+                    // we use a trick way like this:
+                    return ExpressionUtils.falseOrNull(left);
                 } else if (comparisonPredicate instanceof NullSafeEqual) {
                     return BooleanLiteral.of(false);
                 } else if (comparisonPredicate instanceof GreaterThan
@@ -320,10 +311,95 @@ private static Expression processIntegerDecimalLiteralComparison(
         return comparisonPredicate;
     }
 
+    private static Expression processTypeRangeLimitComparison(ComparisonPredicate cp, Expression left,
+            NumericLiteral right) {
+        BigDecimal typeMinValue = null;
+        BigDecimal typeMaxValue = null;
+        // cmp float like have lost precision, for example float.max_value + 0.01 still eval to float.max_value
+        if (left.getDataType().isIntegerLikeType() || left.getDataType().isDecimalV3Type()) {
+            Optional<Pair<BigDecimal, BigDecimal>> minMaxOpt =
+                    TypeCoercionUtils.getDataTypeMinMaxValue(left.getDataType());
+            if (minMaxOpt.isPresent()) {
+                typeMinValue = minMaxOpt.get().first;
+                typeMaxValue = minMaxOpt.get().second;
+            }
+        }
+
+        // cast(child as dataType2) range should be:
+        //  [ max(childDataType.min_value, dataType2.min_value), min(childDataType.max_value, dataType2.max_value)]
+        if (left instanceof Cast) {
+            left = ((Cast) left).child();
+            if (left.getDataType().isIntegerLikeType() || left.getDataType().isDecimalV3Type()) {
+                Optional<Pair<BigDecimal, BigDecimal>> minMaxOpt =
+                        TypeCoercionUtils.getDataTypeMinMaxValue(left.getDataType());
+                if (minMaxOpt.isPresent()) {
+                    if (typeMinValue == null || typeMinValue.compareTo(minMaxOpt.get().first) < 0) {
+                        typeMinValue = minMaxOpt.get().first;
+                    }
+                    if (typeMaxValue == null || typeMaxValue.compareTo(minMaxOpt.get().second) > 0) {
+                        typeMaxValue = minMaxOpt.get().second;
+                    }
+                }
+            }
+        }
+
+        if (typeMinValue == null || typeMaxValue == null) {
+            return cp;
+        }
+        BigDecimal literal = new BigDecimal(right.getStringValue());
+        int cmpMin = literal.compareTo(typeMinValue);
+        int cmpMax = literal.compareTo(typeMaxValue);
+        if (cp instanceof EqualTo) {
+            if (cmpMin < 0 || cmpMax > 0) {
+                return ExpressionUtils.falseOrNull(left);
+            }
+        } else if (cp instanceof NullSafeEqual) {
+            if (cmpMin < 0 || cmpMax > 0) {
+                return BooleanLiteral.of(false);
+            }
+        } else if (cp instanceof GreaterThan) {
+            if (cmpMin < 0) {
+                return ExpressionUtils.trueOrNull(left);
+            }
+            if (cmpMax >= 0) {
+                return ExpressionUtils.falseOrNull(left);
+            }
+        } else if (cp instanceof GreaterThanEqual) {
+            if (cmpMin <= 0) {
+                return ExpressionUtils.trueOrNull(left);
+            }
+            if (cmpMax == 0) {
+                return new EqualTo(cp.left(), cp.right());
+            }
+            if (cmpMax > 0) {
+                return ExpressionUtils.falseOrNull(left);
+            }
+        } else if (cp instanceof LessThan) {
+            if (cmpMin <= 0) {
+                return ExpressionUtils.falseOrNull(left);
+            }
+            if (cmpMax > 0) {
+                return ExpressionUtils.trueOrNull(left);
+            }
+        } else if (cp instanceof LessThanEqual) {
+            if (cmpMin < 0) {
+                return ExpressionUtils.falseOrNull(left);
+            }
+            if (cmpMin == 0) {
+                return new EqualTo(cp.left(), cp.right());
+            }
+            if (cmpMax >= 0) {
+                return ExpressionUtils.trueOrNull(left);
+            }
+        }
+        return cp;
+    }
+
     private static IntegerLikeLiteral convertDecimalToIntegerLikeLiteral(BigDecimal decimal) {
-        Preconditions.checkArgument(
-                decimal.scale() <= 0 && decimal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0,
-                "decimal literal must have 0 scale and smaller than Long.MAX_VALUE");
+        Preconditions.checkArgument(decimal.scale() <= 0
+                && decimal.compareTo(new BigDecimal(Long.MIN_VALUE)) >= 0
+                && decimal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0,
+                "decimal literal must have 0 scale and in range [Long.MIN_VALUE, Long.MAX_VALUE]");
         long val = decimal.longValue();
         if (val >= Byte.MIN_VALUE && val <= Byte.MAX_VALUE) {
             return new TinyIntLiteral((byte) val);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 22b681a6246d92..25637d1b816656 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -260,6 +260,22 @@ public static Expression or(Collection<Expression> expressions) {
         }
     }
 
+    public static Expression falseOrNull(Expression expression) {
+        if (expression.nullable()) {
+            return new And(new IsNull(expression), new NullLiteral(BooleanType.INSTANCE));
+        } else {
+            return BooleanLiteral.FALSE;
+        }
+    }
+
+    public static Expression trueOrNull(Expression expression) {
+        if (expression.nullable()) {
+            return new Or(new Not(new IsNull(expression)), new NullLiteral(BooleanType.INSTANCE));
+        } else {
+            return BooleanLiteral.TRUE;
+        }
+    }
+
     /**
      * Use AND/OR to combine expressions together.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
index 603a891d2d2a49..1da4353d20da33 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
@@ -21,6 +21,7 @@
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.annotation.Developing;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Add;
@@ -116,6 +117,7 @@
 import com.google.common.collect.ImmutableList.Builder;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -1773,6 +1775,48 @@ private static Expression processDecimalV3BinaryArithmetic(BinaryArithmetic bina
                 castIfNotSameType(right, dt2));
     }
 
+    /**
+     * get min and max value of a data type
+     *
+     * @param dataType specific data type
+     * @return min and max values pair
+     */
+    public static Optional<Pair<BigDecimal, BigDecimal>> getDataTypeMinMaxValue(DataType dataType) {
+        if (dataType.isTinyIntType()) {
+            return Optional.of(Pair.of(new BigDecimal(Byte.MIN_VALUE), new BigDecimal(Byte.MAX_VALUE)));
+        } else if (dataType.isSmallIntType()) {
+            return Optional.of(Pair.of(new BigDecimal(Short.MIN_VALUE), new BigDecimal(Short.MAX_VALUE)));
+        } else if (dataType.isIntegerType()) {
+            return Optional.of(Pair.of(new BigDecimal(Integer.MIN_VALUE), new BigDecimal(Integer.MAX_VALUE)));
+        } else if (dataType.isBigIntType()) {
+            return Optional.of(Pair.of(new BigDecimal(Long.MIN_VALUE), new BigDecimal(Long.MAX_VALUE)));
+        } else if (dataType.isLargeIntType()) {
+            return Optional.of(Pair.of(new BigDecimal(LargeIntType.MIN_VALUE), new BigDecimal(LargeIntType.MAX_VALUE)));
+        } else if (dataType.isFloatType()) {
+            return Optional.of(Pair.of(BigDecimal.valueOf(-Float.MAX_VALUE), new BigDecimal(Float.MAX_VALUE)));
+        } else if (dataType.isDoubleType()) {
+            return Optional.of(Pair.of(BigDecimal.valueOf(-Double.MAX_VALUE), new BigDecimal(Double.MAX_VALUE)));
+        } else if (dataType.isDecimalV3Type()) {
+            DecimalV3Type type = (DecimalV3Type) dataType;
+            int precision = type.getPrecision();
+            int scale = type.getScale();
+            if (scale >= 0) {
+                StringBuilder sb = new StringBuilder();
+                sb.append(StringUtils.repeat('9', precision - scale));
+                if (sb.length() == 0) {
+                    sb.append('0');
+                }
+                if (scale > 0) {
+                    sb.append('.');
+                    sb.append(StringUtils.repeat('9', scale));
+                }
+                return Optional.of(Pair.of(new BigDecimal("-" + sb.toString()), new BigDecimal(sb.toString())));
+            }
+        }
+
+        return Optional.empty();
+    }
+
     private static boolean supportCompare(DataType dataType) {
         if (dataType.isArrayType()) {
             return true;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java
index 84ebd7c7250198..9202c1e202469d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java
@@ -17,10 +17,12 @@
 
 package org.apache.doris.nereids.rules.expression.rules;
 
+import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.rules.expression.ExpressionRewriteTestHelper;
 import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor;
 import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.GreaterThan;
@@ -38,17 +40,29 @@
 import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
 import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal;
 import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
+import org.apache.doris.nereids.types.BigIntType;
 import org.apache.doris.nereids.types.BooleanType;
+import org.apache.doris.nereids.types.DataType;
 import org.apache.doris.nereids.types.DateTimeV2Type;
 import org.apache.doris.nereids.types.DecimalV3Type;
 import org.apache.doris.nereids.types.DoubleType;
+import org.apache.doris.nereids.types.IntegerType;
+import org.apache.doris.nereids.types.SmallIntType;
+import org.apache.doris.nereids.types.TinyIntType;
+import org.apache.doris.nereids.util.ExpressionUtils;
 
 import com.google.common.collect.ImmutableList;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.List;
 
 class SimplifyComparisonPredicateTest extends ExpressionRewriteTestHelper {
     @Test
@@ -233,63 +247,292 @@ void testDecimalV3Literal() {
         Assertions.assertEquals(BooleanLiteral.FALSE, rewrittenExpression);
 
         // > right literal should round floor
-        leftChild = new DecimalV3Literal(new BigDecimal("1.24"));
+        leftChild = new DecimalV3Literal(new BigDecimal("10.24"));
         left = new Cast(leftChild, DecimalV3Type.createDecimalV3Type(5, 3));
         right = new DecimalV3Literal(new BigDecimal("12.345"));
         expression = new GreaterThan(left, right);
         rewrittenExpression = executor.rewrite(expression, context);
-        Assertions.assertInstanceOf(Cast.class, rewrittenExpression.child(0));
         Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(4, 2),
                 rewrittenExpression.child(0).getDataType());
         Assertions.assertInstanceOf(DecimalV3Literal.class, rewrittenExpression.child(1));
         Assertions.assertEquals(new BigDecimal("12.34"), ((DecimalV3Literal) rewrittenExpression.child(1)).getValue());
 
         // <= right literal should round floor
-        leftChild = new DecimalV3Literal(new BigDecimal("1.24"));
+        leftChild = new DecimalV3Literal(new BigDecimal("10.24"));
         left = new Cast(leftChild, DecimalV3Type.createDecimalV3Type(5, 3));
         right = new DecimalV3Literal(new BigDecimal("12.345"));
         expression = new LessThanEqual(left, right);
         rewrittenExpression = executor.rewrite(expression, context);
-        Assertions.assertInstanceOf(Cast.class, rewrittenExpression.child(0));
         Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(4, 2),
                 rewrittenExpression.child(0).getDataType());
         Assertions.assertInstanceOf(DecimalV3Literal.class, rewrittenExpression.child(1));
         Assertions.assertEquals(new BigDecimal("12.34"), ((DecimalV3Literal) rewrittenExpression.child(1)).getValue());
 
         // >= right literal should round ceiling
-        leftChild = new DecimalV3Literal(new BigDecimal("1.24"));
+        leftChild = new DecimalV3Literal(new BigDecimal("10.24"));
         left = new Cast(leftChild, DecimalV3Type.createDecimalV3Type(5, 3));
         right = new DecimalV3Literal(new BigDecimal("12.345"));
         expression = new GreaterThanEqual(left, right);
         rewrittenExpression = executor.rewrite(expression, context);
-        Assertions.assertInstanceOf(Cast.class, rewrittenExpression.child(0));
         Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(4, 2),
                 rewrittenExpression.child(0).getDataType());
         Assertions.assertInstanceOf(DecimalV3Literal.class, rewrittenExpression.child(1));
         Assertions.assertEquals(new BigDecimal("12.35"), ((DecimalV3Literal) rewrittenExpression.child(1)).getValue());
 
         // < right literal should round ceiling
-        leftChild = new DecimalV3Literal(new BigDecimal("1.24"));
+        leftChild = new DecimalV3Literal(new BigDecimal("10.24"));
         left = new Cast(leftChild, DecimalV3Type.createDecimalV3Type(5, 3));
         right = new DecimalV3Literal(new BigDecimal("12.345"));
         expression = new LessThan(left, right);
         rewrittenExpression = executor.rewrite(expression, context);
-        Assertions.assertInstanceOf(Cast.class, rewrittenExpression.child(0));
         Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(4, 2),
                 rewrittenExpression.child(0).getDataType());
         Assertions.assertInstanceOf(DecimalV3Literal.class, rewrittenExpression.child(1));
         Assertions.assertEquals(new BigDecimal("12.35"), ((DecimalV3Literal) rewrittenExpression.child(1)).getValue());
 
         // left's child range smaller than right literal
-        leftChild = new DecimalV3Literal(new BigDecimal("1234.12"));
+        leftChild = new DecimalV3Literal(new BigDecimal("12340.12"));
         left = new Cast(leftChild, DecimalV3Type.createDecimalV3Type(10, 5));
         right = new DecimalV3Literal(new BigDecimal("12345.12000"));
         expression = new EqualTo(left, right);
         rewrittenExpression = executor.rewrite(expression, context);
-        Assertions.assertInstanceOf(Cast.class, rewrittenExpression.child(0));
         Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(7, 2),
                 rewrittenExpression.child(0).getDataType());
         Assertions.assertInstanceOf(DecimalV3Literal.class, rewrittenExpression.child(1));
         Assertions.assertEquals(new BigDecimal("12345.12"), ((DecimalV3Literal) rewrittenExpression.child(1)).getValue());
     }
+
+    private enum RangeLimitResult {
+        TRUE, // eval to true
+        FALSE, // eval to false
+        EQUALS, // eval to equals
+        NO_CHANGE_CP // no change cmp type
+    }
+
+    @Test
+    void testTypeRangeLimit() {
+        executor = new ExpressionRuleExecutor(ImmutableList.of(
+            bottomUp(SimplifyComparisonPredicate.INSTANCE)
+        ));
+
+        checkTypeRangeLimit(TinyIntType.INSTANCE,
+                ImmutableList.of(
+                        Pair.of(new SmallIntLiteral((short) -129), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-129")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-128.1")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-1000.1")), null),
+                        Pair.of(new DoubleLiteral(-129.0), new SmallIntLiteral((short) -129)),
+                        Pair.of(new DoubleLiteral(-128.1), new DecimalV3Literal(new BigDecimal("-128.1")))),
+                ImmutableList.of(
+                        Pair.of(new TinyIntLiteral((byte) -128), null),
+                        Pair.of(new SmallIntLiteral((short) -128), null),
+                        Pair.of(new IntegerLiteral(-128), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-128")), new TinyIntLiteral((byte) -128)),
+                        Pair.of(new DoubleLiteral(-128.0), new TinyIntLiteral((byte) -128))),
+                ImmutableList.of(
+                        Pair.of(new TinyIntLiteral((byte) -127), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-127")), new TinyIntLiteral((byte) -127)),
+                        Pair.of(new DoubleLiteral(-127.0), new TinyIntLiteral((byte) -127)),
+                        Pair.of(new TinyIntLiteral((byte) 126), null),
+                        Pair.of(new DoubleLiteral(126.0), new TinyIntLiteral((byte) 126))),
+                ImmutableList.of(
+                        Pair.of(new TinyIntLiteral((byte) 127), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("127")), new TinyIntLiteral((byte) 127)),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("127.00")), new TinyIntLiteral((byte) 127)),
+                        Pair.of(new DoubleLiteral(127.0), new TinyIntLiteral((byte) 127))),
+                ImmutableList.of(
+                        Pair.of(new SmallIntLiteral((short) 128), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("128.02")), null),
+                        Pair.of(new DoubleLiteral(128.0), new SmallIntLiteral((short) 128)),
+                        Pair.of(new DoubleLiteral(127.1), new DecimalV3Literal(new BigDecimal("127.1")))));
+
+        checkTypeRangeLimit(SmallIntType.INSTANCE,
+                ImmutableList.of(
+                        Pair.of(new IntegerLiteral(-32769), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-32769")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-32768.1")), null),
+                        Pair.of(new DoubleLiteral(-32769.0), new IntegerLiteral(-32769)),
+                        Pair.of(new DoubleLiteral(-32769.1), new DecimalV3Literal(new BigDecimal("-32769.1")))),
+                ImmutableList.of(
+                        Pair.of(new SmallIntLiteral((short) -32768), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-32768")), new SmallIntLiteral((short) -32768)),
+                        Pair.of(new DoubleLiteral(-32768.0), new SmallIntLiteral((short) -32768))),
+                ImmutableList.of(
+                        Pair.of(new SmallIntLiteral((short) -32767), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-32767")), new SmallIntLiteral((short) -32767)),
+                        Pair.of(new DoubleLiteral(-32767.0), new SmallIntLiteral((short) -32767)),
+                        Pair.of(new SmallIntLiteral((short) 32766), null),
+                        Pair.of(new DoubleLiteral(32766.0), new SmallIntLiteral((short) 32766))),
+                ImmutableList.of(
+                        Pair.of(new SmallIntLiteral((short) 32767), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("32767")), new SmallIntLiteral((short) 32767)),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("32767.00")), new SmallIntLiteral((short) 32767)),
+                        Pair.of(new DoubleLiteral(32767.0), new SmallIntLiteral((short) 32767))),
+                ImmutableList.of(
+                        Pair.of(new IntegerLiteral(32768), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("32768.02")), null),
+                        Pair.of(new DoubleLiteral(32768.0), new IntegerLiteral(32768)),
+                        Pair.of(new DoubleLiteral(32768.1), new DecimalV3Literal(new BigDecimal("32768.1")))));
+
+        checkTypeRangeLimit(IntegerType.INSTANCE,
+                ImmutableList.of(
+                        Pair.of(new BigIntLiteral(-2147483649L), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-2147483649")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-2147483649.1")), null),
+                        Pair.of(new DoubleLiteral(-2147483649.0), new BigIntLiteral(-2147483649L)),
+                        Pair.of(new DoubleLiteral(-2147483649.1), new DecimalV3Literal(new BigDecimal("-2147483649.1")))),
+                ImmutableList.of(
+                        Pair.of(new IntegerLiteral(-2147483648), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-2147483648.0")), new IntegerLiteral(-2147483648)),
+                        Pair.of(new DoubleLiteral(-2147483648.0), new IntegerLiteral(-2147483648))),
+                ImmutableList.of(
+                        Pair.of(new TinyIntLiteral((byte) 0), null),
+                        Pair.of(new SmallIntLiteral((short) 0), null),
+                        Pair.of(new IntegerLiteral(-2147483647), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-2147483647")), new IntegerLiteral(-2147483647)),
+                        Pair.of(new DoubleLiteral(-2147483647.0), new IntegerLiteral(-2147483647)),
+                        Pair.of(new IntegerLiteral(2147483646), null),
+                        Pair.of(new DoubleLiteral(2147483646.0), new IntegerLiteral(2147483646))),
+                ImmutableList.of(
+                        Pair.of(new IntegerLiteral(2147483647), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("2147483647")), new IntegerLiteral(2147483647)),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("2147483647.00")), new IntegerLiteral(2147483647)),
+                        Pair.of(new DoubleLiteral(2147483647.0), new IntegerLiteral(2147483647))),
+                ImmutableList.of(
+                        Pair.of(new BigIntLiteral(2147483648L), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("2147483648.02")), null),
+                        Pair.of(new DoubleLiteral(2147483648.0), new BigIntLiteral(2147483648L)),
+                        Pair.of(new DoubleLiteral(2147483647.1), new DecimalV3Literal(new BigDecimal("2147483647.1")))));
+
+        checkTypeRangeLimit(BigIntType.INSTANCE,
+                ImmutableList.of(
+                        Pair.of(new LargeIntLiteral(new BigInteger("-9223372036854775809")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-9223372036854775809")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-9223372036854775808.1")), null),
+                        Pair.of(new DoubleLiteral(-9223372036854775809.0), new LargeIntLiteral(new BigInteger("-9223372036854775809"))),
+                        Pair.of(new DoubleLiteral(-9223372036854775808.1), new DecimalV3Literal(new BigDecimal("-9223372036854775808.1")))),
+                ImmutableList.of(
+                        Pair.of(new BigIntLiteral(-9223372036854775808L), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-9223372036854775808")), new BigIntLiteral(-9223372036854775808L))),
+                ImmutableList.of(
+                        Pair.of(new BigIntLiteral(-9223372036854775807L), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-9223372036854775807")), new BigIntLiteral(-9223372036854775807L)),
+                        Pair.of(new DoubleLiteral(-9223372036854775000.0), new BigIntLiteral(-9223372036854775000L)),
+                        Pair.of(new BigIntLiteral(9223372036854775806L), null),
+                        Pair.of(new DoubleLiteral(9223372036854775000.0), new BigIntLiteral(9223372036854775000L))),
+                ImmutableList.of(
+                        Pair.of(new BigIntLiteral(9223372036854775807L), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("9223372036854775807")), new BigIntLiteral(9223372036854775807L)),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("9223372036854775807.00")), new BigIntLiteral(9223372036854775807L))),
+                ImmutableList.of(
+                        Pair.of(new LargeIntLiteral(new BigInteger("9223372036854775808")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("9223372036854775807.02")), null),
+                        Pair.of(new DoubleLiteral(9223372036854775808.0), new LargeIntLiteral(new BigInteger("9223372036854775808"))),
+                        Pair.of(new DoubleLiteral(9223372036854775807.1), new DecimalV3Literal(new BigDecimal("9223372036854775807.1")))));
+
+        checkTypeRangeLimit(DecimalV3Type.createDecimalV3Type(5, 2),
+                ImmutableList.of(
+                        Pair.of(new IntegerLiteral(-1000), null),
+                        Pair.of(new DoubleLiteral(-1000.1), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-999.999")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-1000.00")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-1000.0123")), null)),
+                ImmutableList.of(
+                        Pair.of(new DecimalV3Literal(new BigDecimal("-999.99")), null)),
+                ImmutableList.of(
+                        Pair.of(new DecimalV3Literal(new BigDecimal("100.4")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("100")), null)),
+                ImmutableList.of(
+                        Pair.of(new DecimalV3Literal(new BigDecimal("999.99")), null)),
+                ImmutableList.of(
+                        Pair.of(new IntegerLiteral(1000), null),
+                        Pair.of(new DoubleLiteral(1000.1), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("1000")), null),
+                        Pair.of(new DecimalV3Literal(new BigDecimal("999.999")), null)));
+    }
+
+    // each expr list item is: pair<origin right literal,  rewritten right literal>
+    // if rewritten right literal = null, then rewritten right literal = origin right literal
+    void checkTypeRangeLimit(DataType dataType, List<Pair<Expression, Expression>> lessThanMinExpr,
+            List<Pair<Expression, Expression>> minExpr, List<Pair<Expression, Expression>> betweenMinMaxExpr,
+            List<Pair<Expression, Expression>> maxExpr, List<Pair<Expression, Expression>> greaterThanMaxExpr) {
+        // due to ComparisonPredicate constructor require not null left and right child,
+        // use a dummyExpr as ComparisonPredicate's child
+        Expression dummyExpr = new SmallIntLiteral((short) 100);
+        // cp -> list of cp with lessThanMinExpr, minExpr, betweenMinMaxExpr, maxExpr, greaterThanMaxExpr
+        List<Pair<ComparisonPredicate, List<RangeLimitResult>>> cmpResults = ImmutableList.of(
+                Pair.of(new EqualTo(dummyExpr, dummyExpr), ImmutableList.of(
+                        RangeLimitResult.FALSE, RangeLimitResult.NO_CHANGE_CP, RangeLimitResult.NO_CHANGE_CP,
+                        RangeLimitResult.NO_CHANGE_CP, RangeLimitResult.FALSE)),
+                Pair.of(new NullSafeEqual(dummyExpr, dummyExpr), ImmutableList.of(
+                        RangeLimitResult.FALSE, RangeLimitResult.NO_CHANGE_CP, RangeLimitResult.NO_CHANGE_CP,
+                        RangeLimitResult.NO_CHANGE_CP, RangeLimitResult.FALSE)),
+                Pair.of(new GreaterThan(dummyExpr, dummyExpr), ImmutableList.of(
+                        RangeLimitResult.TRUE, RangeLimitResult.NO_CHANGE_CP, RangeLimitResult.NO_CHANGE_CP,
+                        RangeLimitResult.FALSE, RangeLimitResult.FALSE)),
+                Pair.of(new GreaterThanEqual(dummyExpr, dummyExpr), ImmutableList.of(
+                        RangeLimitResult.TRUE, RangeLimitResult.TRUE, RangeLimitResult.NO_CHANGE_CP,
+                        RangeLimitResult.EQUALS, RangeLimitResult.FALSE)),
+                Pair.of(new LessThan(dummyExpr, dummyExpr), ImmutableList.of(
+                        RangeLimitResult.FALSE, RangeLimitResult.FALSE, RangeLimitResult.NO_CHANGE_CP,
+                        RangeLimitResult.NO_CHANGE_CP, RangeLimitResult.TRUE)),
+                Pair.of(new LessThanEqual(dummyExpr, dummyExpr), ImmutableList.of(
+                        RangeLimitResult.FALSE, RangeLimitResult.EQUALS, RangeLimitResult.NO_CHANGE_CP,
+                        RangeLimitResult.TRUE, RangeLimitResult.TRUE))
+        );
+
+        for (Pair<ComparisonPredicate, List<RangeLimitResult>> cmpResult : cmpResults) {
+            ComparisonPredicate cp = cmpResult.first;
+            List<RangeLimitResult> result = cmpResult.second;
+            checkTypeRangeLimitWithComparison(dataType, cp, lessThanMinExpr, result.get(0));
+            checkTypeRangeLimitWithComparison(dataType, cp, minExpr, result.get(1));
+            checkTypeRangeLimitWithComparison(dataType, cp, betweenMinMaxExpr, result.get(2));
+            checkTypeRangeLimitWithComparison(dataType, cp, maxExpr, result.get(3));
+            checkTypeRangeLimitWithComparison(dataType, cp, greaterThanMaxExpr, result.get(4));
+        }
+    }
+
+    void checkTypeRangeLimitWithComparison(DataType dataType, ComparisonPredicate cp,
+            List<Pair<Expression, Expression>> exprs, RangeLimitResult result) {
+        Expression slot = new SlotReference("slot", dataType, true);
+        for (Pair<Expression, Expression> pair : exprs) {
+            Expression right = pair.first;
+            Expression rewriteRight = pair.second;
+            if (rewriteRight == null) {
+                rewriteRight = right;
+            }
+            Expression left = slot;
+            if (!left.getDataType().equals(right.getDataType())) {
+                left = new Cast(slot, right.getDataType());
+            }
+            Expression originExpr = cp.withChildren(left, right);
+            Expression rewrittenExpr = executor.rewrite(originExpr, context);
+            Expression expectExpr = null;
+            // System.out.println("slot type: " + slot.getDataType() + ", literal type: " + right.getDataType());
+            // System.out.println("origin expr: " + originExpr);
+            // System.out.println("rewrite expr: " + rewrittenExpr);
+            switch (result) {
+                case TRUE:
+                    expectExpr = cp instanceof NullSafeEqual ? BooleanLiteral.TRUE
+                            : ExpressionUtils.trueOrNull(slot);
+                    break;
+                case FALSE:
+                    expectExpr = cp instanceof NullSafeEqual ? BooleanLiteral.FALSE
+                        : ExpressionUtils.falseOrNull(slot);
+                    break;
+                case EQUALS:
+                    Expression expectLeft = slot.getDataType().equals(rewriteRight.getDataType()) ? slot : left;
+                    expectExpr = new EqualTo(expectLeft, rewriteRight);
+                    break;
+                case NO_CHANGE_CP:
+                    Assertions.assertInstanceOf(cp.getClass(), rewrittenExpr);
+                    break;
+                default:
+                    Assertions.assertTrue(false);
+            }
+            if (expectExpr != null) {
+                Assertions.assertEquals(expectExpr, rewrittenExpr);
+            }
+        }
+    }
 }
diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
index 524559cabeb34d..920289be66c360 100644
--- a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
+++ b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
@@ -262,15 +262,7 @@ PhysicalResultSink
 
 -- !const_value_and_join_column_type16 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_tinyint as SMALLINT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_tinyint as SMALLINT) = 32767))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type17 --
 PhysicalResultSink
@@ -414,27 +406,11 @@ PhysicalResultSink
 
 -- !const_value_and_join_column_type32 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_tinyint as INT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_tinyint as INT) = 32768))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type33 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_smallint as INT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_smallint as INT) = 32768))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type34 --
 PhysicalResultSink
@@ -566,39 +542,15 @@ PhysicalResultSink
 
 -- !const_value_and_join_column_type48 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_tinyint as BIGINT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_tinyint as BIGINT) = 214748364799))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type49 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_smallint as BIGINT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_smallint as BIGINT) = 214748364799))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type50 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_int as BIGINT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_int as BIGINT) = 214748364799))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type51 --
 PhysicalResultSink
@@ -718,51 +670,19 @@ PhysicalResultSink
 
 -- !const_value_and_join_column_type64 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_tinyint as LARGEINT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_tinyint as LARGEINT) = 922337203685477580722))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type65 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_smallint as LARGEINT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_smallint as LARGEINT) = 922337203685477580722))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type66 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_int as LARGEINT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_int as LARGEINT) = 922337203685477580722))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type67 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_bigint as LARGEINT) = t.c1)) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_bigint as LARGEINT) = 922337203685477580722))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type68 --
 PhysicalResultSink
@@ -814,15 +734,7 @@ PhysicalResultSink
 
 -- !const_value_and_join_column_type72 --
 PhysicalResultSink
---PhysicalProject
-----hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_decimal as DOUBLE) = expr_cast(c1 as DOUBLE))) otherCondition=()
-------PhysicalLimit[GLOBAL]
---------PhysicalLimit[LOCAL]
-----------PhysicalProject
-------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
-------PhysicalProject
---------filter((cast(d_decimal as DOUBLE) = 9.223372036854776E20))
-----------PhysicalOlapScan[test_types]
+--PhysicalEmptyRelation
 
 -- !const_value_and_join_column_type73 --
 PhysicalResultSink
diff --git a/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out b/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out
index 288c30bb28c1cf..fb99fe6169c1d9 100644
--- a/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out
+++ b/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out
@@ -374,23 +374,23 @@ PhysicalResultSink
 -- !infer9 --
 PhysicalResultSink
 --hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=()
-----filter((cast(id as BIGINT) = 2147483648))
+----filter((t1.id = 12345))
 ------PhysicalOlapScan[t1]
-----filter((cast(id as BIGINT) = 2147483648))
+----filter((t2.id = 12345))
 ------PhysicalOlapScan[t2]
 
 -- !infer10 --
 PhysicalResultSink
 --hashJoin[INNER_JOIN] hashCondition=((expr_cast(id as SMALLINT) = expr_cast(id as SMALLINT))) otherCondition=()
-----filter((cast(id as BIGINT) = 2147483648))
+----filter((t1.id = 12345))
 ------PhysicalOlapScan[t1]
 ----PhysicalOlapScan[t2]
 
 -- !infer11 --
 PhysicalResultSink
 --hashJoin[INNER_JOIN] hashCondition=((expr_cast(id as LARGEINT) = expr_cast(id as LARGEINT))) otherCondition=()
-----filter((cast(id as BIGINT) = 2147483648))
+----filter((t1.id = 12345))
 ------PhysicalOlapScan[t1]
-----filter((cast(id as BIGINT) = 2147483648))
+----filter((t2.id = 12345))
 ------PhysicalOlapScan[t2]
 
diff --git a/regression-test/suites/nereids_rules_p0/predicate_infer/infer_predicate.groovy b/regression-test/suites/nereids_rules_p0/predicate_infer/infer_predicate.groovy
index af985ecd7ee10e..6237d5d66e8ef3 100644
--- a/regression-test/suites/nereids_rules_p0/predicate_infer/infer_predicate.groovy
+++ b/regression-test/suites/nereids_rules_p0/predicate_infer/infer_predicate.groovy
@@ -294,18 +294,17 @@ suite("infer_predicate") {
         explain shape plan select * from t1 join t2 on t1.id != t2.id where t1.id = 1;
     """
 
-    // 测试 infer predicate 是否能推出正确类型， 2147483648 超过了 Int32 的最大值, 但是不超过 Int64 的最大值，用这个值测试类型是否能推导正确
     qt_infer9 """
-        explain shape plan select * from (select * from t1 where t1.id = 2147483648) t1 join t2 on t1.id = t2.id;
+        explain shape plan select * from (select * from t1 where t1.id = 12345) t1 join t2 on t1.id = t2.id;
     """
 
     // 测试 cast = cast
     qt_infer10 """
-        explain shape plan select * from (select * from t1 where t1.id = 2147483648) t1 join t2 on cast(t1.id as smallint) = cast(t2.id as smallint);
+        explain shape plan select * from (select * from t1 where t1.id = 12345) t1 join t2 on cast(t1.id as smallint) = cast(t2.id as smallint);
     """
 
       // 测试 cast = cast
     qt_infer11 """
-        explain shape plan select * from (select * from t1 where t1.id = 2147483648) t1 join t2 on cast(t1.id as largeint) = cast(t2.id as largeint);
+        explain shape plan select * from (select * from t1 where t1.id = 12345) t1 join t2 on cast(t1.id as largeint) = cast(t2.id as largeint);
     """
 }

From 559a2f99bc85f3e089c04d630493b4315acfcdc6 Mon Sep 17 00:00:00 2001
From: zzzxl <yangsiyu@selectdb.com>
Date: Fri, 13 Dec 2024 20:37:11 +0800
Subject: [PATCH 323/399] [opt](inverted index) Add
 NumInvertedIndexRemoteIOTotal statistics in profile (#44863)

Add NumInvertedIndexRemoteIOTotal to count the number of remote IO
operations in the inverted index
Related PR: https://github.com/apache/doris/pull/43542
---
 be/src/io/cache/block_file_cache_profile.h          |  5 +++++
 be/src/io/cache/cached_remote_file_reader.cpp       |  8 ++++++--
 be/src/io/cache/cached_remote_file_reader.h         |  3 ++-
 be/src/io/io_common.h                               |  2 ++
 .../segment_v2/inverted_index_compound_reader.cpp   | 11 +++++++++--
 .../segment_v2/inverted_index_file_reader.cpp       | 13 ++++++++++---
 .../rowset/segment_v2/inverted_index_file_reader.h  |  5 +++--
 .../rowset/segment_v2/inverted_index_fs_directory.h |  1 +
 .../rowset/segment_v2/inverted_index_reader.cpp     | 10 +++++++---
 .../fault_injection_p0/test_index_io_context.groovy |  1 -
 10 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h
index 19d7f4139f7f15..54118d5094cd52 100644
--- a/be/src/io/cache/block_file_cache_profile.h
+++ b/be/src/io/cache/block_file_cache_profile.h
@@ -75,6 +75,7 @@ struct FileCacheProfile {
 struct FileCacheProfileReporter {
     RuntimeProfile::Counter* num_local_io_total = nullptr;
     RuntimeProfile::Counter* num_remote_io_total = nullptr;
+    RuntimeProfile::Counter* num_inverted_index_remote_io_total = nullptr;
     RuntimeProfile::Counter* local_io_timer = nullptr;
     RuntimeProfile::Counter* bytes_scanned_from_cache = nullptr;
     RuntimeProfile::Counter* bytes_scanned_from_remote = nullptr;
@@ -90,6 +91,8 @@ struct FileCacheProfileReporter {
                                                           cache_profile, 1);
         num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumRemoteIOTotal", TUnit::UNIT,
                                                            cache_profile, 1);
+        num_inverted_index_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(
+                profile, "NumInvertedIndexRemoteIOTotal", TUnit::UNIT, cache_profile, 1);
         local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LocalIOUseTimer", cache_profile, 1);
         remote_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "RemoteIOUseTimer", cache_profile, 1);
         write_cache_io_timer =
@@ -107,6 +110,8 @@ struct FileCacheProfileReporter {
     void update(const FileCacheStatistics* statistics) const {
         COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total);
         COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total);
+        COUNTER_UPDATE(num_inverted_index_remote_io_total,
+                       statistics->num_inverted_index_remote_io_total);
         COUNTER_UPDATE(local_io_timer, statistics->local_io_timer);
         COUNTER_UPDATE(remote_io_timer, statistics->remote_io_timer);
         COUNTER_UPDATE(write_cache_io_timer, statistics->write_cache_io_timer);
diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp
index c9a273c5d368a6..f16e0019b6dcc5 100644
--- a/be/src/io/cache/cached_remote_file_reader.cpp
+++ b/be/src/io/cache/cached_remote_file_reader.cpp
@@ -126,7 +126,7 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t*
     ReadStatistics stats;
     auto defer_func = [&](int*) {
         if (io_ctx->file_cache_stats) {
-            _update_state(stats, io_ctx->file_cache_stats);
+            _update_state(stats, io_ctx->file_cache_stats, io_ctx->is_inverted_index);
             io::FileCacheProfile::instance().update(io_ctx->file_cache_stats);
         }
     };
@@ -312,7 +312,8 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t*
 }
 
 void CachedRemoteFileReader::_update_state(const ReadStatistics& read_stats,
-                                           FileCacheStatistics* statis) const {
+                                           FileCacheStatistics* statis,
+                                           bool is_inverted_index) const {
     if (statis == nullptr) {
         return;
     }
@@ -320,6 +321,9 @@ void CachedRemoteFileReader::_update_state(const ReadStatistics& read_stats,
         statis->num_local_io_total++;
         statis->bytes_read_from_local += read_stats.bytes_read;
     } else {
+        if (is_inverted_index) {
+            statis->num_inverted_index_remote_io_total++;
+        }
         statis->num_remote_io_total++;
         statis->bytes_read_from_remote += read_stats.bytes_read;
     }
diff --git a/be/src/io/cache/cached_remote_file_reader.h b/be/src/io/cache/cached_remote_file_reader.h
index b3efb83c0803c8..685414cfa3aba6 100644
--- a/be/src/io/cache/cached_remote_file_reader.h
+++ b/be/src/io/cache/cached_remote_file_reader.h
@@ -76,7 +76,8 @@ class CachedRemoteFileReader final : public FileReader {
         int64_t local_read_timer = 0;
         int64_t local_write_timer = 0;
     };
-    void _update_state(const ReadStatistics& stats, FileCacheStatistics* state) const;
+    void _update_state(const ReadStatistics& stats, FileCacheStatistics* state,
+                       bool is_inverted_index) const;
 };
 
 } // namespace doris::io
diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h
index 80a594473dc376..4acc0538b7ef4f 100644
--- a/be/src/io/io_common.h
+++ b/be/src/io/io_common.h
@@ -38,6 +38,7 @@ namespace io {
 struct FileCacheStatistics {
     int64_t num_local_io_total = 0;
     int64_t num_remote_io_total = 0;
+    int64_t num_inverted_index_remote_io_total = 0;
     int64_t local_io_timer = 0;
     int64_t bytes_read_from_local = 0;
     int64_t bytes_read_from_remote = 0;
@@ -60,6 +61,7 @@ struct IOContext {
     int64_t expiration_time = 0;
     const TUniqueId* query_id = nullptr;             // Ref
     FileCacheStatistics* file_cache_stats = nullptr; // Ref
+    bool is_inverted_index = false;
 };
 
 } // namespace io
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
index 60006ea84550a2..f1b2b0eaedd4fd 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp
@@ -96,12 +96,19 @@ void CSIndexInput::readInternal(uint8_t* b, const int32_t len) {
     if (start + len > _length) {
         _CLTHROWA(CL_ERR_IO, "read past EOF");
     }
-    base->setIoContext(_io_ctx);
+
+    if (_io_ctx) {
+        base->setIoContext(_io_ctx);
+    }
+
     base->setIndexFile(_is_index_file);
     base->seek(fileOffset + start);
     bool read_from_buffer = true;
     base->readBytes(b, len, read_from_buffer);
-    base->setIoContext(nullptr);
+
+    if (_io_ctx) {
+        base->setIoContext(nullptr);
+    }
 }
 
 CSIndexInput::~CSIndexInput() = default;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
index 8d480829a0cd37..813a78f2a3fa86 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
@@ -27,21 +27,27 @@
 
 namespace doris::segment_v2 {
 
-Status InvertedIndexFileReader::init(int32_t read_buffer_size) {
+Status InvertedIndexFileReader::init(int32_t read_buffer_size, const io::IOContext* io_ctx) {
     if (!_inited) {
         _read_buffer_size = read_buffer_size;
         if (_storage_format >= InvertedIndexStorageFormatPB::V2) {
-            auto st = _init_from(read_buffer_size);
+            auto st = _init_from(read_buffer_size, io_ctx);
             if (!st.ok()) {
                 return st;
             }
         }
         _inited = true;
+    } else {
+        if (_storage_format == InvertedIndexStorageFormatPB::V2) {
+            if (_stream) {
+                _stream->setIoContext(io_ctx);
+            }
+        }
     }
     return Status::OK();
 }
 
-Status InvertedIndexFileReader::_init_from(int32_t read_buffer_size) {
+Status InvertedIndexFileReader::_init_from(int32_t read_buffer_size, const io::IOContext* io_ctx) {
     auto index_file_full_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix);
 
     std::unique_lock<std::shared_mutex> lock(_mutex); // Lock for writing
@@ -76,6 +82,7 @@ Status InvertedIndexFileReader::_init_from(int32_t read_buffer_size) {
                     err.what());
         }
         _stream = std::unique_ptr<CL_NS(store)::IndexInput>(index_input);
+        _stream->setIoContext(io_ctx);
 
         // 3. read file
         int32_t version = _stream->readInt(); // Read version number
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
index 443d40cfaf0d4f..ed6ee85e7d7bf1 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
@@ -58,7 +58,8 @@ class InvertedIndexFileReader {
               _storage_format(storage_format),
               _idx_file_info(idx_file_info) {}
 
-    Status init(int32_t read_buffer_size = config::inverted_index_read_buffer_size);
+    Status init(int32_t read_buffer_size = config::inverted_index_read_buffer_size,
+                const io::IOContext* io_ctx = nullptr);
     Result<std::unique_ptr<DorisCompoundReader>> open(const TabletIndex* index_meta) const;
     void debug_file_entries();
     std::string get_index_file_cache_key(const TabletIndex* index_meta) const;
@@ -70,7 +71,7 @@ class InvertedIndexFileReader {
     int64_t get_inverted_file_size() const { return _stream == nullptr ? 0 : _stream->length(); }
 
 private:
-    Status _init_from(int32_t read_buffer_size);
+    Status _init_from(int32_t read_buffer_size, const io::IOContext* io_ctx);
     Result<std::unique_ptr<DorisCompoundReader>> _open(int64_t index_id,
                                                        const std::string& index_suffix) const;
 
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
index dde436054cd35b..41d9fb48356299 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h
@@ -180,6 +180,7 @@ class DorisFSDirectory::FSIndexInput : public lucene::store::BufferedIndexInput
             : BufferedIndexInput(buffer_size) {
         this->_pos = 0;
         this->_handle = std::move(handle);
+        _io_ctx.is_inverted_index = true;
     }
 
 protected:
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 889fee1fc87ef9..b40f9121125207 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -121,7 +121,8 @@ Status InvertedIndexReader::read_null_bitmap(const io::IOContext* io_ctx,
 
         if (!dir) {
             // TODO: ugly code here, try to refact.
-            auto st = _inverted_index_file_reader->init(config::inverted_index_read_buffer_size);
+            auto st = _inverted_index_file_reader->init(config::inverted_index_read_buffer_size,
+                                                        io_ctx);
             if (!st.ok()) {
                 LOG(WARNING) << st;
                 return st;
@@ -137,7 +138,6 @@ Status InvertedIndexReader::read_null_bitmap(const io::IOContext* io_ctx,
                 InvertedIndexDescriptor::get_temporary_null_bitmap_file_name();
         if (dir->fileExists(null_bitmap_file_name)) {
             null_bitmap_in = dir->openInput(null_bitmap_file_name);
-            null_bitmap_in->setIoContext(io_ctx);
             size_t null_bitmap_size = null_bitmap_in->length();
             faststring buf;
             buf.resize(null_bitmap_size);
@@ -180,7 +180,8 @@ Status InvertedIndexReader::handle_searcher_cache(
         SCOPED_RAW_TIMER(&stats->inverted_index_searcher_open_timer);
         IndexSearcherPtr searcher;
 
-        auto st = _inverted_index_file_reader->init(config::inverted_index_read_buffer_size);
+        auto st =
+                _inverted_index_file_reader->init(config::inverted_index_read_buffer_size, io_ctx);
         if (!st.ok()) {
             LOG(WARNING) << st;
             return st;
@@ -211,6 +212,9 @@ Status InvertedIndexReader::create_index_searcher(lucene::store::Directory* dir,
     auto searcher_result = DORIS_TRY(index_searcher_builder->get_index_searcher(dir));
     *searcher = searcher_result;
 
+    // When the meta information has been read, the ioContext needs to be reset to prevent it from being used by other queries.
+    static_cast<DorisCompoundReader*>(dir)->getDorisIndexInput()->setIoContext(nullptr);
+
     // NOTE: before mem_tracker hook becomes active, we caculate reader memory size by hand.
     mem_tracker->consume(index_searcher_builder->get_reader_size());
     return Status::OK();
diff --git a/regression-test/suites/fault_injection_p0/test_index_io_context.groovy b/regression-test/suites/fault_injection_p0/test_index_io_context.groovy
index 9e9a2674897445..f5759e7cfc7b77 100644
--- a/regression-test/suites/fault_injection_p0/test_index_io_context.groovy
+++ b/regression-test/suites/fault_injection_p0/test_index_io_context.groovy
@@ -80,7 +80,6 @@ suite("test_index_io_context", "nonConcurrent") {
 
         try {
             GetDebugPoint().enableDebugPointForAllBEs("DorisFSDirectory::FSIndexInput::readInternal")
-
             qt_sql """ select count() from ${tableName1} where request match_any 'ticket_quest_bg2.jpg'; """
             qt_sql """ select count() from ${tableName1} where request match_any 'ticket_quest_bg2.jpg'; """
             qt_sql """ select count() from ${tableName1} where request match_any 'ticket_quest_bg2.jpg'; """

From 12dcb17eedbca64e854a067d20fd04c0218eb152 Mon Sep 17 00:00:00 2001
From: Tiewei Fang <fangtiewei@selectdb.com>
Date: Sun, 15 Dec 2024 10:48:42 +0800
Subject: [PATCH 324/399] [fix](Export) Cancel the export job when the ring
 buffer have no enough slot (#45244)

Problem Summary:

If there happen exceptions in `addMemoryTask`, we must update the state
of the export job to `CANCELLED` because we have added this export job
in `ExportMgr`.
---
 .../main/java/org/apache/doris/load/ExportMgr.java   | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index 94ae436ee6d87d..ed6ee29bb9c969 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -122,8 +122,16 @@ public void addExportJobAndRegisterTask(ExportJob job) throws Exception {
                     job.getBrokerDesc());
         }
         // ATTN: Must add task after edit log, otherwise the job may finish before adding job.
-        for (int i = 0; i < job.getCopiedTaskExecutors().size(); i++) {
-            Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(job.getCopiedTaskExecutors().get(i));
+        try {
+            for (int i = 0; i < job.getCopiedTaskExecutors().size(); i++) {
+                Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(job.getCopiedTaskExecutors().get(i));
+            }
+        } catch (Exception e) {
+            // If there happens exceptions in `addMemoryTask`
+            // we must update the state of export job to `CANCELLED`
+            // because we have added this export in `ExportMgr`
+            job.updateExportJobState(ExportJobState.CANCELLED, 0L, null,
+                    ExportFailMsg.CancelType.RUN_FAIL, e.getMessage());
         }
         LOG.info("add export job. {}", job);
     }

From 39f47311a0fca08b35f2ce640c1b63adff9a7957 Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Sun, 15 Dec 2024 11:04:11 +0800
Subject: [PATCH 325/399] [Fix](hudi) Fix some errors caused by resolving code
 conflicts (#45355)

### What problem does this PR solve?
Problem Summary:
Fix some errors caused by resolving code conflicts
---
 .../org/apache/doris/datasource/hudi/source/HudiScanNode.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
index e1dfaa40aefce1..486fdea74a00bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
@@ -125,8 +125,8 @@ public class HudiScanNode extends HiveScanNode {
      */
     public HudiScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv,
             Optional<TableScanParams> scanParams, Optional<IncrementalRelation> incrementalRelation,
-            SessionVariable sv) {
-        super(id, desc, "HUDI_SCAN_NODE", StatisticalType.HUDI_SCAN_NODE, needCheckColumnPriv, sv);
+            SessionVariable sessionVariable) {
+        super(id, desc, "HUDI_SCAN_NODE", StatisticalType.HUDI_SCAN_NODE, needCheckColumnPriv, sessionVariable);
         isCowTable = hmsTable.isHoodieCowTable();
         if (LOG.isDebugEnabled()) {
             if (isCowTable) {

From 06efb40f33b8e1b48ca50cca500b1afd866ad69e Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Sun, 15 Dec 2024 11:24:41 +0800
Subject: [PATCH 326/399] [fix](s3) improve error msg (#45360)

---
 be/src/io/fs/err_utils.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/be/src/io/fs/err_utils.cpp b/be/src/io/fs/err_utils.cpp
index 6552d454824796..e9bed7f5887dc3 100644
--- a/be/src/io/fs/err_utils.cpp
+++ b/be/src/io/fs/err_utils.cpp
@@ -122,13 +122,13 @@ Status s3fs_error(const Aws::S3::S3Error& err, std::string_view msg) {
     using namespace Aws::Http;
     switch (err.GetResponseCode()) {
     case HttpResponseCode::NOT_FOUND:
-        return Status::Error<NOT_FOUND, false>("{}: {} {} type={}, request_id={}", msg,
-                                               err.GetExceptionName(), err.GetMessage(),
+        return Status::Error<NOT_FOUND, false>("{}: {} {} code=NOT_FOUND, type={}, request_id={}",
+                                               msg, err.GetExceptionName(), err.GetMessage(),
                                                err.GetErrorType(), err.GetRequestId());
     case HttpResponseCode::FORBIDDEN:
-        return Status::Error<PERMISSION_DENIED, false>("{}: {} {} type={}, request_id={}", msg,
-                                                       err.GetExceptionName(), err.GetMessage(),
-                                                       err.GetErrorType(), err.GetRequestId());
+        return Status::Error<PERMISSION_DENIED, false>(
+                "{}: {} {} code=FORBIDDEN, type={}, request_id={}", msg, err.GetExceptionName(),
+                err.GetMessage(), err.GetErrorType(), err.GetRequestId());
     default:
         return Status::Error<ErrorCode::INTERNAL_ERROR, false>(
                 "{}: {} {} code={} type={}, request_id={}", msg, err.GetExceptionName(),

From 1523b6a9beb7a4686629d7873db441ce42c37783 Mon Sep 17 00:00:00 2001
From: zhiqiang <hezhiqiang@selectdb.com>
Date: Mon, 16 Dec 2024 01:26:34 +0800
Subject: [PATCH 327/399] [opt] Optimization for short circuit of CompoundPred
 (#45422)

### What problem does this PR solve?

Related PR: https://github.com/apache/doris/pull/45414

Problem Summary:

https://github.com/apache/doris/pull/45414 optimized short circuit of
CompoundPred. But for some cases, it will introduce perf recession since
AND/OR process of CompoundPred is not as efficient as Function AND/OR.

Previous
```sql
mysqlslap -hd3 -uroot -P9130  --create-schema=test_db -c 10 -i 100 -q "SELECT count(k) FROM sbtest1 WHERE k BETWEEN 10809931 AND 16922183 OR k BETWEEN 17788920 AND 27291942 OR k BETWEEN 23293962 AND 24940261 OR k BETWEEN 28108000 AND 28870202 OR k BETWEEN 23014347 AND 26008115 OR k BETWEEN 17817024 AND 29749077 OR k BETWEEN 8776291 AND 19869309 OR k BETWEEN 12846851 AND 15917660 OR k BETWEEN 21899521 AND 25614482 OR k BETWEEN 4850578 AND 8454295;"
Benchmark
    Average number of seconds to run all queries: 0.315 seconds
    Minimum number of seconds to run all queries: 0.307 seconds
    Maximum number of seconds to run all queries: 0.328 seconds
    Number of clients running queries: 10
    Average number of queries per client: 1
```
Now
```sql
mysqlslap -hd3 -uroot -P9130  --create-schema=test_db -c 10 -i 100 -q "SELECT count(k) FROM sbtest1 WHERE k BETWEEN 10809931 AND 16922183 OR k BETWEEN 17788920 AND 27291942 OR k BETWEEN 23293962 AND 24940261 OR k BETWEEN 28108000 AND 28870202 OR k BETWEEN 23014347 AND 26008115 OR k BETWEEN 17817024 AND 29749077 OR k BETWEEN 8776291 AND 19869309 OR k BETWEEN 12846851 AND 15917660 OR k BETWEEN 21899521 AND 25614482 OR k BETWEEN 4850578 AND 8454295;"
Benchmark
	Average number of seconds to run all queries: 0.222 seconds
	Minimum number of seconds to run all queries: 0.215 seconds
	Maximum number of seconds to run all queries: 0.263 seconds
	Number of clients running queries: 10
	Average number of queries per client: 1
```
BTW, the performance recession only occurs when compound predicate can
be short circuited in most cases.
Data distribution
```sql
mysql> desc sbtest1;
+-------+--------------+------+-------+---------+-------+
| Field | Type         | Null | Key   | Default | Extra |
+-------+--------------+------+-------+---------+-------+
| id    | int          | No   | true  | NULL    |       |
| k     | int          | Yes  | false | NULL    | NONE  |
| c     | varchar(360) | Yes  | false | NULL    | NONE  |
| pad   | varchar(180) | Yes  | false | NULL    | NONE  |
+-------+--------------+------+-------+---------+-------+
4 rows in set (0.00 sec)

mysql> select count(*) from sbtest1;
+----------+
| count(*) |
+----------+
|  5069205 |
+----------+
1 row in set (0.02 sec)

mysql> SELECT count(k) FROM sbtest1 WHERE k BETWEEN 10809931 AND 16922183;
+----------+
| count(k) |
+----------+
|  4726682 |
+----------+
1 row in set (0.02 sec)

mysql> SELECT count(k) FROM sbtest1 WHERE k BETWEEN 10809931 AND 16922183 OR k BETWEEN 17788920 AND 27291942 OR k BETWEEN 23293962 AND 24940261 OR k BETWEEN 28108000 AND 28870202 OR k BETWEEN 23014347 AND 26008115 OR k BETWEEN 17817024 AND 29749077 OR k BETWEEN 8776291 AND 19869309 OR k BETWEEN 12846851 AND 15917660 OR k BETWEEN 21899521 AND 25614482 OR k BETWEEN 4850578 AND 8454295;
+----------+
| count(k) |
+----------+
|  5066731 |
+----------+
1 row in set (0.06 sec)
```

TODO: Why code is not vectorized.
---
 be/src/vec/exprs/vcompound_pred.h | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/be/src/vec/exprs/vcompound_pred.h b/be/src/vec/exprs/vcompound_pred.h
index 8c65e6c8adbfef..88f3e474b58da3 100644
--- a/be/src/vec/exprs/vcompound_pred.h
+++ b/be/src/vec/exprs/vcompound_pred.h
@@ -234,24 +234,27 @@ class VCompoundPred : public VectorizedFnCall {
         auto vector_vector_null = [&]<bool is_and_op>() {
             auto col_res = ColumnUInt8::create(size);
             auto col_nulls = ColumnUInt8::create(size);
+
             auto* __restrict res_datas = assert_cast<ColumnUInt8*>(col_res)->get_data().data();
             auto* __restrict res_nulls = assert_cast<ColumnUInt8*>(col_nulls)->get_data().data();
             ColumnPtr temp_null_map = nullptr;
             // maybe both children are nullable / or one of children is nullable
-            lhs_null_map = create_null_map_column(temp_null_map, lhs_null_map);
-            rhs_null_map = create_null_map_column(temp_null_map, rhs_null_map);
+            auto* __restrict lhs_null_map_tmp = create_null_map_column(temp_null_map, lhs_null_map);
+            auto* __restrict rhs_null_map_tmp = create_null_map_column(temp_null_map, rhs_null_map);
+            auto* __restrict lhs_data_column_tmp = lhs_data_column;
+            auto* __restrict rhs_data_column_tmp = rhs_data_column;
 
             if constexpr (is_and_op) {
                 for (size_t i = 0; i < size; ++i) {
-                    res_nulls[i] = apply_and_null(lhs_data_column[i], lhs_null_map[i],
-                                                  rhs_data_column[i], rhs_null_map[i]);
-                    res_datas[i] = lhs_data_column[i] & rhs_data_column[i];
+                    res_nulls[i] = apply_and_null(lhs_data_column_tmp[i], lhs_null_map_tmp[i],
+                                                  rhs_data_column_tmp[i], rhs_null_map_tmp[i]);
+                    res_datas[i] = lhs_data_column_tmp[i] & rhs_data_column_tmp[i];
                 }
             } else {
                 for (size_t i = 0; i < size; ++i) {
-                    res_nulls[i] = apply_or_null(lhs_data_column[i], lhs_null_map[i],
-                                                 rhs_data_column[i], rhs_null_map[i]);
-                    res_datas[i] = lhs_data_column[i] | rhs_data_column[i];
+                    res_nulls[i] = apply_or_null(lhs_data_column_tmp[i], lhs_null_map_tmp[i],
+                                                 rhs_data_column_tmp[i], rhs_null_map_tmp[i]);
+                    res_datas[i] = lhs_data_column_tmp[i] | rhs_data_column_tmp[i];
                 }
             }
             auto result_column = ColumnNullable::create(std::move(col_res), std::move(col_nulls));

From b74b5c69aacbcdcc8715c8ba0f09aee0f7cc13b7 Mon Sep 17 00:00:00 2001
From: Sun Chenyang <sunchenyang@selectdb.com>
Date: Mon, 16 Dec 2024 10:42:28 +0800
Subject: [PATCH 328/399] [fix] (inverted index) fix index file in gc binlogs
 (#44994)

Problem Summary:
1. remove duplicate function `get_segment_index_filepath`
2. fix gc binlogs
---
 be/src/http/action/download_binlog_action.cpp |  15 +-
 be/src/olap/snapshot_manager.cpp              |   6 +-
 be/src/olap/tablet.cpp                        |  47 ++---
 be/src/olap/tablet.h                          |   5 -
 .../task/engine_storage_migration_task.cpp    |   5 +-
 .../common/inverted_index_gc_binlogs_test.cpp | 174 ++++++++++++++++++
 6 files changed, 213 insertions(+), 39 deletions(-)
 create mode 100644 be/test/olap/rowset/segment_v2/inverted_index/common/inverted_index_gc_binlogs_test.cpp

diff --git a/be/src/http/action/download_binlog_action.cpp b/be/src/http/action/download_binlog_action.cpp
index 372f840401c4ad..4bb8b8b70dd722 100644
--- a/be/src/http/action/download_binlog_action.cpp
+++ b/be/src/http/action/download_binlog_action.cpp
@@ -144,8 +144,19 @@ void handle_get_segment_index_file(StorageEngine& engine, HttpRequest* req,
         const auto& rowset_id = get_http_param(req, kRowsetIdParameter);
         const auto& segment_index = get_http_param(req, kSegmentIndexParameter);
         const auto& segment_index_id = req->param(kSegmentIndexIdParameter);
-        segment_index_file_path =
-                tablet->get_segment_index_filepath(rowset_id, segment_index, segment_index_id);
+        auto segment_file_path = tablet->get_segment_filepath(rowset_id, segment_index);
+        if (tablet->tablet_schema()->get_inverted_index_storage_format() ==
+            InvertedIndexStorageFormatPB::V1) {
+            // now CCR not support for variant + index v1
+            constexpr std::string_view index_suffix = "";
+            segment_index_file_path = InvertedIndexDescriptor::get_index_file_path_v1(
+                    InvertedIndexDescriptor::get_index_file_path_prefix(segment_file_path),
+                    std::stoll(segment_index_id), index_suffix);
+        } else {
+            DCHECK(segment_index_id == "-1");
+            segment_index_file_path = InvertedIndexDescriptor::get_index_file_path_v2(
+                    InvertedIndexDescriptor::get_index_file_path_prefix(segment_file_path));
+        }
         is_acquire_md5 = !req->param(kAcquireMD5Parameter).empty();
     } catch (const std::exception& e) {
         HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, e.what());
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index 67205835b53947..8202feb68c65b5 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -700,8 +700,10 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet
                     InvertedIndexStorageFormatPB::V1) {
                     for (const auto& index : tablet_schema.inverted_indexes()) {
                         auto index_id = index->index_id();
-                        auto index_file = ref_tablet->get_segment_index_filepath(
-                                rowset_id, segment_index, index_id);
+                        auto index_file = InvertedIndexDescriptor::get_index_file_path_v1(
+                                InvertedIndexDescriptor::get_index_file_path_prefix(
+                                        segment_file_path),
+                                index_id, index->get_index_suffix());
                         auto snapshot_segment_index_file_path =
                                 fmt::format("{}/{}_{}_{}.binlog-index", schema_full_path, rowset_id,
                                             segment_index, index_id);
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 705d0ad53097e8..c7919b3f8dca24 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2615,30 +2615,6 @@ std::string Tablet::get_segment_filepath(std::string_view rowset_id, int64_t seg
     return fmt::format("{}/_binlog/{}_{}.dat", _tablet_path, rowset_id, segment_index);
 }
 
-std::string Tablet::get_segment_index_filepath(std::string_view rowset_id,
-                                               std::string_view segment_index,
-                                               std::string_view index_id) const {
-    auto format = _tablet_meta->tablet_schema()->get_inverted_index_storage_format();
-    if (format == doris::InvertedIndexStorageFormatPB::V1) {
-        return fmt::format("{}/_binlog/{}_{}_{}.idx", _tablet_path, rowset_id, segment_index,
-                           index_id);
-    } else {
-        return fmt::format("{}/_binlog/{}_{}.idx", _tablet_path, rowset_id, segment_index);
-    }
-}
-
-std::string Tablet::get_segment_index_filepath(std::string_view rowset_id, int64_t segment_index,
-                                               int64_t index_id) const {
-    auto format = _tablet_meta->tablet_schema()->get_inverted_index_storage_format();
-    if (format == doris::InvertedIndexStorageFormatPB::V1) {
-        return fmt::format("{}/_binlog/{}_{}_{}.idx", _tablet_path, rowset_id, segment_index,
-                           index_id);
-    } else {
-        DCHECK(index_id == -1);
-        return fmt::format("{}/_binlog/{}_{}.idx", _tablet_path, rowset_id, segment_index);
-    }
-}
-
 std::vector<std::string> Tablet::get_binlog_filepath(std::string_view binlog_version) const {
     const auto& [rowset_id, num_segments] = get_binlog_info(binlog_version);
     std::vector<std::string> binlog_filepath;
@@ -2683,10 +2659,25 @@ void Tablet::gc_binlogs(int64_t version) {
 
         // add binlog segment files and index files
         for (int64_t i = 0; i < num_segments; ++i) {
-            wait_for_deleted_binlog_files.emplace_back(get_segment_filepath(rowset_id, i));
-            for (const auto& index : this->tablet_schema()->inverted_indexes()) {
-                wait_for_deleted_binlog_files.emplace_back(
-                        get_segment_index_filepath(rowset_id, i, index->index_id()));
+            auto segment_file_path = get_segment_filepath(rowset_id, i);
+            wait_for_deleted_binlog_files.emplace_back(segment_file_path);
+
+            // index files
+            if (tablet_schema()->has_inverted_index()) {
+                if (tablet_schema()->get_inverted_index_storage_format() ==
+                    InvertedIndexStorageFormatPB::V1) {
+                    for (const auto& index : tablet_schema()->inverted_indexes()) {
+                        auto index_file = InvertedIndexDescriptor::get_index_file_path_v1(
+                                InvertedIndexDescriptor::get_index_file_path_prefix(
+                                        segment_file_path),
+                                index->index_id(), index->get_index_suffix());
+                        wait_for_deleted_binlog_files.emplace_back(index_file);
+                    }
+                } else {
+                    auto index_file = InvertedIndexDescriptor::get_index_file_path_v2(
+                            InvertedIndexDescriptor::get_index_file_path_prefix(segment_file_path));
+                    wait_for_deleted_binlog_files.emplace_back(index_file);
+                }
             }
         }
     };
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 40b911d6391b9b..d00476f044191c 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -441,11 +441,6 @@ class Tablet final : public BaseTablet {
     std::string get_segment_filepath(std::string_view rowset_id,
                                      std::string_view segment_index) const;
     std::string get_segment_filepath(std::string_view rowset_id, int64_t segment_index) const;
-    std::string get_segment_index_filepath(std::string_view rowset_id,
-                                           std::string_view segment_index,
-                                           std::string_view index_id) const;
-    std::string get_segment_index_filepath(std::string_view rowset_id, int64_t segment_index,
-                                           int64_t index_id) const;
     bool can_add_binlog(uint64_t total_binlog_size) const;
     void gc_binlogs(int64_t version);
     Status ingest_binlog_metas(RowsetBinlogMetasPB* metas_pb);
diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp
index a300e6e0f09fa3..210aa6a8c56f08 100644
--- a/be/src/olap/task/engine_storage_migration_task.cpp
+++ b/be/src/olap/task/engine_storage_migration_task.cpp
@@ -409,8 +409,9 @@ Status EngineStorageMigrationTask::_copy_index_and_data_files(
                 InvertedIndexStorageFormatPB::V1) {
                 for (const auto& index : tablet_schema.inverted_indexes()) {
                     auto index_id = index->index_id();
-                    auto index_file =
-                            _tablet->get_segment_index_filepath(rowset_id, segment_index, index_id);
+                    auto index_file = InvertedIndexDescriptor::get_index_file_path_v1(
+                            InvertedIndexDescriptor::get_index_file_path_prefix(segment_file_path),
+                            index_id, index->get_index_suffix());
                     auto snapshot_segment_index_file_path =
                             fmt::format("{}/{}_{}_{}.binlog-index", full_path, rowset_id,
                                         segment_index, index_id);
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/common/inverted_index_gc_binlogs_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/common/inverted_index_gc_binlogs_test.cpp
new file mode 100644
index 00000000000000..84f82b8f46e302
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/inverted_index/common/inverted_index_gc_binlogs_test.cpp
@@ -0,0 +1,174 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include "olap/rowset/beta_rowset_writer.h"
+#include "olap/rowset/rowset_factory.h"
+#include "olap/rowset/rowset_meta_manager.h"
+#include "olap/storage_engine.h"
+
+namespace doris {
+
+using namespace doris::vectorized;
+
+constexpr static uint32_t MAX_PATH_LEN = 1024;
+constexpr static std::string_view dest_dir = "./ut_dir/inverted_index_test";
+constexpr static std::string_view tmp_dir = "./ut_dir/tmp";
+static int64_t inc_id = 0;
+
+class IndexGcBinglogsTest : public ::testing::Test {
+protected:
+    void SetUp() override {
+        // absolute dir
+        char buffer[MAX_PATH_LEN];
+        EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
+        _curreent_dir = std::string(buffer);
+        _absolute_dir = _curreent_dir + std::string(dest_dir);
+        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
+        EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok());
+
+        // tmp dir
+        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok());
+        EXPECT_TRUE(io::global_local_filesystem()->create_directory(tmp_dir).ok());
+        std::vector<StorePath> paths;
+        paths.emplace_back(std::string(tmp_dir), 1024000000);
+        auto tmp_file_dirs = std::make_unique<segment_v2::TmpFileDirs>(paths);
+        EXPECT_TRUE(tmp_file_dirs->init().ok());
+        ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs));
+
+        // storage engine
+        doris::EngineOptions options;
+        auto engine = std::make_unique<StorageEngine>(options);
+        _engine_ref = engine.get();
+        _data_dir = std::make_unique<DataDir>(*_engine_ref, _absolute_dir);
+        static_cast<void>(_data_dir->update_capacity());
+        EXPECT_TRUE(_data_dir->init(true).ok());
+        ExecEnv::GetInstance()->set_storage_engine(std::move(engine));
+
+        // tablet_schema
+        TabletSchemaPB schema_pb;
+        _schema_pb.set_keys_type(KeysType::DUP_KEYS);
+        _schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
+
+        construct_column(_schema_pb.add_column(), _schema_pb.add_index(), 10000, "key_index", 0,
+                         "INT", "key");
+        construct_column(_schema_pb.add_column(), _schema_pb.add_index(), 10001, "v1_index", 1,
+                         "STRING", "v1");
+    }
+    void TearDown() override {
+        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok());
+        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok());
+        _engine_ref = nullptr;
+        ExecEnv::GetInstance()->set_storage_engine(nullptr);
+    }
+
+    void construct_column(ColumnPB* column_pb, TabletIndexPB* tablet_index, int64_t index_id,
+                          const std::string& index_name, int32_t col_unique_id,
+                          const std::string& column_type, const std::string& column_name,
+                          bool parser = false) {
+        column_pb->set_unique_id(col_unique_id);
+        column_pb->set_name(column_name);
+        column_pb->set_type(column_type);
+        column_pb->set_is_key(false);
+        column_pb->set_is_nullable(true);
+        tablet_index->set_index_id(index_id);
+        tablet_index->set_index_name(index_name);
+        tablet_index->set_index_type(IndexType::INVERTED);
+        tablet_index->add_col_unique_id(col_unique_id);
+        if (parser) {
+            auto* properties = tablet_index->mutable_properties();
+            (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_UNICODE;
+        }
+    }
+
+    RowsetWriterContext rowset_writer_context() {
+        RowsetWriterContext context;
+        RowsetId rowset_id;
+        rowset_id.init(inc_id);
+        context.rowset_id = rowset_id;
+        context.rowset_type = BETA_ROWSET;
+        context.data_dir = _data_dir.get();
+        context.rowset_state = VISIBLE;
+        context.tablet_schema = _tablet_schema;
+        context.tablet_path = _tablet->tablet_path();
+        context.version = Version(inc_id, inc_id);
+        context.max_rows_per_segment = 200;
+        inc_id++;
+        return context;
+    }
+
+    IndexGcBinglogsTest() = default;
+    ~IndexGcBinglogsTest() override = default;
+
+private:
+    TabletSchemaSPtr _tablet_schema = nullptr;
+    StorageEngine* _engine_ref = nullptr;
+    std::unique_ptr<DataDir> _data_dir = nullptr;
+    TabletSharedPtr _tablet = nullptr;
+    std::string _absolute_dir;
+    std::string _curreent_dir;
+    TabletSchemaPB _schema_pb;
+};
+
+TEST_F(IndexGcBinglogsTest, gc_binlogs_test) {
+    auto test = [&](InvertedIndexStorageFormatPB format) {
+        TabletSchemaPB schema_pb = _schema_pb;
+        _schema_pb.set_inverted_index_storage_format(format);
+        _tablet_schema.reset(new TabletSchema);
+        _tablet_schema->init_from_pb(schema_pb);
+        TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema));
+
+        tablet_meta->set_tablet_uid(TabletUid(20, 20));
+        tablet_meta.get()->_tablet_id = 200;
+        _tablet.reset(new Tablet(*_engine_ref, tablet_meta, _data_dir.get()));
+        EXPECT_TRUE(_tablet->init().ok());
+        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok());
+        EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok());
+
+        RowsetSharedPtr rowset;
+        const auto& res =
+                RowsetFactory::create_rowset_writer(*_engine_ref, rowset_writer_context(), false);
+        EXPECT_TRUE(res.has_value()) << res.error();
+        const auto& rowset_writer = res.value();
+
+        Block block = _tablet_schema->create_block();
+        auto columns = block.mutate_columns();
+
+        vectorized::Field key = 10;
+        vectorized::Field v1 = "v1";
+        columns[0]->insert(key);
+        columns[1]->insert(v1);
+
+        EXPECT_TRUE(rowset_writer->add_block(&block).ok());
+        EXPECT_TRUE(rowset_writer->flush().ok());
+        EXPECT_TRUE(rowset_writer->build(rowset).ok());
+        EXPECT_TRUE(rowset->add_to_binlog().ok());
+        EXPECT_TRUE(_tablet->add_rowset(rowset).ok());
+        EXPECT_TRUE(RowsetMetaManager::save(_data_dir->get_meta(), _tablet->tablet_uid(),
+                                            rowset->rowset_id(),
+                                            rowset->rowset_meta()->get_rowset_pb(), true)
+                            .ok());
+        _tablet->save_meta();
+        _tablet->gc_binlogs(0);
+    };
+    test(InvertedIndexStorageFormatPB::V1);
+    test(InvertedIndexStorageFormatPB::V2);
+}
+
+} // namespace doris

From 0c97e0470f20a85f27d9d63673f1f3b44a82f164 Mon Sep 17 00:00:00 2001
From: HappenLee <happenlee@selectdb.com>
Date: Mon, 16 Dec 2024 10:52:10 +0800
Subject: [PATCH 329/399] [Exec](expr) Opt the compound pred performace
 (#45414)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

before：
```
 mysqlslap -hd3 -uroot -P9130  --create-schema=test_db2 -c 10 -i 500 -q "SELECT count(k) FROM sbtest1_dup WHERE k BETWEEN 4850578 AND 8454295 OR k BETWEEN 8776291 AND 29749077;"
Benchmark
	Average number of seconds to run all queries: 0.041 seconds
	Minimum number of seconds to run all queries: 0.037 seconds
	Maximum number of seconds to run all queries: 0.115 seconds
	Number of clients running queries: 10
	Average number of queries per client: 1
```

after：
```
mysqlslap -hd3 -uroot -P9030  --create-schema=test_db -c 10 -i 500 -q "SELECT count(k) FROM sbtest1 WHERE k BETWEEN 4850578 AND 8454295 OR k BETWEEN 8776291 AND 29749077;"
Benchmark
	Average number of seconds to run all queries: 0.029 seconds
	Minimum number of seconds to run all queries: 0.027 seconds
	Maximum number of seconds to run all queries: 0.034 seconds
	Number of clients running queries: 10
	Average number of queries per client: 1
```
---
 be/src/vec/exprs/vcompound_pred.h | 82 ++++++++++++++++++++++---------
 1 file changed, 60 insertions(+), 22 deletions(-)

diff --git a/be/src/vec/exprs/vcompound_pred.h b/be/src/vec/exprs/vcompound_pred.h
index 88f3e474b58da3..8c6832a7a35a37 100644
--- a/be/src/vec/exprs/vcompound_pred.h
+++ b/be/src/vec/exprs/vcompound_pred.h
@@ -158,12 +158,15 @@ class VCompoundPred : public VectorizedFnCall {
         if (_can_fast_execute && fast_execute(context, block, result_column_id)) {
             return Status::OK();
         }
-        if (get_num_children() == 1 || !_all_child_is_compound_and_not_const()) {
+        if (get_num_children() == 1 || _has_const_child()) {
             return VectorizedFnCall::execute(context, block, result_column_id);
         }
 
         int lhs_id = -1;
         int rhs_id = -1;
+        bool lhs_mem_can_reuse = _children[0]->is_compound_predicate();
+        bool rhs_mem_can_reuse = _children[1]->is_compound_predicate();
+
         RETURN_IF_ERROR(_children[0]->execute(context, block, &lhs_id));
         ColumnPtr lhs_column =
                 block->get_by_position(lhs_id).column->convert_to_full_column_if_const();
@@ -210,13 +213,22 @@ class VCompoundPred : public VectorizedFnCall {
             return Status::OK();
         };
 
-        auto return_result_column_id = [&](ColumnPtr res_column, int res_id) -> int {
+        auto return_result_column_id = [&](ColumnPtr res_column, int res_id,
+                                           bool mem_reuse) -> int {
+            if (!mem_reuse) {
+                res_column = res_column->clone_resized(size);
+            }
+
             if (result_is_nullable && !res_column->is_nullable()) {
                 auto result_column =
                         ColumnNullable::create(res_column, ColumnUInt8::create(size, 0));
                 res_id = block->columns();
                 block->insert({std::move(result_column), _data_type, _expr_name});
+            } else if (!mem_reuse) {
+                res_id = block->columns();
+                block->insert({std::move(res_column), _data_type, _expr_name});
             }
+
             return res_id;
         };
 
@@ -231,6 +243,33 @@ class VCompoundPred : public VectorizedFnCall {
             return null_map_data;
         };
 
+        auto vector_vector = [&]<bool is_and_op>() {
+            if (lhs_mem_can_reuse) {
+                *result_column_id = lhs_id;
+            } else if (rhs_mem_can_reuse) {
+                *result_column_id = rhs_id;
+
+                auto tmp_column = rhs_data_column;
+                rhs_data_column = lhs_data_column;
+                lhs_data_column = tmp_column;
+            } else {
+                *result_column_id = block->columns();
+
+                auto col_res = lhs_column->clone_resized(size);
+                lhs_data_column = assert_cast<ColumnUInt8*>(col_res.get())->get_data().data();
+                block->insert({std::move(col_res), _data_type, _expr_name});
+            }
+
+            if constexpr (is_and_op) {
+                for (size_t i = 0; i < size; ++i) {
+                    lhs_data_column[i] &= rhs_data_column[i];
+                }
+            } else {
+                for (size_t i = 0; i < size; ++i) {
+                    lhs_data_column[i] |= rhs_data_column[i];
+                }
+            }
+        };
         auto vector_vector_null = [&]<bool is_and_op>() {
             auto col_res = ColumnUInt8::create(size);
             auto col_nulls = ColumnUInt8::create(size);
@@ -269,28 +308,28 @@ class VCompoundPred : public VectorizedFnCall {
             //2. nullable column: null map all is not null
             if ((lhs_all_false && !lhs_is_nullable) || (lhs_all_false && lhs_all_is_not_null)) {
                 // false and any = false, return lhs
-                *result_column_id = return_result_column_id(lhs_column, lhs_id);
+                *result_column_id = return_result_column_id(lhs_column, lhs_id, lhs_mem_can_reuse);
             } else {
                 RETURN_IF_ERROR(get_rhs_colum());
 
                 if ((lhs_all_true && !lhs_is_nullable) ||    //not null column
                     (lhs_all_true && lhs_all_is_not_null)) { //nullable column
                     // true and any = any, return rhs
-                    *result_column_id = return_result_column_id(rhs_column, rhs_id);
+                    *result_column_id =
+                            return_result_column_id(rhs_column, rhs_id, rhs_mem_can_reuse);
                 } else if ((rhs_all_false && !rhs_is_nullable) ||
                            (rhs_all_false && rhs_all_is_not_null)) {
                     // any and false = false, return rhs
-                    *result_column_id = return_result_column_id(rhs_column, rhs_id);
+                    *result_column_id =
+                            return_result_column_id(rhs_column, rhs_id, rhs_mem_can_reuse);
                 } else if ((rhs_all_true && !rhs_is_nullable) ||
                            (rhs_all_true && rhs_all_is_not_null)) {
                     // any and true = any, return lhs
-                    *result_column_id = return_result_column_id(lhs_column, lhs_id);
+                    *result_column_id =
+                            return_result_column_id(lhs_column, lhs_id, lhs_mem_can_reuse);
                 } else {
                     if (!result_is_nullable) {
-                        *result_column_id = lhs_id;
-                        for (size_t i = 0; i < size; i++) {
-                            lhs_data_column[i] &= rhs_data_column[i];
-                        }
+                        vector_vector.template operator()<true>();
                     } else {
                         vector_vector_null.template operator()<true>();
                     }
@@ -301,26 +340,26 @@ class VCompoundPred : public VectorizedFnCall {
             // false or NULL ----> NULL
             if ((lhs_all_true && !lhs_is_nullable) || (lhs_all_true && lhs_all_is_not_null)) {
                 // true or any = true, return lhs
-                *result_column_id = return_result_column_id(lhs_column, lhs_id);
+                *result_column_id = return_result_column_id(lhs_column, lhs_id, lhs_mem_can_reuse);
             } else {
                 RETURN_IF_ERROR(get_rhs_colum());
                 if ((lhs_all_false && !lhs_is_nullable) || (lhs_all_false && lhs_all_is_not_null)) {
                     // false or any = any, return rhs
-                    *result_column_id = return_result_column_id(rhs_column, rhs_id);
+                    *result_column_id =
+                            return_result_column_id(rhs_column, rhs_id, rhs_mem_can_reuse);
                 } else if ((rhs_all_true && !rhs_is_nullable) ||
                            (rhs_all_true && rhs_all_is_not_null)) {
                     // any or true = true, return rhs
-                    *result_column_id = return_result_column_id(rhs_column, rhs_id);
+                    *result_column_id =
+                            return_result_column_id(rhs_column, rhs_id, rhs_mem_can_reuse);
                 } else if ((rhs_all_false && !rhs_is_nullable) ||
                            (rhs_all_false && rhs_all_is_not_null)) {
                     // any or false = any, return lhs
-                    *result_column_id = return_result_column_id(lhs_column, lhs_id);
+                    *result_column_id =
+                            return_result_column_id(lhs_column, lhs_id, lhs_mem_can_reuse);
                 } else {
                     if (!result_is_nullable) {
-                        *result_column_id = lhs_id;
-                        for (size_t i = 0; i < size; i++) {
-                            lhs_data_column[i] |= rhs_data_column[i];
-                        }
+                        vector_vector.template operator()<false>();
                     } else {
                         vector_vector_null.template operator()<false>();
                     }
@@ -345,10 +384,9 @@ class VCompoundPred : public VectorizedFnCall {
         return (l_null & r_null) | (r_null & (r_null ^ a)) | (l_null & (l_null ^ b));
     }
 
-    bool _all_child_is_compound_and_not_const() const {
-        return std::ranges::all_of(_children, [](const VExprSPtr& arg) -> bool {
-            return arg->is_compound_predicate() && !arg->is_constant();
-        });
+    bool _has_const_child() const {
+        return std::ranges::any_of(_children,
+                                   [](const VExprSPtr& arg) -> bool { return arg->is_constant(); });
     }
 
     std::pair<uint8*, uint8*> _get_raw_data_and_null_map(ColumnPtr column,

From 26b2aab80039dbb7831f2d7bd90ea39918b37d89 Mon Sep 17 00:00:00 2001
From: zhangstar333 <zhangsida@selectdb.com>
Date: Mon, 16 Dec 2024 11:10:39 +0800
Subject: [PATCH 330/399] [Bug](function) fix repeat function result length is
 invalid and cause overflow (#45288)

### What problem does this PR solve?
Problem Summary:
 `repeat('a', k1)`, if k1 is nullable column
when execute repeat function, input column is nested column of k1
so if k1 is NULL value, the nested value maybe not inited and invalid,
cause the result length is large and overflow
---
 be/src/vec/functions/function_string.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h
index 6e4a18fdd3100e..fba8b914eee682 100644
--- a/be/src/vec/functions/function_string.h
+++ b/be/src/vec/functions/function_string.h
@@ -1525,6 +1525,9 @@ class FunctionStringRepeat : public IFunction {
     static FunctionPtr create() { return std::make_shared<FunctionStringRepeat>(); }
     String get_name() const override { return name; }
     size_t get_number_of_arguments() const override { return 2; }
+    // should set NULL value of nested data to default,
+    // as iff it's not inited and invalid, the repeat result of length is so large cause overflow
+    bool need_replace_null_data_to_default() const override { return true; }
 
     DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
         return make_nullable(std::make_shared<DataTypeString>());

From 4a914fe1b88c9477b95414ef48880e3f59e0f854 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@selectdb.com>
Date: Mon, 16 Dec 2024 11:17:01 +0800
Subject: [PATCH 331/399] [Fix](case) remove useless test (#45409)

---
 be/src/agent/workload_group_listener.cpp      |  1 +
 be/src/agent/workload_group_listener.h        |  3 +-
 .../test_workload_sched_policy.groovy         | 57 -------------------
 3 files changed, 3 insertions(+), 58 deletions(-)

diff --git a/be/src/agent/workload_group_listener.cpp b/be/src/agent/workload_group_listener.cpp
index 7b688b7dcdf6ef..0cd5a3ee1ac748 100644
--- a/be/src/agent/workload_group_listener.cpp
+++ b/be/src/agent/workload_group_listener.cpp
@@ -17,6 +17,7 @@
 
 #include "agent/workload_group_listener.h"
 
+#include "runtime/exec_env.h"
 #include "runtime/workload_group/workload_group.h"
 #include "runtime/workload_group/workload_group_manager.h"
 #include "util/mem_info.h"
diff --git a/be/src/agent/workload_group_listener.h b/be/src/agent/workload_group_listener.h
index f596535908d079..9578a36f70d63e 100644
--- a/be/src/agent/workload_group_listener.h
+++ b/be/src/agent/workload_group_listener.h
@@ -20,10 +20,11 @@
 #include <glog/logging.h>
 
 #include "agent/topic_listener.h"
-#include "runtime/exec_env.h"
 
 namespace doris {
 
+class ExecEnv;
+
 class WorkloadGroupListener : public TopicListener {
 public:
     ~WorkloadGroupListener() {}
diff --git a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
index 06e10688397689..ca18c5a17e0f56 100644
--- a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
+++ b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
@@ -235,61 +235,4 @@ suite("test_workload_sched_policy") {
 
     sql "drop user test_alter_policy_user"
     sql "drop workload policy test_alter_policy"
-
-    // daemon thread alter test
-    def thread1 = new Thread({
-        def startTime = System.currentTimeMillis()
-        def curTime = System.currentTimeMillis()
-        def totalTime = 30 * 60 * 1000 // 30min
-
-        connect('test_policy_user', '12345', context.config.jdbcUrl) {
-            sql "set workload_group=policy_group"
-            boolean flag = false
-            long lastTime = System.currentTimeMillis()
-
-            while (curTime - startTime <= totalTime) {
-                if (curTime - lastTime > 20000) {
-                    if (flag) {
-                        connect('root', '', context.config.jdbcUrl) {
-                            sql "alter workload policy test_cancel_query_policy properties('workload_group'='policy_group2');"
-                            sql "alter workload policy test_cancel_query_policy2 properties('workload_group'='policy_group');"
-                        }
-                        flag = false
-                    } else {
-                        connect('root', '', context.config.jdbcUrl) {
-                            sql "alter workload policy test_cancel_query_policy properties('workload_group'='policy_group');"
-                            sql "alter workload policy test_cancel_query_policy2 properties('workload_group'='policy_group2');"
-                        }
-                        flag = true
-                    }
-                    lastTime = System.currentTimeMillis()
-                }
-                try {
-                    sql "select k0 as policy_test_tag,k1,k2,k3,k4,k5,k6,count(distinct k13) from regression_test_load_p0_insert.baseall group by k0,k1,k2,k3,k4,k5,k6"
-                } catch (Exception e) {
-                    boolean ret = e.getMessage().contains("cancelled by workload policy")
-                    if (!ret) {
-                        logger.info("policy_test_tag " + e.getMessage())
-                    }
-                    assertTrue(ret, "policy daemon check failed")
-                }
-
-                try {
-                    sql "select count(1) as policy_test_tag from regression_test_load_p0_insert.baseall"
-                } catch (Exception e) {
-                    boolean ret = e.getMessage().contains("cancelled by workload policy")
-                    if (!ret) {
-                        logger.info("policy_test_tag " + e.getMessage())
-                    }
-                    assertTrue(ret, "policy daemon check failed")
-                }
-
-                Thread.sleep(1000)
-                curTime = System.currentTimeMillis()
-            }
-        }
-    })
-    thread1.setDaemon(true)
-    thread1.start()
-
 }
\ No newline at end of file

From 8058085c5b822066e5c806570ff2756fd4ae07ba Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Mon, 16 Dec 2024 11:31:16 +0800
Subject: [PATCH 332/399] [feat](nereids) add is_merge tag for data sink
 (#45196)

### What problem does this PR solve?
set is_merge true, if its corresponding exchange node contains sortInfo.
BE use this flag to accelerate data transferring.
it is better to determine this by FE than that by BE.
for example, if data spill to disk, the node before sink is scan, not sort.
---
 .../nereids/glue/translator/PhysicalPlanTranslator.java  | 6 ++++++
 .../src/main/java/org/apache/doris/planner/DataSink.java | 9 +++++++++
 .../java/org/apache/doris/planner/DataStreamSink.java    | 4 ++++
 gensrc/thrift/DataSinks.thrift                           | 1 +
 4 files changed, 20 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index d66ff3ebb7649d..f0fa59977a1902 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -2147,6 +2147,9 @@ public PlanFragment visitPhysicalQuickSort(PhysicalQuickSort<? extends Plan> sor
             }
             SortNode sortNode = (SortNode) inputFragment.getPlanRoot().getChild(0);
             ((ExchangeNode) inputFragment.getPlanRoot()).setMergeInfo(sortNode.getSortInfo());
+            if (inputFragment.hasChild(0) && inputFragment.getChild(0).getSink() != null) {
+                inputFragment.getChild(0).getSink().setMerge(true);
+            }
             sortNode.setMergeByExchange();
             sortNode.setChildrenDistributeExprLists(distributeExprLists);
         }
@@ -2198,6 +2201,9 @@ public PlanFragment visitPhysicalTopN(PhysicalTopN<? extends Plan> topN, PlanTra
             ExchangeNode exchangeNode = (ExchangeNode) inputFragment.getPlanRoot();
             exchangeNode.setChildrenDistributeExprLists(distributeExprLists);
             exchangeNode.setMergeInfo(((SortNode) exchangeNode.getChild(0)).getSortInfo());
+            if (inputFragment.hasChild(0) && inputFragment.getChild(0).getSink() != null) {
+                inputFragment.getChild(0).getSink().setMerge(true);
+            }
             exchangeNode.setLimit(topN.getLimit());
             exchangeNode.setOffset(topN.getOffset());
             ((SortNode) exchangeNode.getChild(0)).setMergeByExchange();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java
index 8d6daa2f8b72b4..320fcae9eb0f1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java
@@ -39,6 +39,7 @@
 public abstract class DataSink {
     // Fragment that this DataSink belongs to. Set by the PlanFragment enclosing this sink.
     protected PlanFragment fragment;
+    protected boolean isMerge = false;
 
     /**
      * Return an explain string for the DataSink. Each line of the explain will be
@@ -77,4 +78,12 @@ public static DataSink createDataSink(TableIf table) throws AnalysisException {
             throw new AnalysisException("Unknown table type " + table.getType());
         }
     }
+
+    public boolean isMerge() {
+        return isMerge;
+    }
+
+    public void setMerge(boolean merge) {
+        isMerge = merge;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java
index ef42190fa25004..ceda692aa4b01f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataStreamSink.java
@@ -176,6 +176,9 @@ public String getExplainString(String prefix, TExplainLevel explainLevel) {
             strBuilder.append(prefix).append("  PROJECTION TUPLE: ").append(outputTupleDesc.getId());
             strBuilder.append("\n");
         }
+        if (isMerge) {
+            strBuilder.append("IS_MERGE: true\n");
+        }
 
         return strBuilder.toString();
     }
@@ -234,6 +237,7 @@ protected TDataSink toThrift() {
                 tStreamSink.addToTabletSinkExprs(expr.treeToThrift());
             }
         }
+        tStreamSink.setIsMerge(isMerge);
         tStreamSink.setTabletSinkTxnId(tabletSinkTxnId);
         result.setStreamSink(tStreamSink);
         return result;
diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift
index ed7ccee69cd9a1..03a23c2c532ac3 100644
--- a/gensrc/thrift/DataSinks.thrift
+++ b/gensrc/thrift/DataSinks.thrift
@@ -189,6 +189,7 @@ struct TDataStreamSink {
   11: optional i64 tablet_sink_txn_id
   12: optional Types.TTupleId tablet_sink_tuple_id
   13: optional list<Exprs.TExpr> tablet_sink_exprs
+  14: optional bool is_merge
 }
 
 struct TMultiCastDataStreamSink {

From fe8fa8709415025d0da484942e003f9446fecf1b Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Mon, 16 Dec 2024 12:11:09 +0800
Subject: [PATCH 333/399] [Bug](function) fix wrong result on group_concat with
 distinct+order_by+nullable (#45313)

fix wrong result on group_concat with distinct+order_by+nullable
---
 .../aggregate_functions/aggregate_function.h  |  6 +-
 .../aggregate_function_distinct.h             | 16 ++++-
 .../aggregate_function_null.h                 | 39 ++++++++++--
 .../aggregate_function_sort.h                 |  4 +-
 .../nereids_p0/aggregate/agg_group_concat.out |  4 ++
 .../aggregate/agg_group_concat.groovy         | 61 +++++++++++--------
 6 files changed, 91 insertions(+), 39 deletions(-)
 create mode 100644 regression-test/data/nereids_p0/aggregate/agg_group_concat.out

diff --git a/be/src/vec/aggregate_functions/aggregate_function.h b/be/src/vec/aggregate_functions/aggregate_function.h
index e0ec2bef62fc2a..d761d40c4c932c 100644
--- a/be/src/vec/aggregate_functions/aggregate_function.h
+++ b/be/src/vec/aggregate_functions/aggregate_function.h
@@ -20,6 +20,8 @@
 
 #pragma once
 
+#include <utility>
+
 #include "common/exception.h"
 #include "common/status.h"
 #include "util/defer_op.h"
@@ -81,7 +83,7 @@ using ConstAggregateDataPtr = const char*;
   */
 class IAggregateFunction {
 public:
-    IAggregateFunction(const DataTypes& argument_types_) : argument_types(argument_types_) {}
+    IAggregateFunction(DataTypes argument_types_) : argument_types(std::move(argument_types_)) {}
 
     /// Get main function name.
     virtual String get_name() const = 0;
@@ -225,7 +227,7 @@ class IAggregateFunction {
 
     virtual void set_version(const int version_) { version = version_; }
 
-    virtual AggregateFunctionPtr transmit_to_stable() { return nullptr; }
+    virtual IAggregateFunction* transmit_to_stable() { return nullptr; }
 
     /// Verify function signature
     virtual Status verify_result_type(const bool without_key, const DataTypes& argument_types,
diff --git a/be/src/vec/aggregate_functions/aggregate_function_distinct.h b/be/src/vec/aggregate_functions/aggregate_function_distinct.h
index 46450394627474..a5515145d9d2ad 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_distinct.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_distinct.h
@@ -341,12 +341,22 @@ class AggregateFunctionDistinct
 
     DataTypePtr get_return_type() const override { return nested_func->get_return_type(); }
 
-    AggregateFunctionPtr transmit_to_stable() override {
-        return AggregateFunctionPtr(new AggregateFunctionDistinct<Data, true>(
-                nested_func, IAggregateFunction::argument_types));
+    IAggregateFunction* transmit_to_stable() override {
+        return new AggregateFunctionDistinct<Data, true>(nested_func,
+                                                         IAggregateFunction::argument_types);
     }
 };
 
+template <typename T>
+struct FunctionStableTransfer {
+    using FunctionStable = T;
+};
+
+template <template <bool stable> typename Data>
+struct FunctionStableTransfer<AggregateFunctionDistinct<Data, false>> {
+    using FunctionStable = AggregateFunctionDistinct<Data, true>;
+};
+
 } // namespace doris::vectorized
 
 #include "common/compile_check_end.h"
diff --git a/be/src/vec/aggregate_functions/aggregate_function_null.h b/be/src/vec/aggregate_functions/aggregate_function_null.h
index b3fa3b8230dc52..b46bcbe353777f 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_null.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_null.h
@@ -25,6 +25,7 @@
 #include "common/logging.h"
 #include "common/status.h"
 #include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/aggregate_functions/aggregate_function_distinct.h"
 #include "vec/columns/column_nullable.h"
 #include "vec/common/assert_cast.h"
 #include "vec/data_types/data_type_nullable.h"
@@ -166,7 +167,7 @@ class AggregateFunctionNullBaseInline : public IAggregateFunctionHelper<Derived>
 
     void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
         if constexpr (result_is_nullable) {
-            ColumnNullable& to_concrete = assert_cast<ColumnNullable&>(to);
+            auto& to_concrete = assert_cast<ColumnNullable&>(to);
             if (get_flag(place)) {
                 nested_function->insert_result_into(nested_place(place),
                                                     to_concrete.get_nested_column());
@@ -198,7 +199,7 @@ class AggregateFunctionNullUnaryInline final
 
     void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num,
              Arena* arena) const override {
-        const ColumnNullable* column =
+        const auto* column =
                 assert_cast<const ColumnNullable*, TypeCheckOnRelease::DISABLE>(columns[0]);
         if (!column->is_null_at(row_num)) {
             this->set_flag(place);
@@ -207,6 +208,19 @@ class AggregateFunctionNullUnaryInline final
         }
     }
 
+    IAggregateFunction* transmit_to_stable() override {
+        auto f = AggregateFunctionNullBaseInline<
+                         NestFuction, result_is_nullable,
+                         AggregateFunctionNullUnaryInline<NestFuction, result_is_nullable>>::
+                         nested_function->transmit_to_stable();
+        if (!f) {
+            return nullptr;
+        }
+        return new AggregateFunctionNullUnaryInline<
+                typename FunctionStableTransfer<NestFuction>::FunctionStable, result_is_nullable>(
+                f, IAggregateFunction::argument_types);
+    }
+
     void add_batch(size_t batch_size, AggregateDataPtr* __restrict places, size_t place_offset,
                    const IColumn** columns, Arena* arena, bool agg_many) const override {
         const auto* column = assert_cast<const ColumnNullable*>(columns[0]);
@@ -236,7 +250,7 @@ class AggregateFunctionNullUnaryInline final
 
     void add_batch_single_place(size_t batch_size, AggregateDataPtr place, const IColumn** columns,
                                 Arena* arena) const override {
-        const ColumnNullable* column = assert_cast<const ColumnNullable*>(columns[0]);
+        const auto* column = assert_cast<const ColumnNullable*>(columns[0]);
         bool has_null = column->has_null();
 
         if (has_null) {
@@ -253,7 +267,7 @@ class AggregateFunctionNullUnaryInline final
 
     void add_batch_range(size_t batch_begin, size_t batch_end, AggregateDataPtr place,
                          const IColumn** columns, Arena* arena, bool has_null) override {
-        const ColumnNullable* column = assert_cast<const ColumnNullable*>(columns[0]);
+        const auto* column = assert_cast<const ColumnNullable*>(columns[0]);
 
         if (has_null) {
             for (size_t i = batch_begin; i <= batch_end; ++i) {
@@ -283,13 +297,13 @@ class AggregateFunctionNullVariadicInline final
                       nested_function_, arguments),
               number_of_arguments(arguments.size()) {
         if (number_of_arguments == 1) {
-            throw doris::Exception(
+            throw Exception(
                     ErrorCode::INTERNAL_ERROR,
                     "Logical error: single argument is passed to AggregateFunctionNullVariadic");
         }
 
         if (number_of_arguments > MAX_ARGS) {
-            throw doris::Exception(
+            throw Exception(
                     ErrorCode::INTERNAL_ERROR,
                     "Maximum number of arguments for aggregate function with Nullable types is {}",
                     size_t(MAX_ARGS));
@@ -300,6 +314,19 @@ class AggregateFunctionNullVariadicInline final
         }
     }
 
+    IAggregateFunction* transmit_to_stable() override {
+        auto f = AggregateFunctionNullBaseInline<
+                         NestFuction, result_is_nullable,
+                         AggregateFunctionNullVariadicInline<NestFuction, result_is_nullable>>::
+                         nested_function->transmit_to_stable();
+        if (!f) {
+            return nullptr;
+        }
+        return new AggregateFunctionNullVariadicInline<
+                typename FunctionStableTransfer<NestFuction>::FunctionStable, result_is_nullable>(
+                f, IAggregateFunction::argument_types);
+    }
+
     void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num,
              Arena* arena) const override {
         /// This container stores the columns we really pass to the nested function.
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sort.h b/be/src/vec/aggregate_functions/aggregate_function_sort.h
index a0a05cc7ddd397..c025c42495cf1b 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sort.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_sort.h
@@ -134,8 +134,8 @@ class AggregateFunctionSort
               _arguments(arguments),
               _sort_desc(sort_desc),
               _state(state) {
-        if (auto f = _nested_func->transmit_to_stable(); f) {
-            _nested_func = f;
+        if (auto* f = _nested_func->transmit_to_stable(); f) {
+            _nested_func = AggregateFunctionPtr(f);
         }
         for (const auto& type : _arguments) {
             _block.insert({type, ""});
diff --git a/regression-test/data/nereids_p0/aggregate/agg_group_concat.out b/regression-test/data/nereids_p0/aggregate/agg_group_concat.out
new file mode 100644
index 00000000000000..b57409a6192bad
--- /dev/null
+++ b/regression-test/data/nereids_p0/aggregate/agg_group_concat.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test --
+abc,abcd,eee
+
diff --git a/regression-test/suites/nereids_p0/aggregate/agg_group_concat.groovy b/regression-test/suites/nereids_p0/aggregate/agg_group_concat.groovy
index c8e244008cd044..6df485ad9ada14 100644
--- a/regression-test/suites/nereids_p0/aggregate/agg_group_concat.groovy
+++ b/regression-test/suites/nereids_p0/aggregate/agg_group_concat.groovy
@@ -75,30 +75,39 @@ suite("agg_group_concat") {
         exception "doesn't support order by expression"
     }
 
-    sql """select multi_distinct_sum(kint) from agg_group_concat_table;"""
-
-    sql """select group_concat(distinct kstr order by kint), group_concat(distinct kstr2 order by kbint) from agg_group_concat_table;"""
-    sql """select multi_distinct_group_concat(kstr order by kint), multi_distinct_group_concat(kstr2 order by kbint) from agg_group_concat_table;"""
-    sql """select group_concat(distinct kstr), group_concat(distinct kstr2) from agg_group_concat_table;"""
-    sql """select multi_distinct_group_concat(kstr), multi_distinct_group_concat(kstr2) from agg_group_concat_table;"""
-
-    sql """select group_concat(distinct kstr order by kint), group_concat(distinct kstr2 order by kbint) from agg_group_concat_table group by kbint;"""
-    sql """select multi_distinct_group_concat(kstr order by kint), multi_distinct_group_concat(kstr2 order by kbint) from agg_group_concat_table group by kbint;"""
-    sql """select group_concat(distinct kstr), group_concat(distinct kstr2) from agg_group_concat_table group by kbint;"""
-    sql """select multi_distinct_group_concat(kstr), multi_distinct_group_concat(kstr2) from agg_group_concat_table group by kbint;"""
-
-    sql """select group_concat(distinct kstr order by kbint), group_concat(distinct kstr2 order by kint) from agg_group_concat_table group by kint;"""
-    sql """select multi_distinct_group_concat(kstr order by kbint), multi_distinct_group_concat(kstr2 order by kint) from agg_group_concat_table group by kint;"""
-    sql """select group_concat(distinct kstr), group_concat(distinct kstr2) from agg_group_concat_table group by kint;"""
-    sql """select multi_distinct_group_concat(kstr), multi_distinct_group_concat(kstr2) from agg_group_concat_table group by kint;"""
-
-    sql """select group_concat(distinct kstr order by kint), group_concat(kstr2 order by kbint) from agg_group_concat_table;"""
-    sql """select multi_distinct_group_concat(kstr order by kint), group_concat(kstr2 order by kbint) from agg_group_concat_table;"""
-    sql """select group_concat(distinct kstr), group_concat(kstr2) from agg_group_concat_table;"""
-    sql """select multi_distinct_group_concat(kstr), group_concat(kstr2) from agg_group_concat_table;"""
-
-    sql """select group_concat(distinct kstr order by kint), group_concat(kstr2 order by kbint) from agg_group_concat_table group by kbint;"""
-    sql """select multi_distinct_group_concat(kstr order by kint), group_concat(kstr2 order by kbint) from agg_group_concat_table group by kbint;"""
-    sql """select group_concat(distinct kstr), group_concat(kstr2) from agg_group_concat_table group by kbint;"""
-    sql """select multi_distinct_group_concat(kstr), group_concat(kstr2) from agg_group_concat_table group by kbint;"""
+    sql """select multi_distinct_sum(kint) from agg_group_concat_table order by 1;"""
+
+    sql """select group_concat(distinct kstr order by kint), group_concat(distinct kstr2 order by kbint) from agg_group_concat_table order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr order by kint), multi_distinct_group_concat(kstr2 order by kbint) from agg_group_concat_table order by 1,2;"""
+    sql """select group_concat(distinct kstr), group_concat(distinct kstr2) from agg_group_concat_table order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr), multi_distinct_group_concat(kstr2) from agg_group_concat_table order by 1,2;"""
+
+    sql """select group_concat(distinct kstr order by kint), group_concat(distinct kstr2 order by kbint) from agg_group_concat_table group by kbint order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr order by kint), multi_distinct_group_concat(kstr2 order by kbint) from agg_group_concat_table group by kbint order by 1,2;"""
+    sql """select group_concat(distinct kstr), group_concat(distinct kstr2) from agg_group_concat_table group by kbint order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr), multi_distinct_group_concat(kstr2) from agg_group_concat_table group by kbint order by 1,2;"""
+
+    sql """select group_concat(distinct kstr order by kbint), group_concat(distinct kstr2 order by kint) from agg_group_concat_table group by kint order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr order by kbint), multi_distinct_group_concat(kstr2 order by kint) from agg_group_concat_table group by kint order by 1,2;"""
+    sql """select group_concat(distinct kstr), group_concat(distinct kstr2) from agg_group_concat_table group by kint order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr), multi_distinct_group_concat(kstr2) from agg_group_concat_table group by kint order by 1,2;"""
+
+    sql """select group_concat(distinct kstr order by kint), group_concat(kstr2 order by kbint) from agg_group_concat_table order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr order by kint), group_concat(kstr2 order by kbint) from agg_group_concat_table order by 1,2;"""
+    sql """select group_concat(distinct kstr), group_concat(kstr2) from agg_group_concat_table order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr), group_concat(kstr2) from agg_group_concat_table order by 1,2;"""
+
+    sql """select group_concat(distinct kstr order by kint), group_concat(kstr2 order by kbint) from agg_group_concat_table group by kbint order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr order by kint), group_concat(kstr2 order by kbint) from agg_group_concat_table group by kbint order by 1,2;"""
+    sql """select group_concat(distinct kstr), group_concat(kstr2) from agg_group_concat_table group by kbint order by 1,2;"""
+    sql """select multi_distinct_group_concat(kstr), group_concat(kstr2) from agg_group_concat_table group by kbint order by 1,2;"""
+
+    sql "drop table if exists test_distinct_multi"
+    sql """
+    create table test_distinct_multi(a int, b int, c int, d varchar(10), e date) distributed by hash(a) properties('replication_num'='1'); 
+    """
+    sql """
+    insert into test_distinct_multi values(1,2,3,'abc','2024-01-02'),(1,2,4,'abc','2024-01-03'),(2,2,4,'abcd','2024-01-02'),(1,2,3,'abcd','2024-01-04'),(1,2,4,'eee','2024-02-02'),(2,2,4,'abc','2024-01-02'); 
+    """
+    qt_test "select group_concat( distinct d order by d) from test_distinct_multi order by 1; "
 }
\ No newline at end of file

From 376706b59d867388751bfed5c0b367b738aef471 Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Mon, 16 Dec 2024 14:07:03 +0800
Subject: [PATCH 334/399] [fix](statistics)Fix skip analyze empty table case.
 (#45441)

### What problem does this PR solve?

Still drop empty table stale stats when all columns are healthy.
Fix skip analyze empty table case.

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None
---
 .../apache/doris/statistics/StatisticsAutoCollector.java  | 3 ---
 regression-test/suites/statistics/test_analyze_mv.groovy  | 8 ++++----
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 6c1fd1ba95951d..5a231db24d3798 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -144,9 +144,6 @@ protected void processOneJob(TableIf table, Set<Pair<String, String>> columns,
         columns = columns.stream().filter(
                 c -> StatisticsUtil.needAnalyzeColumn(table, c) || StatisticsUtil.isLongTimeColumn(table, c))
             .collect(Collectors.toSet());
-        if (columns.isEmpty()) {
-            return;
-        }
         AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns, priority);
         if (analyzeJob == null) {
             return;
diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy b/regression-test/suites/statistics/test_analyze_mv.groovy
index 657eb235453808..3a8f7335375aa6 100644
--- a/regression-test/suites/statistics/test_analyze_mv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mv.groovy
@@ -714,11 +714,11 @@ suite("test_analyze_mv") {
     empty_test = sql """show column stats mvTestDup"""
     for (int i = 0; i < 100; i++) {
         empty_test = sql """show column stats mvTestDup"""
-        if (empty_test.size() != 0) {
-            logger.info("async delete is not finished yet.")
-            Thread.sleep(1000)
+        if (empty_test.size() == 0) {
+            break
         }
-        break
+        logger.info("async delete stats is not finished yet.")
+        Thread.sleep(2000)
     }
     assertEquals(0, empty_test.size())
     // ** End of embedded test

From 830b74cf029b4e2dd00ff02ac79bea324262e7aa Mon Sep 17 00:00:00 2001
From: Dongyang Li <lidongyang@selectdb.com>
Date: Mon, 16 Dec 2024 14:14:47 +0800
Subject: [PATCH 335/399] [chore](ci) tmp remove required of performance
 (#45448)

---
 .asf.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.asf.yaml b/.asf.yaml
index 7a7d845e4c9bb0..e3d516b35c19a5 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -63,7 +63,6 @@ github:
           - COMPILE (DORIS_COMPILE)
           - Need_2_Approval
           - Cloud UT (Doris Cloud UT)
-          - performance (Doris Performance)
 
       required_pull_request_reviews:
         dismiss_stale_reviews: true

From b0ab7f1c9eb83f853c06bc792dd28e612aea38cc Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Mon, 16 Dec 2024 12:03:45 +0530
Subject: [PATCH 336/399] [Enhancement] (nereids)implement
 alterCatalogCommentCommand in nereids (#45160)

Issue Number: close #42790
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../apache/doris/datasource/CatalogMgr.java   | 20 ++++---
 .../nereids/parser/LogicalPlanBuilder.java    |  9 +++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/AlterCatalogCommand.java   | 58 +++++++++++++++++++
 .../commands/AlterCatalogCommentCommand.java  | 58 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../test_alter_catalog_comment_command.out    |  7 +++
 .../test_alter_catalog_comment_command.groovy | 54 +++++++++++++++++
 9 files changed, 206 insertions(+), 8 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterCatalogCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterCatalogCommentCommand.java
 create mode 100644 regression-test/data/nereids_p0/test_alter_catalog_comment_command.out
 create mode 100644 regression-test/suites/nereids_p0/test_alter_catalog_comment_command.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 274a354b63bc03..044ed7771a4aa1 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -206,6 +206,7 @@ supportedAlterStatement
     | ALTER WORKLOAD POLICY name=identifierOrText
         properties=propertyClause?                                                  #alterWorkloadPolicy
     | ALTER SQL_BLOCK_RULE name=identifier properties=propertyClause?               #alterSqlBlockRule
+    | ALTER CATALOG name=identifier MODIFY COMMENT comment=STRING_LITERAL           #alterCatalogComment
     ;
 
 supportedDropStatement
@@ -582,7 +583,6 @@ unsupportedAlterStatement
     | ALTER CATALOG name=identifier RENAME newName=identifier                       #alterCatalogRename
     | ALTER CATALOG name=identifier SET PROPERTIES
         LEFT_PAREN propertyItemList RIGHT_PAREN                                     #alterCatalogProperties
-    | ALTER CATALOG name=identifier MODIFY COMMENT comment=STRING_LITERAL           #alterCatalogComment
     | ALTER RESOURCE name=identifierOrText properties=propertyClause?               #alterResource
     | ALTER COLOCATE GROUP name=multipartIdentifier
         SET LEFT_PAREN propertyItemList RIGHT_PAREN                                 #alterColocateGroup
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 1d1a44be7b4834..070d31c071e930 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -314,17 +314,16 @@ public void alterCatalogName(AlterCatalogNameStmt stmt) throws UserException {
         }
     }
 
-    /**
-     * Modify the catalog comment to a new one and write the meta log.
-     */
-    public void alterCatalogComment(AlterCatalogCommentStmt stmt) throws UserException {
+    public void alterCatalogComment(String catalogName, String comment) throws UserException {
         writeLock();
         try {
-            CatalogIf catalog = nameToCatalog.get(stmt.getCatalogName());
+            CatalogIf catalog = nameToCatalog.get(catalogName);
             if (catalog == null) {
-                throw new DdlException("No catalog found with name: " + stmt.getCatalogName());
+                throw new DdlException("No catalog found with name: " + catalogName);
             }
-            CatalogLog log = CatalogFactory.createCatalogLog(catalog.getId(), stmt);
+            CatalogLog log = new CatalogLog();
+            log.setCatalogId(catalog.getId());
+            log.setComment(comment);
             replayAlterCatalogComment(log);
             Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_COMMENT, log);
         } finally {
@@ -332,6 +331,13 @@ public void alterCatalogComment(AlterCatalogCommentStmt stmt) throws UserExcepti
         }
     }
 
+    /**
+     * Modify the catalog comment to a new one and write the meta log.
+     */
+    public void alterCatalogComment(AlterCatalogCommentStmt stmt) throws UserException {
+        alterCatalogComment(stmt.getCatalogName(), stmt.getComment());
+    }
+
     /**
      * Modify the catalog property and write the meta log.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 56de50d620eb6d..e6fee894b41fa9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -60,6 +60,7 @@
 import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext;
 import org.apache.doris.nereids.DorisParser.AliasQueryContext;
 import org.apache.doris.nereids.DorisParser.AliasedQueryContext;
+import org.apache.doris.nereids.DorisParser.AlterCatalogCommentContext;
 import org.apache.doris.nereids.DorisParser.AlterMTMVContext;
 import org.apache.doris.nereids.DorisParser.AlterRoleContext;
 import org.apache.doris.nereids.DorisParser.AlterSqlBlockRuleContext;
@@ -482,6 +483,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
+import org.apache.doris.nereids.trees.plans.commands.AlterCatalogCommentCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterRoleCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterSqlBlockRuleCommand;
@@ -4824,6 +4826,13 @@ public LogicalPlan visitShowWhitelist(ShowWhitelistContext ctx) {
         return new ShowWhiteListCommand();
     }
 
+    @Override
+    public LogicalPlan visitAlterCatalogComment(AlterCatalogCommentContext ctx) {
+        String catalogName = stripQuotes(ctx.name.getText());
+        String comment = stripQuotes(ctx.comment.getText());
+        return new AlterCatalogCommentCommand(catalogName, comment);
+    }
+
     @Override
     public LogicalPlan visitShowDynamicPartition(ShowDynamicPartitionContext ctx) {
         String dbName = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 204d4b07111bec..e8ce4b3e2d88db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -200,6 +200,7 @@ public enum PlanType {
     DROP_USER_COMMAND,
     DROP_WORKLOAD_GROUP_NAME,
     DROP_WORKLOAD_POLICY_COMMAND,
+    ALTER_CATALOG_COMMENT_COMMAND,
     ALTER_SQL_BLOCK_RULE_COMMAND,
     SHOW_BACKENDS_COMMAND,
     SHOW_BLOCK_RULE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterCatalogCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterCatalogCommand.java
new file mode 100644
index 00000000000000..83ef02e8890375
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterCatalogCommand.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.qe.ConnectContext;
+
+import java.util.Objects;
+
+/**
+ * Base class for AlterCatalog commands.
+ */
+public abstract class AlterCatalogCommand extends AlterCommand {
+    protected final String catalogName;
+
+    public AlterCatalogCommand(PlanType type, String catalogName) {
+        super(type);
+        this.catalogName = Objects.requireNonNull(catalogName, "Catalog name cannot be null");
+    }
+
+    protected void validate(ConnectContext ctx) throws Exception {
+        Util.checkCatalogAllRules(catalogName);
+
+        if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv(ctx, catalogName, PrivPredicate.ALTER)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED,
+                    ctx.getQualifiedUser(), catalogName);
+        }
+
+        if (catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+            throw new Exception("Internal catalog cannot be altered.");
+        }
+    }
+
+    public String getCatalogName() {
+        return catalogName;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterCatalogCommentCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterCatalogCommentCommand.java
new file mode 100644
index 00000000000000..eafe1f7c2c1c4d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterCatalogCommentCommand.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+import java.util.Objects;
+
+/**
+ * Represents the command for ALTER CATALOG MODIFY COMMENT.
+ */
+public class AlterCatalogCommentCommand extends AlterCatalogCommand {
+
+    private final String comment;
+
+    public AlterCatalogCommentCommand(String catalogName, String comment) {
+        super(PlanType.ALTER_CATALOG_COMMENT_COMMAND, catalogName);
+        this.comment = Objects.requireNonNull(comment, "Comment cannot be null");
+    }
+
+    @Override
+    public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // Validate the catalog name
+        if (Strings.isNullOrEmpty(comment)) {
+            throw new AnalysisException("New comment is not set.");
+        }
+
+        // Fetch and modify the catalog's comment
+        Env.getCurrentEnv().getCatalogMgr().alterCatalogComment(catalogName, comment);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAlterCatalogCommentCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index e2ac1c863e2f5a..0341ed2f57d7ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -22,6 +22,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
+import org.apache.doris.nereids.trees.plans.commands.AlterCatalogCommentCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterJobStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterRoleCommand;
@@ -324,6 +325,10 @@ default R visitAlterViewCommand(AlterViewCommand alterViewCommand, C context) {
         return visitCommand(alterViewCommand, context);
     }
 
+    default R visitAlterCatalogCommentCommand(AlterCatalogCommentCommand alterCatalogCommentCommand, C context) {
+        return visitCommand(alterCatalogCommentCommand, context);
+    }
+
     default R visitDropCatalogRecycleBinCommand(DropCatalogRecycleBinCommand dropCatalogRecycleBinCommand, C context) {
         return visitCommand(dropCatalogRecycleBinCommand, context);
     }
diff --git a/regression-test/data/nereids_p0/test_alter_catalog_comment_command.out b/regression-test/data/nereids_p0/test_alter_catalog_comment_command.out
new file mode 100644
index 00000000000000..646f4cb278174b
--- /dev/null
+++ b/regression-test/data/nereids_p0/test_alter_catalog_comment_command.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !cmd --
+test_alter_catalog_comment	\nCREATE CATALOG `test_alter_catalog_comment`\nCOMMENT "Initial catalog comment"\n PROPERTIES (\n"type" = "es",\n"hosts" = "http://127.0.0.1:9200"\n);
+
+-- !cmd --
+test_alter_catalog_comment	\nCREATE CATALOG `test_alter_catalog_comment`\nCOMMENT "Updated catalog comment"\n PROPERTIES (\n"type" = "es",\n"hosts" = "http://127.0.0.1:9200"\n);
+
diff --git a/regression-test/suites/nereids_p0/test_alter_catalog_comment_command.groovy b/regression-test/suites/nereids_p0/test_alter_catalog_comment_command.groovy
new file mode 100644
index 00000000000000..389675aff9af23
--- /dev/null
+++ b/regression-test/suites/nereids_p0/test_alter_catalog_comment_command.groovy
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_alter_catalog_comment_command", "nereids_p0") {
+    def catalogName = "test_alter_catalog_comment"
+    def catalogProperties = "\"type\"=\"es\", \"hosts\"=\"http://127.0.0.1:9200\""
+
+    try {
+        // Drop catalog if it already exists
+        sql("DROP CATALOG IF EXISTS ${catalogName}")
+
+        // Create a new catalog
+        sql(
+            """
+            CREATE CATALOG ${catalogName} 
+            COMMENT 'Initial catalog comment'
+            PROPERTIES (${catalogProperties})
+            """
+        )
+        // Verify the catalog was created
+        checkNereidsExecute("""show create catalog ${catalogName}""")
+        qt_cmd("""show create catalog ${catalogName}""")
+
+
+        // Alter the catalog comment
+        checkNereidsExecute(
+            """
+            ALTER CATALOG ${catalogName} MODIFY COMMENT 'Updated catalog comment'
+            """
+        )
+        // Verify the comment was changed
+        checkNereidsExecute("""show create catalog ${catalogName}""")
+        qt_cmd("""show create catalog ${catalogName}""")
+
+    } finally {
+        // Clean up
+        sql("DROP CATALOG IF EXISTS ${catalogName}")
+    }
+}

From 60c3de7d7d2cbf66a4eb05978aba29c5191f5bb4 Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Mon, 16 Dec 2024 14:52:36 +0800
Subject: [PATCH 337/399] [fix](mysql-buffer) fix special buffer size with
 nested type (#45126)

### What problem does this PR solve?
in special table with sepcial data insert into
we could make sql with mysql return Malformed packet
```
mysql> select * from use1r;
ERROR 2027 (HY000): Malformed packet
```
this pr can resolve this problem
---
 be/src/util/mysql_row_buffer.cpp              |   6 +-
 .../test_special_buffer_before_nested.out     |   4 +
 .../test_special_buffer_before_nested.groovy  | 181 ++++++++++++++++++
 3 files changed, 190 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/data/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.out
 create mode 100644 regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy

diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp
index 3e20a2d9de72fe..4fd7de13753a95 100644
--- a/be/src/util/mysql_row_buffer.cpp
+++ b/be/src/util/mysql_row_buffer.cpp
@@ -107,7 +107,11 @@ MysqlRowBuffer<is_binary_format>::~MysqlRowBuffer() {
 template <bool is_binary_format>
 void MysqlRowBuffer<is_binary_format>::open_dynamic_mode() {
     if (!_dynamic_mode) {
-        *_pos++ = NEXT_EIGHT_BYTE;
+        // if _pos now exactly at the end of _buf memory,
+        // we should reserve 1 byte for _dynamic_mode flag byte to avoid *pos = 254
+        // cause _dynamic_mode flag byte be overwritten
+        reserve(1);
+        *_pos++ = NEXT_EIGHT_BYTE; // *_pos = 254 ; _pos++
         // write length when dynamic mode close
         _len_pos = (_pos - _buf);
         _pos = _pos + 8;
diff --git a/regression-test/data/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.out b/regression-test/data/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.out
new file mode 100644
index 00000000000000..8bc27332c196b8
--- /dev/null
+++ b/regression-test/data/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+71111112	2024-06-22T00:55:50	prem	\N	Sarkar sultan	\N	amorynanwaas525@gmail.com	en	1	PK	\N	Asia/Karachi	\N	https://aaa.dddddd.cccc/asgte/bbbccc/1997-11-18/abcdefghigklmnopk0f4c9cbff3b2aac.png	1	\N	\N	\N	\N	1	\N	21	\N	\N	\N	\N	1	\N	\N	\N	R+F-M-	\N	\N	\N	\N	\N	\N	\N	\N	wlg	\N	Google	\N	\N	\N	bb851bcb725b4fc8b2495bde4898e5f5	2024-12-01T11:17:42	\N	PK	\N	2024-06-24T12:52:56	\N	google	\N	0	\N	20	1	["154.81.244.112", "154.80.91.113", "154.80.72.110", "154.80.73.110", "103.131.212.96", "154.80.64.213", "103.131.212.19", "154.80.116.29", "154.80.117.215", "103.131.212.45", "154.80.115.43", "154.80.78.149", "154.80.65.149", "154.80.83.110", "154.80.67.46", "154.81.243.198", "154.80.83.118", "154.81.244.140", "154.80.105.177", "154.80.123.254", "154.80.99.141", "154.80.109.50", "154.80.100.66", "103.131.213.174", "154.80.122.66", "103.131.213.12", "154.81.243.209", "154.80.106.40", "154.80.76.77", "154.80.97.51", "154.80.112.44", "103.131.212.26", "154.81.245.155", "154.81.244.250", "154.80.71.72", "154.80.87.41", "103.131.213.24", "154.80.110.133", "154.80.109.201", "154.80.111.49", "154.80.77.246", "154.80.81.80", "154.80.64.80", "103.131.212.59", "154.80.95.80", "154.81.243.224", "154.80.64.195", "103.131.213.221", "154.80.75.211", "154.81.244.45", "154.80.69.218", "154.80.110.153", "154.80.99.153", "154.80.106.79", "103.131.212.109", "154.80.88.1", "103.131.213.82", "154.81.241.197", "154.80.107.58", "154.80.81.173", "103.131.213.185", "154.80.66.84", "103.131.212.101", "154.80.120.21", "154.80.65.158", "103.131.213.117", "103.131.213.104", "103.131.213.215", "103.131.212.252", "103.131.213.228", "103.131.212.48", "154.81.247.69", "103.131.213.78", "103.131.212.54", "103.131.213.58", "103.131.212.170", "103.131.213.234", "103.131.212.83", "154.81.244.150", "103.131.213.218", "103.131.212.120", "103.131.212.32", "103.131.213.1", "103.131.213.246", "103.131.213.21", "103.131.212.35", "103.131.212.146", "103.131.213.26", "103.131.212.80", "103.131.212.115", "103.131.212.123", "103.131.213.255", "103.131.213.247", "103.131.213.92", "103.131.213.9", "103.131.213.136", "103.131.212.192", "103.131.212.97", "103.131.212.126", "154.80.84.103", "103.131.213.159", "103.131.213.68", "103.131.212.66", "103.131.212.10", "154.80.71.170", "154.80.80.122", "154.80.109.41", "154.80.103.59", "103.131.213.89", "103.131.213.143", "154.80.78.13", "103.131.213.147", "154.80.118.77", "103.131.212.135", "154.81.245.71", "154.80.84.246", "154.80.79.246", "154.80.103.50", "154.80.87.158", "103.131.213.177", "103.131.213.16", "154.80.117.48", "103.131.213.88", "154.80.77.88", "103.131.213.73", "154.80.72.8", "154.80.87.104", "103.131.212.175", "154.80.114.89", "103.131.212.183", "103.131.213.130", "154.80.64.173", "103.131.212.236", "103.131.212.8", "103.131.212.2", "103.131.212.107", "103.131.213.17", "103.131.212.70", "103.131.212.37", "154.80.106.117", "103.131.213.149", "103.131.212.232", "103.131.213.238", "154.80.80.62", "154.80.83.83", "103.131.213.55", "103.131.213.204", "154.80.112.54", "154.80.104.6", "103.131.212.229", "154.80.90.250", "103.131.212.145", "154.80.85.250", "103.131.213.112", "154.80.84.15", "103.131.212.223", "103.131.213.84", "154.80.67.207", "154.80.67.205", "103.131.212.187", "154.80.110.200", "103.131.212.221", "154.80.75.178", "103.131.212.200", "154.80.83.226", "103.131.213.13", "103.131.213.252", "103.131.212.234", "103.131.213.18", "154.80.106.193", "103.131.212.157", "154.80.107.227", "154.80.111.155", "154.80.107.27", "103.131.212.142", "154.80.101.91", "103.131.213.154", "103.131.212.63", "103.131.213.250", "154.80.94.62", "154.80.69.143", "154.80.74.47", "154.80.81.47", "154.80.109.147", "103.131.213.231", "154.80.84.236", "154.80.120.207", "154.81.247.207", "154.80.118.95"]	154.80.118.95	154.81.244.112	["7e5007a7-09e3-4ffa-888b-4487a654c39b"]	7e5007a7-09e3-4ffa-888b-4487a654c39b	7e5007a7-09e3-4ffa-888b-4487a654c39b	vivo_vivo%2B1814	Android_Android_8.1.0	41001	US	\N	\N	\N	wlg	3.4.6	3.3.5	com.video.welive	com.video.welive	EN_wlg_zt_ray_gg_ac_2.5_Sup_all_240409_vo	\N	21174044377	\N	\N	\N	\N	\N	b3202c4487fe9728e68d748350b84f86	\N	ray	\N	zt	\N	wlg	\N	mix-EN-0409	\N	\N	\N	\N	\N	\N	\N	\N	\N	EN	\N	\N	Google	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	[59, 4]	[17, 1]	\N	\N	\N	\N	\N	\N	\N	1	2024-12-02T09:50:52
+
diff --git a/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy
new file mode 100644
index 00000000000000..b7a8572b898b2b
--- /dev/null
+++ b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy
@@ -0,0 +1,181 @@
+import org.apache.commons.lang3.StringUtils
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_special_buffer_before_nested", "p0") {
+    // note: this column order cannot be changed which with data should trigger mysql row buffer exactly reach 4096 before
+    // open_dynamic_mode to write array
+    sql """ DROP TABLE IF EXISTS test_special_buffer_before_nested """
+    sql """ CREATE TABLE `test_special_buffer_before_nested` (
+              `account_id` bigint NOT NULL,
+              `register_time` datetime NULL,
+              `nickname` varchar(1024) NULL,
+              `facebook_name` varchar(1024) NULL,
+              `google_name` varchar(1024) NULL,
+              `twitter_name` varchar(1024) NULL,
+              `email` varchar(1024) NULL,
+              `language` varchar(1024) NULL,
+              `gender` int NULL,
+              `country` varchar(1024) NULL,
+              `phone` varchar(1024) NULL,
+              `timezone` varchar(1024) NULL,
+              `sponsor` varchar(1024) NULL,
+              `avatar` varchar(65533) NULL,
+              `certification` int NULL,
+              `suspicious_user` int NULL,
+              `audit_user` int NULL,
+              `tt_audit_user` int NULL,
+              `auto_invite_bind` int NULL,
+              `third_party_payment` int NULL,
+              `official_payment` int NULL,
+              `user_security_level` int NULL,
+              `vip_end_time` datetime NULL,
+              `agent` int NULL,
+              `user_group_level` int NULL,
+              `nearby_toggle` int NULL,
+              `open_msg_notification` int NULL,
+              `recall_time` datetime NULL,
+              `first_recharge_time` datetime NULL,
+              `total_recharge_amount` double NULL,
+              `user_rfm_level` varchar(1024) NULL,
+              `recharge_credit_amount` double NULL,
+              `recharge_credit_count` int NULL,
+              `recharge_credit_no_callback_count` int NULL,
+              `total_withdraw_amount` double NULL,
+              `withdraw_list` array<int> NULL,
+              `user_charm_level` int NULL,
+              `user_charm_history_level` int NULL,
+              `user_auto_greetings_status` int NULL,
+              `channel` varchar(1024) NULL,
+              `bundle_id` varchar(1024) NULL,
+              `platform` varchar(1024) NULL,
+              `campaign_group_name` varchar(1024) NULL,
+              `similar_group_id` varchar(1024) NULL,
+              `cancel_account_time` datetime NULL,
+              `log_seq` varchar(1024) NULL,
+              `date_time` datetime NULL,
+              `time_millis` bigint NULL,
+              `first_country` varchar(1024) NULL,
+              `invite_time` datetime NULL,
+              `certification_time` datetime NULL,
+              `certification_url` varchar(65533) NULL,
+              `login_type` varchar(1024) NULL,
+              `vip_start_time` datetime NULL,
+              `vip_type` int NULL,
+              `agent_time` datetime NULL,
+              `recall_count` int NULL,
+              `wake_count` int NULL,
+              `client_ips` array<varchar(1024)> NULL,
+              `client_ip` varchar(1024) NULL,
+              `first_client_ip` varchar(1024) NULL,
+              `device_ids` array<varchar(1024)> NULL,
+              `device_id` varchar(1024) NULL,
+              `first_device_id` varchar(1024) NULL,
+              `phone_model` varchar(1024) NULL,
+              `os_version` varchar(1024) NULL,
+              `mcc_mnc` varchar(1024) NULL,
+              `sys_region` varchar(1024) NULL,
+              `carrier_region` varchar(1024) NULL,
+              `cloud_id` varchar(1024) NULL,
+              `device_name` varchar(1024) NULL,
+              `first_channel` varchar(1024) NULL,
+              `version` varchar(1024) NULL,
+              `first_version` varchar(1024) NULL,
+              `app_name` varchar(1024) NULL,
+              `first_app_name` varchar(1024) NULL,
+              `ad_campaign` varchar(65533) NULL,
+              `last_ad_campaign` varchar(65533) NULL,
+              `ad_campaign_id` varchar(65533) NULL,
+              `last_ad_campaign_id` varchar(65533) NULL,
+              `campaign` varchar(65533) NULL,
+              `last_campaign` varchar(65533) NULL,
+              `campaign_time` datetime NULL,
+              `last_campaign_time` datetime NULL,
+              `ad_name` varchar(65533) NULL,
+              `last_ad_name` varchar(65533) NULL,
+              `ad_advertiser` varchar(1024) NULL,
+              `last_ad_advertiser` varchar(1024) NULL,
+              `ad_advertiser_team` varchar(1024) NULL,
+              `last_ad_advertiser_team` varchar(1024) NULL,
+              `ad_app` varchar(1024) NULL,
+              `last_ad_app` varchar(1024) NULL,
+              `ad_group` varchar(65533) NULL,
+              `last_ad_group` varchar(65533) NULL,
+              `adid` varchar(65533) NULL,
+              `last_adid` varchar(65533) NULL,
+              `creative` varchar(65533) NULL,
+              `last_creative` varchar(65533) NULL,
+              `match_type` varchar(1024) NULL,
+              `last_match_type` varchar(1024) NULL,
+              `network` varchar(1024) NULL,
+              `last_network` varchar(1024) NULL,
+              `ad_country` varchar(1024) NULL,
+              `last_ad_country` varchar(1024) NULL,
+              `last_platform` varchar(1024) NULL,
+              `ad_platform` varchar(1024) NULL,
+              `last_ad_platform` varchar(1024) NULL,
+              `install_begin_time` datetime NULL,
+              `last_install_begin_time` datetime NULL,
+              `installed_at` datetime NULL,
+              `last_installed_at` datetime NULL,
+              `install_finish_time` datetime NULL,
+              `last_install_finish_time` datetime NULL,
+              `click_time` datetime NULL,
+              `last_click_time` datetime NULL,
+              `referral_time` datetime NULL,
+              `last_referral_time` datetime NULL,
+              `registration_cost` double NULL,
+              `abtest_sub` array<int> NULL,
+              `abtest_id` array<int> NULL,
+              `block_level_upgrade` int NULL,
+              `passcard_grade` int NULL,
+              `passcard_grade_end_time` datetime NULL,
+              `passcard_pay` array<datetime> NULL,
+              `passcard_number` datetime NULL,
+              `vcast_grade_charm` int NULL,
+              `vcast_grade_wealth` int NULL,
+              `im_chat_violation` int NULL,
+              `process_time` datetime NULL DEFAULT CURRENT_TIMESTAMP
+            ) ENGINE=OLAP
+            UNIQUE KEY(`account_id`)
+            DISTRIBUTED BY HASH(`account_id`) BUCKETS 64
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "min_load_replica_num" = "-1",
+            "is_being_synced" = "false",
+            "storage_medium" = "hdd",
+            "storage_format" = "V2",
+            "inverted_index_storage_format" = "V1",
+            "enable_unique_key_merge_on_write" = "true",
+            "light_schema_change" = "true",
+            "store_row_column" = "true",
+            "row_store_page_size" = "16384",
+            "disable_auto_compaction" = "false",
+            "enable_single_replica_compaction" = "false",
+            "enable_mow_light_delete" = "false"
+            );
+            """
+    sql """ truncate table test_special_buffer_before_nested """
+    sql """ insert into test_special_buffer_before_nested values(71111112,'2024-06-22 00:55:50','prem',NULL,'Sarkar sultan',NULL,'amorynanwaas525@gmail.com','en',1,'PK',NULL,'Asia/Karachi',NULL,'https://aaa.dddddd.cccc/asgte/bbbccc/1997-11-18/abcdefghigklmnopk0f4c9cbff3b2aac.png',1
+            ,NULL,NULL,NULL,NULL,1,NULL,21,NULL,NULL,NULL,NULL,1,NULL,NULL,NULL,'R+F-M-',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'wlg',NULL,'Google',NULL,NULL,NULL,'bb851bcb725b4fc8b2495bde4898e5f5','2024-12-01 11:17:42',NULL,'PK',NULL,'2024-06-24 12:52:56',
+            NULL,'google',NULL,0,NULL,20,1,'["154.81.244.112", "154.80.91.113", "154.80.72.110", "154.80.73.110", "103.131.212.96", "154.80.64.213", "103.131.212.19", "154.80.116.29", "154.80.117.215", "103.131.212.45", "154.80.115.43", "154.80.78.149", "154.80.65.149", "154.80.83.110", "154.80.67.46", "154.81.243.198", "154.80.83.118", "154.81.244.140", "154.80.105.177", "154.80.123.254", "154.80.99.141", "154.80.109.50", "154.80.100.66", "103.131.213.174", "154.80.122.66", "103.131.213.12", "154.81.243.209", "154.80.106.40", "154.80.76.77", "154.80.97.51", "154.80.112.44", "103.131.212.26", "154.81.245.155", "154.81.244.250", "154.80.71.72", "154.80.87.41", "103.131.213.24", "154.80.110.133", "154.80.109.201", "154.80.111.49", "154.80.77.246", "154.80.81.80", "154.80.64.80", "103.131.212.59", "154.80.95.80", "154.81.243.224", "154.80.64.195", "103.131.213.221", "154.80.75.211", "154.81.244.45", "154.80.69.218", "154.80.110.153", "154.80.99.153", "154.80.106.79", "103.131.212.109", "154.80.88.1", "103.131.213.82", "154.81.241.197", "154.80.107.58", "154.80.81.173", "103.131.213.185", "154.80.66.84", "103.131.212.101", "154.80.120.21", "154.80.65.158", "103.131.213.117", "103.131.213.104", "103.131.213.215", "103.131.212.252", "103.131.213.228", "103.131.212.48", "154.81.247.69", "103.131.213.78", "103.131.212.54", "103.131.213.58", "103.131.212.170", "103.131.213.234", "103.131.212.83", "154.81.244.150", "103.131.213.218", "103.131.212.120", "103.131.212.32", "103.131.213.1", "103.131.213.246", "103.131.213.21", "103.131.212.35", "103.131.212.146", "103.131.213.26", "103.131.212.80", "103.131.212.115", "103.131.212.123", "103.131.213.255", "103.131.213.247", "103.131.213.92", "103.131.213.9", "103.131.213.136", "103.131.212.192", "103.131.212.97", "103.131.212.126", "154.80.84.103", "103.131.213.159", "103.131.213.68", "103.131.212.66", "103.131.212.10", "154.80.71.170", "154.80.80.122", "154.80.109.41", "154.80.103.59", "103.131.213.89", "103.131.213.143", "154.80.78.13", "103.131.213.147", "154.80.118.77", "103.131.212.135", "154.81.245.71", "154.80.84.246", "154.80.79.246", "154.80.103.50", "154.80.87.158", "103.131.213.177", "103.131.213.16", "154.80.117.48", "103.131.213.88", "154.80.77.88", "103.131.213.73", "154.80.72.8", "154.80.87.104", "103.131.212.175", "154.80.114.89", "103.131.212.183", "103.131.213.130", "154.80.64.173", "103.131.212.236", "103.131.212.8", "103.131.212.2", "103.131.212.107", "103.131.213.17", "103.131.212.70", "103.131.212.37", "154.80.106.117", "103.131.213.149", "103.131.212.232", "103.131.213.238", "154.80.80.62", "154.80.83.83", "103.131.213.55", "103.131.213.204", "154.80.112.54", "154.80.104.6", "103.131.212.229", "154.80.90.250", "103.131.212.145", "154.80.85.250", "103.131.213.112", "154.80.84.15", "103.131.212.223", "103.131.213.84", "154.80.67.207", "154.80.67.205", "103.131.212.187", "154.80.110.200", "103.131.212.221", "154.80.75.178", "103.131.212.200", "154.80.83.226", "103.131.213.13", "103.131.213.252", "103.131.212.234", "103.131.213.18", "154.80.106.193", "103.131.212.157", "154.80.107.227", "154.80.111.155", "154.80.107.27", "103.131.212.142", "154.80.101.91", "103.131.213.154", "103.131.212.63", "103.131.213.250", "154.80.94.62", "154.80.69.143", "154.80.74.47", "154.80.81.47", "154.80.109.147", "103.131.213.231", "154.80.84.236", "154.80.120.207", "154.81.247.207", "154.80.118.95"]',
+            '154.80.118.95','154.81.244.112','["7e5007a7-09e3-4ffa-888b-4487a654c39b"]','7e5007a7-09e3-4ffa-888b-4487a654c39b','7e5007a7-09e3-4ffa-888b-4487a654c39b','vivo_vivo%2B1814','Android_Android_8.1.0','41001','US',NULL,NULL,NULL,'wlg','3.4.6','3.3.5','com.video.welive','com.video.welive',
+            'EN_wlg_zt_ray_gg_ac_2.5_Sup_all_240409_vo',NULL,'21174044377',NULL,NULL,NULL,NULL,NULL,'b3202c4487fe9728e68d748350b84f86',NULL,'ray',NULL,'zt',NULL,'wlg',NULL,'mix-EN-0409',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'EN',NULL,NULL,'Google',
+            NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'[59, 4]','[17, 1]',NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,'2024-12-02 09:50:52'); """
+    qt_sql """ select * from  test_special_buffer_before_nested """
+}

From 9231b7d50e6df0987bebac9e66feae950cb4368d Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Mon, 16 Dec 2024 15:22:28 +0800
Subject: [PATCH 338/399] [opt](exec)Optimize SetOperator by removing
 unnecessary maintained elements. (#45195)

### What problem does this PR solve?

Previously, the Set operator needed to maintain a RowRefListWithFlag,
but in fact, only a "visited" flag is needed.

before
```
mysql [test]>select count(u64) from( select u64 from nums1 except DISTINCT select u64 from nums2 )t;
+------------+
| count(u64) |
+------------+
|          0 |
+------------+
1 row in set (3.97 sec)
```
now
```
mysql [test]>select count(u64) from( select u64 from nums1 except DISTINCT select u64 from nums2 )t;
+------------+
| count(u64) |
+------------+
|          0 |
+------------+
1 row in set (3.01 sec)
```
### Release note
---
 be/src/pipeline/common/set_utils.h            |  16 +-
 be/src/pipeline/exec/join/join_op.h           | 182 ------------------
 .../exec/join/process_hash_table_probe.h      |   1 -
 .../pipeline/exec/set_probe_sink_operator.cpp |   4 +-
 be/src/pipeline/exec/set_source_operator.cpp  |  12 +-
 be/src/pipeline/exec/set_source_operator.h    |   4 +-
 .../common/hash_table/hash_table_set_probe.h  |   2 +-
 7 files changed, 21 insertions(+), 200 deletions(-)
 delete mode 100644 be/src/pipeline/exec/join/join_op.h

diff --git a/be/src/pipeline/common/set_utils.h b/be/src/pipeline/common/set_utils.h
index 2caf5b7d0b814c..38a82a501ff534 100644
--- a/be/src/pipeline/common/set_utils.h
+++ b/be/src/pipeline/common/set_utils.h
@@ -20,13 +20,20 @@
 #include <variant>
 #include <vector>
 
-#include "pipeline/exec/join/join_op.h"
 #include "vec/common/hash_table/hash_map_util.h"
 
 namespace doris {
 
+struct RowRefWithFlag {
+    bool visited;
+    uint32_t row_num = 0;
+    RowRefWithFlag() = default;
+    RowRefWithFlag(size_t row_num_count, bool is_visited = false)
+            : visited(is_visited), row_num(row_num_count) {}
+};
+
 template <typename T>
-using SetData = PHHashMap<T, RowRefListWithFlags, HashCRC32<T>>;
+using SetData = PHHashMap<T, RowRefWithFlag, HashCRC32<T>>;
 
 template <typename T>
 using SetFixedKeyHashTableContext = vectorized::MethodKeysFixed<SetData<T>>;
@@ -39,9 +46,8 @@ using SetPrimaryTypeHashTableContextNullable = vectorized::MethodSingleNullableC
         vectorized::MethodOneNumber<T, vectorized::DataWithNullKey<SetData<T>>>>;
 
 using SetSerializedHashTableContext =
-        vectorized::MethodSerialized<PHHashMap<StringRef, RowRefListWithFlags>>;
-using SetMethodOneString =
-        vectorized::MethodStringNoCache<PHHashMap<StringRef, RowRefListWithFlags>>;
+        vectorized::MethodSerialized<PHHashMap<StringRef, RowRefWithFlag>>;
+using SetMethodOneString = vectorized::MethodStringNoCache<PHHashMap<StringRef, RowRefWithFlag>>;
 
 using SetHashTableVariants =
         std::variant<std::monostate, SetSerializedHashTableContext, SetMethodOneString,
diff --git a/be/src/pipeline/exec/join/join_op.h b/be/src/pipeline/exec/join/join_op.h
deleted file mode 100644
index f3bd47a911eb32..00000000000000
--- a/be/src/pipeline/exec/join/join_op.h
+++ /dev/null
@@ -1,182 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-#include "vec/common/arena.h"
-#include "vec/common/columns_hashing.h"
-#include "vec/core/block.h"
-
-namespace doris {
-/**
- * Now we have different kinds of RowRef for join operation. Overall, RowRef is the base class and
- * the class inheritance is below:
- *                                         RowRef
- *                                           |
- *          ---------------------------------------------------------
- *          |                          |                            |
- *   RowRefListWithFlag           RowRefList                 RowRefWithFlag
- *                                                                 |
- *                                                         RowRefListWithFlags
- *
- *  RowRef is a basic representation for a row which contains only row_num and block_offset.
- *
- *  RowRefList is a list of many RowRefs. It used for join operations which doesn't need any flags to represent whether a row has already been visited.
- *
- *  RowRefListWithFlag is a list of many RowRefs and an extra visited flag. It used for join operations which all RowRefs in a list has the same visited flag.
- *
- *  RowRefWithFlag is a basic representation for a row with an extra visited flag.
- *
- *  RowRefListWithFlags is a list of many RowRefWithFlags. This means each row will have different visited flags. It's used for join operation which has `other_conjuncts`.
- */
-struct RowRef {
-    uint32_t row_num = 0;
-
-    RowRef() = default;
-    RowRef(size_t row_num_count) : row_num(row_num_count) {}
-    void clear() {};
-};
-
-struct RowRefWithFlag : public RowRef {
-    bool visited;
-
-    RowRefWithFlag() = default;
-    RowRefWithFlag(size_t row_num_count, bool is_visited = false)
-            : RowRef(row_num_count), visited(is_visited) {}
-    void clear() {};
-};
-
-/// Portion of RowRefs, 16 * (MAX_SIZE + 1) bytes sized.
-template <typename RowRefType>
-struct Batch {
-    static constexpr uint32_t MAX_SIZE = 7; /// Adequate values are 3, 7, 15, 31.
-
-    uint8_t size = 0; /// It's smaller than size_t but keeps align in Arena.
-    Batch<RowRefType>* next = nullptr;
-    RowRefType row_refs[MAX_SIZE];
-
-    Batch(Batch<RowRefType>* parent) : next(parent) {}
-
-    bool full() const { return size == MAX_SIZE; }
-
-    Batch<RowRefType>* insert(RowRefType&& row_ref, vectorized::Arena& pool) {
-        if (full()) {
-            auto batch = pool.alloc<Batch<RowRefType>>();
-            *batch = Batch<RowRefType>(this);
-            batch->insert(std::move(row_ref), pool);
-            return batch;
-        }
-
-        row_refs[size++] = std::move(row_ref);
-        return this;
-    }
-};
-
-template <typename RowRefListType>
-class ForwardIterator {
-public:
-    using RowRefType = typename RowRefListType::RowRefType;
-    ForwardIterator() : root(nullptr), first(false), batch(nullptr), position(0) {}
-
-    ForwardIterator(RowRefListType* begin)
-            : root(begin), first(true), batch((&root->next)), position(0) {}
-
-    RowRefType& operator*() {
-        if (first) {
-            return *root;
-        }
-        return batch->operator[](position);
-    }
-
-    RowRefType* operator->() { return &(**this); }
-
-    void operator++() {
-        if (first) {
-            first = false;
-            return;
-        }
-
-        if (batch && position < batch->size()) {
-            ++position;
-        }
-    }
-
-    bool ok() const { return first || (batch && position < batch->size()); }
-
-private:
-    RowRefListType* root = nullptr;
-    bool first;
-    std::vector<RowRefType>* batch = nullptr;
-    size_t position;
-};
-
-struct RowRefList : RowRef {
-    using RowRefType = RowRef;
-
-    RowRefList() = default;
-    RowRefList(size_t row_num_) : RowRef(row_num_) {}
-
-    ForwardIterator<RowRefList> begin() { return {this}; }
-
-    /// insert element after current one
-    void insert(RowRefType&& row_ref, vectorized::Arena& pool) { next.emplace_back(row_ref); }
-
-    void clear() { next.clear(); }
-
-private:
-    friend class ForwardIterator<RowRefList>;
-    std::vector<RowRefType> next;
-};
-
-struct RowRefListWithFlag : RowRef {
-    using RowRefType = RowRef;
-
-    RowRefListWithFlag() = default;
-    RowRefListWithFlag(size_t row_num_) : RowRef(row_num_) {}
-
-    ForwardIterator<RowRefListWithFlag> begin() { return {this}; }
-
-    /// insert element after current one
-    void insert(RowRefType&& row_ref, vectorized::Arena& pool) { next.emplace_back(row_ref); }
-
-    void clear() { next.clear(); }
-
-    bool visited = false;
-
-private:
-    friend class ForwardIterator<RowRefListWithFlag>;
-    std::vector<RowRefType> next;
-};
-
-struct RowRefListWithFlags : RowRefWithFlag {
-    using RowRefType = RowRefWithFlag;
-
-    RowRefListWithFlags() = default;
-    RowRefListWithFlags(size_t row_num_) : RowRefWithFlag(row_num_) {}
-
-    ForwardIterator<RowRefListWithFlags> begin() { return {this}; }
-
-    /// insert element after current one
-    void insert(RowRefType&& row_ref, vectorized::Arena& pool) { next.emplace_back(row_ref); }
-
-    void clear() { next.clear(); }
-
-private:
-    friend class ForwardIterator<RowRefListWithFlags>;
-    std::vector<RowRefType> next;
-};
-
-} // namespace doris
diff --git a/be/src/pipeline/exec/join/process_hash_table_probe.h b/be/src/pipeline/exec/join/process_hash_table_probe.h
index 14e0edd977f57b..91fd82f0644939 100644
--- a/be/src/pipeline/exec/join/process_hash_table_probe.h
+++ b/be/src/pipeline/exec/join/process_hash_table_probe.h
@@ -19,7 +19,6 @@
 
 #include <vector>
 
-#include "join_op.h"
 #include "vec/columns/column.h"
 #include "vec/columns/columns_number.h"
 #include "vec/common/arena.h"
diff --git a/be/src/pipeline/exec/set_probe_sink_operator.cpp b/be/src/pipeline/exec/set_probe_sink_operator.cpp
index 4c250d5603b499..9ca87db925a44a 100644
--- a/be/src/pipeline/exec/set_probe_sink_operator.cpp
+++ b/be/src/pipeline/exec/set_probe_sink_operator.cpp
@@ -231,7 +231,7 @@ void SetProbeSinkOperatorX<is_intersect>::_refresh_hash_table(
                                 local_state._shared_state->valid_element_in_hash_tbl);
                         while (iter != iter_end) {
                             auto& mapped = iter->get_second();
-                            auto it = mapped.begin();
+                            auto* it = &mapped;
 
                             if constexpr (is_intersect) {
                                 if (it->visited) {
@@ -249,7 +249,7 @@ void SetProbeSinkOperatorX<is_intersect>::_refresh_hash_table(
                     } else if (is_intersect) {
                         while (iter != iter_end) {
                             auto& mapped = iter->get_second();
-                            auto it = mapped.begin();
+                            auto* it = &mapped;
                             it->visited = false;
                             ++iter;
                         }
diff --git a/be/src/pipeline/exec/set_source_operator.cpp b/be/src/pipeline/exec/set_source_operator.cpp
index 91c98288d8b8e1..b31ec157bb877e 100644
--- a/be/src/pipeline/exec/set_source_operator.cpp
+++ b/be/src/pipeline/exec/set_source_operator.cpp
@@ -128,7 +128,7 @@ Status SetSourceOperatorX<is_intersect>::_get_data_in_hashtable(
     auto block_size = 0;
 
     auto add_result = [&local_state, &block_size, this](auto value) {
-        auto it = value.begin();
+        auto* it = &value;
         if constexpr (is_intersect) {
             if (it->visited) { //intersected: have done probe, so visited values it's the result
                 _add_result_columns(local_state, value, block_size);
@@ -147,8 +147,8 @@ Status SetSourceOperatorX<is_intersect>::_get_data_in_hashtable(
 
     *eos = iter == hash_table_ctx.hash_table->end();
     if (*eos && hash_table_ctx.hash_table->has_null_key_data()) {
-        auto value = hash_table_ctx.hash_table->template get_null_key_data<RowRefListWithFlags>();
-        if constexpr (std::is_same_v<RowRefListWithFlags, std::decay_t<decltype(value)>>) {
+        auto value = hash_table_ctx.hash_table->template get_null_key_data<RowRefWithFlag>();
+        if constexpr (std::is_same_v<RowRefWithFlag, std::decay_t<decltype(value)>>) {
             add_result(value);
         }
     }
@@ -168,15 +168,13 @@ Status SetSourceOperatorX<is_intersect>::_get_data_in_hashtable(
 
 template <bool is_intersect>
 void SetSourceOperatorX<is_intersect>::_add_result_columns(
-        SetSourceLocalState<is_intersect>& local_state, RowRefListWithFlags& value,
-        int& block_size) {
+        SetSourceLocalState<is_intersect>& local_state, RowRefWithFlag& value, int& block_size) {
     auto& build_col_idx = local_state._shared_state->build_col_idx;
     auto& build_block = local_state._shared_state->build_block;
 
-    auto it = value.begin();
     for (auto idx = build_col_idx.begin(); idx != build_col_idx.end(); ++idx) {
         auto& column = *build_block.get_by_position(idx->second).column;
-        local_state._mutable_cols[idx->first]->insert_from(column, it->row_num);
+        local_state._mutable_cols[idx->first]->insert_from(column, value.row_num);
     }
     block_size++;
 }
diff --git a/be/src/pipeline/exec/set_source_operator.h b/be/src/pipeline/exec/set_source_operator.h
index 976ffde3bf23ea..d881e9277fb7b6 100644
--- a/be/src/pipeline/exec/set_source_operator.h
+++ b/be/src/pipeline/exec/set_source_operator.h
@@ -83,8 +83,8 @@ class SetSourceOperatorX final : public OperatorX<SetSourceLocalState<is_interse
                                   HashTableContext& hash_table_ctx, vectorized::Block* output_block,
                                   const int batch_size, bool* eos);
 
-    void _add_result_columns(SetSourceLocalState<is_intersect>& local_state,
-                             RowRefListWithFlags& value, int& block_size);
+    void _add_result_columns(SetSourceLocalState<is_intersect>& local_state, RowRefWithFlag& value,
+                             int& block_size);
     const size_t _child_quantity;
 };
 #include "common/compile_check_end.h"
diff --git a/be/src/vec/common/hash_table/hash_table_set_probe.h b/be/src/vec/common/hash_table/hash_table_set_probe.h
index ded9dcaa7b1024..943aa5cfe3775d 100644
--- a/be/src/vec/common/hash_table/hash_table_set_probe.h
+++ b/be/src/vec/common/hash_table/hash_table_set_probe.h
@@ -38,7 +38,7 @@ struct HashTableProbe {
         for (int probe_index = 0; probe_index < _probe_rows; probe_index++) {
             auto find_result = hash_table_ctx.find(key_getter, probe_index);
             if (find_result.is_found()) { //if found, marked visited
-                auto it = find_result.get_mapped().begin();
+                auto* it = &find_result.get_mapped();
                 if (!(it->visited)) {
                     it->visited = true;
                     if constexpr (is_intersected) { //intersected

From f49e609ceaa37d4b29cf3ee4562a3c47576c5f04 Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Mon, 16 Dec 2024 15:34:39 +0800
Subject: [PATCH 339/399] [feature](mtmv)Support iceberg partition refresh.
 (#44726)

### What problem does this PR solve?

Previously, when using Iceberg to create MTMV, it was not possible to
perceive changes in partition lists and data, so only ```refresh
materialized view mv1 complete ```could be used to force full refresh.

This PR obtains the partition list of Iceberg, the last update time of
the partition, and the latest snapshotId of the table.

Therefore, MTMV can be partition based on Iceberg tables and perceive
changes in data, automatically refreshing partitions

For now, we only support single partition column table and the partition
transform must one of hour, day, month or year.
Will support Identity transform soon.

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None
---
 .../doris/catalog/RangePartitionItem.java     |   7 +-
 .../iceberg/IcebergExternalTable.java         | 439 +++++++++++++++++-
 .../datasource/iceberg/IcebergPartition.java  |  82 ++++
 .../iceberg/IcebergPartitionInfo.java         |  71 +++
 .../iceberg/IcebergSchemaCacheValue.java      |  50 ++
 .../doris/job/extensions/mtmv/MTMVTask.java   |   7 +
 .../apache/doris/mtmv/MTMVPartitionUtil.java  |   2 +-
 .../apache/doris/mtmv/MTMVRelatedTableIf.java |   9 +
 .../iceberg/IcebergExternalTableTest.java     | 238 ++++++++++
 .../iceberg/IcebergPartitionInfoTest.java     |  53 +++
 .../data/mtmv_p0/test_iceberg_mtmv.out        |  98 ++++
 .../suites/mtmv_p0/test_iceberg_mtmv.groovy   | 147 ++++++
 12 files changed, 1199 insertions(+), 4 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheValue.java
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java
index b5e90634b8bbb1..690ab88991bd16 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionItem.java
@@ -60,9 +60,14 @@ public boolean isDefaultPartition() {
 
     @Override
     public PartitionKeyDesc toPartitionKeyDesc() {
-        return PartitionKeyDesc.createFixed(
+        if (partitionKeyRange.hasLowerBound()) {
+            return PartitionKeyDesc.createFixed(
                 PartitionInfo.toPartitionValue(partitionKeyRange.lowerEndpoint()),
                 PartitionInfo.toPartitionValue(partitionKeyRange.upperEndpoint()));
+        } else {
+            // For null partition value.
+            return PartitionKeyDesc.createLessThan(PartitionInfo.toPartitionValue(partitionKeyRange.upperEndpoint()));
+        }
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
index feded88ea326f0..e259399f63740b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
@@ -17,9 +17,24 @@
 
 package org.apache.doris.datasource.iceberg;
 
+import org.apache.doris.analysis.PartitionValue;
 import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.PartitionType;
+import org.apache.doris.catalog.RangePartitionItem;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.SchemaCacheValue;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
+import org.apache.doris.mtmv.MTMVBaseTableIf;
+import org.apache.doris.mtmv.MTMVRefreshContext;
+import org.apache.doris.mtmv.MTMVRelatedTableIf;
+import org.apache.doris.mtmv.MTMVSnapshotIf;
+import org.apache.doris.mtmv.MTMVVersionSnapshot;
 import org.apache.doris.statistics.AnalysisInfo;
 import org.apache.doris.statistics.BaseAnalysisTask;
 import org.apache.doris.statistics.ExternalAnalysisTask;
@@ -28,13 +43,53 @@
 import org.apache.doris.thrift.TTableDescriptor;
 import org.apache.doris.thrift.TTableType;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
+import com.google.common.collect.Sets;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.MetadataTableType;
+import org.apache.iceberg.MetadataTableUtils;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionsTable;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
 import org.apache.iceberg.Table;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.StructProjection;
 
+import java.io.IOException;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.Month;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 
-public class IcebergExternalTable extends ExternalTable {
+public class IcebergExternalTable extends ExternalTable implements MTMVRelatedTableIf, MTMVBaseTableIf {
+
+    public static final String YEAR = "year";
+    public static final String MONTH = "month";
+    public static final String DAY = "day";
+    public static final String HOUR = "hour";
+    public static final String IDENTITY = "identity";
+    public static final int PARTITION_DATA_ID_START = 1000; // org.apache.iceberg.PartitionSpec
+
+    private Table table;
+    private List<Column> partitionColumns;
+    private boolean isValidRelatedTableCached = false;
+    private boolean isValidRelatedTable = false;
 
     public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) {
         super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE);
@@ -51,9 +106,50 @@ protected synchronized void makeSureInitialized() {
         }
     }
 
+    @VisibleForTesting
+    public void setTable(Table table) {
+        this.table = table;
+    }
+
+    @VisibleForTesting
+    public void setPartitionColumns(List<Column> partitionColumns) {
+        this.partitionColumns = partitionColumns;
+    }
+
     @Override
     public Optional<SchemaCacheValue> initSchema() {
-        return Optional.of(new SchemaCacheValue(IcebergUtils.getSchema(catalog, dbName, name)));
+        table = IcebergUtils.getIcebergTable(catalog, dbName, name);
+        List<Column> schema = IcebergUtils.getSchema(catalog, dbName, name);
+        Snapshot snapshot = table.currentSnapshot();
+        if (snapshot == null) {
+            LOG.debug("Table {} is empty", name);
+            return Optional.of(new IcebergSchemaCacheValue(schema, null, -1, null));
+        }
+        long snapshotId = snapshot.snapshotId();
+        partitionColumns = null;
+        IcebergPartitionInfo partitionInfo = null;
+        if (isValidRelatedTable()) {
+            PartitionSpec spec = table.spec();
+            partitionColumns = Lists.newArrayList();
+
+            // For iceberg table, we only support table with 1 partition column as RelatedTable.
+            // So we use spec.fields().get(0) to get the partition column.
+            Types.NestedField col = table.schema().findField(spec.fields().get(0).sourceId());
+            for (Column c : schema) {
+                if (c.getName().equalsIgnoreCase(col.name())) {
+                    partitionColumns.add(c);
+                    break;
+                }
+            }
+            Preconditions.checkState(partitionColumns.size() == 1,
+                    "Support 1 partition column for iceberg table, but found " + partitionColumns.size());
+            try {
+                partitionInfo = loadPartitionInfo();
+            } catch (AnalysisException e) {
+                LOG.warn("Failed to load iceberg table {} partition info.", name, e);
+            }
+        }
+        return Optional.of(new IcebergSchemaCacheValue(schema, partitionColumns, snapshotId, partitionInfo));
     }
 
     @Override
@@ -90,4 +186,343 @@ public long fetchRowCount() {
     public Table getIcebergTable() {
         return IcebergUtils.getIcebergTable(getCatalog(), getDbName(), getName());
     }
+
+    @Override
+    public void beforeMTMVRefresh(MTMV mtmv) throws DdlException {
+        Env.getCurrentEnv().getRefreshManager()
+            .refreshTable(getCatalog().getName(), getDbName(), getName(), true);
+    }
+
+    @Override
+    public Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot) {
+        return Maps.newHashMap(getPartitionInfoFromCache().getNameToPartitionItem());
+    }
+
+    private IcebergPartitionInfo getPartitionInfoFromCache() {
+        makeSureInitialized();
+        Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
+        if (!schemaCacheValue.isPresent()) {
+            return new IcebergPartitionInfo();
+        }
+        return ((IcebergSchemaCacheValue) schemaCacheValue.get()).getPartitionInfo();
+    }
+
+    @Override
+    public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
+        makeSureInitialized();
+        return isValidRelatedTable() ? PartitionType.RANGE : PartitionType.UNPARTITIONED;
+    }
+
+    @Override
+    public Set<String> getPartitionColumnNames(Optional<MvccSnapshot> snapshot) throws DdlException {
+        return getPartitionColumnsFromCache().stream().map(Column::getName).collect(Collectors.toSet());
+    }
+
+    @Override
+    public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
+        return getPartitionColumnsFromCache();
+    }
+
+    private List<Column> getPartitionColumnsFromCache() {
+        makeSureInitialized();
+        Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
+        return schemaCacheValue
+                .map(cacheValue -> ((IcebergSchemaCacheValue) cacheValue).getPartitionColumns())
+                .orElseGet(Lists::newArrayList);
+    }
+
+    @Override
+    public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
+                                               Optional<MvccSnapshot> snapshot) throws AnalysisException {
+        long latestSnapshotId = getPartitionInfoFromCache().getLatestSnapshotId(partitionName);
+        if (latestSnapshotId <= 0) {
+            throw new AnalysisException("can not find partition: " + partitionName);
+        }
+        return new MTMVVersionSnapshot(latestSnapshotId);
+    }
+
+    @Override
+    public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
+            throws AnalysisException {
+        return new MTMVVersionSnapshot(getLatestSnapshotIdFromCache());
+    }
+
+    public long getLatestSnapshotIdFromCache() throws AnalysisException {
+        makeSureInitialized();
+        Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
+        if (!schemaCacheValue.isPresent()) {
+            throw new AnalysisException("Can't find schema cache of table " + name);
+        }
+        return ((IcebergSchemaCacheValue) schemaCacheValue.get()).getSnapshotId();
+    }
+
+    @Override
+    public boolean isPartitionColumnAllowNull() {
+        return true;
+    }
+
+    /**
+     * For now, we only support single partition column Iceberg table as related table.
+     * The supported transforms now are YEAR, MONTH, DAY and HOUR.
+     * And the column couldn't change to another column during partition evolution.
+     */
+    @Override
+    public boolean isValidRelatedTable() {
+        if (isValidRelatedTableCached) {
+            return isValidRelatedTable;
+        }
+        isValidRelatedTable = false;
+        Set<String> allFields = Sets.newHashSet();
+        for (PartitionSpec spec : table.specs().values()) {
+            if (spec == null) {
+                isValidRelatedTableCached = true;
+                return false;
+            }
+            List<PartitionField> fields = spec.fields();
+            if (fields.size() != 1) {
+                isValidRelatedTableCached = true;
+                return false;
+            }
+            PartitionField partitionField = spec.fields().get(0);
+            String transformName = partitionField.transform().toString();
+            if (!YEAR.equals(transformName)
+                    && !MONTH.equals(transformName)
+                    && !DAY.equals(transformName)
+                    && !HOUR.equals(transformName)) {
+                isValidRelatedTableCached = true;
+                return false;
+            }
+            allFields.add(table.schema().findColumnName(partitionField.sourceId()));
+        }
+        isValidRelatedTableCached = true;
+        isValidRelatedTable = allFields.size() == 1;
+        return isValidRelatedTable;
+    }
+
+    protected IcebergPartitionInfo loadPartitionInfo() throws AnalysisException {
+        List<IcebergPartition> icebergPartitions = loadIcebergPartition();
+        Map<String, IcebergPartition> nameToPartition = Maps.newHashMap();
+        Map<String, PartitionItem> nameToPartitionItem = Maps.newHashMap();
+        for (IcebergPartition partition : icebergPartitions) {
+            nameToPartition.put(partition.getPartitionName(), partition);
+            String transform = table.specs().get(partition.getSpecId()).fields().get(0).transform().toString();
+            Range<PartitionKey> partitionRange = getPartitionRange(partition.getPartitionValues().get(0), transform);
+            PartitionItem item = new RangePartitionItem(partitionRange);
+            nameToPartitionItem.put(partition.getPartitionName(), item);
+        }
+        Map<String, Set<String>> partitionNameMap = mergeOverlapPartitions(nameToPartitionItem);
+        return new IcebergPartitionInfo(nameToPartitionItem, nameToPartition, partitionNameMap);
+    }
+
+    public List<IcebergPartition> loadIcebergPartition() {
+        PartitionsTable partitionsTable = (PartitionsTable) MetadataTableUtils
+                .createMetadataTableInstance(table, MetadataTableType.PARTITIONS);
+        List<IcebergPartition> partitions = Lists.newArrayList();
+        try (CloseableIterable<FileScanTask> tasks = partitionsTable.newScan().planFiles()) {
+            for (FileScanTask task : tasks) {
+                CloseableIterable<StructLike> rows = task.asDataTask().rows();
+                for (StructLike row : rows) {
+                    partitions.add(generateIcebergPartition(row));
+                }
+            }
+        } catch (IOException e) {
+            LOG.warn("Failed to get Iceberg table {} partition info.", name, e);
+        }
+        return partitions;
+    }
+
+    public IcebergPartition generateIcebergPartition(StructLike row) {
+        // row format :
+        // 0. partitionData,
+        // 1. spec_id,
+        // 2. record_count,
+        // 3. file_count,
+        // 4. total_data_file_size_in_bytes,
+        // 5. position_delete_record_count,
+        // 6. position_delete_file_count,
+        // 7. equality_delete_record_count,
+        // 8. equality_delete_file_count,
+        // 9. last_updated_at,
+        // 10. last_updated_snapshot_id
+        Preconditions.checkState(!table.spec().fields().isEmpty(), table.name() + " is not a partition table.");
+        int specId = row.get(1, Integer.class);
+        PartitionSpec partitionSpec = table.specs().get(specId);
+        StructProjection partitionData = row.get(0, StructProjection.class);
+        StringBuilder sb = new StringBuilder();
+        List<String> partitionValues = Lists.newArrayList();
+        List<String> transforms = Lists.newArrayList();
+        for (int i = 0; i < partitionSpec.fields().size(); ++i) {
+            PartitionField partitionField = partitionSpec.fields().get(i);
+            Class<?> fieldClass = partitionSpec.javaClasses()[i];
+            int fieldId = partitionField.fieldId();
+            // Iceberg partition field id starts at PARTITION_DATA_ID_START,
+            // So we can get the field index in partitionData using fieldId - PARTITION_DATA_ID_START
+            int index = fieldId - PARTITION_DATA_ID_START;
+            Object o = partitionData.get(index, fieldClass);
+            String fieldValue = o == null ? null : o.toString();
+            String fieldName = partitionField.name();
+            sb.append(fieldName);
+            sb.append("=");
+            sb.append(fieldValue);
+            sb.append("/");
+            partitionValues.add(fieldValue);
+            transforms.add(partitionField.transform().toString());
+        }
+        if (sb.length() > 0) {
+            sb.delete(sb.length() - 1, sb.length());
+        }
+        String partitionName = sb.toString();
+        long recordCount = row.get(2, Long.class);
+        long fileCount = row.get(3, Integer.class);
+        long fileSizeInBytes = row.get(4, Long.class);
+        long lastUpdateTime = row.get(9, Long.class);
+        long lastUpdateSnapShotId = row.get(10, Long.class);
+        return new IcebergPartition(partitionName, specId, recordCount, fileSizeInBytes, fileCount,
+                lastUpdateTime, lastUpdateSnapShotId, partitionValues, transforms);
+    }
+
+    @VisibleForTesting
+    public Range<PartitionKey> getPartitionRange(String value, String transform)
+            throws AnalysisException {
+        // For NULL value, create a lessThan partition for it.
+        if (value == null) {
+            PartitionKey nullKey = PartitionKey.createPartitionKey(
+                    Lists.newArrayList(new PartitionValue("0000-01-02")), partitionColumns);
+            return Range.lessThan(nullKey);
+        }
+        LocalDateTime epoch = Instant.EPOCH.atZone(ZoneId.of("UTC")).toLocalDateTime();
+        LocalDateTime target;
+        LocalDateTime lower;
+        LocalDateTime upper;
+        long longValue = Long.parseLong(value);
+        switch (transform) {
+            case HOUR:
+                target = epoch.plusHours(longValue);
+                lower = LocalDateTime.of(target.getYear(), target.getMonth(), target.getDayOfMonth(),
+                        target.getHour(), 0, 0);
+                upper = lower.plusHours(1);
+                break;
+            case DAY:
+                target = epoch.plusDays(longValue);
+                lower = LocalDateTime.of(target.getYear(), target.getMonth(), target.getDayOfMonth(), 0, 0, 0);
+                upper = lower.plusDays(1);
+                break;
+            case MONTH:
+                target = epoch.plusMonths(longValue);
+                lower = LocalDateTime.of(target.getYear(), target.getMonth(), 1, 0, 0, 0);
+                upper = lower.plusMonths(1);
+                break;
+            case YEAR:
+                target = epoch.plusYears(longValue);
+                lower = LocalDateTime.of(target.getYear(), Month.JANUARY, 1, 0, 0, 0);
+                upper = lower.plusYears(1);
+                break;
+            default:
+                throw new RuntimeException("Unsupported transform " + transform);
+        }
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+        Column c = partitionColumns.get(0);
+        Preconditions.checkState(c.getDataType().isDateType(), "Only support date type partition column");
+        if (c.getType().isDate() || c.getType().isDateV2()) {
+            formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+        }
+        PartitionValue lowerValue = new PartitionValue(lower.format(formatter));
+        PartitionValue upperValue = new PartitionValue(upper.format(formatter));
+        PartitionKey lowKey = PartitionKey.createPartitionKey(Lists.newArrayList(lowerValue), partitionColumns);
+        PartitionKey upperKey =  PartitionKey.createPartitionKey(Lists.newArrayList(upperValue), partitionColumns);
+        return Range.closedOpen(lowKey, upperKey);
+    }
+
+    /**
+     * Merge overlapped iceberg partitions into one Doris partition.
+     */
+    public Map<String, Set<String>> mergeOverlapPartitions(Map<String, PartitionItem> originPartitions) {
+        List<Map.Entry<String, PartitionItem>> entries = sortPartitionMap(originPartitions);
+        Map<String, Set<String>> map = Maps.newHashMap();
+        for (int i = 0; i < entries.size() - 1; i++) {
+            Range<PartitionKey> firstValue = entries.get(i).getValue().getItems();
+            String firstKey = entries.get(i).getKey();
+            Range<PartitionKey> secondValue = entries.get(i + 1).getValue().getItems();
+            String secondKey = entries.get(i + 1).getKey();
+            // If the first entry enclose the second one, remove the second entry and keep a record in the return map.
+            // So we can track the iceberg partitions those contained by one Doris partition.
+            while (i < entries.size() && firstValue.encloses(secondValue)) {
+                originPartitions.remove(secondKey);
+                map.putIfAbsent(firstKey, Sets.newHashSet(firstKey));
+                String finalSecondKey = secondKey;
+                map.computeIfPresent(firstKey, (key, value) -> {
+                    value.add(finalSecondKey);
+                    return value;
+                });
+                i++;
+                if (i >= entries.size() - 1) {
+                    break;
+                }
+                secondValue = entries.get(i + 1).getValue().getItems();
+                secondKey = entries.get(i + 1).getKey();
+            }
+        }
+        return map;
+    }
+
+    /**
+     * Sort the given map entries by PartitionItem Range(LOW, HIGH)
+     * When comparing two ranges, the one with smaller LOW value is smaller than the other one.
+     * If two ranges have same values of LOW, the one with larger HIGH value is smaller.
+     *
+     * For now, we only support year, month, day and hour,
+     * so it is impossible to have two partially intersect partitions.
+     * One range is either enclosed by another or has no intersection at all with another.
+     *
+     *
+     * For example, we have these 4 ranges:
+     * [10, 20), [30, 40), [0, 30), [10, 15)
+     *
+     * After sort, they become:
+     * [0, 30), [10, 20), [10, 15), [30, 40)
+     */
+    public List<Map.Entry<String, PartitionItem>> sortPartitionMap(Map<String, PartitionItem> originPartitions) {
+        List<Map.Entry<String, PartitionItem>> entries = new ArrayList<>(originPartitions.entrySet());
+        entries.sort(new RangeComparator());
+        return entries;
+    }
+
+    public static class RangeComparator implements Comparator<Map.Entry<String, PartitionItem>> {
+        @Override
+        public int compare(Map.Entry<String, PartitionItem> p1, Map.Entry<String, PartitionItem> p2) {
+            PartitionItem value1 = p1.getValue();
+            PartitionItem value2 = p2.getValue();
+            if (value1 instanceof RangePartitionItem && value2 instanceof RangePartitionItem) {
+                Range<PartitionKey> items1 = value1.getItems();
+                Range<PartitionKey> items2 = value2.getItems();
+                if (!items1.hasLowerBound()) {
+                    return -1;
+                }
+                if (!items2.hasLowerBound()) {
+                    return 1;
+                }
+                PartitionKey upper1 = items1.upperEndpoint();
+                PartitionKey lower1 = items1.lowerEndpoint();
+                PartitionKey upper2 = items2.upperEndpoint();
+                PartitionKey lower2 = items2.lowerEndpoint();
+                int compareLow = lower1.compareTo(lower2);
+                return compareLow == 0 ? upper2.compareTo(upper1) : compareLow;
+            }
+            return 0;
+        }
+    }
+
+    @VisibleForTesting
+    public boolean isValidRelatedTableCached() {
+        return isValidRelatedTableCached;
+    }
+
+    @VisibleForTesting
+    public boolean validRelatedTableCache() {
+        return isValidRelatedTable;
+    }
+
+    public void setIsValidRelatedTableCached(boolean isCached) {
+        this.isValidRelatedTableCached = isCached;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java
new file mode 100644
index 00000000000000..cccc6244a0d0cc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import java.util.List;
+
+public class IcebergPartition {
+    private final String partitionName;
+    private final List<String> partitionValues;
+    private final int specId;
+    private final long recordCount;
+    private final long fileSizeInBytes;
+    private final long fileCount;
+    private final long lastUpdateTime;
+    private final long lastSnapshotId;
+    private final List<String> transforms;
+
+    public IcebergPartition(String partitionName, int specId, long recordCount, long fileSizeInBytes, long fileCount,
+                            long lastUpdateTime, long lastSnapshotId, List<String> partitionValues,
+                            List<String> transforms) {
+        this.partitionName = partitionName;
+        this.specId = specId;
+        this.recordCount = recordCount;
+        this.fileSizeInBytes = fileSizeInBytes;
+        this.fileCount = fileCount;
+        this.lastUpdateTime = lastUpdateTime;
+        this.lastSnapshotId = lastSnapshotId;
+        this.partitionValues = partitionValues;
+        this.transforms = transforms;
+    }
+
+    public String getPartitionName() {
+        return partitionName;
+    }
+
+    public int getSpecId() {
+        return specId;
+    }
+
+    public long getRecordCount() {
+        return recordCount;
+    }
+
+    public long getFileSizeInBytes() {
+        return fileSizeInBytes;
+    }
+
+    public long getFileCount() {
+        return fileCount;
+    }
+
+    public long getLastUpdateTime() {
+        return lastUpdateTime;
+    }
+
+    public long getLastSnapshotId() {
+        return lastSnapshotId;
+    }
+
+    public List<String> getPartitionValues() {
+        return partitionValues;
+    }
+
+    public List<String> getTransforms() {
+        return transforms;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java
new file mode 100644
index 00000000000000..9edb2137f4f389
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.catalog.PartitionItem;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+import java.util.Set;
+
+public class IcebergPartitionInfo {
+    private final Map<String, PartitionItem> nameToPartitionItem;
+    private final Map<String, IcebergPartition> nameToIcebergPartition;
+    private final Map<String, Set<String>> nameToIcebergPartitionNames;
+
+    public IcebergPartitionInfo() {
+        this.nameToPartitionItem = Maps.newHashMap();
+        this.nameToIcebergPartition = Maps.newHashMap();
+        this.nameToIcebergPartitionNames = Maps.newHashMap();
+    }
+
+    public IcebergPartitionInfo(Map<String, PartitionItem> nameToPartitionItem,
+                               Map<String, IcebergPartition> nameToIcebergPartition,
+                                Map<String, Set<String>> nameToIcebergPartitionNames) {
+        this.nameToPartitionItem = nameToPartitionItem;
+        this.nameToIcebergPartition = nameToIcebergPartition;
+        this.nameToIcebergPartitionNames = nameToIcebergPartitionNames;
+    }
+
+    public Map<String, PartitionItem> getNameToPartitionItem() {
+        return nameToPartitionItem;
+    }
+
+    public Map<String, IcebergPartition> getNameToIcebergPartition() {
+        return nameToIcebergPartition;
+    }
+
+    public long getLatestSnapshotId(String partitionName) {
+        Set<String> icebergPartitionNames = nameToIcebergPartitionNames.get(partitionName);
+        if (icebergPartitionNames == null) {
+            return nameToIcebergPartition.get(partitionName).getLastSnapshotId();
+        }
+        long latestSnapshotId = 0;
+        long latestUpdateTime = -1;
+        for (String name : icebergPartitionNames) {
+            IcebergPartition partition = nameToIcebergPartition.get(name);
+            long lastUpdateTime = partition.getLastUpdateTime();
+            if (latestUpdateTime < lastUpdateTime) {
+                latestUpdateTime = lastUpdateTime;
+                latestSnapshotId = partition.getLastSnapshotId();
+            }
+        }
+        return latestSnapshotId;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheValue.java
new file mode 100644
index 00000000000000..e1fde8049fe1ad
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheValue.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.datasource.SchemaCacheValue;
+
+import java.util.List;
+
+public class IcebergSchemaCacheValue extends SchemaCacheValue {
+
+    private final List<Column> partitionColumns;
+    private final IcebergPartitionInfo partitionInfo;
+    private final long snapshotId;
+
+    public IcebergSchemaCacheValue(List<Column> schema, List<Column> partitionColumns,
+                                  long snapshotId, IcebergPartitionInfo partitionInfo) {
+        super(schema);
+        this.partitionColumns = partitionColumns;
+        this.snapshotId = snapshotId;
+        this.partitionInfo = partitionInfo;
+    }
+
+    public List<Column> getPartitionColumns() {
+        return partitionColumns;
+    }
+
+    public IcebergPartitionInfo getPartitionInfo() {
+        return partitionInfo;
+    }
+
+    public long getSnapshotId() {
+        return snapshotId;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index 353e024ed97348..c1002faf4078b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -43,6 +43,7 @@
 import org.apache.doris.mtmv.MTMVRefreshContext;
 import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod;
 import org.apache.doris.mtmv.MTMVRefreshPartitionSnapshot;
+import org.apache.doris.mtmv.MTMVRelatedTableIf;
 import org.apache.doris.mtmv.MTMVRelation;
 import org.apache.doris.mtmv.MTMVUtil;
 import org.apache.doris.nereids.StatementContext;
@@ -182,6 +183,12 @@ public void run() throws JobException {
             this.relation = MTMVPlanUtil.generateMTMVRelation(mtmv, ctx);
             beforeMTMVRefresh();
             if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) {
+                MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable();
+                if (!relatedTable.isValidRelatedTable()) {
+                    throw new JobException("MTMV " + mtmv.getName() + "'s related table " + relatedTable.getName()
+                        + " is not a valid related table anymore, stop refreshing."
+                        + " e.g. Table has multiple partition columns or including not supported transform functions.");
+                }
                 MTMVPartitionUtil.alignMvPartition(mtmv);
             }
             MTMVRefreshContext context = MTMVRefreshContext.buildContext(mtmv);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 92436b063e0b49..9597378c488cfc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -317,7 +317,7 @@ public static boolean isSyncWithPartitions(MTMVRefreshContext context, String mt
         if (!relatedTable.needAutoRefresh()) {
             return true;
         }
-        // check if partitions of related table if changed
+        // check if partitions of related table is changed
         Set<String> snapshotPartitions = mtmv.getRefreshSnapshot().getSnapshotPartitions(mtmvPartitionName);
         if (!Objects.equals(relatedPartitionNames, snapshotPartitions)) {
             return false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
index c4261aa78f10be..821595d7dffd06 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
@@ -115,4 +115,13 @@ default boolean needAutoRefresh() {
      * @return
      */
     boolean isPartitionColumnAllowNull();
+
+    /**
+     * If the table is supported as related table.
+     * For example, an Iceberg table may become unsupported after partition revolution.
+     * @return
+     */
+    default boolean isValidRelatedTable() {
+        return true;
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java
new file mode 100644
index 00000000000000..80d0a7c2429df3
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.RangePartitionItem;
+import org.apache.doris.common.AnalysisException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
+import mockit.Expectations;
+import mockit.Mocked;
+import mockit.Verifications;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.transforms.Days;
+import org.apache.iceberg.transforms.Hours;
+import org.apache.iceberg.transforms.Months;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class IcebergExternalTableTest {
+
+    @Test
+    public void testIsSupportedPartitionTable(@Mocked org.apache.iceberg.Table icebergTable,
+                                              @Mocked PartitionSpec spec,
+                                              @Mocked PartitionField field,
+                                              @Mocked Schema schema) {
+        IcebergExternalTable table = new IcebergExternalTable(1, "1", "2", null);
+        Map<Integer, PartitionSpec> specs = Maps.newHashMap();
+        // Test null
+        specs.put(0, null);
+        new Expectations() {{
+                icebergTable.specs();
+                result = specs;
+            }};
+        table.setTable(icebergTable);
+        Assertions.assertFalse(table.isValidRelatedTableCached());
+        Assertions.assertFalse(table.isValidRelatedTable());
+        new Verifications() {{
+                icebergTable.specs();
+                times = 1;
+            }};
+        Assertions.assertTrue(table.isValidRelatedTableCached());
+        Assertions.assertFalse(table.validRelatedTableCache());
+
+        // Test spec fields are empty.
+        specs.put(0, spec);
+        table.setIsValidRelatedTableCached(false);
+        Assertions.assertFalse(table.isValidRelatedTableCached());
+        new Expectations() {{
+                icebergTable.specs();
+                result = specs;
+            }};
+        List<PartitionField> fields = Lists.newArrayList();
+        new Expectations() {{
+                spec.fields();
+                result = fields;
+            }};
+        Assertions.assertFalse(table.isValidRelatedTable());
+        new Verifications() {{
+                spec.fields();
+                times = 1;
+            }};
+        Assertions.assertTrue(table.isValidRelatedTableCached());
+        Assertions.assertFalse(table.validRelatedTableCache());
+
+        // Test true
+        fields.add(field);
+        table.setIsValidRelatedTableCached(false);
+        Assertions.assertFalse(table.isValidRelatedTableCached());
+        new Expectations() {
+            {
+                icebergTable.schema();
+                result = schema;
+
+                schema.findColumnName(anyInt);
+                result = "col1";
+            }
+        };
+        new Expectations() {{
+                field.transform();
+                result = new Hours();
+            }};
+        Assertions.assertTrue(table.isValidRelatedTable());
+        Assertions.assertTrue(table.isValidRelatedTableCached());
+        Assertions.assertTrue(table.validRelatedTableCache());
+        new Verifications() {{
+                schema.findColumnName(anyInt);
+                times = 1;
+            }};
+        new Expectations() {{
+                field.transform();
+                result = new Days();
+            }};
+        table.setIsValidRelatedTableCached(false);
+        Assertions.assertFalse(table.isValidRelatedTableCached());
+        Assertions.assertTrue(table.isValidRelatedTable());
+        new Expectations() {{
+                field.transform();
+                result = new Months();
+            }};
+        table.setIsValidRelatedTableCached(false);
+        Assertions.assertFalse(table.isValidRelatedTableCached());
+        Assertions.assertTrue(table.isValidRelatedTable());
+        Assertions.assertTrue(table.isValidRelatedTableCached());
+        Assertions.assertTrue(table.validRelatedTableCache());
+    }
+
+    @Test
+    public void testGetPartitionRange() throws AnalysisException {
+        IcebergExternalTable table = new IcebergExternalTable(1, "1", "2", null);
+        Column c = new Column("ts", PrimitiveType.DATETIMEV2);
+        List<Column> partitionColumns = Lists.newArrayList(c);
+        table.setPartitionColumns(partitionColumns);
+
+        // Test null partition value
+        Range<PartitionKey> nullRange = table.getPartitionRange(null, "hour");
+        Assertions.assertFalse(nullRange.hasLowerBound());
+        Assertions.assertEquals("0000-01-02 00:00:00",
+                nullRange.upperEndpoint().getPartitionValuesAsStringList().get(0));
+
+        // Test hour transform.
+        Range<PartitionKey> hour = table.getPartitionRange("100", "hour");
+        PartitionKey lowKey = hour.lowerEndpoint();
+        PartitionKey upKey = hour.upperEndpoint();
+        Assertions.assertEquals("1970-01-05 04:00:00", lowKey.getPartitionValuesAsStringList().get(0));
+        Assertions.assertEquals("1970-01-05 05:00:00", upKey.getPartitionValuesAsStringList().get(0));
+
+        // Test day transform.
+        Range<PartitionKey> day = table.getPartitionRange("100", "day");
+        lowKey = day.lowerEndpoint();
+        upKey = day.upperEndpoint();
+        Assertions.assertEquals("1970-04-11 00:00:00", lowKey.getPartitionValuesAsStringList().get(0));
+        Assertions.assertEquals("1970-04-12 00:00:00", upKey.getPartitionValuesAsStringList().get(0));
+
+        // Test month transform.
+        Range<PartitionKey> month = table.getPartitionRange("100", "month");
+        lowKey = month.lowerEndpoint();
+        upKey = month.upperEndpoint();
+        Assertions.assertEquals("1978-05-01 00:00:00", lowKey.getPartitionValuesAsStringList().get(0));
+        Assertions.assertEquals("1978-06-01 00:00:00", upKey.getPartitionValuesAsStringList().get(0));
+
+        // Test year transform.
+        Range<PartitionKey> year = table.getPartitionRange("100", "year");
+        lowKey = year.lowerEndpoint();
+        upKey = year.upperEndpoint();
+        Assertions.assertEquals("2070-01-01 00:00:00", lowKey.getPartitionValuesAsStringList().get(0));
+        Assertions.assertEquals("2071-01-01 00:00:00", upKey.getPartitionValuesAsStringList().get(0));
+
+        // Test unsupported transform
+        Exception exception = Assertions.assertThrows(RuntimeException.class, () -> {
+            table.getPartitionRange("100", "bucket");
+        });
+        Assertions.assertEquals("Unsupported transform bucket", exception.getMessage());
+    }
+
+    @Test
+    public void testSortRange() throws AnalysisException {
+        IcebergExternalTable table = new IcebergExternalTable(1, "1", "2", null);
+        Column c = new Column("c", PrimitiveType.DATETIMEV2);
+        table.setPartitionColumns(Lists.newArrayList(c));
+        PartitionItem nullRange = new RangePartitionItem(table.getPartitionRange(null, "hour"));
+        PartitionItem year1970 = new RangePartitionItem(table.getPartitionRange("0", "year"));
+        PartitionItem year1971 = new RangePartitionItem(table.getPartitionRange("1", "year"));
+        PartitionItem month197002 = new RangePartitionItem(table.getPartitionRange("1", "month"));
+        PartitionItem month197103 = new RangePartitionItem(table.getPartitionRange("14", "month"));
+        PartitionItem month197204 = new RangePartitionItem(table.getPartitionRange("27", "month"));
+        PartitionItem day19700202 = new RangePartitionItem(table.getPartitionRange("32", "day"));
+        PartitionItem day19730101 = new RangePartitionItem(table.getPartitionRange("1096", "day"));
+        Map<String, PartitionItem> map = Maps.newHashMap();
+        map.put("nullRange", nullRange);
+        map.put("year1970", year1970);
+        map.put("year1971", year1971);
+        map.put("month197002", month197002);
+        map.put("month197103", month197103);
+        map.put("month197204", month197204);
+        map.put("day19700202", day19700202);
+        map.put("day19730101", day19730101);
+        List<Map.Entry<String, PartitionItem>> entries = table.sortPartitionMap(map);
+        Assertions.assertEquals(8, entries.size());
+        Assertions.assertEquals("nullRange", entries.get(0).getKey());
+        Assertions.assertEquals("year1970", entries.get(1).getKey());
+        Assertions.assertEquals("month197002", entries.get(2).getKey());
+        Assertions.assertEquals("day19700202", entries.get(3).getKey());
+        Assertions.assertEquals("year1971", entries.get(4).getKey());
+        Assertions.assertEquals("month197103", entries.get(5).getKey());
+        Assertions.assertEquals("month197204", entries.get(6).getKey());
+        Assertions.assertEquals("day19730101", entries.get(7).getKey());
+
+        Map<String, Set<String>> stringSetMap = table.mergeOverlapPartitions(map);
+        Assertions.assertEquals(2, stringSetMap.size());
+        Assertions.assertTrue(stringSetMap.containsKey("year1970"));
+        Assertions.assertTrue(stringSetMap.containsKey("year1971"));
+
+        Set<String> names1970 = stringSetMap.get("year1970");
+        Assertions.assertEquals(3, names1970.size());
+        Assertions.assertTrue(names1970.contains("year1970"));
+        Assertions.assertTrue(names1970.contains("month197002"));
+        Assertions.assertTrue(names1970.contains("day19700202"));
+
+        Set<String> names1971 = stringSetMap.get("year1971");
+        Assertions.assertEquals(2, names1971.size());
+        Assertions.assertTrue(names1971.contains("year1971"));
+        Assertions.assertTrue(names1971.contains("month197103"));
+
+        Assertions.assertEquals(5, map.size());
+        Assertions.assertTrue(map.containsKey("nullRange"));
+        Assertions.assertTrue(map.containsKey("year1970"));
+        Assertions.assertTrue(map.containsKey("year1971"));
+        Assertions.assertTrue(map.containsKey("month197204"));
+        Assertions.assertTrue(map.containsKey("day19730101"));
+    }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java
new file mode 100644
index 00000000000000..74c8c3f6954a97
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+import java.util.Set;
+
+public class IcebergPartitionInfoTest {
+
+    @Test
+    public void testGetLatestSnapshotId() {
+        IcebergPartition p1 = new IcebergPartition("p1", 0, 0, 0, 0, 1, 101, null, null);
+        IcebergPartition p2 = new IcebergPartition("p2", 0, 0, 0, 0, 2, 102, null, null);
+        IcebergPartition p3 = new IcebergPartition("p3", 0, 0, 0, 0, 3, 103, null, null);
+        Map<String, IcebergPartition> nameToIcebergPartition = Maps.newHashMap();
+        nameToIcebergPartition.put(p1.getPartitionName(), p1);
+        nameToIcebergPartition.put(p2.getPartitionName(), p2);
+        nameToIcebergPartition.put(p3.getPartitionName(), p3);
+        Map<String, Set<String>> nameToIcebergPartitionNames = Maps.newHashMap();
+        Set<String> names = Sets.newHashSet();
+        names.add("p1");
+        names.add("p2");
+        nameToIcebergPartitionNames.put("p1", names);
+
+        IcebergPartitionInfo info = new IcebergPartitionInfo(null, nameToIcebergPartition, nameToIcebergPartitionNames);
+        long snapshot1 = info.getLatestSnapshotId("p1");
+        long snapshot2 = info.getLatestSnapshotId("p2");
+        long snapshot3 = info.getLatestSnapshotId("p3");
+        Assertions.assertEquals(102, snapshot1);
+        Assertions.assertEquals(102, snapshot2);
+        Assertions.assertEquals(103, snapshot3);
+    }
+}
diff --git a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
index d0b47e27fffae5..c9d9799da81300 100644
--- a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
@@ -5,3 +5,101 @@
 -- !mtmv --
 1000
 
+-- !test_ts_refresh1 --
+2024-10-26T01:02:03	1
+2024-10-27T01:02:03	2
+2024-10-27T21:02:03	3
+
+-- !test_ts_refresh2 --
+2024-10-26T01:02:03	1
+2024-10-27T01:02:03	2
+2024-10-27T21:02:03	3
+2024-10-26T21:02:03	4
+
+-- !test_ts_refresh3 --
+2024-10-26T01:02:03	1
+2024-10-27T01:02:03	2
+2024-10-27T21:02:03	3
+2024-10-26T21:02:03	4
+2024-10-26T01:22:03	5
+
+-- !test_ts_refresh4 --
+2024-10-26T01:02:03	1
+2024-10-27T01:02:03	2
+2024-10-27T21:02:03	3
+2024-10-26T21:02:03	4
+2024-10-26T01:22:03	5
+2024-10-27T01:12:03	6
+
+-- !test_ts_refresh5 --
+2024-10-26T01:02:03	1
+2024-10-27T01:02:03	2
+2024-10-27T21:02:03	3
+2024-10-26T21:02:03	4
+2024-10-26T01:22:03	5
+2024-10-27T01:12:03	6
+
+-- !test_ts_refresh6 --
+2024-10-26T01:02:03	1
+2024-10-27T01:02:03	2
+2024-10-27T21:02:03	3
+2024-10-26T21:02:03	4
+2024-10-26T01:22:03	5
+2024-10-27T01:12:03	6
+2024-10-28T01:22:03	7
+
+-- !test_ts_refresh_null --
+2024-10-26T01:02:03	1
+2024-10-27T01:02:03	2
+2024-10-27T21:02:03	3
+2024-10-26T21:02:03	4
+2024-10-26T01:22:03	5
+2024-10-27T01:12:03	6
+2024-10-28T01:22:03	7
+\N	8
+
+-- !unpartition --
+2024-10-26T01:02:03	1
+2024-10-27T01:02:03	2
+2024-10-27T21:02:03	3
+2024-10-26T21:02:03	4
+2024-10-26T01:22:03	5
+2024-10-27T01:12:03	6
+2024-10-28T01:22:03	7
+\N	8
+
+-- !test_d_refresh1 --
+2024-08-26	1
+2024-09-17	2
+2024-09-27	3
+
+-- !test_d_refresh2 --
+2024-08-26	1
+2024-09-17	2
+2024-09-27	3
+2024-09-01	4
+
+-- !test_d_refresh3 --
+2024-08-26	1
+2024-09-17	2
+2024-09-27	3
+2024-09-01	4
+2024-08-22	5
+
+-- !test_d_refresh4 --
+2024-08-26	1
+2024-09-17	2
+2024-09-27	3
+2024-09-01	4
+2024-08-22	5
+2024-09-30	6
+
+-- !test_d_refresh5 --
+2024-08-26	1
+2024-09-17	2
+2024-09-27	3
+2024-09-01	4
+2024-08-22	5
+2024-09-30	6
+2024-10-28	7
+
diff --git a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
index 3004f1677bded6..59cf1173acb46b 100644
--- a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
@@ -65,5 +65,152 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_
         sql """drop materialized view if exists ${mvName};"""
         sql """ drop catalog if exists ${catalog_name} """
     }
+
+    // Test partition refresh.
+    // Use hms catalog to avoid rest catalog fail to write caused by sqlite database file locked.
+    if (enabled != null && enabled.equalsIgnoreCase("true")) {
+        String hivePrefix = "hive2";
+        String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
+        String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort")
+        String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}"
+        String warehouse = "${default_fs}/warehouse"
+
+        String catalog_name = "iceberg_mtmv_catalog_hms";
+        String mvUnpartition = "test_iceberg_unpartition"
+        String mvName1 = "test_iceberg_mtmv_ts"
+        String mvName2 = "test_iceberg_mtmv_d"
+        String dbName = "regression_test_mtmv_partition_p0"
+        String icebergDb = "iceberg_mtmv_partition"
+        String icebergTable1 = "tstable"
+        String icebergTable2 = "dtable"
+        sql """drop catalog if exists ${catalog_name} """
+        sql """create catalog if not exists ${catalog_name} properties (
+            'type'='iceberg',
+            'iceberg.catalog.type'='hms',
+            'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+            'fs.defaultFS' = '${default_fs}',
+            'warehouse' = '${warehouse}',
+            'use_meta_cache' = 'true'
+        );"""
+
+        sql """switch internal"""
+        sql """drop database if exists ${dbName}"""
+        sql """create database if not exists ${dbName}"""
+        sql """use ${dbName}"""
+
+        sql """drop table if exists ${catalog_name}.${icebergDb}.${icebergTable1}"""
+        sql """drop table if exists ${catalog_name}.${icebergDb}.${icebergTable2}"""
+        sql """create database if not exists ${catalog_name}.${icebergDb}"""
+        sql """
+            CREATE TABLE ${catalog_name}.${icebergDb}.${icebergTable1} (
+              ts DATETIME,
+              value INT)
+            ENGINE=iceberg
+            PARTITION BY LIST (DAY(ts)) ();
+        """
+        sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-26 01:02:03', 1), ('2024-10-27 01:02:03', 2), ('2024-10-27 21:02:03', 3)"""
+        sql """CREATE MATERIALIZED VIEW ${mvName1} BUILD DEFERRED REFRESH AUTO ON MANUAL partition by(`ts`) DISTRIBUTED BY RANDOM BUCKETS 2  PROPERTIES ('replication_num' = '1') as SELECT * FROM ${catalog_name}.${icebergDb}.${icebergTable1}"""
+        sql """REFRESH MATERIALIZED VIEW ${mvName1} complete"""
+        waitingMTMVTaskFinishedByMvName(mvName1)
+        qt_test_ts_refresh1 "select * from ${mvName1} order by value"
+
+        sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-26 21:02:03', 4)"""
+        sql """REFRESH MATERIALIZED VIEW ${mvName1} auto"""
+        waitingMTMVTaskFinishedByMvName(mvName1)
+        qt_test_ts_refresh2 """select * from ${mvName1} order by value"""
+
+        sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-26 01:22:03', 5), ('2024-10-27 01:12:03', 6);"""
+        sql """REFRESH MATERIALIZED VIEW ${mvName1} partitions(p_20241026000000_20241027000000);"""
+        waitingMTMVTaskFinishedByMvName(mvName1)
+        qt_test_ts_refresh3 """select * from ${mvName1} order by value"""
+
+        sql """REFRESH MATERIALIZED VIEW ${mvName1} auto"""
+        waitingMTMVTaskFinishedByMvName(mvName1)
+        qt_test_ts_refresh4 """select * from ${mvName1} order by value"""
+
+        sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-28 01:22:03', 7);"""
+        sql """REFRESH MATERIALIZED VIEW ${mvName1} partitions(p_20241026000000_20241027000000);"""
+        waitingMTMVTaskFinishedByMvName(mvName1)
+        qt_test_ts_refresh5 """select * from ${mvName1} order by value"""
+
+        sql """REFRESH MATERIALIZED VIEW ${mvName1} auto"""
+        waitingMTMVTaskFinishedByMvName(mvName1)
+        qt_test_ts_refresh6 """select * from ${mvName1} order by value"""
+
+        sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values (null, 8);"""
+        sql """REFRESH MATERIALIZED VIEW ${mvName1} auto"""
+        waitingMTMVTaskFinishedByMvName(mvName1)
+        qt_test_ts_refresh_null """select * from ${mvName1} order by value"""
+
+        def showPartitionsResult = sql """show partitions from ${mvName1}"""
+        logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+        assertTrue(showPartitionsResult.toString().contains("p_20241026000000_20241027000000"))
+        assertTrue(showPartitionsResult.toString().contains("p_20241027000000_20241028000000"))
+        assertTrue(showPartitionsResult.toString().contains("p_20241028000000_20241029000000"))
+
+        sql """drop materialized view if exists ${mvName1};"""
+
+        // Test unpartitioned mtmv
+        sql """
+            CREATE MATERIALIZED VIEW ${mvUnpartition}
+                BUILD DEFERRED REFRESH AUTO ON MANUAL
+                DISTRIBUTED BY RANDOM BUCKETS 2
+                PROPERTIES ('replication_num' = '1')
+                AS
+                SELECT * FROM ${catalog_name}.${icebergDb}.${icebergTable1};
+        """
+        sql """
+            REFRESH MATERIALIZED VIEW ${mvUnpartition} complete
+        """
+        def jobName = getJobName(dbName, mvUnpartition);
+        waitingMTMVTaskFinished(jobName)
+        qt_unpartition "SELECT * FROM ${mvUnpartition} order by value"
+
+        sql """drop materialized view if exists ${mvUnpartition};"""
+        sql """drop table if exists ${catalog_name}.${icebergDb}.${icebergTable1}"""
+
+
+        sql """
+            CREATE TABLE ${catalog_name}.${icebergDb}.${icebergTable2} (
+              d DATE,
+              value INT)
+            ENGINE=iceberg
+            PARTITION BY LIST (MONTH(d)) ();
+        """
+        sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-08-26', 1), ('2024-09-17', 2), ('2024-09-27', 3);"""
+        sql """CREATE MATERIALIZED VIEW ${mvName2} BUILD DEFERRED REFRESH AUTO ON MANUAL partition by(`d`) DISTRIBUTED BY RANDOM BUCKETS 2  PROPERTIES ('replication_num' = '1') as SELECT * FROM ${catalog_name}.${icebergDb}.${icebergTable2}"""
+        sql """REFRESH MATERIALIZED VIEW ${mvName2} complete"""
+        waitingMTMVTaskFinishedByMvName(mvName2)
+        qt_test_d_refresh1 "select * from ${mvName2} order by value"
+
+        sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-09-01', 4);"""
+        sql """REFRESH MATERIALIZED VIEW ${mvName2} auto"""
+        waitingMTMVTaskFinishedByMvName(mvName2)
+        qt_test_d_refresh2 "select * from ${mvName2} order by value"
+
+        sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-08-22', 5), ('2024-09-30', 6);"""
+        sql """REFRESH MATERIALIZED VIEW ${mvName2} partitions(p_20240801_20240901);"""
+        waitingMTMVTaskFinishedByMvName(mvName2)
+        qt_test_d_refresh3 "select * from ${mvName2} order by value"
+        sql """REFRESH MATERIALIZED VIEW ${mvName2} partitions(p_20240901_20241001);"""
+        waitingMTMVTaskFinishedByMvName(mvName2)
+        qt_test_d_refresh4 "select * from ${mvName2} order by value"
+
+        sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-10-28', 7);"""
+        sql """REFRESH MATERIALIZED VIEW ${mvName2} auto"""
+        waitingMTMVTaskFinishedByMvName(mvName2)
+        qt_test_d_refresh5 "select * from ${mvName2} order by value"
+
+        showPartitionsResult = sql """show partitions from ${mvName2}"""
+        logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+        assertTrue(showPartitionsResult.toString().contains("p_20240801_20240901"))
+        assertTrue(showPartitionsResult.toString().contains("p_20240901_20241001"))
+        assertTrue(showPartitionsResult.toString().contains("p_20241001_20241101"))
+
+        sql """drop materialized view if exists ${mvName2};"""
+        sql """drop table if exists ${catalog_name}.${icebergDb}.${icebergTable2}"""
+
+        sql """ drop catalog if exists ${catalog_name} """
+    }
 }
 

From 91c475e0f4ace1f31ffefa56af7eb437f2b61a9d Mon Sep 17 00:00:00 2001
From: lihangyu <lihangyu@selectdb.com>
Date: Mon, 16 Dec 2024 15:49:18 +0800
Subject: [PATCH 340/399] [Fix](ShortCircuit) fix prepared statement with
 partial arguments prepared (#45371)

We should record the placehold id map to both real Expr and the slot of
conjuncts.Otherwise the info is lost, and lead to the conjuncts updated
in wrong order(`updateScanNodeConjuncts`)
---
 .../doris/nereids/StatementContext.java       |  6 ++
 .../rules/analysis/ExpressionAnalyzer.java    | 21 +++++
 .../apache/doris/qe/PointQueryExecutor.java   | 40 +++++---
 .../data/point_query_p0/test_point_query.out  | 30 ++++++
 .../point_query_p0/test_point_query.groovy    | 92 +++++++++++++++----
 5 files changed, 157 insertions(+), 32 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index f21600ac585763..4d60b06e7c5cc1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -132,6 +132,8 @@ public class StatementContext implements Closeable {
     private final IdGenerator<PlaceholderId> placeHolderIdGenerator = PlaceholderId.createGenerator();
     // relation id to placeholders for prepared statement, ordered by placeholder id
     private final Map<PlaceholderId, Expression> idToPlaceholderRealExpr = new TreeMap<>();
+    // map placeholder id to comparison slot, which will used to replace conjuncts directly
+    private final Map<PlaceholderId, SlotReference> idToComparisonSlot = new TreeMap<>();
 
     // collect all hash join conditions to compute node connectivity in join graph
     private final List<Expression> joinFilters = new ArrayList<>();
@@ -399,6 +401,10 @@ public Map<PlaceholderId, Expression> getIdToPlaceholderRealExpr() {
         return idToPlaceholderRealExpr;
     }
 
+    public Map<PlaceholderId, SlotReference> getIdToComparisonSlot() {
+        return idToComparisonSlot;
+    }
+
     public Map<CTEId, List<Pair<Multimap<Slot, Slot>, Group>>> getCteIdToConsumerGroup() {
         return cteIdToConsumerGroup;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
index 6abbbda447c8ec..8056f396cd3d07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
@@ -24,6 +24,7 @@
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.Pair;
 import org.apache.doris.common.util.Util;
+import org.apache.doris.mysql.MysqlCommand;
 import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.SqlCacheContext;
 import org.apache.doris.nereids.StatementContext;
@@ -77,6 +78,7 @@
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
 import org.apache.doris.nereids.trees.expressions.typecoercion.ImplicitCastInputTypes;
+import org.apache.doris.nereids.trees.plans.PlaceholderId;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.types.ArrayType;
@@ -583,10 +585,29 @@ public Expression visitPlaceholder(Placeholder placeholder, ExpressionRewriteCon
         return visit(realExpr, context);
     }
 
+    // Register prepared statement placeholder id to related slot in comparison predicate.
+    // Used to replace expression in ShortCircuit plan
+    private void registerPlaceholderIdToSlot(ComparisonPredicate cp,
+                    ExpressionRewriteContext context, Expression left, Expression right) {
+        if (ConnectContext.get() != null
+                    && ConnectContext.get().getCommand() == MysqlCommand.COM_STMT_EXECUTE) {
+            // Used to replace expression in ShortCircuit plan
+            if (cp.right() instanceof Placeholder && left instanceof SlotReference) {
+                PlaceholderId id = ((Placeholder) cp.right()).getPlaceholderId();
+                context.cascadesContext.getStatementContext().getIdToComparisonSlot().put(id, (SlotReference) left);
+            } else if (cp.left() instanceof Placeholder && right instanceof SlotReference) {
+                PlaceholderId id = ((Placeholder) cp.left()).getPlaceholderId();
+                context.cascadesContext.getStatementContext().getIdToComparisonSlot().put(id, (SlotReference) right);
+            }
+        }
+    }
+
     @Override
     public Expression visitComparisonPredicate(ComparisonPredicate cp, ExpressionRewriteContext context) {
         Expression left = cp.left().accept(this, context);
         Expression right = cp.right().accept(this, context);
+        // Used to replace expression in ShortCircuit plan
+        registerPlaceholderIdToSlot(cp, context, left, right);
         cp = (ComparisonPredicate) cp.withChildren(left, right);
         return TypeCoercionUtils.processComparisonPredicate(cp);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
index 9e4030b768b911..b1bf3e227f0f0a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
@@ -31,7 +31,9 @@
 import org.apache.doris.mysql.MysqlCommand;
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.plans.PlaceholderId;
 import org.apache.doris.planner.OlapScanNode;
 import org.apache.doris.proto.InternalService;
 import org.apache.doris.proto.InternalService.KeyTuple;
@@ -59,12 +61,12 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
 
 public class PointQueryExecutor implements CoordInterface {
     private static final Logger LOG = LogManager.getLogger(PointQueryExecutor.class);
@@ -142,33 +144,45 @@ public static void directExecuteShortCircuitQuery(StmtExecutor executor,
         Preconditions.checkNotNull(preparedStmtCtx.shortCircuitQueryContext);
         ShortCircuitQueryContext shortCircuitQueryContext = preparedStmtCtx.shortCircuitQueryContext.get();
         // update conjuncts
-        List<Expr> conjunctVals = statementContext.getIdToPlaceholderRealExpr().values().stream().map(
-                        expression -> (
-                                (Literal) expression).toLegacyLiteral())
-                .collect(Collectors.toList());
-        if (conjunctVals.size() != preparedStmtCtx.command.placeholderCount()) {
+        Map<String, Expr> colNameToConjunct = Maps.newHashMap();
+        for (Entry<PlaceholderId, SlotReference> entry : statementContext.getIdToComparisonSlot().entrySet()) {
+            String colName = entry.getValue().getColumn().get().getName();
+            Expr conjunctVal = ((Literal)  statementContext.getIdToPlaceholderRealExpr()
+                    .get(entry.getKey())).toLegacyLiteral();
+            colNameToConjunct.put(colName, conjunctVal);
+        }
+        if (colNameToConjunct.size() != preparedStmtCtx.command.placeholderCount()) {
             throw new AnalysisException("Mismatched conjuncts values size with prepared"
                     + "statement parameters size, expected "
                     + preparedStmtCtx.command.placeholderCount()
-                    + ", but meet " + conjunctVals.size());
+                    + ", but meet " + colNameToConjunct.size());
         }
-        updateScanNodeConjuncts(shortCircuitQueryContext.scanNode, conjunctVals);
+        updateScanNodeConjuncts(shortCircuitQueryContext.scanNode, colNameToConjunct);
         // short circuit plan and execution
         executor.executeAndSendResult(false, false,
                 shortCircuitQueryContext.analzyedQuery, executor.getContext()
                         .getMysqlChannel(), null, null);
     }
 
-    private static void updateScanNodeConjuncts(OlapScanNode scanNode, List<Expr> conjunctVals) {
-        for (int i = 0; i < conjunctVals.size(); ++i) {
-            BinaryPredicate binaryPredicate = (BinaryPredicate) scanNode.getConjuncts().get(i);
+    private static void updateScanNodeConjuncts(OlapScanNode scanNode,
+                Map<String, Expr> colNameToConjunct) {
+        for (Expr conjunct : scanNode.getConjuncts()) {
+            BinaryPredicate binaryPredicate = (BinaryPredicate) conjunct;
+            SlotRef slot = null;
+            int updateChildIdx = 0;
             if (binaryPredicate.getChild(0) instanceof LiteralExpr) {
-                binaryPredicate.setChild(0, conjunctVals.get(i));
+                slot = (SlotRef) binaryPredicate.getChildWithoutCast(1);
             } else if (binaryPredicate.getChild(1) instanceof LiteralExpr) {
-                binaryPredicate.setChild(1, conjunctVals.get(i));
+                slot = (SlotRef) binaryPredicate.getChildWithoutCast(0);
+                updateChildIdx = 1;
             } else {
                 Preconditions.checkState(false, "Should contains literal in " + binaryPredicate.toSqlImpl());
             }
+            // not a placeholder to replace
+            if (!colNameToConjunct.containsKey(slot.getColumnName())) {
+                continue;
+            }
+            binaryPredicate.setChild(updateChildIdx, colNameToConjunct.get(slot.getColumnName()));
         }
     }
 
diff --git a/regression-test/data/point_query_p0/test_point_query.out b/regression-test/data/point_query_p0/test_point_query.out
index 1cc4142e39f306..55c797578208db 100644
--- a/regression-test/data/point_query_p0/test_point_query.out
+++ b/regression-test/data/point_query_p0/test_point_query.out
@@ -160,3 +160,33 @@
 -- !sql --
 -10	20	aabc	update val
 
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
+-- !point_select --
+user_guid	feature	sk	feature_value	2021-01-01T00:00
+
diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy b/regression-test/suites/point_query_p0/test_point_query.groovy
index 7c3a255b10eb73..a15b48084ca223 100644
--- a/regression-test/suites/point_query_p0/test_point_query.groovy
+++ b/regression-test/suites/point_query_p0/test_point_query.groovy
@@ -27,32 +27,30 @@ suite("test_point_query", "nonConcurrent") {
             logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
         }
     }
+    def user = context.config.jdbcUser
+    def password = context.config.jdbcPassword
+    def realDb = "regression_test_serving_p0"
+    // Parse url
+    String jdbcUrl = context.config.jdbcUrl
+    String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3)
+    def sql_ip = urlWithoutSchema.substring(0, urlWithoutSchema.indexOf(":"))
+    def sql_port
+    if (urlWithoutSchema.indexOf("/") >= 0) {
+        // e.g: jdbc:mysql://locahost:8080/?a=b
+        sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1, urlWithoutSchema.indexOf("/"))
+    } else {
+        // e.g: jdbc:mysql://locahost:8080
+        sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1)
+    }
+    // set server side prepared statement url
+    def prepare_url = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb + "?&useServerPrepStmts=true"
     try {
         set_be_config.call("disable_storage_row_cache", "false")
-        // nereids do not support point query now
         sql "set global enable_fallback_to_original_planner = false"
         sql """set global enable_nereids_planner=true"""
-        def user = context.config.jdbcUser
-        def password = context.config.jdbcPassword
-        def realDb = "regression_test_serving_p0"
         def tableName = realDb + ".tbl_point_query"
         sql "CREATE DATABASE IF NOT EXISTS ${realDb}"
 
-        // Parse url
-        String jdbcUrl = context.config.jdbcUrl
-        String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3)
-        def sql_ip = urlWithoutSchema.substring(0, urlWithoutSchema.indexOf(":"))
-        def sql_port
-        if (urlWithoutSchema.indexOf("/") >= 0) {
-            // e.g: jdbc:mysql://locahost:8080/?a=b
-            sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1, urlWithoutSchema.indexOf("/"))
-        } else {
-            // e.g: jdbc:mysql://locahost:8080
-            sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1)
-        }
-        // set server side prepared statement url
-        def prepare_url = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb + "?&useServerPrepStmts=true"
-
         def generateString = {len ->
             def str = ""
             for (int i = 0; i < len; i++) {
@@ -330,4 +328,60 @@ suite("test_point_query", "nonConcurrent") {
     qt_sql "select * from table_3821461 where col1 = 10 and col2 = 20 and loc3 = 'aabc';"
     sql "update table_3821461 set value = 'update value' where col1 = -10 or col1 = 20;"
     qt_sql """select * from table_3821461 where col1 = -10 and col2 = 20 and loc3 = 'aabc'"""
+
+    sql "DROP TABLE IF EXISTS test_partial_prepared_statement"
+    sql """
+        CREATE TABLE `test_partial_prepared_statement` (
+          `user_guid` varchar(64) NOT NULL,
+          `feature` varchar(256) NOT NULL,
+          `sk` varchar(256) NOT NULL,
+          `feature_value` text NULL,
+          `data_time` datetime NOT NULL
+        ) ENGINE=OLAP
+        UNIQUE KEY(`user_guid`, `feature`, `sk`)
+        DISTRIBUTED BY HASH(`user_guid`) BUCKETS 32
+        PROPERTIES (
+        "enable_unique_key_merge_on_write" = "true",
+        "light_schema_change" = "true",
+        "function_column.sequence_col" = "data_time",
+        "store_row_column" = "true",
+        "replication_num" = "1",
+        "row_store_page_size" = "16384"
+        );
+    """
+    sql "insert into test_partial_prepared_statement values ('user_guid', 'feature', 'sk','feature_value', '2021-01-01 00:00:00')"
+    def result2 = connect(user, password, prepare_url) {
+        def partial_prepared_stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=true) */ * from regression_test_point_query_p0.test_partial_prepared_statement where sk = 'sk' and user_guid = 'user_guid' and  feature = ? "
+        assertEquals(partial_prepared_stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
+        partial_prepared_stmt.setString(1, "feature")
+        qe_point_select partial_prepared_stmt
+        qe_point_select partial_prepared_stmt
+
+        partial_prepared_stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=true) */ * from regression_test_point_query_p0.test_partial_prepared_statement where user_guid = ? and  feature = 'feature' and sk = ?"
+        assertEquals(partial_prepared_stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
+        partial_prepared_stmt.setString(1, "user_guid")
+        partial_prepared_stmt.setString(2, "sk")
+        qe_point_select partial_prepared_stmt
+        qe_point_select partial_prepared_stmt
+
+        partial_prepared_stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=true) */ * from regression_test_point_query_p0.test_partial_prepared_statement where ? = user_guid and sk = 'sk'  and  feature = 'feature' "
+        assertEquals(partial_prepared_stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
+        partial_prepared_stmt.setString(1, "user_guid")
+        qe_point_select partial_prepared_stmt
+        qe_point_select partial_prepared_stmt
+
+        partial_prepared_stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=true) */ * from regression_test_point_query_p0.test_partial_prepared_statement where ? = user_guid and sk = 'sk'  and  feature = ? "
+        assertEquals(partial_prepared_stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
+        partial_prepared_stmt.setString(1, "user_guid")
+        partial_prepared_stmt.setString(2, "feature")
+        qe_point_select partial_prepared_stmt
+        qe_point_select partial_prepared_stmt
+
+        partial_prepared_stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=true) */ * from regression_test_point_query_p0.test_partial_prepared_statement where  sk = ? and  feature = ? and 'user_guid' = user_guid"
+        assertEquals(partial_prepared_stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement);
+        partial_prepared_stmt.setString(1, "sk")
+        partial_prepared_stmt.setString(2, "feature")
+        qe_point_select partial_prepared_stmt
+        qe_point_select partial_prepared_stmt
+    }
 } 
\ No newline at end of file

From 8cba7a2672ee99febfa5787d7b2d1340fd00d855 Mon Sep 17 00:00:00 2001
From: FreeOnePlus <54164178+FreeOnePlus@users.noreply.github.com>
Date: Mon, 16 Dec 2024 15:59:55 +0800
Subject: [PATCH 341/399] [feat](docker)Add a BE ENV item 'SKIP_CHECK_ULIMIT'
 for Docker to start quickly (#45267)

In the storage_engine.cpp of the BE process, it is mandatory to check
that the `ulimit` value must be greater than 60,000. When starting with
Docker or Docker-Compose, it is necessary to preemptively change the
corresponding value on the host machine. This change is very unfriendly
to Docker, as it loses its unique advantage of being able to "build
quickly and anywhere," and cannot become a fundamental capability for
rapid startup. Therefore, a new environment variable has been added to
control whether to skip this check value. The default value is false,
which enforces the check of this value. When set to true, it skips the
check and starts directly.
---
 be/src/olap/storage_engine.cpp | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index e00b5b595e20dc..24cda8232f115c 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -463,6 +463,16 @@ Status StorageEngine::_check_file_descriptor_number() {
                      << ", use default configuration instead.";
         return Status::OK();
     }
+    if (getenv("SKIP_CHECK_ULIMIT") == nullptr) {
+        LOG(INFO) << "will check 'ulimit' value.";
+    } else if (std::string(getenv("SKIP_CHECK_ULIMIT")) == "true") {
+        LOG(INFO) << "the 'ulimit' value check is skipped"
+                  << ", the SKIP_CHECK_ULIMIT env value is " << getenv("SKIP_CHECK_ULIMIT");
+        return Status::OK();
+    } else {
+        LOG(INFO) << "the SKIP_CHECK_ULIMIT env value is " << getenv("SKIP_CHECK_ULIMIT")
+                  << ", will check ulimit value.";
+    }
     if (l.rlim_cur < config::min_file_descriptor_number) {
         LOG(ERROR) << "File descriptor number is less than " << config::min_file_descriptor_number
                    << ". Please use (ulimit -n) to set a value equal or greater than "

From 5c2bfe7ec06c683cbd705382beea2c0c53acf42a Mon Sep 17 00:00:00 2001
From: abmdocrt <lianyukang@selectdb.com>
Date: Mon, 16 Dec 2024 16:14:32 +0800
Subject: [PATCH 342/399] [Fix](test) Fix cumu compaction with delete case fail
 (#45442)

After https://github.com/apache/doris/pull/44576, defining global
variables in test groovy files is disabled.
---
 .../compaction/test_cumu_compaction_with_delete.groovy    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
index 7c6be0b177ce1e..05ab96a628a7ec 100644
--- a/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
+++ b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
@@ -50,7 +50,7 @@ suite("test_cumu_compaction_with_delete") {
             sql """ delete from ${tableName} where user_id = 1"""
         }
 
-        now = System.currentTimeMillis()
+        def now = System.currentTimeMillis()
 
         while(true){
             if(check_cumu_point(100)){
@@ -58,7 +58,7 @@ suite("test_cumu_compaction_with_delete") {
             }
             Thread.sleep(1000)
         }
-        time_diff = System.currentTimeMillis() - now
+        def time_diff = System.currentTimeMillis() - now
         logger.info("time_diff:" + time_diff)
         assertTrue(time_diff<200*1000)
 
@@ -99,7 +99,7 @@ suite("test_cumu_compaction_with_delete") {
             sql """ delete from ${tableName} where user_id = 1"""
         }
 
-        now = System.currentTimeMillis()
+        def now = System.currentTimeMillis()
 
         while(true){
             if(check_cumu_point(100)){
@@ -107,7 +107,7 @@ suite("test_cumu_compaction_with_delete") {
             }
             Thread.sleep(1000)
         }
-        time_diff = System.currentTimeMillis() - now
+        def time_diff = System.currentTimeMillis() - now
         logger.info("time_diff:" + time_diff)
         assertTrue(time_diff>=200*1000)
 

From 79eddb78c08567be905994a87e20c243b4689c45 Mon Sep 17 00:00:00 2001
From: Jerry Hu <hushenggang@selectdb.com>
Date: Mon, 16 Dec 2024 17:15:17 +0800
Subject: [PATCH 343/399] [opt](join) Check the property of nullable from
 intermediate row (#45017)

---
 .../pipeline/exec/hashjoin_probe_operator.cpp | 29 ++++++++++++++-----
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
index 7c663b256832ed..7c98a4c9f97763 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
@@ -17,6 +17,8 @@
 
 #include "hashjoin_probe_operator.h"
 
+#include <gen_cpp/PlanNodes_types.h>
+
 #include <string>
 
 #include "common/cast_set.h"
@@ -616,21 +618,34 @@ Status HashJoinProbeOperatorX::open(RuntimeState* state) {
     size_t idx = 0;
     for (const auto* slot : slots_to_check) {
         auto data_type = slot->get_data_type_ptr();
-        auto target_data_type = idx < right_col_idx ? _left_table_data_types[idx]
-                                                    : _right_table_data_types[idx - right_col_idx];
+        const auto slot_on_left = idx < right_col_idx;
+        auto target_data_type = slot_on_left ? _left_table_data_types[idx]
+                                             : _right_table_data_types[idx - right_col_idx];
         ++idx;
         if (data_type->equals(*target_data_type)) {
             continue;
         }
 
-        auto data_type_non_nullable = vectorized::remove_nullable(data_type);
-        if (data_type_non_nullable->equals(*target_data_type)) {
+        /// For outer join(left/right/full), the non-nullable columns may be converted to nullable.
+        const auto accept_nullable_not_match =
+                _join_op == TJoinOp::FULL_OUTER_JOIN ||
+                (slot_on_left ? _join_op == TJoinOp::RIGHT_OUTER_JOIN
+                              : _join_op == TJoinOp::LEFT_OUTER_JOIN);
+
+        if (accept_nullable_not_match) {
+            auto data_type_non_nullable = vectorized::remove_nullable(data_type);
+            if (data_type_non_nullable->equals(*target_data_type)) {
+                continue;
+            }
+        } else if (data_type->equals(*target_data_type)) {
             continue;
         }
 
-        return Status::InternalError("intermediate slot({}) data type not match: '{}' vs '{}'",
-                                     slot->id(), data_type->get_name(),
-                                     _left_table_data_types[idx]->get_name());
+        return Status::InternalError(
+                "Join node(id={}, OP={}) intermediate slot({}, #{})'s on {} table data type not "
+                "match: '{}' vs '{}'",
+                _node_id, _join_op, slot->col_name(), slot->id(), (slot_on_left ? "left" : "right"),
+                data_type->get_name(), target_data_type->get_name());
     }
 
     _build_side_child.reset();

From cf6dba79c48c40775a40749d730c69916c5c900a Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Mon, 16 Dec 2024 14:50:33 +0530
Subject: [PATCH 344/399] [Enhancement] (nereids)implement CreateCatalogCommand
 in nereids (#45150)

Issue Number: close #42594
---
 .../org/apache/doris/nereids/DorisParser.g4   |   8 +-
 .../doris/datasource/CatalogFactory.java      |  10 ++
 .../apache/doris/datasource/CatalogMgr.java   |  33 ++--
 .../nereids/parser/LogicalPlanBuilder.java    |  14 ++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../plans/commands/CreateCatalogCommand.java  | 144 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../test_create_catalog_command.out           |   4 +
 .../test_create_catalog_command.groovy        |  43 ++++++
 9 files changed, 249 insertions(+), 13 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java
 create mode 100644 regression-test/data/nereids_p0/test_create_catalog_command.out
 create mode 100644 regression-test/suites/nereids_p0/test_create_catalog_command.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 044ed7771a4aa1..bc91a667eda44a 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -185,7 +185,10 @@ supportedCreateStatement
         (WITH ROLLUP (rollupNames=identifierList)?)?                      #createTableLike
     | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?    #createRole
     | CREATE WORKLOAD GROUP (IF NOT EXISTS)?
-        name=identifierOrText properties=propertyClause?                        #createWorkloadGroup
+        name=identifierOrText properties=propertyClause?                        #createWorkloadGroup          
+    | CREATE CATALOG (IF NOT EXISTS)? catalogName=identifier
+        (WITH RESOURCE resourceName=identifier)?
+        (COMMENT STRING_LITERAL)? properties=propertyClause?                    #createCatalog           
     | CREATE ROW POLICY (IF NOT EXISTS)? name=identifier
         ON table=multipartIdentifier
         AS type=(RESTRICTIVE | PERMISSIVE)
@@ -744,9 +747,6 @@ analyzeProperties
 unsupportedCreateStatement
     : CREATE (DATABASE | SCHEMA) (IF NOT EXISTS)? name=multipartIdentifier
         properties=propertyClause?                                              #createDatabase
-    | CREATE CATALOG (IF NOT EXISTS)? catalogName=identifier
-        (WITH RESOURCE resourceName=identifier)?
-        (COMMENT STRING_LITERAL)? properties=propertyClause?                    #createCatalog
     | CREATE (GLOBAL | SESSION | LOCAL)?
         (TABLES | AGGREGATE)? FUNCTION (IF NOT EXISTS)?
         functionIdentifier LEFT_PAREN functionArguments? RIGHT_PAREN
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
index 43ac17839c9606..984d29cae4923a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
@@ -37,6 +37,7 @@
 import org.apache.doris.datasource.paimon.PaimonExternalCatalogFactory;
 import org.apache.doris.datasource.test.TestExternalCatalog;
 import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalCatalogFactory;
+import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
 
 import com.google.common.base.Strings;
 import org.apache.logging.log4j.LogManager;
@@ -83,6 +84,15 @@ public static CatalogIf createFromLog(CatalogLog log) throws DdlException {
                 log.getComment(), log.getProps(), true);
     }
 
+    /**
+     * create the catalog instance from CreateCatalogCommand.
+     */
+    public static CatalogIf createFromCommand(long catalogId, CreateCatalogCommand cmd)
+            throws DdlException {
+        return createCatalog(catalogId, cmd.getCatalogName(), cmd.getResource(),
+                cmd.getComment(), cmd.getProperties(), false);
+    }
+
     /**
      * create the catalog instance from creating statement.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 070d31c071e930..0203aa7020b090 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -50,6 +50,7 @@
 import org.apache.doris.datasource.hive.HMSExternalCatalog;
 import org.apache.doris.datasource.hive.HMSExternalTable;
 import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
 import org.apache.doris.persist.OperationType;
 import org.apache.doris.persist.gson.GsonPostProcessable;
 import org.apache.doris.persist.gson.GsonUtils;
@@ -235,20 +236,16 @@ private void readUnlock() {
         lock.readLock().unlock();
     }
 
-    /**
-     * Create and hold the catalog instance and write the meta log.
-     */
-    public void createCatalog(CreateCatalogStmt stmt) throws UserException {
-        long id = Env.getCurrentEnv().getNextId();
-        CatalogIf catalog = CatalogFactory.createFromStmt(id, stmt);
+    private void createCatalogImpl(CatalogIf catalog, String catalogName,
+            boolean ifNotExists) throws UserException {
         writeLock();
         try {
             if (nameToCatalog.containsKey(catalog.getName())) {
-                if (stmt.isSetIfNotExists()) {
-                    LOG.warn("Catalog {} is already exist.", stmt.getCatalogName());
+                if (ifNotExists) {
+                    LOG.warn("Catalog {} is already exist.", catalogName);
                     return;
                 }
-                throw new DdlException("Catalog had already exist with name: " + stmt.getCatalogName());
+                throw new DdlException("Catalog had already exist with name: " + catalogName);
             }
             createCatalogInternal(catalog, false);
             Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_CREATE_CATALOG, catalog.constructEditLog());
@@ -257,6 +254,24 @@ public void createCatalog(CreateCatalogStmt stmt) throws UserException {
         }
     }
 
+    /**
+     * Create and hold the catalog instance and write the meta log.
+     */
+    public void createCatalog(CreateCatalogCommand cmd) throws UserException {
+        long id = Env.getCurrentEnv().getNextId();
+        CatalogIf catalog = CatalogFactory.createFromCommand(id, cmd);
+        createCatalogImpl(catalog, cmd.getCatalogName(), cmd.isSetIfNotExists());
+    }
+
+    /**
+     * Create and hold the catalog instance and write the meta log.
+     */
+    public void createCatalog(CreateCatalogStmt stmt) throws UserException {
+        long id = Env.getCurrentEnv().getNextId();
+        CatalogIf catalog = CatalogFactory.createFromStmt(id, stmt);
+        createCatalogImpl(catalog, stmt.getCatalogName(), stmt.isSetIfNotExists());
+    }
+
     /**
      * Remove the catalog instance by name and write the meta log.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index e6fee894b41fa9..94c2757e560b24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -95,6 +95,7 @@
 import org.apache.doris.nereids.DorisParser.ComplexColTypeListContext;
 import org.apache.doris.nereids.DorisParser.ComplexDataTypeContext;
 import org.apache.doris.nereids.DorisParser.ConstantContext;
+import org.apache.doris.nereids.DorisParser.CreateCatalogContext;
 import org.apache.doris.nereids.DorisParser.CreateEncryptkeyContext;
 import org.apache.doris.nereids.DorisParser.CreateFileContext;
 import org.apache.doris.nereids.DorisParser.CreateMTMVContext;
@@ -500,6 +501,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.Constraint;
+import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
@@ -4843,6 +4845,18 @@ public LogicalPlan visitShowDynamicPartition(ShowDynamicPartitionContext ctx) {
         return new ShowDynamicPartitionCommand(dbName);
     }
 
+    @Override
+    public LogicalPlan visitCreateCatalog(CreateCatalogContext ctx) {
+        String catalogName = ctx.catalogName.getText();
+        boolean ifNotExists = ctx.IF() != null;
+        String resourceName = ctx.resourceName == null ? null : (ctx.resourceName.getText());
+        String comment = ctx.STRING_LITERAL() == null ? null : stripQuotes(ctx.STRING_LITERAL().getText());
+        Map<String, String> properties = ctx.propertyClause() != null
+                                    ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+
+        return new CreateCatalogCommand(catalogName, ifNotExists, resourceName, comment, properties);
+    }
+
     @Override
     public LogicalPlan visitRecoverDatabase(RecoverDatabaseContext ctx) {
         String dbName = ctx.name.getText();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index e8ce4b3e2d88db..26b8ed4c0c9365 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -252,6 +252,7 @@ public enum PlanType {
     REPLAY_COMMAND,
     CREATE_ENCRYPTKEY_COMMAND,
     CREATE_WORKLOAD_GROUP_COMMAND,
+    CREATE_CATALOG_COMMAND,
     CREATE_FILE_COMMAND,
     CREATE_ROUTINE_LOAD_COMMAND,
     SHOW_TABLE_CREATION_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java
new file mode 100644
index 00000000000000..22c1285a0556fd
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.PrintableMap;
+import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Command to create a catalog in the Nereids planner.
+ */
+public class CreateCatalogCommand extends Command implements ForwardWithSync, NeedAuditEncryption {
+    private final String catalogName;
+    private final boolean ifNotExists;
+    private final String resourceName;
+    private final String comment;
+    private final Map<String, String> properties;
+
+    /**
+    * Constructor
+    */
+    public CreateCatalogCommand(String catalogName, boolean ifNotExists, String resourceName, String comment,
+                                Map<String, String> properties) {
+        super(PlanType.CREATE_CATALOG_COMMAND);
+        this.catalogName = Objects.requireNonNull(catalogName, "Catalog name cannot be null");
+        this.ifNotExists = ifNotExists;
+        this.resourceName = resourceName;
+        this.comment = comment;
+        this.properties = properties == null ? Maps.newHashMap() : properties;
+    }
+
+    private void validate(ConnectContext ctx) throws AnalysisException {
+        Util.checkCatalogAllRules(catalogName);
+        if (catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+            throw new AnalysisException("Internal catalog name can't be create.");
+        }
+
+        if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv(
+                ConnectContext.get(), catalogName, PrivPredicate.CREATE)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED,
+                    ctx.getQualifiedUser(), catalogName);
+        }
+
+        if (Config.disallow_create_catalog_with_resource && !Strings.isNullOrEmpty(resourceName)) {
+            throw new AnalysisException("Create catalog with resource is deprecated and is not allowed."
+                    + " You can set `disallow_create_catalog_with_resource=false` in fe.conf"
+                    + " to enable it temporarily.");
+        }
+
+        String currentDateTime = LocalDateTime.now(ZoneId.systemDefault()).toString().replace("T", " ");
+        properties.put(ExternalCatalog.CREATE_TIME, currentDateTime);
+        PropertyAnalyzer.checkCatalogProperties(properties, false);
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        Env.getCurrentEnv().getCatalogMgr().createCatalog(this);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCreateCatalogCommand(this, context);
+    }
+
+    public String getCatalogName() {
+        return catalogName;
+    }
+
+    public boolean isSetIfNotExists() {
+        return ifNotExists;
+    }
+
+    public String getResource() {
+        return resourceName;
+    }
+
+    public String getComment() {
+        return comment;
+    }
+
+    public Map<String, String> getProperties() {
+        return properties;
+    }
+
+    @Override
+    public boolean needAuditEncryption() {
+        return true;
+    }
+
+    @Override
+    public String toSql() {
+        StringBuilder stringBuilder = new StringBuilder();
+        stringBuilder.append("CREATE CATALOG ").append("`").append(catalogName).append("`");
+        if (!Strings.isNullOrEmpty(resourceName)) {
+            stringBuilder.append(" WITH RESOURCE `").append(resourceName).append("`");
+        }
+        if (!Strings.isNullOrEmpty(comment)) {
+            stringBuilder.append("\nCOMMENT \"").append(comment).append("\"");
+        }
+        if (properties.size() > 0) {
+            stringBuilder.append("\nPROPERTIES (\n");
+            stringBuilder.append(new PrintableMap<>(properties, "=", true, true, true));
+            stringBuilder.append("\n)");
+        }
+        return stringBuilder.toString();
+    }
+}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 0341ed2f57d7ec..efb81ae7ee037f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -38,6 +38,7 @@
 import org.apache.doris.nereids.trees.plans.commands.CancelWarmUpJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
 import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
@@ -305,6 +306,10 @@ default R visitShowProcedureStatusCommand(ShowProcedureStatusCommand showProcedu
         return visitCommand(showProcedureStatusCommand, context);
     }
 
+    default R visitCreateCatalogCommand(CreateCatalogCommand createCatalogCommand, C context) {
+        return visitCommand(createCatalogCommand, context);
+    }
+
     default R visitShowWarningErrorsCommand(ShowWarningErrorsCommand showWarningErrorsCommand, C context) {
         return visitCommand(showWarningErrorsCommand, context);
     }
diff --git a/regression-test/data/nereids_p0/test_create_catalog_command.out b/regression-test/data/nereids_p0/test_create_catalog_command.out
new file mode 100644
index 00000000000000..b3ff20b67db426
--- /dev/null
+++ b/regression-test/data/nereids_p0/test_create_catalog_command.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !cmd --
+test_create_catalog	\nCREATE CATALOG `test_create_catalog`\nCOMMENT "Test catalog for regression"\n PROPERTIES (\n"type" = "es",\n"hosts" = "http://127.0.0.1:9200"\n);
+
diff --git a/regression-test/suites/nereids_p0/test_create_catalog_command.groovy b/regression-test/suites/nereids_p0/test_create_catalog_command.groovy
new file mode 100644
index 00000000000000..be28de0a246dd3
--- /dev/null
+++ b/regression-test/suites/nereids_p0/test_create_catalog_command.groovy
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_create_catalog_command", "nereids_p0") {
+    def catalogName = "test_create_catalog"
+    def resourceName = "test_resource"
+    def catalogProperties = "\"type\"=\"es\", \"hosts\"=\"http://127.0.0.1:9200\""
+
+    try {
+        // Drop catalog if it already exists
+        sql("DROP CATALOG IF EXISTS ${catalogName}")
+
+        // Create a new catalog
+        checkNereidsExecute(
+            """
+            CREATE CATALOG ${catalogName} 
+            COMMENT 'Test catalog for regression'
+            PROPERTIES (${catalogProperties})
+            """
+        )
+
+        // Verify the catalog was created
+        checkNereidsExecute("""show create catalog ${catalogName}""")
+        qt_cmd("""show create catalog ${catalogName}""")
+    } finally {
+        // Clean up
+        sql("DROP CATALOG IF EXISTS ${catalogName}")
+    }
+}

From aee98d89b4f1cd91d3b8223ab142ce7004ac8f41 Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Mon, 16 Dec 2024 18:12:35 +0800
Subject: [PATCH 345/399] [fix](schema-change) fix array/map/struct in
 schema-change not-null to null will make core (#45305)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

if we create table with array/map/struct column which defined not null
then we alter column to null, in fact it can be work ,
but when we select it ,here may meet core below
```
mysql> create table etest ( `id` VARCHAR(50) NOT NULL COMMENT '主键id',  `second_categories` ARRAY < VARCHAR(255)> NOT NULL COMMENT '二级分类') ENGINE = OLAP UNIQUE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1  PROPERTIES (  'replication_allocation' = 'tag.location.default: 1',  'enable_unique_key_merge_on_write' = 'true',  'light_schema_change' = 'true', 'disable_auto_compaction'='true');
Query OK, 0 rows affected (0.12 sec)

mysql> insert into etest values (3, ['aaa', 'bbb', 'ccc']);
Query OK, 1 row affected (0.22 sec)
{'label':'label_e30fad022ff5423f_9c74c9fc9eb886a7', 'status':'VISIBLE', 'txnId':'1025'}

mysql> alter table etest  modify column `second_categories` array<varchar(255)> NULL COMMENT '二级分类';
Query OK, 0 rows affected (0.10 sec)

mysql> show alter table column;
+-------+-----------+-------------------------+-------------------------+-----------+---------+---------------+---------------+---------------+----------+------+----------+---------+
| JobId | TableName | CreateTime              | FinishTime              | IndexName | IndexId | OriginIndexId | SchemaVersion | TransactionId | State    | Msg  | Progress | Timeout |
+-------+-----------+-------------------------+-------------------------+-----------+---------+---------------+---------------+---------------+----------+------+----------+---------+
| 11109 | mtest     | 2024-12-10 20:46:04.491 | 2024-12-10 20:46:05.731 | mtest     | 11110   | 11093         | 1:1857912462  | 1007          | FINISHED |      | NULL     | 2592000 |
| 11156 | btest     | 2024-12-10 21:07:07.575 | 2024-12-10 21:07:08.901 | btest     | 11157   | 11147         | 1:70379151    | 1018          | FINISHED |      | NULL     | 2592000 |
| 11178 | ctest     | 2024-12-10 21:10:52.656 | 2024-12-10 21:10:53.931 | ctest     | 11179   | 11162         | 1:1462395050  | 1022          | FINISHED |      | NULL     | 2592000 |
| 11189 | dtest     | 2024-12-10 21:22:17.019 | 2024-12-10 21:22:18.029 | dtest     | 11190   | 11184         | 1:1194829729  | 1024          | FINISHED |      | NULL     | 2592000 |
| 11200 | etest     | 2024-12-10 21:30:06.902 | 2024-12-10 21:30:08.094 | etest     | 11201   | 11195         | 1:18626462    | 1026          | FINISHED |      | NULL     | 2592000 |
+-------+-----------+-------------------------+-------------------------+-----------+---------+---------------+---------------+---------------+----------+------+----------+---------+
5 rows in set (0.10 sec)

mysql> select * from etest;
ERROR 1105 (HY000): errCode = 2, detailMessage = There is no scanNode Backend available.[10048: not alive]
```

be core:
```
StdoutLogger 2024-12-10 21:09:52,967 Start time: 2024年 12月 10日 星期二 21:09:52 CST
INFO: java_cmd /mnt/disk2/wangqiannan/tool/jdk-17.0.10/bin/java
INFO: jdk_version 17
Java HotSpot(TM) 64-Bit Server VM warning: Option CriticalJNINatives was deprecated in version 16.0 and will likely be removed in a future release.
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/mnt/disk2/wangqiannan/amory/master/doris/output/be/lib/java_extensions/preload-extensions/preload-extensions-jar-with-dependencies.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/mnt/disk2/wangqiannan/amory/master/doris/output/be/lib/java_extensions/java-udf/java-udf-jar-with-dependencies.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/mnt/disk2/wangqiannan/amory/master/doris/output/be/lib/hadoop_hdfs/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Reload4jLoggerFactory]
[WARNING!] /sys/kernel/mm/transparent_hugepage/enabled: [always] madvise never, Doris not recommend turning on THP, which may cause the BE process to use more memory and cannot be freed in time. Turn off THP: `echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled`
*** Query id: 5a013c428ba942dd-afd4481a8e7ec4b3 ***
*** is nereids: 1 ***
*** tablet id: 0 ***
*** Aborted at 1733836267 (unix time) try "date -d @1733836267" if you are using GNU date ***
*** Current BE git commitID: 443e87e203 ***
*** SIGSEGV address not mapped to object (@0x0) received by PID 3270216 (TID 3276799 OR 0x7f0aae76a700) from PID 0; stack trace: ***
 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /mnt/disk2/wangqiannan/amory/master/doris/be/src/common/signal_handler.h:421
 1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0] in /mnt/disk2/wangqiannan/tool/jdk-17.0.10/lib/server/libjvm.so
 2# JVM_handle_linux_signal in /mnt/disk2/wangqiannan/tool/jdk-17.0.10/lib/server/libjvm.so
 3# 0x00007F10737305B0 in /lib64/libc.so.6
 4# doris::segment_v2::ArrayFileColumnIterator::next_batch(unsigned long*, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>&, bool*) at /mnt/disk2/wangqiannan/amory/master/doris/be/src/olap/rowset/segment_v2/column_reader.cpp:999
 5# doris::segment_v2::ArrayFileColumnIterator::read_by_rowids(unsigned int const*, unsigned long, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>&) at /mnt/disk2/wangqiannan/amory/master/doris/be/src/olap/rowset/segment_v2/column_reader.cpp:1012
 6# doris::segment_v2::SegmentIterator::_read_columns_by_rowids(std::vector<unsigned int, std::allocator<unsigned int> >&, std::vector<unsigned int, std::allocator<unsigned int> >&, unsigned short*, unsigned long, std::vector<COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>, std::allocator<COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn> > >*) in /mnt/disk2/wangqiannan/amory/master/doris/output/be/lib/doris_be
 7# doris::segment_v2::SegmentIterator::_next_batch_internal(doris::vectorized::Block*) at /mnt/disk2/wangqiannan/amory/master/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:2222
 8# doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*) at /mnt/disk2/wangqiannan/amory/master/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:1908
 9# doris::segment_v2::LazyInitSegmentIterator::next_batch(doris::vectorized::Block*) at /mnt/disk2/wangqiannan/amory/master/doris/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h:45
10# doris::BetaRowsetReader::next_block(doris::vectorized::Block*) at /mnt/disk2/wangqiannan/amory/master/doris/be/src/olap/rowset/beta_rowset_reader.cpp:342
11# doris::vectorized::VCollectIterator::Level0Iterator::refresh_current_row() at /mnt/disk2/wangqiannan/amory/master/doris/be/src/vec/olap/vcollect_iterator.cpp:508
12# doris::vectorized::VCollectIterator::Level0Iterator::ensure_first_row_ref() at /mnt/disk2/wangqiannan/amory/master/doris/be/src/vec/olap/vcollect_iterator.cpp:483
13# doris::vectorized::VCollectIterator::Level1Iterator::ensure_first_row_ref() in /mnt/disk2/wangqiannan/amory/master/doris/output/be/lib/doris_be
14# doris::vectorized::VCollectIterator::build_heap(std::vector<std::shared_ptr<doris::RowsetReader>, std::allocator<std::shared_ptr<doris::RowsetReader> > >&) at /mnt/disk2/wangqiannan/amory/master/doris/be/src/vec/olap/vcollect_iterator.cpp:186
```
---
 .../olap/rowset/segment_v2/column_reader.cpp  | 42 ++++++++--
 .../create_nestedtypes_with_schemachange.out  | 84 +++++++++++++++++++
 ...reate_nestedtypes_with_schemachange.groovy | 57 +++++++++++++
 3 files changed, 177 insertions(+), 6 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index b96cf4f7e6794e..9d5328de869304 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -871,8 +871,18 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
         size_t num_read = *n;
         auto null_map_ptr =
                 static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
-        bool null_signs_has_null = false;
-        RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+        // in not-null to null linked-schemachange mode,
+        // actually we do not change dat data include meta in footer,
+        // so may dst from changed meta which is nullable but old data is not nullable,
+        // if so, we should set null_map to all null by default
+        if (_null_iterator) {
+            bool null_signs_has_null = false;
+            RETURN_IF_ERROR(
+                    _null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+        } else {
+            auto& null_map = assert_cast<vectorized::ColumnUInt8&>(*null_map_ptr);
+            null_map.insert_many_vals(0, num_read);
+        }
         DCHECK(num_read == *n);
     }
     return Status::OK();
@@ -932,8 +942,18 @@ Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumn
         size_t num_read = *n;
         auto null_map_ptr =
                 static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
-        bool null_signs_has_null = false;
-        RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+        // in not-null to null linked-schemachange mode,
+        // actually we do not change dat data include meta in footer,
+        // so may dst from changed meta which is nullable but old data is not nullable,
+        // if so, we should set null_map to all null by default
+        if (_null_iterator) {
+            bool null_signs_has_null = false;
+            RETURN_IF_ERROR(
+                    _null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+        } else {
+            auto& null_map = assert_cast<vectorized::ColumnUInt8&>(*null_map_ptr);
+            null_map.insert_many_vals(0, num_read);
+        }
         DCHECK(num_read == *n);
     }
 
@@ -1086,8 +1106,18 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnP
         auto null_map_ptr =
                 static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
         size_t num_read = *n;
-        bool null_signs_has_null = false;
-        RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+        // in not-null to null linked-schemachange mode,
+        // actually we do not change dat data include meta in footer,
+        // so may dst from changed meta which is nullable but old data is not nullable,
+        // if so, we should set null_map to all null by default
+        if (_null_iterator) {
+            bool null_signs_has_null = false;
+            RETURN_IF_ERROR(
+                    _null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+        } else {
+            auto& null_map = assert_cast<vectorized::ColumnUInt8&>(*null_map_ptr);
+            null_map.insert_many_vals(0, num_read);
+        }
         DCHECK(num_read == *n);
     }
 
diff --git a/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out b/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
index af88eb6b9bb042..5b811095fcbc33 100644
--- a/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
+++ b/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
@@ -44,3 +44,87 @@ col3	array<int>	Yes	false	\N	NONE
 col4	map<int,int>	Yes	false	\N	NONE
 col5	struct<f1:int>	Yes	false	\N	NONE
 
+-- !sql_before --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql --
+col0	bigint	No	true	\N	
+col2	int	No	false	\N	NONE
+col3	array<int>	Yes	false	\N	NONE
+col4	map<int,int>	Yes	false	\N	NONE
+col5	struct<f1:int>	Yes	false	\N	NONE
+col6	variant	Yes	false	\N	NONE
+
+-- !sql_after --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql_before --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql --
+col0	bigint	No	true	\N	
+col2	int	No	false	\N	NONE
+col3	array<int>	Yes	false	\N	NONE
+col4	map<int,int>	No	false	\N	NONE
+col5	struct<f1:int>	No	false	\N	NONE
+col6	variant	No	false	\N	NONE
+
+-- !sql_after --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql_before --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql --
+col0	bigint	No	true	\N	
+col2	int	No	false	\N	NONE
+col3	array<int>	Yes	false	\N	NONE
+col4	map<int,int>	Yes	false	\N	NONE
+col5	struct<f1:int>	Yes	false	\N	NONE
+col6	variant	Yes	false	\N	NONE
+
+-- !sql_after --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql_before --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql --
+col0	bigint	No	true	\N	
+col2	int	No	false	\N	NONE
+col3	array<int>	No	false	\N	NONE
+col4	map<int,int>	Yes	false	\N	NONE
+col5	struct<f1:int>	No	false	\N	NONE
+col6	variant	No	false	\N	NONE
+
+-- !sql_after --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql_before --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql --
+col0	bigint	No	true	\N	
+col2	int	No	false	\N	NONE
+col3	array<int>	Yes	false	\N	NONE
+col4	map<int,int>	Yes	false	\N	NONE
+col5	struct<f1:int>	Yes	false	\N	NONE
+col6	variant	Yes	false	\N	NONE
+
+-- !sql_after --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql_before --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
+-- !sql --
+col0	bigint	No	true	\N	
+col2	int	No	false	\N	NONE
+col3	array<int>	No	false	\N	NONE
+col4	map<int,int>	No	false	\N	NONE
+col5	struct<f1:int>	Yes	false	\N	NONE
+col6	variant	No	false	\N	NONE
+
+-- !sql_after --
+1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
+
diff --git a/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy b/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
index ec0f163821d16a..62b9ab3bb567f4 100644
--- a/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
@@ -62,4 +62,61 @@ suite("create_nestedtypes_with_schemachange", "p0") {
     // struct with other type
     create_nested_table_and_schema_change.call("test_struct_schemachange_1", "STRUCT<f1: varchar(1)>", "col5", "errCode = 2");
 
+    def create_nested_table_and_schema_change_null_trans = {testTablex, nested_type, column_name, notNull2Null ->
+        def null_define = "NULL"
+        if (notNull2Null) {
+            null_define = "NOT NULL"
+        }
+        // create basic type
+        sql "DROP TABLE IF EXISTS $testTablex"
+        sql """ CREATE TABLE IF NOT EXISTS $testTablex (
+                     col0 BIGINT NOT NULL,
+                     col2 int NOT NULL,
+                     col3 array<int> $null_define,
+                     col4 map<int, int> $null_define,
+                     col5 struct<f1: int> $null_define,
+                     col6 variant $null_define
+                )
+                /* mow */
+                UNIQUE KEY(col0) DISTRIBUTED BY HASH(col0) BUCKETS 4 PROPERTIES (
+                  "enable_unique_key_merge_on_write" = "true",
+                  "replication_num" = "1",
+                  'light_schema_change' = 'true', 'disable_auto_compaction'='true'
+                ); """
+        // insert data
+        sql """ INSERT INTO $testTablex VALUES (1, 2, array(1, 2), map(1, 2), named_struct('f1', 1), '{"a": [1,2,3]}')"""
+        // select
+        qt_sql_before "select * from $testTablex"
+
+        if (notNull2Null) {
+            sql "ALTER TABLE $testTablex MODIFY COLUMN $column_name $nested_type NULL"
+            waitForSchemaChangeDone {
+                sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='$testTablex' ORDER BY createtime DESC LIMIT 1 """
+                time 600
+            }
+        } else {
+            // schema change from null to non-nullable is not supported
+            test {
+               sql "ALTER TABLE $testTablex MODIFY COLUMN $column_name $nested_type NOT NULL"
+               exception "Can not change from nullable to non-nullable"
+            }
+        }
+        // desc table
+        qt_sql "DESC $testTablex"
+        qt_sql_after "select * from $testTablex"
+    }
+
+    // array
+    create_nested_table_and_schema_change_null_trans.call("test_array_schemachange_null", "ARRAY<INT>", "col3", false)
+    create_nested_table_and_schema_change_null_trans.call("test_array_schemachange_null1", "ARRAY<INT>", "col3", true)
+    // map
+    create_nested_table_and_schema_change_null_trans.call("test_map_schemachange_null", "Map<INT, INT>", "col4", false)
+    create_nested_table_and_schema_change_null_trans.call("test_map_schemachange_null1", "Map<INT, INT>", "col4", true)
+    // struct
+    create_nested_table_and_schema_change_null_trans.call("test_struct_schemachange_null", "struct<f1: int>", "col5", false)
+    create_nested_table_and_schema_change_null_trans.call("test_struct_schemachange_null1", "struct<f1: int>", "col5", true)
+    // variant
+    // create_nested_table_and_schema_change_null_trans.call("test_v_schemachange_null", "variant", "col6", false)
+    // create_nested_table_and_schema_change_null_trans.call("test_v_schemachange_null1", "variant", "col6", true)
+
 }

From 6b5cef3ea5105f2ff85bb7bb0b711b305dfd54b2 Mon Sep 17 00:00:00 2001
From: yujun <yujun@selectdb.com>
Date: Mon, 16 Dec 2024 18:29:17 +0800
Subject: [PATCH 346/399] [opt](nereids) range inference convert isnull to
 EmptyValue  (#45310)

Problem Summary:

for sql

a > 10 and a < 20 or a > 30 and a < 40 or  a > 60 and a < 50

AddMinMax should opt it as:

(a < 20 or a > 30 or a is null and null) and a > 10 and a < 40

But it fail, because the SimplifyRange will opt it as

a > 10 and a < 20 or a > 30 and a < 40 or  a is null and null

since the 3rd is `a is null and null`, it couldn't get a's range and
suppose a's range is [-00, +00], then the whole expression, it will get
a's range is [-00, +00], to fix this problem, RangeInference need to
convert `a is null and null` to EmptyValue instead of UnknownValue.
---
 .../rules/expression/rules/AddMinMax.java     | 12 +++++-
 .../expression/rules/RangeInference.java      | 26 +++++++++++-
 .../rules/expression/rules/SimplifyRange.java | 30 +++++--------
 .../expression/ExpressionRewriteTest.java     | 42 ++++++++++++++++++-
 4 files changed, 88 insertions(+), 22 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/AddMinMax.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/AddMinMax.java
index 639616244ec0ab..c1efb82bac6efc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/AddMinMax.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/AddMinMax.java
@@ -326,7 +326,17 @@ private Map<Expression, MinMaxValue> getExprMinMaxValues(UnknownValue valueDesc)
         for (int i = 1; i < sourceValues.size(); i++) {
             // process in sourceValues[i]
             Map<Expression, MinMaxValue> minMaxValues = getExprMinMaxValues(sourceValues.get(i));
-            for (Map.Entry<Expression, MinMaxValue> entry : minMaxValues.entrySet()) {
+            // merge values of sourceValues[i] into result.
+            // also keep the value's relative order in sourceValues[i].
+            // for example, if a and b in sourceValues[i], but not in result, then during merging,
+            // a and b will assign a new exprOrderIndex (using nextExprOrderIndex).
+            // if in sourceValues[i], a's exprOrderIndex < b's exprOrderIndex,
+            // then make sure in result, a's new exprOrderIndex < b's new exprOrderIndex.
+            // so that their relative order can preserve.
+            List<Map.Entry<Expression, MinMaxValue>> minMaxValueList = minMaxValues.entrySet().stream()
+                    .sorted((a, b) -> Integer.compare(a.getValue().exprOrderIndex, b.getValue().exprOrderIndex))
+                    .collect(Collectors.toList());
+            for (Map.Entry<Expression, MinMaxValue> entry : minMaxValueList) {
                 Expression expr = entry.getKey();
                 MinMaxValue value = result.get(expr);
                 MinMaxValue otherValue = entry.getValue();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java
index 74099df1123f25..a94444e160709b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java
@@ -25,10 +25,12 @@
 import org.apache.doris.nereids.trees.expressions.GreaterThan;
 import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
 import org.apache.doris.nereids.trees.expressions.InPredicate;
+import org.apache.doris.nereids.trees.expressions.IsNull;
 import org.apache.doris.nereids.trees.expressions.LessThan;
 import org.apache.doris.nereids.trees.expressions.LessThanEqual;
 import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
@@ -130,10 +132,22 @@ private ValueDesc simplify(ExpressionRewriteContext context,
             Expression originExpr, List<Expression> predicates,
             BinaryOperator<ValueDesc> op, boolean isAnd) {
 
+        boolean convertIsNullToEmptyValue = isAnd && predicates.stream().anyMatch(expr -> expr instanceof NullLiteral);
         Multimap<Expression, ValueDesc> groupByReference
                 = Multimaps.newListMultimap(new LinkedHashMap<>(), ArrayList::new);
         for (Expression predicate : predicates) {
-            ValueDesc valueDesc = predicate.accept(this, context);
+            // EmptyValue(a) = IsNull(a) and null,  it doesn't equals to IsNull(a).
+            // Only the and expression contains at least a null literal in its conjunctions,
+            // then EmptyValue(a) can equivalent to IsNull(a).
+            // so for expression and(IsNull(a), IsNull(b), ..., null), a, b can convert to EmptyValue.
+            // What's more, if a is not nullable, then EmptyValue(a) always equals to IsNull(a),
+            // but we don't consider this case here, we should fold IsNull(a) to FALSE using other rule.
+            ValueDesc valueDesc = null;
+            if (convertIsNullToEmptyValue && predicate instanceof IsNull) {
+                valueDesc = new EmptyValue(context, ((IsNull) predicate).child(), predicate);
+            } else {
+                valueDesc = predicate.accept(this, context);
+            }
             List<ValueDesc> valueDescs = (List<ValueDesc>) groupByReference.get(valueDesc.reference);
             valueDescs.add(valueDesc);
         }
@@ -461,6 +475,11 @@ public boolean isAnd() {
 
         @Override
         public ValueDesc union(ValueDesc other) {
+            // for RangeValue/DiscreteValue/UnknownValue, when union with EmptyValue,
+            // call EmptyValue.union(this) => this
+            if (other instanceof EmptyValue) {
+                return other.union(this);
+            }
             Expression originExpr = FoldConstantRuleOnFE.evaluate(
                     ExpressionUtils.or(toExpr, other.toExpr), context);
             return new UnknownValue(context, originExpr,
@@ -469,6 +488,11 @@ public ValueDesc union(ValueDesc other) {
 
         @Override
         public ValueDesc intersect(ValueDesc other) {
+            // for RangeValue/DiscreteValue/UnknownValue, when intersect with EmptyValue,
+            // call EmptyValue.intersect(this) => EmptyValue
+            if (other instanceof EmptyValue) {
+                return other.intersect(this);
+            }
             Expression originExpr = FoldConstantRuleOnFE.evaluate(
                     ExpressionUtils.and(toExpr, other.toExpr), context);
             return new UnknownValue(context, originExpr,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
index 628d94d1dccafd..4666342943a85c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
@@ -25,22 +25,16 @@
 import org.apache.doris.nereids.rules.expression.rules.RangeInference.RangeValue;
 import org.apache.doris.nereids.rules.expression.rules.RangeInference.UnknownValue;
 import org.apache.doris.nereids.rules.expression.rules.RangeInference.ValueDesc;
-import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.GreaterThan;
 import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
 import org.apache.doris.nereids.trees.expressions.InPredicate;
-import org.apache.doris.nereids.trees.expressions.IsNull;
 import org.apache.doris.nereids.trees.expressions.LessThan;
 import org.apache.doris.nereids.trees.expressions.LessThanEqual;
-import org.apache.doris.nereids.trees.expressions.Not;
 import org.apache.doris.nereids.trees.expressions.Or;
-import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
-import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
-import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
 import com.google.common.collect.BoundType;
@@ -52,7 +46,6 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 /**
  * This class implements the function to simplify expression range.
@@ -108,11 +101,7 @@ private Expression getExpression(ValueDesc value) {
 
     private Expression getExpression(EmptyValue value) {
         Expression reference = value.getReference();
-        if (reference.nullable()) {
-            return new And(new IsNull(reference), new NullLiteral(BooleanType.INSTANCE));
-        } else {
-            return BooleanLiteral.FALSE;
-        }
+        return ExpressionUtils.falseOrNull(reference);
     }
 
     private Expression getExpression(RangeValue value) {
@@ -136,11 +125,7 @@ private Expression getExpression(RangeValue value) {
         if (!result.isEmpty()) {
             return ExpressionUtils.and(result);
         } else {
-            if (reference.nullable()) {
-                return new Or(new Not(new IsNull(reference)), new NullLiteral(BooleanType.INSTANCE));
-            } else {
-                return BooleanLiteral.TRUE;
-            }
+            return ExpressionUtils.trueOrNull(reference);
         }
     }
 
@@ -167,8 +152,15 @@ private Expression getExpression(UnknownValue value) {
         if (sourceValues.isEmpty()) {
             return originExpr;
         }
-        List<Expression> sourceExprs = sourceValues.stream().map(sourceValue -> getExpression(sourceValue))
-                .collect(Collectors.toList());
+        List<Expression> sourceExprs = Lists.newArrayListWithExpectedSize(sourceValues.size());
+        for (ValueDesc sourceValue : sourceValues) {
+            Expression expr = getExpression(sourceValue);
+            if (value.isAnd()) {
+                sourceExprs.addAll(ExpressionUtils.extractConjunction(expr));
+            } else {
+                sourceExprs.addAll(ExpressionUtils.extractDisjunction(expr));
+            }
+        }
         Expression result = value.isAnd() ? ExpressionUtils.and(sourceExprs) : ExpressionUtils.or(sourceExprs);
         result = FoldConstantRuleOnFE.evaluate(result, value.getExpressionRewriteContext());
         // ATTN: we must return original expr, because OrToIn is implemented with MutableState,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java
index 296cae5fe4023b..0fcc8043944fef 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java
@@ -277,7 +277,7 @@ void testDeadLoop() {
     }
 
     @Test
-    void testOrAddMinMax() {
+    void testAddMinMax() {
         executor = new ExpressionRuleExecutor(ImmutableList.of(
             bottomUp(
                 AddMinMax.INSTANCE
@@ -336,4 +336,44 @@ void testOrAddMinMax() {
                 "(AA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') or AA < timestamp '2024-01-01 01:00:00' ) and AA <= timestamp '2024-01-03 02:00:00'");
 
     }
+
+    @Test
+    void testSimplifyRangeAndAddMinMax() {
+        executor = new ExpressionRuleExecutor(ImmutableList.of(
+                bottomUp(
+                        SimplifyRange.INSTANCE,
+                        AddMinMax.INSTANCE
+                )
+        ));
+
+        assertRewriteAfterTypeCoercion("ISNULL(TA)", "ISNULL(TA)");
+        assertRewriteAfterTypeCoercion("ISNULL(TA) and null", "ISNULL(TA) and null");
+        assertRewriteAfterTypeCoercion("ISNULL(TA) and ISNULL(TA)", "ISNULL(TA)");
+        assertRewriteAfterTypeCoercion("ISNULL(TA) or ISNULL(TA)", "ISNULL(TA)");
+        assertRewriteAfterTypeCoercion("ISNULL(TA) and TA between 20 and 10", "ISNULL(TA) and null");
+        // assertRewriteAfterTypeCoercion("ISNULL(TA) and TA > 10", "ISNULL(TA) and null"); // should be, but not support now
+        assertRewriteAfterTypeCoercion("ISNULL(TA) and TA > 10 and null", "ISNULL(TA) and null");
+        assertRewriteAfterTypeCoercion("ISNULL(TA) or TA > 10", "ISNULL(TA) or TA > 10");
+        // assertRewriteAfterTypeCoercion("(TA < 30 or TA > 40) and TA between 20 and 10", "TA IS NULL AND NULL"); // should be, but not support because flatten and
+        assertRewriteAfterTypeCoercion("(TA < 30 or TA > 40) and TA is null and null", "TA IS NULL AND NULL");
+        assertRewriteAfterTypeCoercion("(TA < 30 or TA > 40) or TA between 20 and 10", "TA < 30 or TA > 40");
+
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 or TA between 30 and 40 or TA between 60 and 50",
+                "(TA <= 20 or TA >= 30) and TA >= 10 and TA <= 40");
+        // should be, but not support yet, because 'TA is null and null' => UnknownValue(EmptyValue(TA) and null)
+        //assertRewriteAfterTypeCoercion("TA between 10 and 20 or TA between 30 and 40 or TA is null and null",
+        //        "(TA <= 20 or TA >= 30) and TA >= 10 and TA <= 40");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 or TA between 30 and 40 or TA is null and null",
+                "(TA <= 20 or TA >= 30 or TA is null and null) and TA >= 10 and TA <= 40");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 or TA between 30 and 40 or TA is null",
+                "TA >= 10 and TA <= 20 or TA >= 30 and TA <= 40 or TA is null");
+        assertRewriteAfterTypeCoercion("ISNULL(TB) and (TA between 10 and 20 or TA between 30 and 40 or TA between 60 and 50)",
+                "ISNULL(TB) and ((TA <= 20 or TA >= 30) and TA >= 10 and TA <= 40)");
+        assertRewriteAfterTypeCoercion("ISNULL(TB) and (TA between 10 and 20 or TA between 30 and 40 or TA is null)",
+                "ISNULL(TB) and (TA >= 10 and TA <= 20 or TA >= 30 and TA <= 40 or TA is null)");
+        assertRewriteAfterTypeCoercion("TB between 20 and 10 and (TA between 10 and 20 or TA between 30 and 40 or TA between 60 and 50)",
+                "TB IS NULL AND NULL and (TA <= 20 or TA >= 30) and TA >= 10 and TA <= 40");
+        assertRewriteAfterTypeCoercion("TA between 10 and 20 and TB between 10 and 20 or TA between 30 and 40 and TB between 30 and 40 or TA between 60 and 50 and TB between 60 and 50",
+                "(TA <= 20 and TB <= 20 or TA >= 30 and TB >= 30 or TA is null and null and TB is null) and TA >= 10 and TA <= 40 and TB >= 10 and TB <= 40");
+    }
 }

From 4ef95faf3a1c0fae68ca8b98eb7a297832517cb5 Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Mon, 16 Dec 2024 16:05:55 +0530
Subject: [PATCH 347/399] [Enhancement] (nereids)implement showCharsetCommand
 in nereids (#43155)

Issue Number: close #42749
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  7 ++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/ShowCharsetCommand.java    | 67 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../show/test_show_charset_command.out        |  4 ++
 .../show/test_show_charset_command.groovy     | 30 +++++++++
 7 files changed, 115 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCharsetCommand.java
 create mode 100644 regression-test/data/nereids_p0/show/test_show_charset_command.out
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_charset_command.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index bc91a667eda44a..3d797988046108 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -234,6 +234,7 @@ supportedShowStatement
     | SHOW DYNAMIC PARTITION TABLES ((FROM | IN) database=multipartIdentifier)?     #showDynamicPartition
     | SHOW EVENTS ((FROM | IN) database=multipartIdentifier)? wildWhere?            #showEvents
     | SHOW LAST INSERT                                                              #showLastInsert 
+    | SHOW ((CHAR SET) | CHARSET)                                                   #showCharset
     | SHOW DELETE ((FROM | IN) database=multipartIdentifier)?                       #showDelete
     | SHOW ALL? GRANTS                                                              #showGrants
     | SHOW GRANTS FOR userIdentify                                                  #showGrantsForUser
@@ -331,7 +332,6 @@ unsupportedShowStatement
     | SHOW CATALOG name=identifier                                                  #showCatalog
     | SHOW FULL? (COLUMNS | FIELDS) (FROM | IN) tableName=multipartIdentifier
         ((FROM | IN) database=multipartIdentifier)? wildWhere?                      #showColumns
-    | SHOW ((CHAR SET) | CHARSET) wildWhere?                                        #showCharset
     | SHOW COUNT LEFT_PAREN ASTERISK RIGHT_PAREN (WARNINGS | ERRORS)                #showWaringErrorCount
     | SHOW LOAD WARNINGS ((((FROM | IN) database=multipartIdentifier)?
         wildWhere? limitClause?) | (ON url=STRING_LITERAL))                         #showLoadWarings
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 94c2757e560b24..921e3014c9053a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -239,6 +239,7 @@
 import org.apache.doris.nereids.DorisParser.ShowAuthorsContext;
 import org.apache.doris.nereids.DorisParser.ShowBackendsContext;
 import org.apache.doris.nereids.DorisParser.ShowBrokerContext;
+import org.apache.doris.nereids.DorisParser.ShowCharsetContext;
 import org.apache.doris.nereids.DorisParser.ShowCollationContext;
 import org.apache.doris.nereids.DorisParser.ShowConfigContext;
 import org.apache.doris.nereids.DorisParser.ShowConstraintContext;
@@ -550,6 +551,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowAuthorsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBackendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBrokerCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCharsetCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCollationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
@@ -4795,6 +4797,11 @@ public LogicalPlan visitCreateFile(CreateFileContext ctx) {
         return new CreateFileCommand(stripQuotes(ctx.name.getText()), dbName, properties);
     }
 
+    @Override
+    public LogicalPlan visitShowCharset(ShowCharsetContext ctx) {
+        return new ShowCharsetCommand();
+    }
+
     @Override
     public LogicalPlan visitShowFrontends(ShowFrontendsContext ctx) {
         String detail = (ctx.name != null) ? ctx.name.getText() : null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 26b8ed4c0c9365..060dc859fef766 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -205,6 +205,7 @@ public enum PlanType {
     SHOW_BACKENDS_COMMAND,
     SHOW_BLOCK_RULE_COMMAND,
     SHOW_BROKER_COMMAND,
+    SHOW_CHARSET_COMMAND,
     SHOW_COLLATION_COMMAND,
     SHOW_CONFIG_COMMAND,
     SHOW_CREATE_CATALOG_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCharsetCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCharsetCommand.java
new file mode 100644
index 00000000000000..bce3469528174d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCharsetCommand.java
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * Represents the command for SHOW CHARSETG
+ */
+public class ShowCharsetCommand extends ShowCommand {
+    private static final ShowResultSetMetaData CHARSET_META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("Charset", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Description", ScalarType.createVarchar(60)))
+                    .addColumn(new Column("Default collation", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("Maxlen", ScalarType.createVarchar(10)))
+                    .build();
+
+    public ShowCharsetCommand() {
+        super(PlanType.SHOW_CHARSET_COMMAND);
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        List<List<String>> rows = Lists.newArrayList();
+        List<String> row = Lists.newArrayList();
+        // | utf8mb4 | UTF-8 Unicode | utf8mb4_general_ci | 4|
+        row.add(ctx.getSessionVariable().getCharsetServer());
+        row.add("UTF-8 Unicode");
+        row.add(ctx.getSessionVariable().getCollationConnection());
+        row.add("4");
+        rows.add(row);
+
+        return new ShowResultSet(CHARSET_META_DATA, rows);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowCharsetCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index efb81ae7ee037f..dc01cdd6d76bd3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -85,6 +85,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowAuthorsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBackendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBrokerCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCharsetCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCollationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
@@ -578,6 +579,10 @@ default R visitShowReplicaDistributionCommand(ShowReplicaDistributionCommand sho
         return visitCommand(showReplicaDistributedCommand, context);
     }
 
+    default R visitShowCharsetCommand(ShowCharsetCommand showCharsetCommand, C context) {
+        return visitCommand(showCharsetCommand, context);
+    }
+
     default R visitDropWorkloadPolicyCommand(DropWorkloadPolicyCommand dropWorkloadPolicyCommand, C context) {
         return visitCommand(dropWorkloadPolicyCommand, context);
     }
diff --git a/regression-test/data/nereids_p0/show/test_show_charset_command.out b/regression-test/data/nereids_p0/show/test_show_charset_command.out
new file mode 100644
index 00000000000000..dd9ca552bf7daf
--- /dev/null
+++ b/regression-test/data/nereids_p0/show/test_show_charset_command.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !cmd --
+utf8mb4	UTF-8 Unicode	utf8mb4_0900_bin	4
+
diff --git a/regression-test/suites/nereids_p0/show/test_show_charset_command.groovy b/regression-test/suites/nereids_p0/show/test_show_charset_command.groovy
new file mode 100644
index 00000000000000..30ad3631b941c1
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_charset_command.groovy
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_charset_command", "query,charset") {
+    try {
+        // Execute the SHOW CHARSET command and verify the output
+        checkNereidsExecute("SHOW CHARSET")
+        qt_cmd("SHOW CHARSET")
+
+    } catch (Exception e) {
+        // Log any exceptions that occur during testing
+        log.error("Failed to execute SHOW CHARSET command", e)
+        throw e
+    }
+}
+

From 088d2fecaa00a44b903ed4ee4753bc7d75bceedf Mon Sep 17 00:00:00 2001
From: Uniqueyou <wangyixuan@selectdb.com>
Date: Mon, 16 Dec 2024 19:49:52 +0800
Subject: [PATCH 348/399] [Fix](ccr) Incorrectly generated sql with NONE
 aggregate type (#45452)

Execute:

```
alter table tbl enable feature "UPDATE_FLEXIBLE_COLUMNS";
```

The generated SQL in binlog:

```
ALTER TABLE `t` ADD COLUMN `__DORIS_SKIP_BITMAP_COL__` bitmap NONE NOT NULL DEFAULT BITMAP_EMPTY COMMENT "doris skip bitmap hidden column" AFTER `__DORIS_VERSION_COL__`
```

The error message:

```
Error 1105 (HY000): errCode = 2, detailMessage = mismatched input 'NONE' expecting {<EOF>, ';'}(line 1, pos 62)
```
---
 .../main/java/org/apache/doris/analysis/ColumnDef.java    | 2 +-
 .../java/org/apache/doris/analysis/ColumnDefTest.java     | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index d74451a9331e8b..3bfe099dd8da2e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -658,7 +658,7 @@ public String toSql() {
         sb.append("`").append(name).append("` ");
         sb.append(typeDef.toSql()).append(" ");
 
-        if (aggregateType != null) {
+        if (aggregateType != null && aggregateType != AggregateType.NONE) {
             sb.append(aggregateType.name()).append(" ");
         }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
index c8764a0a2e0f88..0a8fcdf780b864 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
@@ -38,6 +38,7 @@ public class ColumnDefTest {
     private TypeDef stringCol;
     private TypeDef floatCol;
     private TypeDef booleanCol;
+    private TypeDef bitmapCol;
     private ConnectContext ctx;
 
     @Before
@@ -46,6 +47,7 @@ public void setUp() {
         stringCol = new TypeDef(ScalarType.createChar(10));
         floatCol = new TypeDef(ScalarType.createType(PrimitiveType.FLOAT));
         booleanCol = new TypeDef(ScalarType.createType(PrimitiveType.BOOLEAN));
+        bitmapCol = new TypeDef(ScalarType.createType(PrimitiveType.BITMAP));
 
         ctx = new ConnectContext();
         new MockUp<ConnectContext>() {
@@ -80,6 +82,12 @@ public void testNormal() throws AnalysisException {
         Assert.assertEquals("10", column.getDefaultValue());
         Assert.assertEquals(AggregateType.SUM, column.getAggregateType());
         Assert.assertEquals("`col` float SUM NOT NULL DEFAULT \"10\" COMMENT \"\"", column.toSql());
+
+        // agg none
+        column = new ColumnDef("col", bitmapCol, false, AggregateType.NONE, false, DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE, "");
+
+        Assert.assertEquals(AggregateType.NONE, column.getAggregateType());
+        Assert.assertEquals("`col` bitmap NOT NULL DEFAULT BITMAP_EMPTY COMMENT \"\"", column.toSql());
     }
 
     @Test

From 86c3c7604b77311924a2d20835bace2767b1e43e Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Mon, 16 Dec 2024 21:54:59 +0800
Subject: [PATCH 349/399] [improvement](statistics)Set enableQueryCache to
 false for statistics queries. (#45474)

### What problem does this PR solve?

Set enableQueryCache to false for statistics queries.

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None
---
 .../java/org/apache/doris/statistics/util/StatisticsUtil.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index dd037617ba6e30..6b50cd32a1e77c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -223,6 +223,8 @@ public static AutoCloseConnectContext buildConnectContext(boolean useFileCacheFo
         sessionVariable.enablePushDownStringMinMax = true;
         sessionVariable.enableUniqueKeyPartialUpdate = false;
         sessionVariable.enableMaterializedViewRewrite = false;
+        sessionVariable.enableQueryCache = false;
+        sessionVariable.enableSqlCache = false;
         connectContext.setEnv(Env.getCurrentEnv());
         connectContext.setDatabase(FeConstants.INTERNAL_DB_NAME);
         connectContext.setQualifiedUser(UserIdentity.ADMIN.getQualifiedUser());

From d670ec221348ae590150e4391c72f2df4ff29ad2 Mon Sep 17 00:00:00 2001
From: wudongliang <46414265+DongLiang-0@users.noreply.github.com>
Date: Tue, 17 Dec 2024 10:28:57 +0800
Subject: [PATCH 350/399] [Enhancement] (nereids)implement
 adminRebalanceDiskCommand in nereids (#45108)

Issue Number: close #42844, #42845
---
 .../org/apache/doris/nereids/DorisParser.g4   |   8 +-
 .../apache/doris/clone/TabletScheduler.java   |   8 ++
 .../nereids/parser/LogicalPlanBuilder.java    |  24 ++++
 .../doris/nereids/trees/plans/PlanType.java   |   2 +
 .../AdminCancelRebalanceDiskCommand.java      | 108 +++++++++++++++++
 .../commands/AdminRebalanceDiskCommand.java   | 109 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  10 ++
 .../test_database_management_auth.groovy      |   4 +-
 .../test_nereids_rebalance_disk.groovy        |  61 ++++++++++
 9 files changed, 328 insertions(+), 6 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCancelRebalanceDiskCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminRebalanceDiskCommand.java
 create mode 100644 regression-test/suites/nereids_p0/ddl/rebalance_disk/test_nereids_rebalance_disk.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 3d797988046108..11796c5df3b5b3 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -495,6 +495,10 @@ unsupportedCancelStatement
 
 supportedAdminStatement
     : ADMIN SHOW REPLICA DISTRIBUTION FROM baseTableRef                             #adminShowReplicaDistribution
+    | ADMIN REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL
+        (COMMA backends+=STRING_LITERAL)* RIGHT_PAREN)?                             #adminRebalanceDisk
+    | ADMIN CANCEL REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL
+        (COMMA backends+=STRING_LITERAL)* RIGHT_PAREN)?                             #adminCancelRebalanceDisk
     | ADMIN DIAGNOSE TABLET tabletId=INTEGER_VALUE                                  #adminDiagnoseTablet
     | ADMIN SHOW REPLICA STATUS FROM baseTableRef (WHERE STATUS EQ|NEQ STRING_LITERAL)?   #adminShowReplicaStatus
     | ADMIN COMPACT TABLE baseTableRef (WHERE TYPE EQ STRING_LITERAL)?              #adminCompactTable
@@ -520,10 +524,6 @@ unsupportedAdminStatement
     | ADMIN CANCEL REPAIR TABLE baseTableRef                                        #adminCancelRepairTable
     | ADMIN SET (FRONTEND | (ALL FRONTENDS)) CONFIG
         (LEFT_PAREN propertyItemList RIGHT_PAREN)? ALL?                             #adminSetFrontendConfig
-    | ADMIN REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL
-        (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?                              #adminRebalanceDisk
-    | ADMIN CANCEL REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL
-        (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?                              #adminCancelRebalanceDisk
     | ADMIN SET TABLE name=multipartIdentifier
         PARTITION VERSION properties=propertyClause?                                #adminSetPartitionVersion
     | ADMIN COPY TABLET tabletId=INTEGER_VALUE properties=propertyClause?           #adminCopyTablet
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
index bb88afd2b6b040..504e1d36a65af3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -317,6 +317,14 @@ public synchronized void rebalanceDisk(AdminRebalanceDiskStmt stmt) {
         diskRebalancer.addPrioBackends(stmt.getBackends(), stmt.getTimeoutS());
     }
 
+    public synchronized void rebalanceDisk(List<Backend> backends, long timeoutS) {
+        diskRebalancer.addPrioBackends(backends, timeoutS);
+    }
+
+    public synchronized void cancelRebalanceDisk(List<Backend> backends) {
+        diskRebalancer.removePrioBackends(backends);
+    }
+
     public synchronized void cancelRebalanceDisk(AdminCancelRebalanceDiskStmt stmt) {
         diskRebalancer.removePrioBackends(stmt.getBackends());
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 921e3014c9053a..085ad4458d0b3d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -50,9 +50,11 @@
 import org.apache.doris.mtmv.MTMVRefreshTriggerInfo;
 import org.apache.doris.nereids.DorisParser;
 import org.apache.doris.nereids.DorisParser.AddConstraintContext;
+import org.apache.doris.nereids.DorisParser.AdminCancelRebalanceDiskContext;
 import org.apache.doris.nereids.DorisParser.AdminCheckTabletsContext;
 import org.apache.doris.nereids.DorisParser.AdminCompactTableContext;
 import org.apache.doris.nereids.DorisParser.AdminDiagnoseTabletContext;
+import org.apache.doris.nereids.DorisParser.AdminRebalanceDiskContext;
 import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext;
 import org.apache.doris.nereids.DorisParser.AdminShowReplicaStatusContext;
 import org.apache.doris.nereids.DorisParser.AdminShowTabletStorageFormatContext;
@@ -481,9 +483,11 @@
 import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
 import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
 import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCancelRebalanceDiskCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminRebalanceDiskCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterCatalogCommentCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
@@ -4724,6 +4728,26 @@ public LogicalPlan visitShowStorageEngines(ShowStorageEnginesContext ctx) {
         return new ShowStorageEnginesCommand();
     }
 
+    @Override
+    public LogicalPlan visitAdminRebalanceDisk(AdminRebalanceDiskContext ctx) {
+        if (ctx.ON() != null) {
+            List<String> backendList = Lists.newArrayList();
+            ctx.backends.forEach(backend -> backendList.add(stripQuotes(backend.getText())));
+            return new AdminRebalanceDiskCommand(backendList);
+        }
+        return new AdminRebalanceDiskCommand();
+    }
+
+    @Override
+    public LogicalPlan visitAdminCancelRebalanceDisk(AdminCancelRebalanceDiskContext ctx) {
+        if (ctx.ON() != null) {
+            List<String> backendList = Lists.newArrayList();
+            ctx.backends.forEach(backend -> backendList.add(stripQuotes(backend.getText())));
+            return new AdminCancelRebalanceDiskCommand(backendList);
+        }
+        return new AdminCancelRebalanceDiskCommand();
+    }
+
     @Override
     public LogicalPlan visitShowDiagnoseTablet(ShowDiagnoseTabletContext ctx) {
         long tabletId = Long.parseLong(ctx.INTEGER_VALUE().getText());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 060dc859fef766..e9488239b8ac2f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -251,6 +251,8 @@ public enum PlanType {
     RECOVER_TABLE_COMMAND,
     RECOVER_PARTITION_COMMAND,
     REPLAY_COMMAND,
+    ADMIN_REBALANCE_DISK_COMMAND,
+    ADMIN_CANCEL_REBALANCE_DISK_COMMAND,
     CREATE_ENCRYPTKEY_COMMAND,
     CREATE_WORKLOAD_GROUP_COMMAND,
     CREATE_CATALOG_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCancelRebalanceDiskCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCancelRebalanceDiskCommand.java
new file mode 100644
index 00000000000000..7f07555cd4b150
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCancelRebalanceDiskCommand.java
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.NetUtils;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.system.Backend;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * admin cancel rebalance disk
+ */
+public class AdminCancelRebalanceDiskCommand extends Command implements NoForward {
+    private static final Logger LOG = LogManager.getLogger(AdminCancelRebalanceDiskCommand.class);
+    private List<String> backends;
+
+    public AdminCancelRebalanceDiskCommand() {
+        super(PlanType.ADMIN_CANCEL_REBALANCE_DISK_COMMAND);
+    }
+
+    public AdminCancelRebalanceDiskCommand(List<String> backends) {
+        super(PlanType.ADMIN_CANCEL_REBALANCE_DISK_COMMAND);
+        this.backends = backends;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+        handleCancelRebalanceDisk();
+    }
+
+    private void handleCancelRebalanceDisk() throws AnalysisException {
+        List<Backend> rebalanceDiskBackends = getNeedRebalanceDiskBackends(backends);
+        if (rebalanceDiskBackends.isEmpty()) {
+            LOG.info("The matching be is empty, no be to cancel rebalance disk.");
+            return;
+        }
+        Env.getCurrentEnv().getTabletScheduler().cancelRebalanceDisk(rebalanceDiskBackends);
+    }
+
+    private List<Backend> getNeedRebalanceDiskBackends(List<String> backends) throws AnalysisException {
+        ImmutableMap<Long, Backend> backendsInfo = Env.getCurrentSystemInfo().getAllBackendsByAllCluster();
+        List<Backend> needRebalanceDiskBackends = Lists.newArrayList();
+        if (backends == null) {
+            needRebalanceDiskBackends.addAll(backendsInfo.values());
+        } else {
+            Map<String, Long> backendsID = new HashMap<>();
+            for (Backend backend : backendsInfo.values()) {
+                backendsID.put(
+                        NetUtils.getHostPortInAccessibleFormat(backend.getHost(), backend.getHeartbeatPort()),
+                        backend.getId());
+            }
+            for (String be : backends) {
+                if (backendsID.containsKey(be)) {
+                    needRebalanceDiskBackends.add(backendsInfo.get(backendsID.get(be)));
+                    backendsID.remove(be);
+                }
+            }
+        }
+        return needRebalanceDiskBackends;
+    }
+
+    @Override
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
+        LOG.info("AdminCancelRebalanceDiskCommand not supported in cloud mode");
+        throw new DdlException("Unsupported operation");
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAdminCancelRebalanceDiskCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminRebalanceDiskCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminRebalanceDiskCommand.java
new file mode 100644
index 00000000000000..d33b81c262ddc9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminRebalanceDiskCommand.java
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.NetUtils;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.system.Backend;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * admin rebalance disk
+ */
+public class AdminRebalanceDiskCommand extends Command implements NoForward {
+    private static final Logger LOG = LogManager.getLogger(AdminRebalanceDiskCommand.class);
+    private final long timeoutS = 24 * 3600; // default 24 hours
+    private List<String> backends;
+
+    public AdminRebalanceDiskCommand() {
+        super(PlanType.ADMIN_REBALANCE_DISK_COMMAND);
+    }
+
+    public AdminRebalanceDiskCommand(List<String> backends) {
+        super(PlanType.ADMIN_REBALANCE_DISK_COMMAND);
+        this.backends = backends;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+        handleRebalanceDisk();
+    }
+
+    private void handleRebalanceDisk() throws AnalysisException {
+        List<Backend> rebalanceDiskBackends = getNeedRebalanceDiskBackends(backends);
+        if (rebalanceDiskBackends.isEmpty()) {
+            LOG.info("The matching be is empty, no be to rebalance disk.");
+            return;
+        }
+        Env.getCurrentEnv().getTabletScheduler().rebalanceDisk(rebalanceDiskBackends, timeoutS);
+    }
+
+    private List<Backend> getNeedRebalanceDiskBackends(List<String> backends) throws AnalysisException {
+        ImmutableMap<Long, Backend> backendsInfo = Env.getCurrentSystemInfo().getAllBackendsByAllCluster();
+        List<Backend> needRebalanceDiskBackends = Lists.newArrayList();
+        if (backends == null) {
+            needRebalanceDiskBackends.addAll(backendsInfo.values());
+        } else {
+            Map<String, Long> backendsID = new HashMap<>();
+            for (Backend backend : backendsInfo.values()) {
+                backendsID.put(
+                        NetUtils.getHostPortInAccessibleFormat(backend.getHost(), backend.getHeartbeatPort()),
+                        backend.getId());
+            }
+            for (String be : backends) {
+                if (backendsID.containsKey(be)) {
+                    needRebalanceDiskBackends.add(backendsInfo.get(backendsID.get(be)));
+                    backendsID.remove(be);
+                }
+            }
+        }
+        return needRebalanceDiskBackends;
+    }
+
+    @Override
+    protected void checkSupportedInCloudMode(ConnectContext ctx) throws DdlException {
+        LOG.info("AdminRebalanceDiskCommand not supported in cloud mode");
+        throw new DdlException("Unsupported operation");
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAdminRebalanceDiskCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index dc01cdd6d76bd3..ff71331b7afc5e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -18,9 +18,11 @@
 package org.apache.doris.nereids.trees.plans.visitor;
 
 import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCancelRebalanceDiskCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminRebalanceDiskCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterCatalogCommentCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterJobStatusCommand;
@@ -502,6 +504,14 @@ default R visitShowFrontendsCommand(ShowFrontendsCommand showFrontendsCommand, C
         return visitCommand(showFrontendsCommand, context);
     }
 
+    default R visitAdminRebalanceDiskCommand(AdminRebalanceDiskCommand adminRebalanceDiskCommand, C context) {
+        return visitCommand(adminRebalanceDiskCommand, context);
+    }
+
+    default R visitAdminCancelRebalanceDiskCommand(AdminCancelRebalanceDiskCommand command, C context) {
+        return visitCommand(command, context);
+    }
+
     default R visitShowDynamicPartitionCommand(ShowDynamicPartitionCommand showDynamicPartitionCommand, C context) {
         return visitCommand(showDynamicPartitionCommand, context);
     }
diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy
index de4a55ad5973cf..fb643d9ee089e2 100644
--- a/regression-test/suites/auth_call/test_database_management_auth.groovy
+++ b/regression-test/suites/auth_call/test_database_management_auth.groovy
@@ -119,11 +119,11 @@ suite("test_database_management_auth","p0,auth_call") {
         }
         test {
             sql """ADMIN REBALANCE DISK;"""
-            exception "denied"
+            exception "${error_in_cloud}"
         }
         test {
             sql """ADMIN CANCEL REBALANCE DISK;"""
-            exception "denied"
+            exception "${error_in_cloud}"
         }
         test {
             sql """UNSET GLOBAL VARIABLE ALL;"""
diff --git a/regression-test/suites/nereids_p0/ddl/rebalance_disk/test_nereids_rebalance_disk.groovy b/regression-test/suites/nereids_p0/ddl/rebalance_disk/test_nereids_rebalance_disk.groovy
new file mode 100644
index 00000000000000..fa32edf3ba279e
--- /dev/null
+++ b/regression-test/suites/nereids_p0/ddl/rebalance_disk/test_nereids_rebalance_disk.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_nereids_rebalance_disk") {
+    //cloud-mode
+    if (isCloudMode()) {
+        def String error_in_cloud = "Unsupported"
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+
+        test {
+            sql """ admin rebalance disk; """
+            exception "${error_in_cloud}"
+        }
+        test {
+            sql """ admin rebalance disk ON ("127.0.0.1:9050"); """
+            exception "${error_in_cloud}"
+        }
+        test {
+            sql """ admin rebalance disk ON ("192.168.0.1:9050", "127.0.0.1:9050", "192.168.0.2:9050"); """
+            exception "${error_in_cloud}"
+        }
+
+        test {
+            sql """ admin cancel rebalance disk;  """
+            exception "${error_in_cloud}"
+        }
+        test {
+            sql """ admin cancel rebalance disk ON ("127.0.0.1:9050"); """
+            exception "${error_in_cloud}"
+        }
+        test {
+            sql """ admin cancel rebalance disk ON ("192.168.0.1:9050", "127.0.0.1:9050", "192.168.0.2:9050"); """
+            exception "${error_in_cloud}"
+        }
+    } else {
+        // can not use qt command since the output change based on cluster and backend ip
+        checkNereidsExecute(""" admin rebalance disk; """)
+        checkNereidsExecute(""" admin rebalance disk ON ("127.0.0.1:9050"); """)
+        checkNereidsExecute(""" admin rebalance disk ON ("192.168.0.1:9050", "127.0.0.1:9050", "192.168.0.2:9050"); """)
+
+        checkNereidsExecute(""" admin cancel rebalance disk; """)
+        checkNereidsExecute(""" admin cancel rebalance disk ON ("127.0.0.1:9050"); """)
+        checkNereidsExecute(""" admin cancel rebalance disk ON ("192.168.0.1:9050", "127.0.0.1:9050", "192.168.0.2:9050"); """)
+    }
+
+}
\ No newline at end of file

From a5bf8a1ded1477b3e1b3fb1b21ecffbcb81d4b36 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Tue, 17 Dec 2024 10:35:58 +0800
Subject: [PATCH 351/399] [Bug](pipeline) make sink operator process eos
 signals after wake_up_early  (#45207)

1. make sink operator process eos signals after wake_up_early
2. set wake_up_early when `pipeline task meet
wake_up_by_downstream`/`sink reach limit`/`sink get eof status`
3. close non-sink operators after sink meet eos
---
 be/src/exprs/runtime_filter.cpp               |   6 +-
 be/src/pipeline/exec/hashjoin_build_sink.cpp  |  50 ++++----
 .../pipeline/exec/hashjoin_probe_operator.cpp |   2 +-
 be/src/pipeline/pipeline.cpp                  |   2 +-
 be/src/pipeline/pipeline_task.cpp             | 116 +++++++++---------
 be/src/pipeline/pipeline_task.h               |   8 +-
 6 files changed, 87 insertions(+), 97 deletions(-)

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 8f297d7074ff12..d1567a8fa79cb4 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -498,12 +498,12 @@ class RuntimePredicateWrapper {
         switch (_filter_type) {
         case RuntimeFilterType::IN_FILTER: {
             if (!_context->hybrid_set) {
-                _context->ignored = true;
+                set_ignored();
                 return Status::OK();
             }
             _context->hybrid_set->insert(wrapper->_context->hybrid_set.get());
             if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) {
-                _context->ignored = true;
+                set_ignored();
                 // release in filter
                 _context->hybrid_set.reset();
             }
@@ -1337,7 +1337,7 @@ void IRuntimeFilter::set_synced_size(uint64_t global_size) {
 }
 
 void IRuntimeFilter::set_ignored() {
-    _wrapper->_context->ignored = true;
+    _wrapper->set_ignored();
 }
 
 bool IRuntimeFilter::get_ignored() {
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index b2a79a941f79e7..19e8493e596a7e 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -135,26 +135,16 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
         }
     }};
 
-    if (!_runtime_filter_slots || _runtime_filters.empty() || state->is_cancelled()) {
+    if (!_runtime_filter_slots || _runtime_filters.empty() || state->is_cancelled() || !_eos) {
         return Base::close(state, exec_status);
     }
 
     try {
-        if (state->get_task()->wake_up_by_downstream()) {
-            if (_should_build_hash_table) {
-                // partitial ignore rf to make global rf work
-                RETURN_IF_ERROR(
-                        _runtime_filter_slots->send_filter_size(state, 0, _finish_dependency));
-                RETURN_IF_ERROR(_runtime_filter_slots->ignore_all_filters());
-            } else {
-                // do not publish filter coz local rf not inited and useless
-                return Base::close(state, exec_status);
-            }
+        if (state->get_task()->wake_up_early()) {
+            // partitial ignore rf to make global rf work or ignore useless rf
+            RETURN_IF_ERROR(_runtime_filter_slots->send_filter_size(state, 0, _finish_dependency));
+            RETURN_IF_ERROR(_runtime_filter_slots->ignore_all_filters());
         } else if (_should_build_hash_table) {
-            if (p._shared_hashtable_controller &&
-                !p._shared_hash_table_context->complete_build_stage) {
-                return Status::InternalError("close before sink meet eos");
-            }
             auto* block = _shared_state->build_block.get();
             uint64_t hash_table_size = block ? block->rows() : 0;
             {
@@ -166,26 +156,25 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
                 SCOPED_TIMER(_runtime_filter_compute_timer);
                 _runtime_filter_slots->insert(block);
             }
-        } else if ((p._shared_hashtable_controller && !p._shared_hash_table_context->signaled) ||
-                   (p._shared_hash_table_context &&
-                    !p._shared_hash_table_context->complete_build_stage)) {
-            throw Exception(ErrorCode::INTERNAL_ERROR, "build_sink::close meet error state");
-        } else {
-            RETURN_IF_ERROR(
-                    _runtime_filter_slots->copy_from_shared_context(p._shared_hash_table_context));
         }
 
         SCOPED_TIMER(_publish_runtime_filter_timer);
         RETURN_IF_ERROR(_runtime_filter_slots->publish(state, !_should_build_hash_table));
     } catch (Exception& e) {
+        bool blocked_by_complete_build_stage = p._shared_hashtable_controller &&
+                                               !p._shared_hash_table_context->complete_build_stage;
+        bool blocked_by_shared_hash_table_signal = !_should_build_hash_table &&
+                                                   p._shared_hashtable_controller &&
+                                                   !p._shared_hash_table_context->signaled;
+
         return Status::InternalError(
-                "rf process meet error: {}, wake_up_by_downstream: {}, should_build_hash_table: "
-                "{}, _finish_dependency: {}, complete_build_stage: {}, shared_hash_table_signaled: "
+                "rf process meet error: {}, wake_up_early: {}, should_build_hash_table: "
+                "{}, _finish_dependency: {}, blocked_by_complete_build_stage: {}, "
+                "blocked_by_shared_hash_table_signal: "
                 "{}",
-                e.to_string(), state->get_task()->wake_up_by_downstream(), _should_build_hash_table,
-                _finish_dependency->debug_string(),
-                p._shared_hash_table_context && !p._shared_hash_table_context->complete_build_stage,
-                p._shared_hashtable_controller && !p._shared_hash_table_context->signaled);
+                e.to_string(), state->get_task()->wake_up_early(), _should_build_hash_table,
+                _finish_dependency->debug_string(), blocked_by_complete_build_stage,
+                blocked_by_shared_hash_table_signal);
     }
     return Base::close(state, exec_status);
 }
@@ -479,7 +468,6 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
     SCOPED_TIMER(local_state.exec_time_counter());
     COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)in_block->rows());
 
-    local_state._eos = eos;
     if (local_state._should_build_hash_table) {
         // If eos or have already met a null value using short-circuit strategy, we do not need to pull
         // data from probe side.
@@ -556,6 +544,9 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
             return _shared_hash_table_context->status;
         }
 
+        RETURN_IF_ERROR(local_state._runtime_filter_slots->copy_from_shared_context(
+                _shared_hash_table_context));
+
         local_state.profile()->add_info_string(
                 "SharedHashTableFrom",
                 print_id(
@@ -581,6 +572,7 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
     }
 
     if (eos) {
+        local_state._eos = true;
         local_state.init_short_circuit_for_probe();
         // Since the comparison of null values is meaningless, null aware left anti/semi join should not output null
         // when the build side is not empty.
diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
index 7c98a4c9f97763..0db525f1bf5222 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
@@ -242,7 +242,7 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc
         // If we use a short-circuit strategy, should return block directly by add additional null data.
         auto block_rows = local_state._probe_block.rows();
         if (local_state._probe_eos && block_rows == 0) {
-            *eos = local_state._probe_eos;
+            *eos = true;
             return Status::OK();
         }
 
diff --git a/be/src/pipeline/pipeline.cpp b/be/src/pipeline/pipeline.cpp
index e4678b7dcf3a83..6c39d361e59c77 100644
--- a/be/src/pipeline/pipeline.cpp
+++ b/be/src/pipeline/pipeline.cpp
@@ -112,7 +112,7 @@ void Pipeline::make_all_runnable() {
     if (_sink->count_down_destination()) {
         for (auto* task : _tasks) {
             if (task) {
-                task->set_wake_up_by_downstream();
+                task->set_wake_up_early();
             }
         }
         for (auto* task : _tasks) {
diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp
index 6814881ac7a300..5ed725010ec364 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -22,6 +22,7 @@
 #include <glog/logging.h>
 #include <stddef.h>
 
+#include <algorithm>
 #include <ostream>
 #include <vector>
 
@@ -223,9 +224,6 @@ bool PipelineTask::_wait_to_start() {
     _blocked_dep = _execution_dep->is_blocked_by(this);
     if (_blocked_dep != nullptr) {
         static_cast<Dependency*>(_blocked_dep)->start_watcher();
-        if (_wake_up_by_downstream) {
-            _eos = true;
-        }
         return true;
     }
 
@@ -233,9 +231,6 @@ bool PipelineTask::_wait_to_start() {
         _blocked_dep = op_dep->is_blocked_by(this);
         if (_blocked_dep != nullptr) {
             _blocked_dep->start_watcher();
-            if (_wake_up_by_downstream) {
-                _eos = true;
-            }
             return true;
         }
     }
@@ -257,9 +252,6 @@ bool PipelineTask::_is_blocked() {
                 _blocked_dep = dep->is_blocked_by(this);
                 if (_blocked_dep != nullptr) {
                     _blocked_dep->start_watcher();
-                    if (_wake_up_by_downstream) {
-                        _eos = true;
-                    }
                     return true;
                 }
             }
@@ -279,9 +271,6 @@ bool PipelineTask::_is_blocked() {
         _blocked_dep = op_dep->is_blocked_by(this);
         if (_blocked_dep != nullptr) {
             _blocked_dep->start_watcher();
-            if (_wake_up_by_downstream) {
-                _eos = true;
-            }
             return true;
         }
     }
@@ -289,15 +278,15 @@ bool PipelineTask::_is_blocked() {
 }
 
 Status PipelineTask::execute(bool* eos) {
-    SCOPED_TIMER(_task_profile->total_time_counter());
-    SCOPED_TIMER(_exec_timer);
-    SCOPED_ATTACH_TASK(_state);
-    _eos = _sink->is_finished(_state) || _eos || _wake_up_by_downstream;
-    *eos = _eos;
     if (_eos) {
-        // If task is waken up by finish dependency, `_eos` is set to true by last execution, and we should return here.
+        *eos = true;
         return Status::OK();
     }
+
+    SCOPED_TIMER(_task_profile->total_time_counter());
+    SCOPED_TIMER(_exec_timer);
+    SCOPED_ATTACH_TASK(_state);
+
     int64_t time_spent = 0;
     DBUG_EXECUTE_IF("fault_inject::PipelineXTask::execute", {
         Status status = Status::Error<INTERNAL_ERROR>("fault_inject pipeline_task execute failed");
@@ -320,27 +309,31 @@ Status PipelineTask::execute(bool* eos) {
     if (_wait_to_start()) {
         return Status::OK();
     }
-    if (_wake_up_by_downstream) {
-        _eos = true;
-        *eos = true;
-        return Status::OK();
-    }
+
     // The status must be runnable
     if (!_opened && !_fragment_context->is_canceled()) {
+        if (_wake_up_early) {
+            *eos = true;
+            _eos = true;
+            return Status::OK();
+        }
         RETURN_IF_ERROR(_open());
     }
 
+    auto set_wake_up_and_dep_ready = [&]() {
+        if (wake_up_early()) {
+            return;
+        }
+        set_wake_up_early();
+        clear_blocking_state();
+    };
+
     _task_profile->add_info_string("TaskState", "Runnable");
     _task_profile->add_info_string("BlockedByDependency", "");
     while (!_fragment_context->is_canceled()) {
         if (_is_blocked()) {
             return Status::OK();
         }
-        if (_wake_up_by_downstream) {
-            _eos = true;
-            *eos = true;
-            return Status::OK();
-        }
 
         /// When a task is cancelled,
         /// its blocking state will be cleared and it will transition to a ready state (though it is not truly ready).
@@ -361,47 +354,47 @@ Status PipelineTask::execute(bool* eos) {
             RETURN_IF_ERROR(_sink->revoke_memory(_state));
             continue;
         }
-        *eos = _eos;
         DBUG_EXECUTE_IF("fault_inject::PipelineXTask::executing", {
             Status status =
                     Status::Error<INTERNAL_ERROR>("fault_inject pipeline_task executing failed");
             return status;
         });
-        // `_dry_run` means sink operator need no more data
         // `_sink->is_finished(_state)` means sink operator should be finished
-        if (_dry_run || _sink->is_finished(_state)) {
-            *eos = true;
-            _eos = true;
-        } else {
+        if (_sink->is_finished(_state)) {
+            set_wake_up_and_dep_ready();
+        }
+
+        // `_dry_run` means sink operator need no more data
+        *eos = wake_up_early() || _dry_run;
+        if (!*eos) {
             SCOPED_TIMER(_get_block_timer);
             _get_block_counter->update(1);
             RETURN_IF_ERROR(_root->get_block_after_projects(_state, block, eos));
         }
 
+        if (*eos) {
+            RETURN_IF_ERROR(close(Status::OK(), false));
+        }
+
         if (_block->rows() != 0 || *eos) {
             SCOPED_TIMER(_sink_timer);
-            Status status = Status::OK();
-            // Define a lambda function to catch sink exception, because sink will check
-            // return error status with EOF, it is special, could not return directly.
-            auto sink_function = [&]() -> Status {
-                Status internal_st;
-                internal_st = _sink->sink(_state, block, *eos);
-                return internal_st;
-            };
-            status = sink_function();
-            if (!status.is<ErrorCode::END_OF_FILE>()) {
-                RETURN_IF_ERROR(status);
+            Status status = _sink->sink(_state, block, *eos);
+
+            if (status.is<ErrorCode::END_OF_FILE>()) {
+                set_wake_up_and_dep_ready();
+            } else if (!status) {
+                return status;
             }
-            *eos = status.is<ErrorCode::END_OF_FILE>() ? true : *eos;
+
             if (*eos) { // just return, the scheduler will do finish work
-                _eos = true;
                 _task_profile->add_info_string("TaskState", "Finished");
+                _eos = true;
                 return Status::OK();
             }
         }
     }
 
-    static_cast<void>(get_task_queue()->push_back(this));
+    RETURN_IF_ERROR(get_task_queue()->push_back(this));
     return Status::OK();
 }
 
@@ -470,17 +463,14 @@ void PipelineTask::finalize() {
     _le_state_map.clear();
 }
 
-Status PipelineTask::close(Status exec_status) {
+Status PipelineTask::close(Status exec_status, bool close_sink) {
     int64_t close_ns = 0;
-    Defer defer {[&]() {
-        if (_task_queue) {
-            _task_queue->update_statistics(this, close_ns);
-        }
-    }};
     Status s;
     {
         SCOPED_RAW_TIMER(&close_ns);
-        s = _sink->close(_state, exec_status);
+        if (close_sink) {
+            s = _sink->close(_state, exec_status);
+        }
         for (auto& op : _operators) {
             auto tem = op->close(_state);
             if (!tem.ok() && s.ok()) {
@@ -489,10 +479,18 @@ Status PipelineTask::close(Status exec_status) {
         }
     }
     if (_opened) {
-        _fresh_profile_counter();
-        COUNTER_SET(_close_timer, close_ns);
+        COUNTER_UPDATE(_close_timer, close_ns);
         COUNTER_UPDATE(_task_profile->total_time_counter(), close_ns);
     }
+
+    if (close_sink && _opened) {
+        _task_profile->add_info_string("WakeUpEarly", wake_up_early() ? "true" : "false");
+        _fresh_profile_counter();
+    }
+
+    if (_task_queue) {
+        _task_queue->update_statistics(this, close_ns);
+    }
     return s;
 }
 
@@ -508,10 +506,10 @@ std::string PipelineTask::debug_string() {
     auto elapsed = _fragment_context->elapsed_time() / 1000000000.0;
     fmt::format_to(debug_string_buffer,
                    "PipelineTask[this = {}, id = {}, open = {}, eos = {}, finish = {}, dry run = "
-                   "{}, elapse time = {}s, _wake_up_by_downstream = {}], block dependency = {}, is "
+                   "{}, elapse time = {}s, _wake_up_early = {}], block dependency = {}, is "
                    "running = {}\noperators: ",
                    (void*)this, _index, _opened, _eos, _finalized, _dry_run, elapsed,
-                   _wake_up_by_downstream.load(),
+                   _wake_up_early.load(),
                    cur_blocked_dep && !_finalized ? cur_blocked_dep->debug_string() : "NULL",
                    is_running());
     for (size_t i = 0; i < _operators.size(); i++) {
diff --git a/be/src/pipeline/pipeline_task.h b/be/src/pipeline/pipeline_task.h
index 4bb062122c0c08..1a31e5954f479c 100644
--- a/be/src/pipeline/pipeline_task.h
+++ b/be/src/pipeline/pipeline_task.h
@@ -61,7 +61,7 @@ class PipelineTask {
 
     // if the pipeline create a bunch of pipeline task
     // must be call after all pipeline task is finish to release resource
-    Status close(Status exec_status);
+    Status close(Status exec_status, bool close_sink = true);
 
     PipelineFragmentContext* fragment_context() { return _fragment_context; }
 
@@ -135,7 +135,7 @@ class PipelineTask {
     int task_id() const { return _index; };
     bool is_finalized() const { return _finalized; }
 
-    void set_wake_up_by_downstream() { _wake_up_by_downstream = true; }
+    void set_wake_up_early() { _wake_up_early = true; }
 
     void clear_blocking_state() {
         _state->get_query_ctx()->get_execution_dependency()->set_always_ready();
@@ -237,7 +237,7 @@ class PipelineTask {
 
     PipelineId pipeline_id() const { return _pipeline->id(); }
 
-    bool wake_up_by_downstream() const { return _wake_up_by_downstream; }
+    bool wake_up_early() const { return _wake_up_early; }
 
 private:
     friend class RuntimeFilterDependency;
@@ -319,7 +319,7 @@ class PipelineTask {
 
     std::atomic<bool> _running = false;
     std::atomic<bool> _eos = false;
-    std::atomic<bool> _wake_up_by_downstream = false;
+    std::atomic<bool> _wake_up_early = false;
 };
 
 } // namespace doris::pipeline

From d0d726e5e8860d825a4fe867ba3031e6d61c5499 Mon Sep 17 00:00:00 2001
From: Jerry Hu <hushenggang@selectdb.com>
Date: Tue, 17 Dec 2024 10:46:24 +0800
Subject: [PATCH 352/399] [fix](function) The parameters after the first of the
 mask function need to be restricted to constants (#45348)

```
*** Query id: 88218e14284c497a-b1a04172e7b896b6 ***
*** is nereids: 1 ***
*** tablet id: 0 ***
*** Aborted at 1733912200 (unix time) try "date -d @1733912200" if you are using GNU date ***
*** Current BE git commitID: 17bcc208e9 ***
*** SIGSEGV invalid permissions for mapped object (@0x7f04d94a0000) received by PID 6347 (TID 8285 OR 0x7f05c4996640) from PID 18446744073060089856; stack trace: ***
terminate called after throwing an instance of 'terminate called recursively
std::system_error'
  what():  Invalid argument
 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/common/signal_handler.h:421
 1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0] in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
 2# JVM_handle_linux_signal in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
 3# 0x00007F09BFC91520 in /lib/x86_64-linux-gnu/libc.so.6
 4# doris::vectorized::FunctionMask::vector_mask(doris::vectorized::ColumnStr<unsigned int> const&, doris::vectorized::ColumnStr<unsigned int>&, char, char, char) at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/functions/function_string.h:986
 5# doris::vectorized::FunctionMask::execute_impl(doris::FunctionContext*, doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long) const in /mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
 6# doris::vectorized::DefaultExecutable::execute_impl(doris::FunctionContext*, doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long) const at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/functions/function.h:461
 7# doris::vectorized::PreparedFunctionImpl::_execute_skipped_constant_deal(doris::FunctionContext*, doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long, bool) const at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/functions/function.cpp:122
 8# doris::vectorized::PreparedFunctionImpl::default_implementation_for_nulls(doris::FunctionContext*, doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long, bool, bool*) const at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/functions/function.cpp:217
 9# doris::vectorized::PreparedFunctionImpl::execute_without_low_cardinality_columns(doris::FunctionContext*, doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long, bool) const at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/functions/function.cpp:244
10# doris::vectorized::PreparedFunctionImpl::execute(doris::FunctionContext*, doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long, bool) const at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/functions/function.cpp:250
11# doris::vectorized::IFunctionBase::execute(doris::FunctionContext*, doris::vectorized::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long, bool) const at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/functions/function.h:194
12# doris::vectorized::VectorizedFnCall::_do_execute(doris::vectorized::VExprContext*, doris::vectorized::Block*, int*, std::vector<unsigned long, std::allocator<unsigned long> >&) in /mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
13# doris::vectorized::VectorizedFnCall::execute(doris::vectorized::VExprContext*, doris::vectorized::Block*, int*) at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/exprs/vectorized_fn_call.cpp:196
14# doris::vectorized::VExprContext::execute(doris::vectorized::Block*, int*) at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/vec/exprs/vexpr_context.cpp:54
15# doris::pipeline::OperatorXBase::do_projections(doris::RuntimeState*, doris::vectorized::Block*, doris::vectorized::Block*) const at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/pipeline/pipeline_x/operator.cpp:259
16# doris::pipeline::OperatorXBase::get_block_after_projects(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/pipeline/pipeline_x/operator.cpp:290
17# doris::pipeline::PipelineXTask::execute(bool*) at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/pipeline/pipeline_x/pipeline_x_task.cpp:346
18# doris::pipeline::TaskScheduler::_do_work(unsigned long) at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/pipeline/task_scheduler.cpp:347
19# doris::ThreadPool::dispatch_thread() in /mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
20# doris::Thread::supervise_thread(void*) at /home/zcp/repo_center/doris_branch-2.1/doris/be/src/util/thread.cpp:499
21# start_thread at ./nptl/pthread_create.c:442
22# 0x00007F09BFD75850 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:83
```
---
 be/src/vec/functions/function_string.h        |  2 ++
 .../correctness_p0/test_mask_function.groovy  | 21 +++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h
index fba8b914eee682..14926e1062c020 100644
--- a/be/src/vec/functions/function_string.h
+++ b/be/src/vec/functions/function_string.h
@@ -702,6 +702,8 @@ class FunctionMask : public IFunction {
 
     size_t get_number_of_arguments() const override { return 0; }
 
+    ColumnNumbers get_arguments_that_are_always_constant() const override { return {1, 2, 3}; }
+
     bool is_variadic() const override { return true; }
 
     Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
diff --git a/regression-test/suites/correctness_p0/test_mask_function.groovy b/regression-test/suites/correctness_p0/test_mask_function.groovy
index b7717ab183ce4a..c1116e633d2eaf 100644
--- a/regression-test/suites/correctness_p0/test_mask_function.groovy
+++ b/regression-test/suites/correctness_p0/test_mask_function.groovy
@@ -76,6 +76,27 @@ suite("test_mask_function") {
         select digital_masking(13812345678);
     """
 
+    test {
+        sql """
+            select mask('abcd', name) from table_mask_test order by id;
+        """
+        exception "Argument at index 1 for function mask must be constant"
+    }
+
+    test {
+        sql """
+            select mask('abcd', '>', name) from table_mask_test order by id;
+        """
+        exception "Argument at index 2 for function mask must be constant"
+    }
+
+    test {
+        sql """
+            select mask('abcd', '>', '<', `name`) from table_mask_test order by id;
+        """
+        exception "Argument at index 3 for function mask must be constant"
+    }
+
     test {
         sql """ select mask_last_n("12345", -100); """
         exception "function mask_last_n only accept non-negative input for 2nd argument but got -100"

From eb769d4096c4d3b072f2edcc3d38d6ec52438e05 Mon Sep 17 00:00:00 2001
From: zzzxl <yangsiyu@selectdb.com>
Date: Tue, 17 Dec 2024 11:04:37 +0800
Subject: [PATCH 353/399] [fix](inverted index) Fix Null Pointer Exception in
 function match (#45456)

Problem Summary:

Using select match_phrase('', '') in the old optimizer causes issues
---
 be/src/vec/functions/match.cpp               |  3 +
 be/test/vec/function/function_match_test.cpp | 58 ++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 be/test/vec/function/function_match_test.cpp

diff --git a/be/src/vec/functions/match.cpp b/be/src/vec/functions/match.cpp
index d6ed3ca944a27a..fa5ef59c47f954 100644
--- a/be/src/vec/functions/match.cpp
+++ b/be/src/vec/functions/match.cpp
@@ -171,6 +171,9 @@ std::vector<std::string> FunctionMatchBase::analyse_query_str_token(
     VLOG_DEBUG << "begin to run " << get_name() << ", parser_type: "
                << inverted_index_parser_type_to_string(inverted_index_ctx->parser_type);
     std::vector<std::string> query_tokens;
+    if (inverted_index_ctx == nullptr) {
+        return query_tokens;
+    }
     if (inverted_index_ctx->parser_type == InvertedIndexParserType::PARSER_NONE) {
         query_tokens.emplace_back(match_query_str);
         return query_tokens;
diff --git a/be/test/vec/function/function_match_test.cpp b/be/test/vec/function/function_match_test.cpp
new file mode 100644
index 00000000000000..6d95a4632884f8
--- /dev/null
+++ b/be/test/vec/function/function_match_test.cpp
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "function_test_util.h"
+#include "olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h"
+#include "vec/functions/match.h"
+
+namespace doris::vectorized {
+
+TEST(FunctionMatchTest, analyse_query_str) {
+    FunctionMatchPhrase func_match_phrase;
+
+    {
+        auto inverted_index_ctx = nullptr;
+        std::vector<std::string> query_tokens =
+                func_match_phrase.analyse_query_str_token(inverted_index_ctx, "a b c", "name");
+        ASSERT_EQ(query_tokens.size(), 0);
+    }
+
+    {
+        auto inverted_index_ctx = std::make_unique<InvertedIndexCtx>();
+        inverted_index_ctx->parser_type = InvertedIndexParserType::PARSER_NONE;
+        std::vector<std::string> query_tokens = func_match_phrase.analyse_query_str_token(
+                inverted_index_ctx.get(), "a b c", "name");
+        ASSERT_EQ(query_tokens.size(), 1);
+    }
+
+    {
+        auto inverted_index_ctx = std::make_unique<InvertedIndexCtx>();
+        inverted_index_ctx->parser_type = InvertedIndexParserType::PARSER_ENGLISH;
+        auto analyzer = doris::segment_v2::inverted_index::InvertedIndexAnalyzer::create_analyzer(
+                inverted_index_ctx.get());
+        inverted_index_ctx->analyzer = analyzer.get();
+        std::vector<std::string> query_tokens = func_match_phrase.analyse_query_str_token(
+                inverted_index_ctx.get(), "a b c", "name");
+        ASSERT_EQ(query_tokens.size(), 3);
+    }
+}
+
+} // namespace doris::vectorized
\ No newline at end of file

From 66c74459ebce77695561b33f1bda56fc58c889c5 Mon Sep 17 00:00:00 2001
From: Sun Chenyang <sunchenyang@selectdb.com>
Date: Tue, 17 Dec 2024 11:05:01 +0800
Subject: [PATCH 354/399] [fix](cloud) fix error in get detached tablet stat
 (#45449)

### What problem does this PR solve?

Related PR: #44120

Problem Summary:

- `detached tablet stats` should maintain a fixed size.
---
 cloud/src/meta-service/keys.h                 |   6 ++
 .../meta_service_tablet_stats.cpp             |  10 +-
 cloud/test/CMakeLists.txt                     |   1 +
 cloud/test/meta_service_tablet_stats_test.cpp | 101 ++++++++++++++++++
 4 files changed, 115 insertions(+), 3 deletions(-)
 create mode 100644 cloud/test/meta_service_tablet_stats_test.cpp

diff --git a/cloud/src/meta-service/keys.h b/cloud/src/meta-service/keys.h
index 855171c1dc648a..c2c9a9dd4daa15 100644
--- a/cloud/src/meta-service/keys.h
+++ b/cloud/src/meta-service/keys.h
@@ -254,6 +254,12 @@ void stats_tablet_num_segs_key(const StatsTabletKeyInfo& in, std::string* out);
 void stats_tablet_index_size_key(const StatsTabletKeyInfo& in, std::string* out);
 void stats_tablet_segment_size_key(const StatsTabletKeyInfo& in, std::string* out);
 static inline std::string stats_tablet_key(const StatsTabletKeyInfo& in) { std::string s; stats_tablet_key(in, &s); return s; }
+static inline std::string stats_tablet_data_size_key(const StatsTabletKeyInfo& in) { std::string s; stats_tablet_data_size_key(in, &s); return s; }
+static inline std::string stats_tablet_num_rows_key(const StatsTabletKeyInfo& in) { std::string s; stats_tablet_num_rows_key(in, &s); return s; }
+static inline std::string stats_tablet_num_rowsets_key(const StatsTabletKeyInfo& in) { std::string s; stats_tablet_num_rowsets_key(in, &s); return s; }
+static inline std::string stats_tablet_num_segs_key(const StatsTabletKeyInfo& in) { std::string s; stats_tablet_num_segs_key(in, &s); return s; }
+static inline std::string stats_tablet_index_size_key(const StatsTabletKeyInfo& in) { std::string s; stats_tablet_index_size_key(in, &s); return s; }
+static inline std::string stats_tablet_segment_size_key(const StatsTabletKeyInfo& in) { std::string s; stats_tablet_segment_size_key(in, &s); return s; }
 
 void job_recycle_key(const JobRecycleKeyInfo& in, std::string* out);
 void job_check_key(const JobRecycleKeyInfo& in, std::string* out);
diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp b/cloud/src/meta-service/meta_service_tablet_stats.cpp
index 4cbf629c9a3185..da556b6c574557 100644
--- a/cloud/src/meta-service/meta_service_tablet_stats.cpp
+++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp
@@ -94,7 +94,8 @@ int get_detached_tablet_stats(const std::vector<std::pair<std::string, std::stri
                               TabletStats& detached_stats) {
     bool unexpected_size = false;
     // clang-format off
-    if (stats_kvs.size() != 7    // aggregated stats and 4 splitted stats: num_rowsets num_segs data_size num_rows index_size segment_size
+    if (stats_kvs.size() != 7    // aggregated stats and 6 splitted stats: num_rowsets num_segs data_size num_rows index_size segment_size
+        && stats_kvs.size() != 5 // aggregated stats and 4 splitted stats: num_rowsets num_segs data_size num_rows
         && stats_kvs.size() != 2 // aggregated stats and 1 splitted stats: num_rowsets
         && stats_kvs.size() != 1 // aggregated stats only (nothing has been imported since created)
         ) {
@@ -148,8 +149,11 @@ int get_detached_tablet_stats(const std::vector<std::pair<std::string, std::stri
     }
 
     if (unexpected_size) {
-        LOG_EVERY_N(WARNING, 100) << "unexpected tablet stats_kvs, it should be 1 or 2 or 5, size="
-                                  << stats_kvs.size() << " suffix=" << ss.str();
+        DCHECK(false) << "unexpected tablet stats_kvs, it should be 1 or 2 or 5 or 7, size="
+                      << stats_kvs.size() << " suffix=" << ss.str();
+        LOG_EVERY_N(WARNING, 100)
+                << "unexpected tablet stats_kvs, it should be 1 or 2 or 5 or 7, size="
+                << stats_kvs.size() << " suffix=" << ss.str();
     }
 
     return 0;
diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt
index c529b0b1c9ef80..51affb8a46f681 100644
--- a/cloud/test/CMakeLists.txt
+++ b/cloud/test/CMakeLists.txt
@@ -25,6 +25,7 @@ add_executable(meta_service_test
    meta_service_test.cpp
    meta_service_job_test.cpp
    meta_service_http_test.cpp
+   meta_service_tablet_stats_test.cpp
    schema_kv_test.cpp
 )
 
diff --git a/cloud/test/meta_service_tablet_stats_test.cpp b/cloud/test/meta_service_tablet_stats_test.cpp
new file mode 100644
index 00000000000000..000d33602e5dc2
--- /dev/null
+++ b/cloud/test/meta_service_tablet_stats_test.cpp
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "meta-service/meta_service_tablet_stats.h"
+
+#include <gen_cpp/cloud.pb.h>
+#include <gtest/gtest.h>
+
+#include "meta-service/codec.h"
+#include "meta-service/keys.h"
+
+namespace doris::cloud {
+
+std::string size_value(int64_t tablet_stat_data_size) {
+    std::string tablet_stat_data_size_value(sizeof(tablet_stat_data_size), '\0');
+    memcpy(tablet_stat_data_size_value.data(), &tablet_stat_data_size,
+           sizeof(tablet_stat_data_size));
+    return tablet_stat_data_size_value;
+}
+
+TEST(MetaServiceTabletStatsTest, test_get_detached_tablet_stats) {
+    std::vector<std::pair<std::string, std::string>> stats_kvs;
+    StatsTabletKeyInfo tablet_key_info {"instance_0", 10000, 10001, 10002, 10003};
+
+    // key->TabletStatsPB
+    TabletStatsPB tablet_stat;
+    std::string tablet_stat_value;
+    auto tablet_stat_key = stats_tablet_key(tablet_key_info);
+    tablet_stat.SerializeToString(&tablet_stat_value);
+    stats_kvs.emplace_back(tablet_stat_key, tablet_stat_value);
+
+    // key->data_size
+    auto tablet_stat_data_size_key = stats_tablet_data_size_key(tablet_key_info);
+    stats_kvs.emplace_back(tablet_stat_data_size_key, size_value(100));
+
+    // key->num_rows
+    auto tablet_stat_num_rows_key = stats_tablet_num_rows_key(tablet_key_info);
+    stats_kvs.emplace_back(tablet_stat_num_rows_key, size_value(10));
+
+    // key->num_rowsets
+    auto tablet_stat_num_rowsets_key = stats_tablet_num_rowsets_key(tablet_key_info);
+    stats_kvs.emplace_back(tablet_stat_num_rowsets_key, size_value(1));
+
+    // key->num_segs
+    auto tablet_stat_num_segs_key = stats_tablet_num_segs_key(tablet_key_info);
+    stats_kvs.emplace_back(tablet_stat_num_segs_key, size_value(1));
+
+    // key->index_size
+    auto tablet_stat_index_size_key = stats_tablet_index_size_key(tablet_key_info);
+    stats_kvs.emplace_back(tablet_stat_index_size_key, size_value(50));
+
+    // key->segment_size
+    auto tablet_stat_segment_size_key = stats_tablet_segment_size_key(tablet_key_info);
+    stats_kvs.emplace_back(tablet_stat_segment_size_key, size_value(50));
+
+    TabletStats res1;
+    int ret = get_detached_tablet_stats(stats_kvs, res1);
+    EXPECT_EQ(ret, 0);
+    EXPECT_EQ(res1.data_size, 100);
+    EXPECT_EQ(res1.num_rows, 10);
+    EXPECT_EQ(res1.num_rowsets, 1);
+    EXPECT_EQ(res1.num_segs, 1);
+    EXPECT_EQ(res1.index_size, 50);
+    EXPECT_EQ(res1.segment_size, 50);
+
+    stats_kvs.resize(5);
+    TabletStats res2;
+    ret = get_detached_tablet_stats(stats_kvs, res2);
+    EXPECT_EQ(ret, 0);
+    EXPECT_EQ(res1.data_size, 100);
+    EXPECT_EQ(res1.num_rows, 10);
+    EXPECT_EQ(res1.num_rowsets, 1);
+    EXPECT_EQ(res1.num_segs, 1);
+
+    stats_kvs.resize(2);
+    TabletStats res3;
+    ret = get_detached_tablet_stats(stats_kvs, res3);
+    EXPECT_EQ(ret, 0);
+    EXPECT_EQ(res1.data_size, 100);
+
+    stats_kvs.resize(1);
+    TabletStats res4;
+    ret = get_detached_tablet_stats(stats_kvs, res4);
+    EXPECT_EQ(ret, 0);
+}
+
+} // namespace doris::cloud

From f7187a92c2a2474da2511e4dabdc7c501343a389 Mon Sep 17 00:00:00 2001
From: zclllyybb <zhaochangle@selectdb.com>
Date: Tue, 17 Dec 2024 11:50:49 +0800
Subject: [PATCH 355/399] [Enhancement](function) Support TIMEV2 type for agg
 function max,min,any (#45437)

Support TIMEV2 type for agg function max,min,any
---
 .../vec/aggregate_functions/aggregate_function_min_max.cpp | 6 ++++++
 regression-test/data/nereids_syntax_p0/any_value.out       | 7 +++++++
 regression-test/suites/nereids_syntax_p0/any_value.groovy  | 3 +++
 3 files changed, 16 insertions(+)
 create mode 100644 regression-test/data/nereids_syntax_p0/any_value.out

diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
index e6e558b7fdc2db..52a6debbe60c2f 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
@@ -23,6 +23,7 @@
 #include "vec/aggregate_functions/aggregate_function_simple_factory.h"
 #include "vec/aggregate_functions/factory_helpers.h"
 #include "vec/aggregate_functions/helpers.h"
+#include "vec/core/types.h"
 #include "vec/data_types/data_type_nullable.h"
 
 namespace doris::vectorized {
@@ -70,6 +71,11 @@ AggregateFunctionPtr create_aggregate_function_single_value(const String& name,
                 AggregateFunctionsSingleValue<Data<SingleValueDataFixed<UInt64>>>>(
                 argument_types, result_is_nullable);
     }
+    if (which.idx == TypeIndex::Time || which.idx == TypeIndex::TimeV2) {
+        return creator_without_type::create<
+                AggregateFunctionsSingleValue<Data<SingleValueDataFixed<Float64>>>>(
+                argument_types, result_is_nullable);
+    }
     if (which.idx == TypeIndex::IPv4) {
         return creator_without_type::create<
                 AggregateFunctionsSingleValue<Data<SingleValueDataFixed<IPv4>>>>(
diff --git a/regression-test/data/nereids_syntax_p0/any_value.out b/regression-test/data/nereids_syntax_p0/any_value.out
new file mode 100644
index 00000000000000..ac9836e0630b33
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/any_value.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql_max --
+99:00:00
+
+-- !sql_min --
+00:00:00
+
diff --git a/regression-test/suites/nereids_syntax_p0/any_value.groovy b/regression-test/suites/nereids_syntax_p0/any_value.groovy
index 3de2b0dd733bf0..c838efe1c1d5bc 100644
--- a/regression-test/suites/nereids_syntax_p0/any_value.groovy
+++ b/regression-test/suites/nereids_syntax_p0/any_value.groovy
@@ -23,4 +23,7 @@ suite("any_value") {
     test {
         sql "select any(s_suppkey), any(s_name), any_value(s_address) from supplier;"
     }
+    qt_sql_max """select max(cast(concat(number, ":00:00") as time)) from numbers("number" = "100");"""
+    qt_sql_min """select min(cast(concat(number, ":00:00") as time)) from numbers("number" = "100");"""
+    sql """select any(cast(concat(number, ":00:00") as time)) from numbers("number" = "100");"""
 }
\ No newline at end of file

From ce5753d3dcb32328c4b86cc084d5a162bdf9f394 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Tue, 17 Dec 2024 11:55:28 +0800
Subject: [PATCH 356/399] [Chore](pipeline) catch exception on task::prepare to
 avoid exception make backend coredump (#45479)

catch exception on task::prepare to avoid exception make backend
coredump
---
 be/src/pipeline/pipeline_fragment_context.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index 8ab0f1d151568d..5ae89db55a45ac 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -35,6 +35,7 @@
 #include "cloud/config.h"
 #include "common/cast_set.h"
 #include "common/config.h"
+#include "common/exception.h"
 #include "common/logging.h"
 #include "common/status.h"
 #include "io/fs/stream_load_pipe.h"
@@ -498,8 +499,8 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag
             if (pipeline_id_to_task.contains(_pipelines[pip_idx]->id())) {
                 auto* task = pipeline_id_to_task[_pipelines[pip_idx]->id()];
                 DCHECK(pipeline_id_to_profile[pip_idx]);
-                RETURN_IF_ERROR(task->prepare(local_params, request.fragment.output_sink,
-                                              _query_ctx.get()));
+                RETURN_IF_ERROR_OR_CATCH_EXCEPTION(task->prepare(
+                        local_params, request.fragment.output_sink, _query_ctx.get()));
             }
         }
         {

From 9c98cbf3adbf5f1b14b4a8ac104d1953c7477340 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Tue, 17 Dec 2024 14:10:41 +0800
Subject: [PATCH 357/399] [fix](restore) Add restore_reset_index_id config
 (#45283)

The inverted index saves the index ID in the file path/header, so the
index id between two clusters must be the same when synced with the
ccr-syncer.

This PR add a new config `restore_reset_index_id` to indicate whether to
reset the index IDs.
---
 .../java/org/apache/doris/common/Config.java  |   9 +
 .../doris/alter/SchemaChangeHandler.java      |  10 +
 .../org/apache/doris/catalog/OlapTable.java   |  14 +-
 .../test_backup_restore_inverted_idx.groovy   | 174 ++++++++++++++++++
 4 files changed, 206 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/suites/backup_restore/test_backup_restore_inverted_idx.groovy

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 42a64a2770ff69..c601a492162958 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1623,6 +1623,15 @@ public class Config extends ConfigBase {
     @ConfField(mutable = false)
     public static boolean enable_restore_snapshot_rpc_compression = true;
 
+    /**
+     * A internal config, to indicate whether to reset the index id when restore olap table.
+     *
+     * The inverted index saves the index id in the file path/header, so the index id between
+     * two clusters must be the same.
+     */
+    @ConfField(mutable = true, masterOnly = true)
+    public static boolean restore_reset_index_id = true;
+
     /**
      * Control the max num of tablets per backup job involved.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 8d257bc2d78d1f..eb1cc09f5066c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2717,6 +2717,7 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
         IndexDef indexDef = alterClause.getIndexDef();
         Set<String> newColset = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
         newColset.addAll(indexDef.getColumns());
+        Set<Long> existedIndexIdSet = Sets.newHashSet();
         for (Index existedIdx : existedIndexes) {
             if (existedIdx.getIndexName().equalsIgnoreCase(indexDef.getIndexName())) {
                 if (indexDef.isSetIfNotExists()) {
@@ -2733,7 +2734,16 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
                         indexDef.getIndexType() + " index for columns (" + String.join(",", indexDef.getColumns())
                                 + " ) already exist.");
             }
+            existedIndexIdSet.add(existedIdx.getIndexId());
         }
+
+        // The restored olap table may not reset the index id, which comes from the upstream,
+        // so we need to check and reset the index id here, to avoid confliction.
+        // See OlapTable.resetIdsForRestore for details.
+        while (existedIndexIdSet.contains(alterIndex.getIndexId())) {
+            alterIndex.setIndexId(Env.getCurrentEnv().getNextId());
+        }
+
         boolean disableInvertedIndexV1ForVariant = olapTable.getInvertedIndexFileStorageFormat()
                         == TInvertedIndexFileStorageFormat.V1 && ConnectContext.get().getSessionVariable()
                                                                         .getDisableInvertedIndexV1ForVaraint();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 92d9aa4e9c7f82..9f1f455ab35495 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -885,7 +885,19 @@ public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restore
         if (this.indexes != null) {
             List<Index> indexes = this.indexes.getIndexes();
             for (Index idx : indexes) {
-                idx.setIndexId(env.getNextId());
+                long newIdxId;
+                if (Config.restore_reset_index_id) {
+                    newIdxId = env.getNextId();
+                } else {
+                    // The index id from the upstream is used, if restore_reset_index_id is not set.
+                    //
+                    // This is because the index id is used as a part of inverted file name/header
+                    // in BE. During restore, the inverted file is copied from the upstream to the
+                    // downstream. If the index id is changed, it might cause the BE to fail to find
+                    // the inverted files.
+                    newIdxId = idx.getIndexId();
+                }
+                idx.setIndexId(newIdxId);
             }
             for (Map.Entry<Long, MaterializedIndexMeta> entry : indexIdToMeta.entrySet()) {
                 entry.getValue().setIndexes(indexes);
diff --git a/regression-test/suites/backup_restore/test_backup_restore_inverted_idx.groovy b/regression-test/suites/backup_restore/test_backup_restore_inverted_idx.groovy
new file mode 100644
index 00000000000000..0dadc99dd21b4f
--- /dev/null
+++ b/regression-test/suites/backup_restore/test_backup_restore_inverted_idx.groovy
@@ -0,0 +1,174 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_backup_restore_inverted_idx", "backup_restore") {
+    String suiteName = "test_backup_restore_inverted_idx"
+    String dbName = "${suiteName}_db"
+    String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
+    String snapshotName = "${suiteName}_snapshot"
+    String tableName = "${suiteName}_table"
+
+    def syncer = getSyncer()
+    syncer.createS3Repository(repoName)
+    sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+
+    sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+    sql """
+        CREATE TABLE ${dbName}.${tableName} (
+            `id` LARGEINT NOT NULL,
+            `value` STRING DEFAULT "",
+            `value1` STRING DEFAULT "",
+            INDEX `idx_value` (`value`) USING INVERTED PROPERTIES ("parser" = "english")
+        )
+        UNIQUE KEY(`id`)
+        PARTITION BY RANGE(`id`)
+        (
+            PARTITION p1 VALUES LESS THAN ("10"),
+            PARTITION p2 VALUES LESS THAN ("20"),
+            PARTITION p3 VALUES LESS THAN ("30"),
+            PARTITION p4 VALUES LESS THAN ("40"),
+            PARTITION p5 VALUES LESS THAN ("50"),
+            PARTITION p6 VALUES LESS THAN ("60"),
+            PARTITION p7 VALUES LESS THAN ("70")
+        )
+        DISTRIBUTED BY HASH(`id`) BUCKETS 2
+        PROPERTIES
+        (
+            "replication_num" = "1"
+        )
+        """
+    List<String> values = []
+    int numRows = 6;
+    for (int j = 0; j <= numRows; ++j) {
+        values.add("(${j}1, \"${j} ${j*10} ${j*100}\", \"${j*11} ${j*12}\")")
+    }
+    sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"
+
+    def indexes = sql_return_maparray "SHOW INDEX FROM ${dbName}.${tableName}"
+    logger.info("current indexes: ${indexes}")
+    assertTrue(indexes.any { it.Key_name == "idx_value" && it.Index_type == "INVERTED" })
+
+    def query_index_id = { indexName ->
+        def res = sql_return_maparray "SHOW TABLETS FROM ${dbName}.${tableName}"
+        def tabletId = res[0].TabletId
+        res = sql_return_maparray "SHOW TABLET ${tabletId}"
+        def dbId = res[0].DbId
+        def tableId = res[0].TableId
+        res = sql_return_maparray """ SHOW PROC "/dbs/${dbId}/${tableId}/indexes" """
+        for (def record in res) {
+            if (record.KeyName == indexName) {
+                return record.IndexId
+            }
+        }
+        throw new Exception("index ${indexName} is not exists")
+    }
+
+    try {
+        sql """ ADMIN SET FRONTEND CONFIG ("restore_reset_index_id" = "false") """
+        sql """
+            BACKUP SNAPSHOT ${dbName}.${snapshotName}
+            TO `${repoName}`
+            ON (`${tableName}`)
+        """
+
+        syncer.waitSnapshotFinish(dbName)
+
+        def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+        assertTrue(snapshot != null)
+
+        def indexId = query_index_id("idx_value")
+        logger.info("the exists index id is ${indexId}")
+
+        sql "DROP TABLE ${dbName}.${tableName}"
+
+        sql """
+            RESTORE SNAPSHOT ${dbName}.${snapshotName}
+            FROM `${repoName}`
+            ON (`${tableName}`)
+            PROPERTIES
+            (
+                "backup_timestamp" = "${snapshot}",
+                "reserve_replica" = "true"
+            )
+        """
+
+        syncer.waitAllRestoreFinish(dbName)
+
+        indexes = sql_return_maparray "SHOW INDEX FROM ${dbName}.${tableName}"
+        logger.info("current indexes: ${indexes}")
+        assertTrue(indexes.any { it.Key_name == "idx_value" && it.Index_type == "INVERTED" })
+
+        def newIndexId = query_index_id("idx_value")
+        assertTrue(newIndexId == indexId, "old index id ${indexId}, new index id ${newIndexId}")
+
+        // 1. query with inverted index
+        sql """ set enable_match_without_inverted_index = false """
+        def res = sql """ SELECT /*+ SET_VAR(inverted_index_skip_threshold = 0) */ * FROM ${dbName}.${tableName} WHERE value MATCH_ANY "10" """
+        assertTrue(res.size() > 0)
+
+        // 2. add partition and query
+        sql """ ALTER TABLE ${dbName}.${tableName} ADD PARTITION p8 VALUES LESS THAN ("80") """
+        sql """ INSERT INTO ${dbName}.${tableName} VALUES (75, "75 750", "76 77") """
+        res = sql """ SELECT /*+ SET_VAR(inverted_index_skip_threshold = 0) */ * FROM ${dbName}.${tableName} WHERE value MATCH_ANY "75" """
+        assertTrue(res.size() > 0)
+
+        // 3. add new index
+        sql """ ALTER TABLE ${dbName}.${tableName}
+            ADD INDEX idx_value1(value1) USING INVERTED PROPERTIES("parser" = "english") """
+
+        indexes = sql_return_maparray """ SHOW INDEX FROM ${dbName}.${tableName} """
+        logger.info("current indexes: ${indexes}")
+        assertTrue(indexes.any { it.Key_name == "idx_value1" && it.Index_type == "INVERTED" })
+
+        // 4. drop old index
+        sql """ ALTER TABLE ${dbName}.${tableName} DROP INDEX idx_value"""
+        indexes = sql_return_maparray """ SHOW INDEX FROM ${dbName}.${tableName} """
+        logger.info("current indexes: ${indexes}")
+        assertFalse(indexes.any { it.Key_name == "idx_value" && it.Index_type == "INVERTED" })
+
+        // 5. query new index with inverted idx
+        sql """ INSERT INTO ${dbName}.${tableName} VALUES(76, "76 760", "12321 121") """
+        sql """ BUILD INDEX idx_value1 ON ${dbName}.${tableName} """
+        def build_index_finished = false
+        for (int i = 0; i < 100; i++) {
+            def build_status = sql_return_maparray """
+                SHOW BUILD INDEX FROM ${dbName} WHERE TableName = "${tableName}" """
+            if (!(build_status.any { it.State != 'FINISHED' })) {
+                build_index_finished = true
+                break
+            }
+            sleep(1000)
+        }
+        if (!build_index_finished) {
+            def build_status = sql_return_maparray """
+                SHOW BUILD INDEX FROM ${dbName} WHERE TableName = "${tableName}" """
+            logger.info("the build index status: ${build_status}")
+            assertTrue(false)
+        }
+        res = sql """ SELECT /*+ SET_VAR(inverted_index_skip_threshold = 0) */ * FROM ${dbName}.${tableName} WHERE value1 MATCH_ANY "12321" """
+        assertTrue(res.size() > 0)
+
+    } finally {
+        sql """ set enable_match_without_inverted_index = true """
+        sql """ ADMIN SET FRONTEND CONFIG ("restore_reset_index_id" = "true") """
+    }
+
+    sql "DROP TABLE ${dbName}.${tableName} FORCE"
+    sql "DROP DATABASE ${dbName} FORCE"
+    sql "DROP REPOSITORY `${repoName}`"
+}
+

From b54d1dcd4375b7c3fde4f8b667a1a1782f7c5279 Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Tue, 17 Dec 2024 14:17:31 +0800
Subject: [PATCH 358/399] [fix](nereids) fix bug when sum0 distinct appear in
 cte (#45447)

### What problem does this PR solve?

Related PR: #45209

Problem Summary:

with tmp as (select * from t1) select * from (select sum0(distinct b) from tmp) t cross join (select sum0(distinct a) from tmp) tt

before this pr, this sql report error:
VSlotRef xxx have invalid slot id.
This pr fix this problem.
---
 .../implementation/AggregateStrategies.java   |  6 +++-
 .../agg_function/sum0_cte.out                 | 21 ++++++++++++++
 .../agg_function/sum0_cte.groovy              | 29 +++++++++++++++++++
 3 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/data/nereids_function_p0/agg_function/sum0_cte.out
 create mode 100644 regression-test/suites/nereids_function_p0/agg_function/sum0_cte.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
index 094882294fa6db..3d25bce0b48b5e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
@@ -54,6 +54,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctCount;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Sum0;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
@@ -1813,6 +1814,8 @@ private AggregateFunction tryConvertToMultiDistinct(AggregateFunction function)
                     function.getArguments().subList(1, function.arity()).toArray(new Expression[0]));
         } else if (function instanceof Sum && function.isDistinct()) {
             return ((Sum) function).convertToMultiDistinct();
+        } else if (function instanceof Sum0 && function.isDistinct()) {
+            return ((Sum0) function).convertToMultiDistinct();
         } else if (function instanceof GroupConcat && function.isDistinct()) {
             return ((GroupConcat) function).convertToMultiDistinct();
         }
@@ -2076,7 +2079,8 @@ private boolean couldConvertToMulti(LogicalAggregate<? extends Plan> aggregate)
             if (!func.isDistinct()) {
                 continue;
             }
-            if (!(func instanceof Count || func instanceof Sum || func instanceof GroupConcat)) {
+            if (!(func instanceof Count || func instanceof Sum || func instanceof GroupConcat
+                    || func instanceof Sum0)) {
                 return false;
             }
             if (func.arity() <= 1) {
diff --git a/regression-test/data/nereids_function_p0/agg_function/sum0_cte.out b/regression-test/data/nereids_function_p0/agg_function/sum0_cte.out
new file mode 100644
index 00000000000000..3872d308917826
--- /dev/null
+++ b/regression-test/data/nereids_function_p0/agg_function/sum0_cte.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sum0_cte --
+78	78
+
+-- !shape --
+PhysicalCteAnchor ( cteId=CTEId#0 )
+--PhysicalCteProducer ( cteId=CTEId#0 )
+----PhysicalOlapScan[fn_test]
+--PhysicalResultSink
+----NestedLoopJoin[CROSS_JOIN]
+------hashAgg[DISTINCT_GLOBAL]
+--------hashAgg[DISTINCT_LOCAL]
+----------hashAgg[GLOBAL]
+------------hashAgg[LOCAL]
+--------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------hashAgg[DISTINCT_GLOBAL]
+--------hashAgg[DISTINCT_LOCAL]
+----------hashAgg[GLOBAL]
+------------hashAgg[LOCAL]
+--------------PhysicalCteConsumer ( cteId=CTEId#0 )
+
diff --git a/regression-test/suites/nereids_function_p0/agg_function/sum0_cte.groovy b/regression-test/suites/nereids_function_p0/agg_function/sum0_cte.groovy
new file mode 100644
index 00000000000000..a6f923baaf0447
--- /dev/null
+++ b/regression-test/suites/nereids_function_p0/agg_function/sum0_cte.groovy
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("sum0_cte") {
+    sql 'use regression_test_nereids_function_p0'
+    sql "set ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    qt_sum0_cte """with tmp as (select * from fn_test)
+    select * from (select sum0(distinct kint) from tmp ) t cross join (select sum0(distinct ksint) from tmp) tt;
+    """
+    qt_shape """
+    explain shape plan
+    with tmp as (select * from fn_test)
+    select * from (select sum0(distinct kint) from tmp ) t cross join (select sum0(distinct ksint) from tmp) tt;
+    """
+}
\ No newline at end of file

From 129bb017bc628076951c7b809eb204578068f8d9 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Tue, 17 Dec 2024 14:30:52 +0800
Subject: [PATCH 359/399] [fix](binlog) Fix linking binlog failure due to
 existing files (#45445)

If the binlog files already exist, the link file will be failed:

```
W20241212 15:56:51.084318 601741 status.h:424] meet error status: [E-242]fail to create hard link. from=/mnt/disk2/VEC_ASAN/doris.HDD/data/60/367148/684012786/020000000001570ce04a2e65a3a57b99d083b08558dfcf81_0.dat, to=/mnt/disk2/VEC_ASAN/doris.HDD/data/60/367148/684012786/_binlog/020000000001570ce04a2e65a3a57b99d083b08558dfcf81_0.dat, errno=17

        0#  doris::BetaRowset::add_to_binlog() at /home/zcp/repo_center/doris_master/doris/be/src/common/status.h:389
        1#  doris::TxnManager::publish_txn(doris::OlapMeta*, long, long, long, doris::UniqueId, doris::Version const&, doris::TabletPublishStatistics*) at /home/zcp/repo_center/doris_master/doris/be/src/olap/txn_manager.cpp:0
        2#  doris::TxnManager::publish_txn(long, std::shared_ptr<doris::Tablet> const&, long, doris::Version const&, doris::TabletPublishStatistics*) at /home/zcp/repo_center/doris_master/doris/be/src/olap/txn_manager.cpp:197
        3#  doris::TabletPublishTxnTask::handle() at /home/zcp/repo_center/doris_master/doris/be/src/olap/task/engine_publish_version_task.cpp:411
        4#  doris::ThreadPool::dispatch_thread() at /home/zcp/repo_center/doris_master/doris/be/src/util/threadpool.cpp:0
        5#  doris::Thread::supervise_thread(void*) at /var/local/ldb-toolchain/bin/../usr/include/pthread.h:562
        6#  ?
        7#  ?
```
---
 be/src/common/status.h                   |  2 +-
 be/src/olap/rowset/beta_rowset.cpp       | 32 ++++++++++++++++--------
 be/src/olap/txn_manager.cpp              |  5 ++--
 be/test/olap/rowset/beta_rowset_test.cpp | 10 ++++++++
 4 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/be/src/common/status.h b/be/src/common/status.h
index 344f82a81b8e25..d059f289402cea 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -570,7 +570,7 @@ class [[nodiscard]] Status {
 // and another thread is call to_string method, it may core, because the _err_msg is an unique ptr and
 // it is deconstructed during copy method.
 // And also we could not use lock, because we need get status frequently to check if it is cancelled.
-// The defaule value is ok.
+// The default value is ok.
 class AtomicStatus {
 public:
     AtomicStatus() : error_st_(Status::OK()) {}
diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp
index cd52deed0c8a4d..2546c87492de84 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -557,10 +557,6 @@ Status BetaRowset::add_to_binlog() {
     }
 
     const auto& fs = io::global_local_filesystem();
-
-    // all segments are in the same directory, so cache binlog_dir without multi times check
-    std::string binlog_dir;
-
     auto segments_num = num_segments();
     VLOG_DEBUG << fmt::format("add rowset to binlog. rowset_id={}, segments_num={}",
                               rowset_id().to_string(), segments_num);
@@ -569,28 +565,34 @@ Status BetaRowset::add_to_binlog() {
     std::vector<string> linked_success_files;
     Defer remove_linked_files {[&]() { // clear linked files if errors happen
         if (!status.ok()) {
-            LOG(WARNING) << "will delete linked success files due to error " << status;
+            LOG(WARNING) << "will delete linked success files due to error "
+                         << status.to_string_no_stack();
             std::vector<io::Path> paths;
             for (auto& file : linked_success_files) {
                 paths.emplace_back(file);
                 LOG(WARNING) << "will delete linked success file " << file << " due to error";
             }
             static_cast<void>(fs->batch_delete(paths));
-            LOG(WARNING) << "done delete linked success files due to error " << status;
+            LOG(WARNING) << "done delete linked success files due to error "
+                         << status.to_string_no_stack();
         }
     }};
 
+    // all segments are in the same directory, so cache binlog_dir without multi times check
+    std::string binlog_dir;
     for (int i = 0; i < segments_num; ++i) {
         auto seg_file = local_segment_path(_tablet_path, rowset_id().to_string(), i);
 
         if (binlog_dir.empty()) {
             binlog_dir = std::filesystem::path(seg_file).parent_path().append("_binlog").string();
 
+            // Delete all existing files in binlog dir, to keep binlog dir clean.
             bool exists = true;
             RETURN_IF_ERROR(fs->exists(binlog_dir, &exists));
-            if (!exists) {
-                RETURN_IF_ERROR(fs->create_directory(binlog_dir));
+            if (exists) {
+                RETURN_IF_ERROR(fs->delete_directory(binlog_dir));
             }
+            RETURN_IF_ERROR(fs->create_directory(binlog_dir));
         }
 
         auto binlog_file =
@@ -614,7 +616,12 @@ Status BetaRowset::add_to_binlog() {
                                           std::filesystem::path(index_file).filename())
                                                  .string();
                 VLOG_DEBUG << "link " << index_file << " to " << binlog_index_file;
-                RETURN_IF_ERROR(fs->link_file(index_file, binlog_index_file));
+                if (!fs->link_file(index_file, binlog_index_file).ok()) {
+                    status = Status::Error<OS_ERROR>(
+                            "fail to create hard link. from={}, to={}, errno={}", index_file,
+                            binlog_index_file, Errno::no());
+                    return status;
+                }
                 linked_success_files.push_back(binlog_index_file);
             }
         } else {
@@ -625,7 +632,12 @@ Status BetaRowset::add_to_binlog() {
                                           std::filesystem::path(index_file).filename())
                                                  .string();
                 VLOG_DEBUG << "link " << index_file << " to " << binlog_index_file;
-                RETURN_IF_ERROR(fs->link_file(index_file, binlog_index_file));
+                if (!fs->link_file(index_file, binlog_index_file).ok()) {
+                    status = Status::Error<OS_ERROR>(
+                            "fail to create hard link. from={}, to={}, errno={}", index_file,
+                            binlog_index_file, Errno::no());
+                    return status;
+                }
                 linked_success_files.push_back(binlog_index_file);
             }
         }
diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp
index d227f53053128b..c54b9c5e8f980f 100644
--- a/be/src/olap/txn_manager.cpp
+++ b/be/src/olap/txn_manager.cpp
@@ -548,8 +548,9 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id,
         if (!status.ok()) {
             return Status::Error<ROWSET_ADD_TO_BINLOG_FAILED>(
                     "add rowset to binlog failed. when publish txn rowset_id: {}, tablet id: {}, "
-                    "txn id: {}",
-                    rowset->rowset_id().to_string(), tablet_id, transaction_id);
+                    "txn id: {}, status: {}",
+                    rowset->rowset_id().to_string(), tablet_id, transaction_id,
+                    status.to_string_no_stack());
         }
     }
 
diff --git a/be/test/olap/rowset/beta_rowset_test.cpp b/be/test/olap/rowset/beta_rowset_test.cpp
index 1ed3a9ed04b2c7..0c3001758f08af 100644
--- a/be/test/olap/rowset/beta_rowset_test.cpp
+++ b/be/test/olap/rowset/beta_rowset_test.cpp
@@ -292,4 +292,14 @@ TEST_F(BetaRowsetTest, ReadTest) {
     }
 }
 
+TEST_F(BetaRowsetTest, AddToBinlogTest) {
+    RowsetMetaSharedPtr rowset_meta = std::make_shared<RowsetMeta>();
+    BetaRowset rowset(nullptr, rowset_meta, "");
+    std::string resource_id = "10000";
+    Status s = rowset.add_to_binlog();
+    ASSERT_TRUE(s.ok()) << "first add_to_binlog(): " << s;
+    s = rowset.add_to_binlog();
+    ASSERT_TRUE(s.ok()) << "second add_to_binlog(): " << s;
+}
+
 } // namespace doris

From 3e500f29124fd703931ca10d9fc46f7f293d327b Mon Sep 17 00:00:00 2001
From: Siyang Tang <tangsiyang@selectdb.com>
Date: Tue, 17 Dec 2024 14:43:59 +0800
Subject: [PATCH 360/399] [fix](export) Make enable_outfile_to_local available
 for export command (#45241)

`enable_outfile_to_local` previously not work to control export command.
---
 .../src/main/java/org/apache/doris/analysis/ExportStmt.java | 6 ++++++
 .../doris/nereids/trees/plans/commands/ExportCommand.java   | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
index 1ca6c4e3f5be4a..cba01d22fca791 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
@@ -47,6 +47,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.UUID;
 
@@ -146,6 +147,11 @@ public boolean needAuditEncryption() {
     public void analyze(Analyzer analyzer) throws UserException {
         super.analyze(analyzer);
 
+        if (!Config.enable_outfile_to_local && Objects.requireNonNull(path)
+                .startsWith(OutFileClause.LOCAL_FILE_PREFIX)) {
+            throw new AnalysisException("`enable_outfile_to_local` = false, exporting file to local fs is disabled.");
+        }
+
         tableRef = analyzer.resolveTableRef(tableRef);
         Preconditions.checkNotNull(tableRef);
         tableRef.analyze(analyzer);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
index c5d61f585862bf..b08c6a6b54afbf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
@@ -140,6 +140,10 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
                     tblName.getDb() + ": " + tblName.getTbl());
         }
 
+        if (!Config.enable_outfile_to_local && path.startsWith(OutFileClause.LOCAL_FILE_PREFIX)) {
+            throw new AnalysisException("`enable_outfile_to_local` = false, exporting file to local fs is disabled.");
+        }
+
         // check phases
         checkAllParameters(ctx, tblName, fileProperties);
 

From 57226bce94712a8e34798da8af7e1f46a7435b51 Mon Sep 17 00:00:00 2001
From: zclllyybb <zhaochangle@selectdb.com>
Date: Tue, 17 Dec 2024 15:19:51 +0800
Subject: [PATCH 361/399] [Fix](show-stmt) Raise an error when show partitions
 with complex predicate  (#45257)

### What problem does this PR solve?
we support `partitions` tvf. so it's not necessary to support this.
raise an error.
Co-authored-by: Pxl <xl@selectdb.com>
---
 .../doris/common/proc/PartitionsProcDir.java  |  9 ++++-
 .../query_p0/show/test_show_partitions.groovy | 39 +++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/suites/query_p0/show/test_show_partitions.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
index c76a4185fca6c6..3fd945c013cd5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
@@ -22,6 +22,7 @@
 import org.apache.doris.analysis.Expr;
 import org.apache.doris.analysis.IntLiteral;
 import org.apache.doris.analysis.LimitElement;
+import org.apache.doris.analysis.LiteralExpr;
 import org.apache.doris.analysis.StringLiteral;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.DataProperty;
@@ -94,11 +95,16 @@ public static boolean filter(String columnName, Comparable element, Map<String,
         if (filterMap == null) {
             return true;
         }
-        Expr subExpr = filterMap.get(columnName.toLowerCase());
+        Expr subExpr = filterMap.get(columnName.toLowerCase()); // predicate on this column.
         if (subExpr == null) {
             return true;
         }
         if (subExpr instanceof BinaryPredicate) {
+            // show partitions only provide very limited filtering capacity in FE. so restrict here.
+            if (!(subExpr.getChild(1) instanceof LiteralExpr)) {
+                throw new AnalysisException("Not Supported. Use `select * from partitions(...)` instead");
+            }
+
             BinaryPredicate binaryPredicate = (BinaryPredicate) subExpr;
             if (subExpr.getChild(1) instanceof StringLiteral
                     && binaryPredicate.getOp() == BinaryPredicate.Operator.EQ) {
@@ -169,6 +175,7 @@ public ProcResult fetchResultByFilter(Map<String, Expr> filterMap, List<OrderByP
             filterPartitionInfos = partitionInfos;
         } else {
             filterPartitionInfos = Lists.newArrayList();
+            // TODO: we should change the order of loops to speed up. use filters to filter column value.
             for (List<Comparable> partitionInfo : partitionInfos) {
                 if (partitionInfo.size() != TITLE_NAMES.size()) {
                     throw new AnalysisException("PartitionInfos.size() " + partitionInfos.size()
diff --git a/regression-test/suites/query_p0/show/test_show_partitions.groovy b/regression-test/suites/query_p0/show/test_show_partitions.groovy
new file mode 100644
index 00000000000000..bf88872029ef72
--- /dev/null
+++ b/regression-test/suites/query_p0/show/test_show_partitions.groovy
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_partitions") {
+    sql "drop table if exists test_show_part"
+    sql """
+        create table test_show_part (
+            month varchar (255),
+            id int,
+            code varchar (255),
+            name varchar (255)
+        ) ENGINE = OLAP DUPLICATE KEY(month, id)
+        AUTO PARTITION BY LIST (month)()
+        distributed by hash (id) buckets auto
+        PROPERTIES(
+        "replication_allocation" = "tag.location.default: 1"
+        )
+    """
+    sql """ insert into test_show_part(month, id, code, name) values ('2024-12', 10001, 'test10001', 'test') """
+    test {
+        sql "show partitions from test_show_part where PartitionName = auto_partition_name('list','2024-12')"
+        exception "Not Supported. Use `select * from partitions(...)` instead"
+    }
+    sql " show partitions from test_show_part where PartitionName = '123'; "
+}
\ No newline at end of file

From 06034959a64ddb732024ac45f2bedca54d1fee15 Mon Sep 17 00:00:00 2001
From: zclllyybb <zhaochangle@selectdb.com>
Date: Tue, 17 Dec 2024 15:51:21 +0800
Subject: [PATCH 362/399] [Fix](function) fix coredump of function
 StDistanceSphere and StAngleSphere (#45508)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: https://github.com/apache/doris/pull/40107

Problem Summary:

In https://github.com/apache/doris/pull/40107 we introduced a wrong
behaviour which would lead to BE crash because of dangling reference.
now fix it.

### Release note

fix BE crash of function StDistanceSphere and StAngleSphere

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [x] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [x] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 be/src/vec/functions/functions_geo.cpp        | 43 ++++++++++++-------
 .../spatial_functions/test_gis_function.out   |  6 +++
 .../test_gis_function.groovy                  | 18 ++++++++
 3 files changed, 51 insertions(+), 16 deletions(-)

diff --git a/be/src/vec/functions/functions_geo.cpp b/be/src/vec/functions/functions_geo.cpp
index 7b833f91a8ddfd..6d75258d146ff7 100644
--- a/be/src/vec/functions/functions_geo.cpp
+++ b/be/src/vec/functions/functions_geo.cpp
@@ -249,15 +249,21 @@ struct StDistanceSphere {
         DCHECK_EQ(arguments.size(), 4);
         auto return_type = block.get_data_type(result);
 
-        const auto* x_lng = check_and_get_column<ColumnFloat64>(
-                block.get_by_position(arguments[0]).column->convert_to_full_column_if_const());
-        const auto* x_lat = check_and_get_column<ColumnFloat64>(
-                block.get_by_position(arguments[1]).column->convert_to_full_column_if_const());
-        const auto* y_lng = check_and_get_column<ColumnFloat64>(
-                block.get_by_position(arguments[2]).column->convert_to_full_column_if_const());
-        const auto* y_lat = check_and_get_column<ColumnFloat64>(
-                block.get_by_position(arguments[3]).column->convert_to_full_column_if_const());
+        ColumnPtr x_lng_origin =
+                block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
+        ColumnPtr x_lat_origin =
+                block.get_by_position(arguments[1]).column->convert_to_full_column_if_const();
+        ColumnPtr y_lng_origin =
+                block.get_by_position(arguments[2]).column->convert_to_full_column_if_const();
+        ColumnPtr y_lat_origin =
+                block.get_by_position(arguments[3]).column->convert_to_full_column_if_const();
+
+        const auto* x_lng = check_and_get_column<ColumnFloat64>(x_lng_origin);
+        const auto* x_lat = check_and_get_column<ColumnFloat64>(x_lat_origin);
+        const auto* y_lng = check_and_get_column<ColumnFloat64>(y_lng_origin);
+        const auto* y_lat = check_and_get_column<ColumnFloat64>(y_lat_origin);
         CHECK(x_lng && x_lat && y_lng && y_lat);
+
         const auto size = x_lng->size();
         auto res = ColumnFloat64::create();
         res->reserve(size);
@@ -290,14 +296,19 @@ struct StAngleSphere {
         DCHECK_EQ(arguments.size(), 4);
         auto return_type = block.get_data_type(result);
 
-        const auto* x_lng = check_and_get_column<ColumnFloat64>(
-                block.get_by_position(arguments[0]).column->convert_to_full_column_if_const());
-        const auto* x_lat = check_and_get_column<ColumnFloat64>(
-                block.get_by_position(arguments[1]).column->convert_to_full_column_if_const());
-        const auto* y_lng = check_and_get_column<ColumnFloat64>(
-                block.get_by_position(arguments[2]).column->convert_to_full_column_if_const());
-        const auto* y_lat = check_and_get_column<ColumnFloat64>(
-                block.get_by_position(arguments[3]).column->convert_to_full_column_if_const());
+        ColumnPtr x_lng_origin =
+                block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
+        ColumnPtr x_lat_origin =
+                block.get_by_position(arguments[1]).column->convert_to_full_column_if_const();
+        ColumnPtr y_lng_origin =
+                block.get_by_position(arguments[2]).column->convert_to_full_column_if_const();
+        ColumnPtr y_lat_origin =
+                block.get_by_position(arguments[3]).column->convert_to_full_column_if_const();
+
+        const auto* x_lng = check_and_get_column<ColumnFloat64>(x_lng_origin);
+        const auto* x_lat = check_and_get_column<ColumnFloat64>(x_lat_origin);
+        const auto* y_lng = check_and_get_column<ColumnFloat64>(y_lng_origin);
+        const auto* y_lat = check_and_get_column<ColumnFloat64>(y_lat_origin);
         CHECK(x_lng && x_lat && y_lng && y_lat);
 
         const auto size = x_lng->size();
diff --git a/regression-test/data/nereids_p0/sql_functions/spatial_functions/test_gis_function.out b/regression-test/data/nereids_p0/sql_functions/spatial_functions/test_gis_function.out
index db1b1ffcae52d3..df93348581bd64 100644
--- a/regression-test/data/nereids_p0/sql_functions/spatial_functions/test_gis_function.out
+++ b/regression-test/data/nereids_p0/sql_functions/spatial_functions/test_gis_function.out
@@ -113,3 +113,9 @@ LINESTRING (1 1, 2 2)
 -- !sql --
 POLYGON ((114.104486 22.547119, 114.093758 22.547753, 114.096504 22.532057, 114.104229 22.539826, 114.106203 22.54268, 114.104486 22.547119))
 
+-- !sql_part_const_dis_sph --
+3335.853035325018
+
+-- !sql_part_const_ang_sph --
+0.030000000000004495
+
diff --git a/regression-test/suites/nereids_p0/sql_functions/spatial_functions/test_gis_function.groovy b/regression-test/suites/nereids_p0/sql_functions/spatial_functions/test_gis_function.groovy
index d50ac65719af03..8c384f51ff7048 100644
--- a/regression-test/suites/nereids_p0/sql_functions/spatial_functions/test_gis_function.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/spatial_functions/test_gis_function.groovy
@@ -71,4 +71,22 @@ suite("test_gis_function") {
     qt_sql "SELECT ST_AsText(ST_GeomFromWKB(ST_AsBinary(ST_GeometryFromText(\"LINESTRING (1 1, 2 2)\"))));"
     qt_sql "SELECT ST_AsText(ST_GeomFromWKB(ST_AsBinary(ST_Polygon(\"POLYGON ((114.104486 22.547119,114.093758 22.547753,114.096504 22.532057,114.104229 22.539826,114.106203 22.542680,114.104486 22.547119))\"))));"
 
+    // test const
+    sql "drop table if exists test_gis_const"
+    sql """
+    CREATE TABLE test_gis_const (
+        `userid` varchar(32) NOT NULL COMMENT '个人账号id',
+        `c_1` double NOT NULL COMMENT '发送时间',
+        `c_2` double NOT NULL COMMENT '信源类型'
+    ) ENGINE=OLAP
+    UNIQUE KEY(`userid`)
+    COMMENT 'OLAP'
+    DISTRIBUTED BY HASH(`userid`) BUCKETS 6
+    PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+    sql "insert into test_gis_const values ('xxxx',78.73,31.5);"
+    qt_sql_part_const_dis_sph "select st_distance_sphere(78.73,31.53,c_1,c_2) from test_gis_const; "
+    qt_sql_part_const_ang_sph "select st_angle_sphere(78.73,31.53,c_1,c_2) from test_gis_const; "
 }

From 2e4d86141a605474980ccb42ea01b2d742dcdf0c Mon Sep 17 00:00:00 2001
From: yiguolei <guolei@selectdb.com>
Date: Tue, 17 Dec 2024 18:47:29 +0800
Subject: [PATCH 363/399] [bugfix](becore) local exchange should check
 receiver's local state to avoid core (#45470)

_local_recvr depdend on pipeline::ExchangeLocalState* _parent to do some
memory counter settings
but it only owns a raw pointer, so that the ExchangeLocalState object
may be deconstructed.
So that I lock the local state to avoid it is deconstruted
---
 be/src/vec/sink/vdata_stream_sender.cpp | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/be/src/vec/sink/vdata_stream_sender.cpp b/be/src/vec/sink/vdata_stream_sender.cpp
index abed61334583b8..f46602e8d67158 100644
--- a/be/src/vec/sink/vdata_stream_sender.cpp
+++ b/be/src/vec/sink/vdata_stream_sender.cpp
@@ -97,8 +97,14 @@ Status Channel::open(RuntimeState* state) {
         auto st = _parent->state()->exec_env()->vstream_mgr()->find_recvr(
                 _fragment_instance_id, _dest_node_id, &_local_recvr);
         if (!st.ok()) {
-            // Recvr not found. Maybe downstream task is finished already.
-            LOG(INFO) << "Recvr is not found : " << st.to_string();
+            // If could not find local receiver, then it means the channel is EOF.
+            // Maybe downstream task is finished already.
+            //if (_receiver_status.ok()) {
+            //    _receiver_status = Status::EndOfFile("local data stream receiver is deconstructed");
+            //}
+            LOG(INFO) << "Query: " << print_id(state->query_id())
+                      << " recvr is not found, maybe downstream task is finished. error st is: "
+                      << st.to_string();
         }
     }
     _be_number = state->be_number();
@@ -193,8 +199,20 @@ Status Channel::send_local_block(Block* block, bool eos, bool can_be_moved) {
     if (is_receiver_eof()) {
         return _receiver_status;
     }
-
     auto receiver_status = _recvr_status();
+    // _local_recvr depdend on pipeline::ExchangeLocalState* _parent to do some memory counter settings
+    // but it only owns a raw pointer, so that the ExchangeLocalState object may be deconstructed.
+    // Lock the fragment context to ensure the runtime state and other objects are not deconstructed
+    TaskExecutionContextSPtr ctx_lock = nullptr;
+    if (receiver_status.ok() && _local_recvr != nullptr) {
+        ctx_lock = _local_recvr->task_exec_ctx();
+        // Do not return internal error, because when query finished, the downstream node
+        // may finish before upstream node. And the object maybe deconstructed. If return error
+        // then the upstream node may report error status to FE, the query is failed.
+        if (ctx_lock == nullptr) {
+            receiver_status = Status::EndOfFile("local data stream receiver is deconstructed");
+        }
+    }
     if (receiver_status.ok()) {
         COUNTER_UPDATE(_parent->local_bytes_send_counter(), block->bytes());
         COUNTER_UPDATE(_parent->local_sent_rows(), block->rows());

From 575d6448f58efb0c0c4aec4603ac4b5708ba9f2a Mon Sep 17 00:00:00 2001
From: Dongyang Li <lidongyang@selectdb.com>
Date: Tue, 17 Dec 2024 19:24:19 +0800
Subject: [PATCH 364/399] [chore](ci) cloud_p0 print running_pipeline_tasks
 (#45533)

---
 regression-test/pipeline/cloud_p0/run.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/regression-test/pipeline/cloud_p0/run.sh b/regression-test/pipeline/cloud_p0/run.sh
index c1e2570047696b..5eabe7d78a487d 100644
--- a/regression-test/pipeline/cloud_p0/run.sh
+++ b/regression-test/pipeline/cloud_p0/run.sh
@@ -13,7 +13,7 @@ fi
 EOF
 ############################# run.sh content ########################################
 # shellcheck source=/dev/null
-# check_tpcds_table_rows, restart_doris, set_session_variable, check_tpcds_result, _monitor_connection
+# _monitor_regression_log, print_running_pipeline_tasks
 source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
 # shellcheck source=/dev/null
 # create_an_issue_comment
@@ -107,6 +107,7 @@ export -f run
 timeout_minutes=$((${repeat_times_from_trigger:-1} * ${BUILD_TIMEOUT_MINUTES:-180}))m
 timeout "${timeout_minutes}" bash -cx run
 exit_flag="$?"
+if print_running_pipeline_tasks; then :; fi
 # shellcheck source=/dev/null
 source "$(cd "${teamcity_build_checkoutDir}" && bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
 

From 95e21ad96cd6130aa4b48d6bdead9d515b4839c8 Mon Sep 17 00:00:00 2001
From: abmdocrt <lianyukang@selectdb.com>
Date: Tue, 17 Dec 2024 19:35:59 +0800
Subject: [PATCH 365/399] [Fix](test) Fix cumu compaction with delete case fail
 (#45523)

---
 .../test_cumu_compaction_with_delete.groovy   | 32 ++++++++++---------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
index 05ab96a628a7ec..7ad5adf069ed7f 100644
--- a/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
+++ b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
@@ -18,13 +18,25 @@
 import org.codehaus.groovy.runtime.IOGroovyMethods
 
 suite("test_cumu_compaction_with_delete") {
-    def tableName = "test_cumu_compaction_with_delete"
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+    def set_be_config = { key, value ->
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+            logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+        }
+    }
+    set_be_config.call("enable_sleep_between_delete_cumu_compaction", "false")
+
+    def tableName = "test_cumu_compaction_with_delete1"
     def check_cumu_point = { cumu_point ->
         def tablets = sql_return_maparray """ show tablets from ${tableName}; """
         int cumuPoint = 0
         for (def tablet in tablets) {
             String tablet_id = tablet.TabletId
-            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            def (code, out, err) = curl("GET", tablet.CompactionStatus)
             logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
             assertEquals(code, 0)
             def tabletJson = parseJson(out.trim())
@@ -56,7 +68,7 @@ suite("test_cumu_compaction_with_delete") {
             if(check_cumu_point(100)){
                 break;
             }
-            Thread.sleep(1000)
+            Thread.sleep(10000)
         }
         def time_diff = System.currentTimeMillis() - now
         logger.info("time_diff:" + time_diff)
@@ -70,19 +82,9 @@ suite("test_cumu_compaction_with_delete") {
         try_sql("DROP TABLE IF EXISTS ${tableName} FORCE")
     }
 
-    def backendId_to_backendIP = [:]
-    def backendId_to_backendHttpPort = [:]
-    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
-
-    def set_be_config = { key, value ->
-        for (String backend_id: backendId_to_backendIP.keySet()) {
-            def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
-            logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
-        }
-    }
-
     try {
         set_be_config.call("enable_sleep_between_delete_cumu_compaction", "true")
+        tableName = "test_cumu_compaction_with_delete2"
         sql """ DROP TABLE IF EXISTS ${tableName} """
         sql """
             CREATE TABLE ${tableName} (
@@ -105,7 +107,7 @@ suite("test_cumu_compaction_with_delete") {
             if(check_cumu_point(100)){
                 break;
             }
-            Thread.sleep(1000)
+            Thread.sleep(10000)
         }
         def time_diff = System.currentTimeMillis() - now
         logger.info("time_diff:" + time_diff)

From 10b2bf28176ccacafe57838e22985d3157526e79 Mon Sep 17 00:00:00 2001
From: Xujian Duan <50550370+DarvenDuan@users.noreply.github.com>
Date: Tue, 17 Dec 2024 19:51:06 +0800
Subject: [PATCH 366/399] [test](regression-test) add a regression test case
 for s3 vault (#45261)

---
 ...test_s3_vault_path_start_with_slash.groovy | 70 +++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 regression-test/suites/vault_p0/default/test_s3_vault_path_start_with_slash.groovy

diff --git a/regression-test/suites/vault_p0/default/test_s3_vault_path_start_with_slash.groovy b/regression-test/suites/vault_p0/default/test_s3_vault_path_start_with_slash.groovy
new file mode 100644
index 00000000000000..704a4b67836ebf
--- /dev/null
+++ b/regression-test/suites/vault_p0/default/test_s3_vault_path_start_with_slash.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_s3_vault_path_start_with_slash", "nonConcurrent") {
+    if (!isCloudMode()) {
+        logger.info("skip ${name} case, because not cloud mode")
+        return
+    }
+
+    if (!enableStoragevault()) {
+        logger.info("skip ${name} case")
+        return
+    }
+
+    def tableName = "table_test_s3_vault_path_start_with_slash"
+    try {
+        def vault_name = "test_s3_vault_path_start_with_slash_vault"
+        sql """
+            CREATE STORAGE VAULT IF NOT EXISTS ${vault_name}
+            PROPERTIES (
+                "type"="S3",
+                "s3.endpoint"="${getS3Endpoint()}",
+                "s3.region" = "${getS3Region()}",
+                "s3.access_key" = "${getS3AK()}",
+                "s3.secret_key" = "${getS3SK()}",
+                "s3.root.path" = "/test_s3_vault_path_start_with_slash_vault",
+                "s3.bucket" = "${getS3BucketName()}",
+                "s3.external_endpoint" = "",
+                "provider" = "${getS3Provider()}",
+                "set_as_default" = "true"
+            );
+        """
+
+        sql "DROP TABLE IF EXISTS ${tableName}"
+        sql """
+                CREATE TABLE ${tableName} (
+                    `key` INT,
+                    value INT
+                ) DUPLICATE KEY (`key`) 
+                DISTRIBUTED BY HASH (`key`) BUCKETS 1
+                PROPERTIES (
+                    "replication_num" = "1",
+                    "storage_vault_name" = "${vault_name}"
+                )
+            """
+
+        sql """ insert into ${tableName} values(1, 1); """
+        sql """ sync;"""
+        def result = sql """ select * from ${tableName}; """
+        logger.info("result:${result}");
+        assertTrue(result.size() == 1)
+        assertTrue(result[0][0].toInteger() == 1)
+    } finally {
+        sql "DROP TABLE IF EXISTS ${tableName}"
+    }
+}

From 13999c85b7e3e9871adcba5228652538fea903aa Mon Sep 17 00:00:00 2001
From: Uniqueyou <wangyixuan@selectdb.com>
Date: Tue, 17 Dec 2024 19:54:09 +0800
Subject: [PATCH 367/399] [Test](binlog) Check correctly for show create
 database (#45472)

```
Exception:
java.lang.IllegalStateException: Check tag 'select' failed:
Check tag 'select' failed, line 1, CHAR result mismatch.
Expect cell is: CREATE DATABASE `test_recover_db`
But real is   : CREATE DATABASE `test_recover_db`\\nPROPERTIES (\\n"binlog.enable" = "true"\\n)
line 1 mismatch
ExpectRow: [test_recover_db, CREATE DATABASE `test_recover_db`]
RealRow  : [test_recover_db, CREATE DATABASE `test_recover_db`
PROPERTIES (
"binlog.enable" = "true"
)]
```
---
 regression-test/data/ddl_p0/test_recover.out  | 216 ------------------
 .../suites/ddl_p0/test_recover.groovy         |  27 ++-
 2 files changed, 18 insertions(+), 225 deletions(-)

diff --git a/regression-test/data/ddl_p0/test_recover.out b/regression-test/data/ddl_p0/test_recover.out
index 149c4c5459b5de..646ec068a0ea6b 100644
--- a/regression-test/data/ddl_p0/test_recover.out
+++ b/regression-test/data/ddl_p0/test_recover.out
@@ -27,33 +27,6 @@ test_recover_tb_new
 test_recover_tb
 test_recover_tb_new
 
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
 -- !select --
 test_recover_tb
 test_recover_tb_2
@@ -89,33 +62,6 @@ test_recover_tb_new
 test_recover_tb
 test_recover_tb_new
 
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
 -- !select --
 test_recover_tb
 test_recover_tb_2
@@ -151,33 +97,6 @@ test_recover_tb_new
 test_recover_tb
 test_recover_tb_new
 
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
 -- !select --
 test_recover_tb
 test_recover_tb_2
@@ -213,33 +132,6 @@ test_recover_tb_new
 test_recover_tb
 test_recover_tb_new
 
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
 -- !select --
 test_recover_tb
 test_recover_tb_2
@@ -275,33 +167,6 @@ test_recover_tb_new
 test_recover_tb
 test_recover_tb_new
 
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
 -- !select --
 test_recover_tb
 test_recover_tb_2
@@ -337,33 +202,6 @@ test_recover_tb_new
 test_recover_tb
 test_recover_tb_new
 
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
 -- !select --
 test_recover_tb
 test_recover_tb_2
@@ -399,33 +237,6 @@ test_recover_tb_new
 test_recover_tb
 test_recover_tb_new
 
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
 -- !select --
 test_recover_tb
 test_recover_tb_2
@@ -461,33 +272,6 @@ test_recover_tb_new
 test_recover_tb
 test_recover_tb_new
 
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
--- !select --
-test_recover_db	CREATE DATABASE `test_recover_db`
-
--- !select --
-test_recover_db_new	CREATE DATABASE `test_recover_db_new`
-
 -- !select --
 test_recover_tb
 test_recover_tb_2
diff --git a/regression-test/suites/ddl_p0/test_recover.groovy b/regression-test/suites/ddl_p0/test_recover.groovy
index 4f17b0f644b0fb..a1c1b1917ae5d3 100644
--- a/regression-test/suites/ddl_p0/test_recover.groovy
+++ b/regression-test/suites/ddl_p0/test_recover.groovy
@@ -273,7 +273,8 @@ suite("test_recover") {
                 
             // test drop/recover db
 
-                qt_select """ SHOW CREATE DATABASE `test_recover_db` """
+                def showDatabase = sql """ SHOW CREATE DATABASE test_recover_db """
+                assertTrue(showDatabase[0][1].contains("CREATE DATABASE `test_recover_db`"))
 
                 sql """
             DROP DATABASE `test_recover_db`
@@ -283,7 +284,8 @@ suite("test_recover") {
             RECOVER DATABASE `test_recover_db`
             """
 
-                qt_select """ SHOW CREATE DATABASE `test_recover_db` """
+                showDatabase = sql """ SHOW CREATE DATABASE test_recover_db """
+                assertTrue(showDatabase[0][1].contains("CREATE DATABASE `test_recover_db`"))
 
                 sql """
             CREATE TABLE `test_recover_db`.`test_recover_tb_1` (
@@ -312,7 +314,8 @@ suite("test_recover") {
             CREATE DATABASE `test_recover_db`
             """
 
-                qt_select """ SHOW CREATE DATABASE `test_recover_db` """
+                showDatabase = sql """ SHOW CREATE DATABASE `test_recover_db` """
+                assertTrue(showDatabase[0][1].contains("CREATE DATABASE `test_recover_db`"))
 
                 sql """
             DROP DATABASE `test_recover_db`
@@ -321,7 +324,8 @@ suite("test_recover") {
             CREATE DATABASE `test_recover_db`
             """
 
-                qt_select """ SHOW CREATE DATABASE `test_recover_db` """
+                showDatabase = sql """ SHOW CREATE DATABASE `test_recover_db` """
+                assertTrue(showDatabase[0][1].contains("CREATE DATABASE `test_recover_db`"))
 
                 sql """
             DROP DATABASE `test_recover_db`
@@ -331,7 +335,8 @@ suite("test_recover") {
             CREATE DATABASE `test_recover_db`
             """
 
-                qt_select """ SHOW CREATE DATABASE `test_recover_db` """
+                showDatabase = sql """ SHOW CREATE DATABASE `test_recover_db` """
+                assertTrue(showDatabase[0][1].contains("CREATE DATABASE `test_recover_db`"))
                 
                 sql """
             CREATE TABLE `test_recover_db`.`test_recover_tb` (
@@ -359,7 +364,8 @@ suite("test_recover") {
             CREATE DATABASE `test_recover_db`
             """
 
-                qt_select """ SHOW CREATE DATABASE `test_recover_db` """
+                showDatabase = sql """ SHOW CREATE DATABASE `test_recover_db` """
+                assertTrue(showDatabase[0][1].contains("CREATE DATABASE `test_recover_db`"))
 
                 sql """
             DROP DATABASE `test_recover_db`
@@ -369,7 +375,8 @@ suite("test_recover") {
             RECOVER DATABASE `test_recover_db` AS `test_recover_db_new`
             """
 
-                qt_select """ SHOW CREATE DATABASE `test_recover_db_new` """
+                showDatabase = sql """ SHOW CREATE DATABASE `test_recover_db_new` """
+                assertTrue(showDatabase[0][1].contains("CREATE DATABASE `test_recover_db_new`"))
 
                 sql """
             CREATE TABLE `test_recover_db_new`.`test_recover_tb_2` (
@@ -413,8 +420,10 @@ suite("test_recover") {
             )
             """
 
-                qt_select """ SHOW CREATE DATABASE `test_recover_db` """
-                qt_select """ SHOW CREATE DATABASE `test_recover_db_new` """
+                showDatabase = sql """ SHOW CREATE DATABASE `test_recover_db` """
+                assertTrue(showDatabase[0][1].contains("CREATE DATABASE `test_recover_db`"))
+                showDatabase = sql """ SHOW CREATE DATABASE `test_recover_db_new` """
+                assertTrue(showDatabase[0][1].contains("CREATE DATABASE `test_recover_db_new`"))
                 res = sql """SHOW CREATE TABLE `test_recover_db`.`test_recover_tb`"""
                 assertTrue(res.size() != 0)
                 qt_select """SHOW TABLES FROM `test_recover_db`"""

From a283f0fab28190d8cb28d8868b7a067033b7cf31 Mon Sep 17 00:00:00 2001
From: Uniqueyou <wangyixuan@selectdb.com>
Date: Tue, 17 Dec 2024 19:54:35 +0800
Subject: [PATCH 368/399] [Fix](ccr) Wrong result of tosql for add column with
 comment (#45319)

ccr increment sync problem, ccr-syncer also will be fixed and add test
case
```
alter table t add column col1 varchar(20) NULL COMMENT "xxx\"xx\""
```
```
[2024-12-11 17:48:30.737]  WARN job sync failed, job: test, err: [normal] Error 1105 (HY000): errCode = 2, detailMessage =
mismatched input 'xx' expecting {<EOF>, ';'}(line 1, pos 64)
exec sql ALTER TABLE `t` ADD COLUMN `col1` varchar(20) NULL COMMENT "xxx"xx"" failed
github.com/selectdb/ccr_syncer/pkg/ccr/base.(*Spec).DbExec
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/base/spec.go:1166
github.com/selectdb/ccr_syncer/pkg/ccr/base.(*Spec).LightningSchemaChange
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/base/spec.go:1242
github.com/selectdb/ccr_syncer/pkg/ccr.(*Job).handleLightningSchemaChange
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/job.go:2107
github.com/selectdb/ccr_syncer/pkg/ccr.(*Job).handleBinlog
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/job.go:2725
github.com/selectdb/ccr_syncer/pkg/ccr.(*Job).handleBinlogs
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/job.go:2662
github.com/selectdb/ccr_syncer/pkg/ccr.(*Job).incrementalSync
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/job.go:2839
github.com/selectdb/ccr_syncer/pkg/ccr.(*Job).dbSync
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/job.go:2900
github.com/selectdb/ccr_syncer/pkg/ccr.(*Job).sync
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/job.go:2917
github.com/selectdb/ccr_syncer/pkg/ccr.(*Job).run
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/job.go:2975
github.com/selectdb/ccr_syncer/pkg/ccr.(*Job).Run
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/job.go:3096
github.com/selectdb/ccr_syncer/pkg/ccr.(*JobManager).runJob.func1
    /mnt/disk2/wangyixuan/ccr-syncer/pkg/ccr/job_manager.go:166
runtime.goexit
    /mnt/disk2/wangyixuan/go_1_23/go/src/runtime/asm_amd64.s:1700 job=test line=ccr/job.go:2980
```
---
 .../org/apache/doris/analysis/ColumnDef.java  |  2 +-
 .../doris/analysis/AddColumnCommentTest.java  | 75 +++++++++++++++++++
 2 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnCommentTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index 3bfe099dd8da2e..404b0060b94b2d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -693,7 +693,7 @@ public String toSql() {
                 sb.append("DEFAULT ").append("NULL").append(" ");
             }
         }
-        sb.append("COMMENT \"").append(comment).append("\"");
+        sb.append("COMMENT \"").append(SqlUtils.escapeQuota(comment)).append("\"");
 
         return sb.toString();
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnCommentTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnCommentTest.java
new file mode 100644
index 00000000000000..1adc4add7e38ac
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AddColumnCommentTest.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.analysis.ColumnDef.DefaultValue;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+
+import com.google.common.collect.Lists;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.List;
+
+public class AddColumnCommentTest {
+    private static Analyzer analyzer;
+
+    @BeforeClass
+    public static void setUp() {
+        analyzer = AccessTestUtil.fetchAdminAnalyzer(false);
+    }
+
+    @Test
+    public void testNormal() throws AnalysisException {
+        List<ColumnDef> columns = Lists.newArrayList();
+        ColumnDef definition = new ColumnDef("col1", new TypeDef(ScalarType.createType(PrimitiveType.VARCHAR)),
+                true,
+                null, false,
+                new DefaultValue(true, "col1_default"), "a");
+        definition.setKeysType(KeysType.DUP_KEYS);
+        columns.add(definition);
+        definition = new ColumnDef("col2", new TypeDef(ScalarType.createType(PrimitiveType.INT)),
+                false,
+                null, false,
+                new DefaultValue(true, 0), "[\"a\", \"b\"]");
+        columns.add(definition);
+        definition = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.VARCHAR)),
+                false,
+                null, false,
+                new DefaultValue(true, "col3_default"), "['a', 'b']");
+        columns.add(definition);
+        definition = new ColumnDef("col4", new TypeDef(ScalarType.createType(PrimitiveType.VARCHAR)),
+                false,
+                null, false,
+                new DefaultValue(true, "col4_default"), "'[123, 456]' and '[789, 10]'");
+        columns.add(definition);
+        AddColumnsClause clause = new AddColumnsClause(columns, null, null);
+        clause.analyze(analyzer);
+        System.out.println(clause.toString());
+        Assert.assertEquals(
+                "ADD COLUMN (`col1` varchar(65533) NOT NULL DEFAULT \"col1_default\" COMMENT \"a\", "
+                        + "`col2` int NOT NULL DEFAULT \"0\" COMMENT \"[\\\"a\\\", \\\"b\\\"]\", "
+                        + "`col3` varchar(65533) NOT NULL DEFAULT \"col3_default\" COMMENT \"['a', 'b']\", "
+                        + "`col4` varchar(65533) NOT NULL DEFAULT \"col4_default\" COMMENT \"'[123, 456]' and '[789, 10]'\")",
+                clause.toString());
+    }
+}

From 7a79d97057405dfff19a51b962859ce3f41d9cd2 Mon Sep 17 00:00:00 2001
From: Siyang Tang <tangsiyang@selectdb.com>
Date: Tue, 17 Dec 2024 20:28:09 +0800
Subject: [PATCH 369/399] [fix](regression-test) Avoid duplicated table name in
 schema change case for ck result in unstable state (#45471)

---
 .../unique_ck/test_schema_change_unique.groovy                | 2 +-
 .../unique_ck/test_unique_schema_value_drop.groovy            | 4 ++--
 .../unique_ck/test_unique_schema_value_modify.groovy          | 4 ++--
 .../unique_ck/test_unique_schema_value_modify1.groovy         | 4 ++--
 .../unique_ck/test_unique_schema_value_modify2.groovy         | 4 ++--
 .../unique_ck/test_unique_schema_value_modify3.groovy         | 4 ++--
 6 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/regression-test/suites/schema_change_p0/unique_ck/test_schema_change_unique.groovy b/regression-test/suites/schema_change_p0/unique_ck/test_schema_change_unique.groovy
index be20986c4af9ae..c82a0f616a09ee 100644
--- a/regression-test/suites/schema_change_p0/unique_ck/test_schema_change_unique.groovy
+++ b/regression-test/suites/schema_change_p0/unique_ck/test_schema_change_unique.groovy
@@ -34,7 +34,7 @@ import org.apache.http.client.methods.CloseableHttpResponse
 import org.apache.http.util.EntityUtils
 
 suite("test_schema_change_unique", "p0") {
-    def tableName3 = "test_all_unique"
+    def tableName3 = "test_all_unique_ck"
 
     def getJobState = { tableName ->
         def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
diff --git a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_drop.groovy b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_drop.groovy
index e465661cdbceec..f3b83dd23709cc 100644
--- a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_drop.groovy
+++ b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_drop.groovy
@@ -16,8 +16,8 @@
 // under the License.
 
 suite("test_unique_schema_value_drop", "p0") {
-    def tbName = "test_unique_model_schema_value_drop"
-    def tbName2 = "test_unique_model_schema_value_drop_1"
+    def tbName = "test_unique_model_schema_value_drop_ck"
+    def tbName2 = "test_unique_model_schema_value_drop_1_ck"
     def on_write = true
     println String.format("current enable_unique_key_merge_on_write is : %s ",on_write)
     sql """ DROP TABLE IF EXISTS ${tbName} """
diff --git a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify.groovy b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify.groovy
index c1d7df20b09df4..f39a7ef9f81995 100644
--- a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify.groovy
+++ b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify.groovy
@@ -16,8 +16,8 @@
 // under the License.
 
 suite("test_unique_schema_value_modify","p0") {
-     def tbName = "unique_model_value_change0"
-     def tbName2 = "unique_model_value_change_0"
+     def tbName = "unique_model_value_change0_ck"
+     def tbName2 = "unique_model_value_change_0_ck"
      def on_write = true
      println String.format("current enable_unique_key_merge_on_write is : %s ",on_write)
      //Test the unique model by adding a value column
diff --git a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify1.groovy b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify1.groovy
index c9d30bff727b6c..f698d8df89e5f2 100644
--- a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify1.groovy
+++ b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify1.groovy
@@ -16,8 +16,8 @@
 // under the License.
 
 suite("test_unique_schema_value_modify1","p0") {
-     def tbName = "test_unique_model_value_change1"
-     def tbName2 = "test_unique_model_value_change_1"
+     def tbName = "test_unique_model_value_change1_ck"
+     def tbName2 = "test_unique_model_value_change_1_ck"
      def on_write = true
      println String.format("current enable_unique_key_merge_on_write is : %s ",on_write)
      //Test the unique model by adding a value column
diff --git a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify2.groovy b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify2.groovy
index 5ba388d98b9c8d..9e03362d9a23c0 100644
--- a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify2.groovy
+++ b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify2.groovy
@@ -16,8 +16,8 @@
 // under the License.
 
 suite("test_unique_schema_value_modify2", "p0") {
-    def tbName = "test_unique_model_value_change2"
-    def tbName2 = "test_unique_model_value_change_2"
+    def tbName = "test_unique_model_value_change2_ck"
+    def tbName2 = "test_unique_model_value_change_2_ck"
     def initTable1 = ""
     def initTableData1 = ""
     def on_write = true
diff --git a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify3.groovy b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify3.groovy
index 0c8e30acc779af..6e1c97567066f1 100644
--- a/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify3.groovy
+++ b/regression-test/suites/schema_change_p0/unique_ck/test_unique_schema_value_modify3.groovy
@@ -16,8 +16,8 @@
 // under the License.
 
 suite("test_unique_schema_value_modify3", "p0") {
-    def tbName = "test_unique_model_value_change3"
-    def tbName2 = "test_unique_model_value_change_3"
+    def tbName = "test_unique_model_value_change3_ck"
+    def tbName2 = "test_unique_model_value_change_3_ck"
     def on_write = true
     println String.format("current enable_unique_key_merge_on_write is : %s ",on_write)
     //Test the unique model by adding a value column

From e88bde1b89083ac5d9afc97e602fdaafeae5faa5 Mon Sep 17 00:00:00 2001
From: Siyang Tang <tangsiyang@selectdb.com>
Date: Tue, 17 Dec 2024 20:28:44 +0800
Subject: [PATCH 370/399] [fix](regression-test) Increase timeout for
 compaction tasks in some sc cases (#45461)

Awaitility default timeout: 10s may be not enough for some compaction
tasks in some schema change test cases.
---
 .../datev2/test_agg_keys_schema_change_datev2.groovy            | 2 +-
 .../suites/schema_change_p0/test_dup_keys_schema_change.groovy  | 2 +-
 .../suites/schema_change_p0/test_uniq_keys_schema_change.groovy | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/regression-test/suites/schema_change_p0/datev2/test_agg_keys_schema_change_datev2.groovy b/regression-test/suites/schema_change_p0/datev2/test_agg_keys_schema_change_datev2.groovy
index 3a92de35b8f967..8afc4fd6c110ab 100644
--- a/regression-test/suites/schema_change_p0/datev2/test_agg_keys_schema_change_datev2.groovy
+++ b/regression-test/suites/schema_change_p0/datev2/test_agg_keys_schema_change_datev2.groovy
@@ -51,7 +51,7 @@ suite("test_agg_keys_schema_change_datev2") {
 
         // wait for all compactions done
         for (String[] tablet in tablets) {
-            Awaitility.await().untilAsserted(() -> {
+            Awaitility.await().atMost(20, TimeUnit.SECONDS).untilAsserted(() -> {
                 String tablet_id = tablet[0]
                 backend_id = tablet[2]
                 (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
diff --git a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy
index 044eaf62258be7..6fe76138cd28ca 100644
--- a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy
@@ -151,7 +151,7 @@ suite ("test_dup_keys_schema_change") {
 
         // wait for all compactions done
         for (String[] tablet in tablets) {
-                Awaitility.await().untilAsserted(() -> {
+                Awaitility.await().atMost(20, TimeUnit.SECONDS).untilAsserted(() -> {
                     String tablet_id = tablet[0]
                     def backend_id = tablet[2]
                     def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
diff --git a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy
index b8dcf97e791582..45c37051b43ec5 100644
--- a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy
@@ -134,7 +134,7 @@ suite ("test_uniq_keys_schema_change") {
 
         // wait for all compactions done
         for (String[] tablet in tablets) {
-                Awaitility.await().untilAsserted(() -> {
+                Awaitility.await().atMost(20, TimeUnit.SECONDS).untilAsserted(() -> {
                     String tablet_id = tablet[0]
                     backend_id = tablet[2]
                     def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)

From 793dd9d19cffc7a2ab6ea8db5b2c341fcd5625a0 Mon Sep 17 00:00:00 2001
From: zhannngchen <zhangchen@selectdb.com>
Date: Tue, 17 Dec 2024 20:46:24 +0800
Subject: [PATCH 371/399] [misc](regression) adjust doc case for
 unique-update.md (#44854)

adjust doc case for unique-update.md
---
 .../data-operate/update/unique-update.md.out  |  6 +-
 .../update/unique-update.md.groovy            | 60 +++++++++++++++----
 2 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/regression-test/data/doc/data-operate/update/unique-update.md.out b/regression-test/data/doc/data-operate/update/unique-update.md.out
index 7f5f43272d9b5b..65683c5f73b474 100644
--- a/regression-test/data/doc/data-operate/update/unique-update.md.out
+++ b/regression-test/data/doc/data-operate/update/unique-update.md.out
@@ -1,4 +1,8 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
-1	100.000000000	To be shipped
+1001	5001	2024-11-24	2024-11-24T14:30	100.00	iPhone 12	New York	100.00	10	false	low
+1002	5002	2024-11-24	2024-11-24T03:30	120.00	iPhone 12	New York	100.00	15	false	medium
+1003	5003	2024-11-24	2024-11-24T10:00	150.00	Samsung S21	Los Angeles	100.00	30	false	medium
+1004	5004	2024-11-24	2024-11-24T16:00	300.00	MacBook Pro	high_risk_region1	200.00	5	false	high
+1005	5005	2024-11-24	2024-11-24T11:00	1100.00	iPad Pro	Chicago	200.00	10	false	high
 
diff --git a/regression-test/suites/doc/data-operate/update/unique-update.md.groovy b/regression-test/suites/doc/data-operate/update/unique-update.md.groovy
index 2cd4f64afce600..582274b053ff84 100644
--- a/regression-test/suites/doc/data-operate/update/unique-update.md.groovy
+++ b/regression-test/suites/doc/data-operate/update/unique-update.md.groovy
@@ -20,23 +20,57 @@ import org.junit.jupiter.api.Assertions;
 suite("docs/data-operate/update/unique-update.md") {
     try {
         multi_sql """
-            DROP TABLE IF EXISTS test_order;
-            CREATE TABLE IF NOT EXISTS test_order
-            (
-                `order_id` BIGINT NOT NULL,
-                `order_amount` DECIMAL(27, 9) NOT NULL,
-                `order_status` VARCHAR(65533)
+            DROP TABLE IF EXISTS transaction_details;
+            CREATE TABLE transaction_details (
+              transaction_id BIGINT NOT NULL,        -- Unique transaction ID
+              user_id BIGINT NOT NULL,               -- User ID
+              transaction_date DATE NOT NULL,        -- Transaction date
+              transaction_time DATETIME NOT NULL,    -- Transaction time
+              transaction_amount DECIMAL(18, 2),     -- Transaction amount
+              transaction_device STRING,             -- Transaction device
+              transaction_region STRING,             -- Transaction region
+              average_daily_amount DECIMAL(18, 2),   -- Average daily transaction amount over the last 3 months
+              recent_transaction_count INT,          -- Number of transactions in the last 7 days
+              has_dispute_history BOOLEAN,           -- Whether there is a dispute history
+              risk_level STRING                      -- Risk level
             )
-            UNIQUE KEY(`order_id`)
-            DISTRIBUTED BY HASH(`order_id`) BUCKETS 1
+            UNIQUE KEY(transaction_id)
+            DISTRIBUTED BY HASH(transaction_id) BUCKETS 10
             PROPERTIES (
-            "replication_allocation" = "tag.location.default: 1"
+              "replication_num" = "3",               -- Number of replicas, default is 3
+              "enable_unique_key_merge_on_write" = "true"  -- Enable MOW mode, support merge update
             );
-            INSERT INTO test_order(order_id, order_amount, order_status) VALUES
-                                  (1       , 100         , 'Pending'   );
+
+            INSERT INTO transaction_details VALUES
+            (1001, 5001, '2024-11-24', '2024-11-24 14:30:00', 100.00, 'iPhone 12', 'New York', 100.00, 10, false, NULL),
+            (1002, 5002, '2024-11-24', '2024-11-24 03:30:00', 120.00, 'iPhone 12', 'New York', 100.00, 15, false, NULL),
+            (1003, 5003, '2024-11-24', '2024-11-24 10:00:00', 150.00, 'Samsung S21', 'Los Angeles', 100.00, 30, false, NULL),
+            (1004, 5004, '2024-11-24', '2024-11-24 16:00:00', 300.00, 'MacBook Pro', 'high_risk_region1', 200.00, 5, false, NULL),
+            (1005, 5005, '2024-11-24', '2024-11-24 11:00:00', 1100.00, 'iPad Pro', 'Chicago', 200.00, 10, false, NULL);
+        """
+        sql """
+           UPDATE transaction_details
+           SET risk_level = CASE
+             -- Transactions with dispute history or in high-risk regions
+             WHEN has_dispute_history = TRUE THEN 'high'
+             WHEN transaction_region IN ('high_risk_region1', 'high_risk_region2') THEN 'high'
+
+             -- Abnormal transaction amount
+             WHEN transaction_amount > 5 * average_daily_amount THEN 'high'
+
+             -- High transaction frequency in the last 7 days
+             WHEN recent_transaction_count > 50 THEN 'high'
+             WHEN recent_transaction_count BETWEEN 20 AND 50 THEN 'medium'
+
+             -- Transactions during non-working hours
+             WHEN HOUR(transaction_time) BETWEEN 2 AND 4 THEN 'medium'
+
+             -- Default risk
+             ELSE 'low'
+           END
+           WHERE transaction_date = '2024-11-24';
         """
-        sql """UPDATE test_order SET order_status = 'To be shipped' WHERE order_id = 1;"""
-        qt_sql "SELECT * FROM test_order"
+        qt_sql "SELECT * FROM transaction_details order by transaction_id"
     } catch (Throwable t) {
         Assertions.fail("examples in docs/data-operate/update/unique-update.md failed to exec, please fix it", t)
     }

From bc6c2e7ecd612f77a3bf050c2f098227ae93dabb Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Tue, 17 Dec 2024 20:53:14 +0800
Subject: [PATCH 372/399] [opt](cloud) Make get tablet stats and update delete
 bitmap update lock be able to be in different fdb txns (#45206)

https://github.com/apache/doris/pull/37670 let the meta service return
the tablet compaction stats along with the
`getDeleteBitmapUpdateLockResponse` to FE to let the BE know whether it
should `sync_rowsets()` due to successful compaction on other BEs on the
same tablet. That PR makes the process of reading tablets' stats and
writing the delete bitmap update lock KV in one fdb txn to achieve the
atomic sematic. However, when a load involves a large number of tablets,
the process of reading tablets' stats may take longer than fdb txn's 5
seconds limitation and cause `TXN_TOO_OLD` error.
This PR re-arrange the process so that the read of tablet stats can be
not necessarily in the same fdb txn with the txn which update the
lock_info.lock_id. In detail, we do as the following:
1. gain the delete bitmap update lock in MS (write delete bitmap update
lock KV)
2. read tablets' stats to get the compaction counts.
3. check if the delete bitmap update lock is still held by the current
load.
---
 cloud/src/meta-service/meta_service.cpp       |  98 +++++--
 cloud/test/meta_service_test.cpp              | 276 +++++++++++++++++-
 .../CloudGlobalTransactionMgr.java            |  22 +-
 3 files changed, 360 insertions(+), 36 deletions(-)

diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index 33c109d19bcc31..26c8c30110a1db 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -1685,6 +1685,7 @@ static bool check_delete_bitmap_lock(MetaServiceCode& code, std::string& msg, st
     std::string lock_val;
     DeleteBitmapUpdateLockPB lock_info;
     auto err = txn->get(lock_key, &lock_val);
+    TEST_SYNC_POINT_CALLBACK("check_delete_bitmap_lock.inject_get_lock_key_err", &err);
     if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
         msg = "lock id key not found";
         code = MetaServiceCode::LOCK_EXPIRED;
@@ -1701,6 +1702,7 @@ static bool check_delete_bitmap_lock(MetaServiceCode& code, std::string& msg, st
         msg = "failed to parse DeleteBitmapUpdateLockPB";
         return false;
     }
+    TEST_SYNC_POINT_CALLBACK("check_delete_bitmap_lock.set_lock_info", &lock_info);
     if (lock_info.lock_id() != lock_id) {
         ss << "lock id not match, locked by lock_id=" << lock_info.lock_id();
         msg = ss.str();
@@ -1877,7 +1879,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
                                               request->lock_id(), request->initiator())) {
                     LOG(WARNING) << "failed to check delete bitmap lock, table_id=" << table_id
                                  << " request lock_id=" << request->lock_id()
-                                 << " request initiator=" << request->initiator() << " msg" << msg;
+                                 << " request initiator=" << request->initiator() << " msg " << msg;
                     return;
                 }
             }
@@ -2150,32 +2152,6 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
         }
     }
 
-    bool require_tablet_stats =
-            request->has_require_compaction_stats() ? request->require_compaction_stats() : false;
-    if (require_tablet_stats) {
-        // this request is from fe when it commits txn for MOW table, we send the compaction stats
-        // along with the GetDeleteBitmapUpdateLockResponse which will be sent to BE later to let
-        // BE eliminate unnecessary sync_rowsets() calls if possible
-        for (const auto& tablet_index : request->tablet_indexes()) {
-            TabletIndexPB idx(tablet_index);
-            TabletStatsPB tablet_stat;
-            internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, false);
-            if (code != MetaServiceCode::OK) {
-                response->clear_base_compaction_cnts();
-                response->clear_cumulative_compaction_cnts();
-                response->clear_cumulative_points();
-                LOG_WARNING(
-                        "failed to get tablet stats when get_delete_bitmap_update_lock, "
-                        "lock_id={}, initiator={}, tablet_id={}",
-                        request->lock_id(), request->initiator(), tablet_index.tablet_id());
-                return;
-            }
-            response->add_base_compaction_cnts(tablet_stat.base_compaction_cnt());
-            response->add_cumulative_compaction_cnts(tablet_stat.cumulative_compaction_cnt());
-            response->add_cumulative_points(tablet_stat.cumulative_point());
-        }
-    }
-
     lock_info.set_lock_id(request->lock_id());
     lock_info.set_expiration(now + request->expiration());
     bool found = false;
@@ -2205,6 +2181,74 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
         msg = ss.str();
         return;
     }
+
+    bool require_tablet_stats =
+            request->has_require_compaction_stats() ? request->require_compaction_stats() : false;
+    if (!require_tablet_stats) return;
+    // this request is from fe when it commits txn for MOW table, we send the compaction stats
+    // along with the GetDeleteBitmapUpdateLockResponse which will be sent to BE later to let
+    // BE eliminate unnecessary sync_rowsets() calls if possible
+
+    // 1. hold the delete bitmap update lock in MS(update lock_info.lock_id to current load's txn id)
+    // 2. read tablets' stats
+    // 3. check whether we still hold the delete bitmap update lock
+    // these steps can be done in different fdb txns
+
+    StopWatch read_stats_sw;
+    err = txn_kv_->create_txn(&txn);
+    if (err != TxnErrorCode::TXN_OK) {
+        code = cast_as<ErrCategory::CREATE>(err);
+        msg = "failed to init txn";
+        return;
+    }
+
+    for (const auto& tablet_idx : request->tablet_indexes()) {
+        TabletStatsPB tablet_stat;
+        std::string stats_key =
+                stats_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(),
+                                  tablet_idx.partition_id(), tablet_idx.tablet_id()});
+        std::string stats_val;
+        TxnErrorCode err = txn->get(stats_key, &stats_val);
+        TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_update_lock.get_compaction_cnts_inject_error",
+                                 &err);
+        if (err == TxnErrorCode::TXN_TOO_OLD) {
+            code = MetaServiceCode::OK;
+            err = txn_kv_->create_txn(&txn);
+            if (err != TxnErrorCode::TXN_OK) {
+                code = cast_as<ErrCategory::CREATE>(err);
+                ss << "failed to init txn when get tablet stats";
+                msg = ss.str();
+                return;
+            }
+            err = txn->get(stats_key, &stats_val);
+        }
+        if (err != TxnErrorCode::TXN_OK) {
+            code = cast_as<ErrCategory::READ>(err);
+            msg = fmt::format("failed to get tablet stats, err={} tablet_id={}", err,
+                              tablet_idx.tablet_id());
+            return;
+        }
+        if (!tablet_stat.ParseFromArray(stats_val.data(), stats_val.size())) {
+            code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+            msg = fmt::format("marformed tablet stats value, key={}", hex(stats_key));
+            return;
+        }
+        response->add_base_compaction_cnts(tablet_stat.base_compaction_cnt());
+        response->add_cumulative_compaction_cnts(tablet_stat.cumulative_compaction_cnt());
+        response->add_cumulative_points(tablet_stat.cumulative_point());
+    }
+
+    read_stats_sw.pause();
+    LOG(INFO) << fmt::format("tablet_idxes.size()={}, read tablet compaction cnts cost={} ms",
+                             request->tablet_indexes().size(), read_stats_sw.elapsed_us() / 1000);
+
+    if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id, table_id, request->lock_id(),
+                                  request->initiator())) {
+        LOG(WARNING) << "failed to check delete bitmap lock after get tablet stats, table_id="
+                     << table_id << " request lock_id=" << request->lock_id()
+                     << " request initiator=" << request->initiator() << " code=" << code
+                     << " msg=" << msg;
+    }
 }
 
 void MetaServiceImpl::remove_delete_bitmap_update_lock(
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index d2dd80f6871e3e..ce5eda9ed7a46c 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -216,6 +216,35 @@ static void update_tmp_rowset(MetaServiceProxy* meta_service,
     if (!arena) delete req;
 }
 
+static void get_delete_bitmap_update_lock(MetaServiceProxy* meta_service,
+                                          GetDeleteBitmapUpdateLockResponse& res, int64_t db_id,
+                                          int64_t table_id, int64_t index_id,
+                                          const std::vector<std::array<int64_t, 2>>& tablet_idxes,
+                                          int64_t expiration, int64_t lock_id, int64_t initiator,
+                                          bool require_compaction_stats) {
+    brpc::Controller cntl;
+    GetDeleteBitmapUpdateLockRequest req;
+    req.set_cloud_unique_id("test_cloud_unique_id");
+    req.set_table_id(table_id);
+    for (const auto& [partition_id, _] : tablet_idxes) {
+        req.add_partition_ids(partition_id);
+    }
+    req.set_expiration(expiration);
+    req.set_lock_id(lock_id);
+    req.set_initiator(initiator);
+    req.set_require_compaction_stats(require_compaction_stats);
+    for (const auto& [partition_id, tablet_id] : tablet_idxes) {
+        TabletIndexPB* idx = req.add_tablet_indexes();
+        idx->set_db_id(db_id);
+        idx->set_table_id(table_id);
+        idx->set_index_id(index_id);
+        idx->set_partition_id(partition_id);
+        idx->set_tablet_id(tablet_id);
+    }
+    meta_service->get_delete_bitmap_update_lock(
+            reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+}
+
 static void insert_rowset(MetaServiceProxy* meta_service, int64_t db_id, const std::string& label,
                           int64_t table_id, int64_t partition_id, int64_t tablet_id) {
     int64_t txn_id = 0;
@@ -230,6 +259,26 @@ static void insert_rowset(MetaServiceProxy* meta_service, int64_t db_id, const s
     commit_txn(meta_service, db_id, txn_id, label);
 }
 
+static void add_tablet_stats(MetaServiceProxy* meta_service, std::string instance_id,
+                             int64_t table_id, int64_t index_id,
+                             const std::vector<std::array<int64_t, 2>>& tablet_idxes) {
+    std::unique_ptr<Transaction> txn;
+    ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+
+    for (const auto& idx : tablet_idxes) {
+        int64_t partition_id = idx[0];
+        int64_t tablet_id = idx[1];
+        std::string stats_key =
+                stats_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id});
+        TabletStatsPB stats;
+        stats.set_base_compaction_cnt(10);
+        stats.set_cumulative_compaction_cnt(20);
+        stats.set_cumulative_point(30);
+        txn->put(stats_key, stats.SerializeAsString());
+    }
+    ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+}
+
 TEST(MetaServiceTest, GetInstanceIdTest) {
     extern std::string get_instance_id(const std::shared_ptr<ResourceManager>& rc_mgr,
                                        const std::string& cloud_unique_id);
@@ -4488,7 +4537,7 @@ TEST(MetaServiceTest, GetTabletStatsTest) {
     EXPECT_EQ(res.tablet_stats(0).segment_size(), 40000);
 }
 
-TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
+TEST(MetaServiceTest, GetDeleteBitmapUpdateLockNoReadStats) {
     auto meta_service = get_meta_service();
 
     brpc::Controller cntl;
@@ -4530,6 +4579,231 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
     ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
 }
 
+TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsNormal) {
+    auto meta_service = get_meta_service();
+
+    std::string instance_id = "test_get_delete_bitmap_update_lock_normal";
+    [[maybe_unused]] auto* sp = SyncPoint::get_instance();
+    std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01, [](int*) {
+        SyncPoint::get_instance()->disable_processing();
+        SyncPoint::get_instance()->clear_all_call_backs();
+    });
+    sp->set_call_back("get_instance_id", [&](auto&& args) {
+        auto* ret = try_any_cast_ret<std::string>(args);
+        ret->first = instance_id;
+        ret->second = true;
+    });
+    sp->enable_processing();
+
+    int64_t db_id = 1000;
+    int64_t table_id = 2001;
+    int64_t index_id = 3001;
+    // [(partition_id, tablet_id)]
+    std::vector<std::array<int64_t, 2>> tablet_idxes {{70001, 12345}, {80001, 3456}, {90001, 6789}};
+
+    add_tablet_stats(meta_service.get(), instance_id, table_id, index_id, tablet_idxes);
+
+    GetDeleteBitmapUpdateLockResponse res;
+    get_delete_bitmap_update_lock(meta_service.get(), res, db_id, table_id, index_id, tablet_idxes,
+                                  5, 999999, -1, true);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
+    ASSERT_EQ(res.base_compaction_cnts().size(), tablet_idxes.size());
+    for (const auto& base_compaction_cnt : res.base_compaction_cnts()) {
+        ASSERT_EQ(base_compaction_cnt, 10);
+    }
+    ASSERT_EQ(res.cumulative_compaction_cnts().size(), tablet_idxes.size());
+    for (const auto& cumu_compaction_cnt : res.cumulative_compaction_cnts()) {
+        ASSERT_EQ(cumu_compaction_cnt, 20);
+    }
+    ASSERT_EQ(res.cumulative_points().size(), tablet_idxes.size());
+    for (const auto& cumulative_point : res.cumulative_points()) {
+        ASSERT_EQ(cumulative_point, 30);
+    }
+}
+
+TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsLockExpired) {
+    auto meta_service = get_meta_service();
+
+    {
+        // 2.1 abnormal path, lock has been expired and taken by another load/compaction during
+        // the reading of tablet stats
+        std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal1";
+        [[maybe_unused]] auto* sp = SyncPoint::get_instance();
+        std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01, [](int*) {
+            SyncPoint::get_instance()->disable_processing();
+            SyncPoint::get_instance()->clear_all_call_backs();
+        });
+        sp->set_call_back("get_instance_id", [&](auto&& args) {
+            auto* ret = try_any_cast_ret<std::string>(args);
+            ret->first = instance_id;
+            ret->second = true;
+        });
+        sp->set_call_back("check_delete_bitmap_lock.set_lock_info", [&](auto&& args) {
+            auto* lock_info = try_any_cast<DeleteBitmapUpdateLockPB*>(args[0]);
+            // simulate that lock_id has been modified by another load
+            lock_info->set_lock_id(345);
+            LOG(INFO) << "change lock_info.lock_id to 345, lock_info=" << lock_info->DebugString();
+        });
+
+        sp->enable_processing();
+
+        int64_t db_id = 1000;
+        int64_t table_id = 2001;
+        int64_t index_id = 3001;
+        // [(partition_id, tablet_id)]
+        std::vector<std::array<int64_t, 2>> tablet_idxes {
+                {70001, 12345}, {80001, 3456}, {90001, 6789}};
+
+        add_tablet_stats(meta_service.get(), instance_id, table_id, index_id, tablet_idxes);
+
+        GetDeleteBitmapUpdateLockResponse res;
+        get_delete_bitmap_update_lock(meta_service.get(), res, db_id, table_id, index_id,
+                                      tablet_idxes, 5, 999999, -1, true);
+        ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+        ASSERT_EQ(res.base_compaction_cnts().size(), 0);
+        ASSERT_EQ(res.cumulative_compaction_cnts().size(), 0);
+        ASSERT_EQ(res.cumulative_points().size(), 0);
+    }
+
+    {
+        // 2.2 abnormal path, lock has been taken by another load/compaction and been released during
+        // the reading of tablet stats
+        std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal2";
+        [[maybe_unused]] auto* sp = SyncPoint::get_instance();
+        std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01, [](int*) {
+            SyncPoint::get_instance()->disable_processing();
+            SyncPoint::get_instance()->clear_all_call_backs();
+        });
+        sp->set_call_back("get_instance_id", [&](auto&& args) {
+            auto* ret = try_any_cast_ret<std::string>(args);
+            ret->first = instance_id;
+            ret->second = true;
+        });
+        sp->set_call_back("check_delete_bitmap_lock.inject_get_lock_key_err", [&](auto&& args) {
+            auto* err = try_any_cast<TxnErrorCode*>(args[0]);
+            // the lock has been taken by another load and been released,
+            // so the delete bitmap update lock KV will be removed
+            *err = TxnErrorCode::TXN_KEY_NOT_FOUND;
+        });
+
+        sp->enable_processing();
+
+        int64_t db_id = 1000;
+        int64_t table_id = 2001;
+        int64_t index_id = 3001;
+        // [(partition_id, tablet_id)]
+        std::vector<std::array<int64_t, 2>> tablet_idxes {
+                {70001, 12345}, {80001, 3456}, {90001, 6789}};
+
+        add_tablet_stats(meta_service.get(), instance_id, table_id, index_id, tablet_idxes);
+
+        GetDeleteBitmapUpdateLockResponse res;
+        get_delete_bitmap_update_lock(meta_service.get(), res, db_id, table_id, index_id,
+                                      tablet_idxes, 5, 999999, -1, true);
+        ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+    }
+}
+
+TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsError) {
+    auto meta_service = get_meta_service();
+
+    {
+        // 2.3 abnormal path, meeting error when reading tablets' stats
+        std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal3";
+        [[maybe_unused]] auto* sp = SyncPoint::get_instance();
+        std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01, [](int*) {
+            SyncPoint::get_instance()->disable_processing();
+            SyncPoint::get_instance()->clear_all_call_backs();
+        });
+        sp->set_call_back("get_instance_id", [&](auto&& args) {
+            auto* ret = try_any_cast_ret<std::string>(args);
+            ret->first = instance_id;
+            ret->second = true;
+        });
+
+        TxnErrorCode injected_error_code {TxnErrorCode::TXN_KEY_NOT_FOUND};
+        sp->set_call_back("get_delete_bitmap_update_lock.get_compaction_cnts_inject_error",
+                          [&](auto&& args) {
+                              auto* err = try_any_cast<TxnErrorCode*>(args[0]);
+                              *err = injected_error_code;
+                          });
+
+        sp->enable_processing();
+
+        int64_t db_id = 1000;
+        int64_t table_id = 2001;
+        int64_t index_id = 3001;
+        // [(partition_id, tablet_id)]
+        std::vector<std::array<int64_t, 2>> tablet_idxes {
+                {70001, 12345}, {80001, 3456}, {90001, 6789}};
+
+        add_tablet_stats(meta_service.get(), instance_id, table_id, index_id, tablet_idxes);
+
+        GetDeleteBitmapUpdateLockResponse res;
+        get_delete_bitmap_update_lock(meta_service.get(), res, db_id, table_id, index_id,
+                                      tablet_idxes, 5, 999999, -1, true);
+        ASSERT_EQ(res.status().code(), MetaServiceCode::KV_TXN_GET_ERR);
+    }
+
+    {
+        // 2.4 abnormal path, meeting TXN_TOO_OLD error when reading tablets' stats,
+        // this should not fail if lock is not expired
+        std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal4";
+        [[maybe_unused]] auto* sp = SyncPoint::get_instance();
+        std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01, [](int*) {
+            SyncPoint::get_instance()->disable_processing();
+            SyncPoint::get_instance()->clear_all_call_backs();
+        });
+        sp->set_call_back("get_instance_id", [&](auto&& args) {
+            auto* ret = try_any_cast_ret<std::string>(args);
+            ret->first = instance_id;
+            ret->second = true;
+        });
+
+        int counter = 0;
+        sp->set_call_back("get_delete_bitmap_update_lock.get_compaction_cnts_inject_error",
+                          [&](auto&& args) {
+                              if (counter++ % 2 == 0) {
+                                  auto* err = try_any_cast<TxnErrorCode*>(args[0]);
+                                  *err = TxnErrorCode::TXN_TOO_OLD;
+                              }
+                          });
+
+        sp->enable_processing();
+
+        int64_t db_id = 1000;
+        int64_t table_id = 2001;
+        int64_t index_id = 3001;
+        // [(partition_id, tablet_id)]
+        std::vector<std::array<int64_t, 2>> tablet_idxes;
+        for (int i = 0; i < 20; i++) {
+            int64_t partition_id = 70000 + i;
+            int64_t tablet_id = 80000 + i;
+            tablet_idxes.push_back({partition_id, tablet_id});
+        }
+
+        add_tablet_stats(meta_service.get(), instance_id, table_id, index_id, tablet_idxes);
+
+        GetDeleteBitmapUpdateLockResponse res;
+        get_delete_bitmap_update_lock(meta_service.get(), res, db_id, table_id, index_id,
+                                      tablet_idxes, 5, 999999, -1, true);
+        ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+        ASSERT_EQ(res.base_compaction_cnts().size(), tablet_idxes.size());
+        for (const auto& base_compaction_cnt : res.base_compaction_cnts()) {
+            ASSERT_EQ(base_compaction_cnt, 10);
+        }
+        ASSERT_EQ(res.cumulative_compaction_cnts().size(), tablet_idxes.size());
+        for (const auto& cumu_compaction_cnt : res.cumulative_compaction_cnts()) {
+            ASSERT_EQ(cumu_compaction_cnt, 20);
+        }
+        ASSERT_EQ(res.cumulative_points().size(), tablet_idxes.size());
+        for (const auto& cumulative_point : res.cumulative_points()) {
+            ASSERT_EQ(cumulative_point, 30);
+        }
+    }
+}
+
 static std::string generate_random_string(int length) {
     std::string char_set = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
     std::random_device rd;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index 97d75403ffcb6e..11a3f05ead70c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -922,14 +922,20 @@ private void getDeleteBitmapUpdateLock(Map<Long, Set<Long>> tableToParttions, lo
             List<Long> respBaseCompactionCnts = response.getBaseCompactionCntsList();
             List<Long> respCumulativeCompactionCnts = response.getCumulativeCompactionCntsList();
             List<Long> respCumulativePoints = response.getCumulativePointsList();
-            if (!respBaseCompactionCnts.isEmpty() && !respCumulativeCompactionCnts.isEmpty()
-                    && !respCumulativePoints.isEmpty()) {
-                for (int i = 0; i < tabletList.size(); i++) {
-                    long tabletId = tabletList.get(i);
-                    baseCompactionCnts.put(tabletId, respBaseCompactionCnts.get(i));
-                    cumulativeCompactionCnts.put(tabletId, respCumulativeCompactionCnts.get(i));
-                    cumulativePoints.put(tabletId, respCumulativePoints.get(i));
-                }
+            int size1 = respBaseCompactionCnts.size();
+            int size2 = respCumulativeCompactionCnts.size();
+            int size3 = respCumulativePoints.size();
+            if (size1 != tabletList.size() || size2 != tabletList.size() || size3 != tabletList.size()) {
+                throw new UserException("The size of returned compaction cnts can't match the size of tabletList, "
+                        + "tabletList.size()=" + tabletList.size() + ", respBaseCompactionCnts.size()=" + size1
+                                + ", respCumulativeCompactionCnts.size()=" + size2 + ", respCumulativePoints.size()="
+                                        + size3);
+            }
+            for (int i = 0; i < tabletList.size(); i++) {
+                long tabletId = tabletList.get(i);
+                baseCompactionCnts.put(tabletId, respBaseCompactionCnts.get(i));
+                cumulativeCompactionCnts.put(tabletId, respCumulativeCompactionCnts.get(i));
+                cumulativePoints.put(tabletId, respCumulativePoints.get(i));
             }
             totalRetryTime += retryTime;
         }

From 47112d4b368c24cec5135b3d2a3347180038628b Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Tue, 17 Dec 2024 20:55:04 +0800
Subject: [PATCH 373/399] [revert](memory) Revert disable Jemalloc Hook
 (#45534)

### What problem does this PR solve?

It is not easy to remove `with-jemalloc-prefix`, which may affect the
compatibility between third-party and old version codes.
Also, will building failed on Mac, it said can't find mallctl symbol.
because jemalloc's default prefix on macOS is "je_", not "".

Maybe can use alias instead of override.

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 be/src/http/default_path_handlers.cpp   |  2 +-
 be/src/runtime/CMakeLists.txt           |  4 +++-
 be/src/runtime/memory/heap_profiler.cpp |  8 ++++----
 be/src/util/mem_info.cpp                |  2 +-
 be/src/util/mem_info.h                  | 10 +++++-----
 be/src/vec/common/allocator.h           |  2 +-
 build.sh                                | 20 +++++++++++++++++++-
 cloud/src/common/CMakeLists.txt         |  6 ++++++
 thirdparty/build-thirdparty.sh          |  7 ++++---
 9 files changed, 44 insertions(+), 17 deletions(-)

diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp
index b26ba67b54625c..04e1121cab63ba 100644
--- a/be/src/http/default_path_handlers.cpp
+++ b/be/src/http/default_path_handlers.cpp
@@ -146,7 +146,7 @@ void memory_info_handler(std::stringstream* output) {
         auto* _opaque = static_cast<std::string*>(opaque);
         _opaque->append(buf);
     };
-    malloc_stats_print(write_cb, &tmp, "a");
+    jemalloc_stats_print(write_cb, &tmp, "a");
     boost::replace_all(tmp, "\n", "<br>");
     (*output) << tmp;
 #else
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index ab380f9711f1ac..a0b3b799a764cb 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -25,7 +25,9 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/runtime")
 
 file(GLOB_RECURSE RUNTIME_FILES CONFIGURE_DEPENDS *.cpp *.cc)
 
-list(REMOVE_ITEM RUNTIME_FILES ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp)
+if (NOT USE_JEMALLOC OR NOT USE_MEM_TRACKER)
+    list(REMOVE_ITEM RUNTIME_FILES ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp)
+endif()
 
 add_library(Runtime STATIC
     ${RUNTIME_FILES}
diff --git a/be/src/runtime/memory/heap_profiler.cpp b/be/src/runtime/memory/heap_profiler.cpp
index 0b0448ce0eaaf6..01ed82f76ef6d1 100644
--- a/be/src/runtime/memory/heap_profiler.cpp
+++ b/be/src/runtime/memory/heap_profiler.cpp
@@ -30,8 +30,8 @@ void HeapProfiler::set_prof_active(bool prof) {
 #ifdef USE_JEMALLOC
     std::lock_guard guard(_mutex);
     try {
-        int err = mallctl("prof.active", nullptr, nullptr, &prof, 1);
-        err |= mallctl("prof.thread_active_init", nullptr, nullptr, &prof, 1);
+        int err = jemallctl("prof.active", nullptr, nullptr, &prof, 1);
+        err |= jemallctl("prof.thread_active_init", nullptr, nullptr, &prof, 1);
         if (err) {
             LOG(WARNING) << "jemalloc heap profiling start failed, " << err;
         } else {
@@ -48,7 +48,7 @@ bool HeapProfiler::get_prof_dump(const std::string& profile_file_name) {
     std::lock_guard guard(_mutex);
     const char* file_name_ptr = profile_file_name.c_str();
     try {
-        int err = mallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*));
+        int err = jemallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*));
         if (err) {
             LOG(WARNING) << "dump heap profile failed, " << err;
             return false;
@@ -93,7 +93,7 @@ bool HeapProfiler::check_heap_profiler() {
 #ifdef USE_JEMALLOC
     size_t value = 0;
     size_t sz = sizeof(value);
-    mallctl("prof.active", &value, &sz, nullptr, 0);
+    jemallctl("prof.active", &value, &sz, nullptr, 0);
     return value;
 #else
     return false;
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index 97e529ac6c72e1..fe9cf84b2aed54 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -101,7 +101,7 @@ void MemInfo::refresh_allocator_mem() {
     // the current epoch number, which might be useful to log as a sanity check.
     uint64_t epoch = 0;
     size_t sz = sizeof(epoch);
-    mallctl("epoch", &epoch, &sz, &epoch, sz);
+    jemallctl("epoch", &epoch, &sz, &epoch, sz);
 
     // Number of extents of the given type in this arena in the bucket corresponding to page size index.
     // Large size class starts at 16384, the extents have three sizes before 16384: 4096, 8192, and 12288, so + 3
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 0e14b64bd8f965..39ae9eb0b79cfb 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -103,7 +103,7 @@ class MemInfo {
 #ifdef USE_JEMALLOC
         size_t value = 0;
         size_t sz = sizeof(value);
-        if (mallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
+        if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
             return value;
         }
 #endif
@@ -114,7 +114,7 @@ class MemInfo {
 #ifdef USE_JEMALLOC
         unsigned value = 0;
         size_t sz = sizeof(value);
-        if (mallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
+        if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
             return value;
         }
 #endif
@@ -146,8 +146,8 @@ class MemInfo {
         if (config::enable_je_purge_dirty_pages) {
             try {
                 // Purge all unused dirty pages for arena <i>, or for all arenas if <i> equals MALLCTL_ARENAS_ALL.
-                int err = mallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(),
-                                  nullptr, nullptr, nullptr, 0);
+                int err = jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(),
+                                    nullptr, nullptr, nullptr, 0);
                 if (err) {
                     LOG(WARNING) << "Jemalloc purge all unused dirty pages failed";
                 }
@@ -166,7 +166,7 @@ class MemInfo {
 #ifdef USE_JEMALLOC
         constexpr size_t TCACHE_LIMIT = (1ULL << 30); // 1G
         if (allocator_cache_mem() - je_dirty_pages_mem() > TCACHE_LIMIT) {
-            int err = mallctl("thread.tcache.flush", nullptr, nullptr, nullptr, 0);
+            int err = jemallctl("thread.tcache.flush", nullptr, nullptr, nullptr, 0);
             if (err) {
                 LOG(WARNING) << "Jemalloc thread.tcache.flush failed";
             }
diff --git a/be/src/vec/common/allocator.h b/be/src/vec/common/allocator.h
index a0ab9c8c0a8207..b05128bc6933cc 100644
--- a/be/src/vec/common/allocator.h
+++ b/be/src/vec/common/allocator.h
@@ -109,7 +109,7 @@ class DefaultMemoryAllocator {
 
     static void release_unused() {
 #if defined(USE_JEMALLOC)
-        mallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), NULL, NULL, NULL, 0);
+        jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), NULL, NULL, NULL, 0);
 #endif // defined(USE_JEMALLOC)
     }
 };
diff --git a/build.sh b/build.sh
index 89a22842b7d100..5a09b7a0a165e3 100755
--- a/build.sh
+++ b/build.sh
@@ -375,7 +375,25 @@ BUILD_TYPE_LOWWER=$(echo "${BUILD_TYPE}" | tr '[:upper:]' '[:lower:]')
 if [[ "${BUILD_TYPE_LOWWER}" == "asan" ]]; then
     USE_JEMALLOC='OFF'
 elif [[ -z "${USE_JEMALLOC}" ]]; then
-    USE_JEMALLOC='ON'
+    if [[ "$(uname -s)" != 'Darwin' ]]; then
+        USE_JEMALLOC='ON'
+    else
+        USE_JEMALLOC='OFF'
+    fi
+fi
+if [[ -f "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" ]]; then
+    # compatible with old thirdparty
+    rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
+    rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a"
+    rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
+    rm -rf "${TP_INCLUDE_DIR}/rocksdb"
+    rm -rf "${TP_LIB_DIR}/librocksdb.a"
+
+    mv "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
+    mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a"
+    mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
+    mv "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" "${TP_LIB_DIR}/librocksdb.a"
+    mv -f "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" "${TP_INCLUDE_DIR}/rocksdb"
 fi
 if [[ -z "${USE_BTHREAD_SCANNER}" ]]; then
     USE_BTHREAD_SCANNER='OFF'
diff --git a/cloud/src/common/CMakeLists.txt b/cloud/src/common/CMakeLists.txt
index 008e8906044d51..429748909bebd9 100644
--- a/cloud/src/common/CMakeLists.txt
+++ b/cloud/src/common/CMakeLists.txt
@@ -15,6 +15,12 @@ set(COMMON_FILES
     network_util.cpp
 )
 
+if (USE_JEMALLOC)
+    set(COMMON_FILES ${COMMON_FILES}
+        jemalloc_hook.cpp
+        )
+endif()
+
 add_library(Common STATIC
     ${COMMON_FILES}
     )
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index a209977769c278..333787cb64d808 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -1529,10 +1529,11 @@ build_jemalloc_doris() {
         WITH_LG_PAGE=''
     fi
 
-    # CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris" "${WITH_LG_PAGE}" \
-    #     --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared
+    # It is not easy to remove `with-jemalloc-prefix`, which may affect the compatibility between third-party and old version codes.
+    # Also, will building failed on Mac, it said can't find mallctl symbol. because jemalloc's default prefix on macOS is "je_", not "".
+    # Maybe can use alias instead of overwrite.
     CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris" "${WITH_LG_PAGE}" \
-        --enable-prof --disable-libdl --disable-shared
+        --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared
 
     make -j "${PARALLEL}"
     make install

From b252c0fa5ddd18defe47f577a93ecfd36fa5b0dd Mon Sep 17 00:00:00 2001
From: Gavin Chou <gavin@selectdb.com>
Date: Tue, 17 Dec 2024 21:01:05 +0800
Subject: [PATCH 374/399] [fix](storage vault) Fix missing use_path_style when
 create storage vault (#45155)

---
 .../meta-service/meta_service_resource.cpp    | 27 ++++++-----
 .../apache/doris/catalog/StorageVault.java    |  4 ++
 .../suites/vault_p0/create/load.groovy        | 48 +++++++++++++++++++
 3 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp
index cc459c090bfd28..f26c2d26200dd3 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -753,6 +753,7 @@ struct ObjectStorageDesc {
     std::string& endpoint;
     std::string& external_endpoint;
     std::string& region;
+    bool& use_path_style;
 };
 
 static int extract_object_storage_info(const AlterObjStoreInfoRequest* request,
@@ -765,7 +766,7 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request,
         msg = "s3 obj info err " + proto_to_json(*request);
         return -1;
     }
-    auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region] = obj_desc;
+    auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style] = obj_desc;
     const auto& obj = request->has_obj() ? request->obj() : request->vault().obj_info();
     // Prepare data
     if (!obj.has_ak() || !obj.has_sk()) {
@@ -791,6 +792,7 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request,
     endpoint = obj.has_endpoint() ? obj.endpoint() : "";
     external_endpoint = obj.has_external_endpoint() ? obj.external_endpoint() : "";
     region = obj.has_region() ? obj.region() : "";
+    use_path_style = obj.use_path_style();
     //  obj size > 1k, refuse
     if (obj.ByteSizeLong() > 1024) {
         code = MetaServiceCode::INVALID_ARGUMENT;
@@ -800,13 +802,13 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request,
     return 0;
 }
 
-static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_info,
+static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_desc,
                                                 const ObjectStoreInfoPB& obj,
                                                 InstanceInfoPB& instance,
                                                 EncryptionInfoPB& encryption_info,
                                                 AkSkPair& cipher_ak_sk_pair) {
     ObjectStoreInfoPB last_item;
-    auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region] = obj_info;
+    auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style] = obj_desc;
     auto now_time = std::chrono::system_clock::now();
     uint64_t time =
             std::chrono::duration_cast<std::chrono::seconds>(now_time.time_since_epoch()).count();
@@ -828,6 +830,7 @@ static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_info,
     last_item.set_region(region);
     last_item.set_provider(obj.provider());
     last_item.set_sse_enabled(instance.sse_enabled());
+    last_item.set_use_path_style(use_path_style);
     return last_item;
 }
 
@@ -836,14 +839,15 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
                                           AlterObjStoreInfoResponse* response,
                                           ::google::protobuf::Closure* done) {
     std::string ak, sk, bucket, prefix, endpoint, external_endpoint, region;
+    bool use_path_style;
     EncryptionInfoPB encryption_info;
     AkSkPair cipher_ak_sk_pair;
     RPC_PREPROCESS(alter_storage_vault);
     switch (request->op()) {
     case AlterObjStoreInfoRequest::ADD_S3_VAULT:
     case AlterObjStoreInfoRequest::DROP_S3_VAULT: {
-        auto tmp_desc =
-                ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region};
+        auto tmp_desc = ObjectStorageDesc {
+                ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style};
         if (0 != extract_object_storage_info(request, code, msg, tmp_desc, encryption_info,
                                              cipher_ak_sk_pair)) {
             return;
@@ -982,8 +986,8 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
             }
         }
         // calc id
-        auto tmp_tuple =
-                ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region};
+        auto tmp_tuple = ObjectStorageDesc {
+                ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style};
         ObjectStoreInfoPB last_item = object_info_pb_factory(tmp_tuple, obj, instance,
                                                              encryption_info, cipher_ak_sk_pair);
         if (instance.storage_vault_names().end() !=
@@ -1126,6 +1130,7 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
                                            AlterObjStoreInfoResponse* response,
                                            ::google::protobuf::Closure* done) {
     std::string ak, sk, bucket, prefix, endpoint, external_endpoint, region;
+    bool use_path_style;
     EncryptionInfoPB encryption_info;
     AkSkPair cipher_ak_sk_pair;
     RPC_PREPROCESS(alter_obj_store_info);
@@ -1133,8 +1138,8 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
     case AlterObjStoreInfoRequest::ADD_OBJ_INFO:
     case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK:
     case AlterObjStoreInfoRequest::UPDATE_AK_SK: {
-        auto tmp_desc =
-                ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region};
+        auto tmp_desc = ObjectStorageDesc {
+                ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style};
         if (0 != extract_object_storage_info(request, code, msg, tmp_desc, encryption_info,
                                              cipher_ak_sk_pair)) {
             return;
@@ -1273,8 +1278,8 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
             }
         }
         // calc id
-        auto tmp_tuple =
-                ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region};
+        auto tmp_tuple = ObjectStorageDesc {
+                ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style};
         ObjectStoreInfoPB last_item = object_info_pb_factory(tmp_tuple, obj, instance,
                                                              encryption_info, cipher_ak_sk_pair);
         instance.add_obj_info()->CopyFrom(last_item);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
index df9310526e4757..9d45ce7bdd8f51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
@@ -23,6 +23,7 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.PropertyConverter;
 import org.apache.doris.qe.ShowResultSetMetaData;
 
 import com.google.common.base.Strings;
@@ -144,6 +145,9 @@ public void setId(String id) {
                 vault.modifyProperties(stmt.getProperties());
                 break;
             case S3:
+                if (!stmt.getProperties().containsKey(PropertyConverter.USE_PATH_STYLE)) {
+                    stmt.getProperties().put(PropertyConverter.USE_PATH_STYLE, "true");
+                }
                 CreateResourceStmt resourceStmt =
                         new CreateResourceStmt(false, ifNotExists, name, stmt.getProperties());
                 resourceStmt.analyzeResourceType();
diff --git a/regression-test/suites/vault_p0/create/load.groovy b/regression-test/suites/vault_p0/create/load.groovy
index 58974856db5e2a..321a58a2170ff3 100644
--- a/regression-test/suites/vault_p0/create/load.groovy
+++ b/regression-test/suites/vault_p0/create/load.groovy
@@ -112,6 +112,54 @@ suite("create_storage_vault", "nonConcurrent") {
         );
     """
 
+    // test path style
+    sql """
+        CREATE STORAGE VAULT IF NOT EXISTS create_s3_vault1
+        PROPERTIES (
+            "type"="S3",
+            "s3.endpoint"="${getS3Endpoint()}",
+            "s3.region" = "${getS3Region()}",
+            "s3.access_key" = "${getS3AK()}",
+            "s3.secret_key" = "${getS3SK()}",
+            "s3.root.path" = "test_create_s3_vault1",
+            "s3.bucket" = "${getS3BucketName()}",
+            "s3.external_endpoint" = "",
+            "use_path_style" = "true",
+            "provider" = "${getS3Provider()}"
+        );
+    """
+
+    // test path style
+    sql """
+        CREATE STORAGE VAULT IF NOT EXISTS create_s3_vault2
+        PROPERTIES (
+            "type"="S3",
+            "s3.endpoint"="${getS3Endpoint()}",
+            "s3.region" = "${getS3Region()}",
+            "s3.access_key" = "${getS3AK()}",
+            "s3.secret_key" = "${getS3SK()}",
+            "s3.root.path" = "test_create_s3_vault2",
+            "s3.bucket" = "${getS3BucketName()}",
+            "s3.external_endpoint" = "",
+            "use_path_style" = "false",
+            "provider" = "${getS3Provider()}"
+        );
+    """
+
+    var result = """show storage vault"""
+    assertTrue(result.size() >= 3);
+    for (int i = 0; i < result.size(); ++i) {
+        if (result[i][0].equals("create_s3_vault")) {
+            assertTrue(result[i][2].contains("use_path_style: true"));
+        }
+        if (result[i][0].equals("create_s3_vault1")) {
+            assertTrue(result[i][2].contains("use_path_style: true"));
+        }
+        if (result[i][0].equals("create_s3_vault2")) {
+            assertTrue(result[i][2].contains("use_path_style: false"));
+        }
+    }
+
     expectExceptionLike({
         sql """
             CREATE STORAGE VAULT create_s3_vault

From 038dd93f4d5e667813240fc839a2cc4ea86157ce Mon Sep 17 00:00:00 2001
From: yagagagaga <zhangminkefromflydish@gmail.com>
Date: Tue, 17 Dec 2024 21:49:06 +0800
Subject: [PATCH 375/399] [chore](version) Show binary version in metrics: fe
 be ms (#45490)

before
```
root@af0e5d234e63:~# curl 127.0.0.1:8030/metrics | grep doris_fe_version

root@af0e5d234e63:~# curl 127.0.0.1:8060/brpc_metrics | grep doris_be_version

root@af0e5d234e63:~#  curl 127.0.0.1:5000/brpc_metrics | grep doris_cloud_version
```

after
```
[root@dbb7fecb3b61 output]# curl 127.0.0.1:8030/metrics | grep doris_fe_version
# TYPE doris_fe_version gauge
doris_fe_version 30004

[root@dbb7fecb3b61 output]# curl 127.0.0.1:8060/brpc_metrics | grep doris_be_version
# TYPE doris_be_version gauge
doris_be_version 30004

[root@dbb7fecb3b61 output]# curl 127.0.0.1:5000/brpc_metrics | grep doris_cloud_version
# TYPE doris_cloud_version gauge
doris_cloud_version 30004
```

Co-authored-by: Gavin Chou <gavineaglechou@gmail.com>
---
 be/src/common/version_internal.cpp            |  5 +++-
 be/src/common/version_internal.h              |  3 ++-
 be/src/util/debug_util.cpp                    | 11 ++++++++
 cloud/src/main.cpp                            |  3 +++
 .../org/apache/doris/metric/MetricRepo.java   | 20 +++++++++++++++
 gensrc/script/gen_build_version.sh            | 25 ++++++++++++++-----
 6 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/be/src/common/version_internal.cpp b/be/src/common/version_internal.cpp
index 1190242b6aa687..55402fab209400 100644
--- a/be/src/common/version_internal.cpp
+++ b/be/src/common/version_internal.cpp
@@ -34,6 +34,9 @@ int doris_build_version_minor() {
 int doris_build_version_patch() {
     return DORIS_BUILD_VERSION_PATCH;
 }
+int doris_build_version_hotfix() {
+    return DORIS_BUILD_VERSION_HOTFIX;
+}
 const char* doris_build_version_rc_version() {
     return DORIS_BUILD_VERSION_RC_VERSION;
 }
@@ -56,4 +59,4 @@ const char* doris_build_info() {
 
 } // namespace version
 
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/common/version_internal.h b/be/src/common/version_internal.h
index 8852d26dba9531..f4deaa15aff545 100644
--- a/be/src/common/version_internal.h
+++ b/be/src/common/version_internal.h
@@ -24,6 +24,7 @@ extern const char* doris_build_version_prefix();
 extern int doris_build_version_major();
 extern int doris_build_version_minor();
 extern int doris_build_version_patch();
+extern int doris_build_version_hotfix();
 extern const char* doris_build_version_rc_version();
 
 extern const char* doris_build_version();
@@ -34,4 +35,4 @@ extern const char* doris_build_info();
 
 } // namespace version
 
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/util/debug_util.cpp b/be/src/util/debug_util.cpp
index 1cf03d2c22d0e1..0856b10c051709 100644
--- a/be/src/util/debug_util.cpp
+++ b/be/src/util/debug_util.cpp
@@ -17,6 +17,7 @@
 
 #include "util/debug_util.h"
 
+#include <bvar/bvar.h>
 #include <gen_cpp/HeartbeatService_types.h>
 #include <gen_cpp/PlanNodes_types.h>
 #include <stdint.h>
@@ -104,6 +105,16 @@ std::string hexdump(const char* buf, int len) {
     return ss.str();
 }
 
+bvar::Status<uint64_t> be_version_metrics("doris_be_version", [] {
+    std::stringstream ss;
+    ss << version::doris_build_version_major() << 0 << version::doris_build_version_minor() << 0
+       << version::doris_build_version_patch();
+    if (version::doris_build_version_hotfix() > 0) {
+        ss << 0 << version::doris_build_version_hotfix();
+    }
+    return std::strtoul(ss.str().c_str(), nullptr, 10);
+}());
+
 std::string PrintThriftNetworkAddress(const TNetworkAddress& add) {
     std::stringstream ss;
     add.printTo(ss);
diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp
index 74e6a8daaf161c..26033cdaad2a07 100644
--- a/cloud/src/main.cpp
+++ b/cloud/src/main.cpp
@@ -150,6 +150,9 @@ bvar::Status<uint64_t> doris_cloud_version_metrics("doris_cloud_version", [] {
     std::stringstream ss;
     ss << DORIS_CLOUD_BUILD_VERSION_MAJOR << 0 << DORIS_CLOUD_BUILD_VERSION_MINOR << 0
        << DORIS_CLOUD_BUILD_VERSION_PATCH;
+    if (DORIS_CLOUD_BUILD_VERSION_HOTFIX > 0) {
+        ss << 0 << DORIS_CLOUD_BUILD_VERSION_HOTFIX;
+    }
     return std::strtoul(ss.str().c_str(), nullptr, 10);
 }());
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index ca731eacf4b0bb..cda73ecbe6221d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -24,6 +24,7 @@
 import org.apache.doris.common.Config;
 import org.apache.doris.common.Pair;
 import org.apache.doris.common.ThreadPoolManager;
+import org.apache.doris.common.Version;
 import org.apache.doris.common.util.NetUtils;
 import org.apache.doris.load.EtlJobType;
 import org.apache.doris.load.loadv2.JobState;
@@ -158,6 +159,25 @@ public static synchronized void init() {
             return;
         }
 
+        // version
+        GaugeMetric<Long> feVersion = new GaugeMetric<Long>("version", MetricUnit.NOUNIT, "") {
+            @Override
+            public Long getValue() {
+                try {
+                    return Long.parseLong("" + Version.DORIS_BUILD_VERSION_MAJOR + "0"
+                                            + Version.DORIS_BUILD_VERSION_MINOR + "0"
+                                            + Version.DORIS_BUILD_VERSION_PATCH
+                                            + (Version.DORIS_BUILD_VERSION_HOTFIX > 0
+                                                ? ("0" + Version.DORIS_BUILD_VERSION_HOTFIX)
+                                                : ""));
+                } catch (Throwable t) {
+                    LOG.warn("failed to init version metrics", t);
+                    return 0L;
+                }
+            }
+        };
+        DORIS_METRIC_REGISTER.addMetrics(feVersion);
+
         // load jobs
         for (EtlJobType jobType : EtlJobType.values()) {
             if (jobType == EtlJobType.UNKNOWN) {
diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh
index ce850850e00128..a85effebd5e3bb 100755
--- a/gensrc/script/gen_build_version.sh
+++ b/gensrc/script/gen_build_version.sh
@@ -31,9 +31,14 @@ build_version_prefix="doris"
 build_version_major=0
 build_version_minor=0
 build_version_patch=0
+build_version_hotfix=0
 build_version_rc_version=""
 
-build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}-${build_version_rc_version}"
+build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}"
+if [[ ${build_version_hotfix} > 0 ]]; then
+    build_version+=".${build_version_hotfix}"
+fi
+build_version+="-${build_version_rc_version}"
 
 # This version is used to check FeMetaVersion is not changed during release
 build_fe_meta_version=0
@@ -125,6 +130,7 @@ public class Version {
   public static final int DORIS_BUILD_VERSION_MAJOR = ${build_version_major};
   public static final int DORIS_BUILD_VERSION_MINOR = ${build_version_minor};
   public static final int DORIS_BUILD_VERSION_PATCH = ${build_version_patch};
+  public static final int DORIS_BUILD_VERSION_HOTFIX = ${build_version_hotfix};
   public static final String DORIS_BUILD_VERSION_RC_VERSION = "${build_version_rc_version}";
 
   public static final String DORIS_BUILD_VERSION = "${build_version}";
@@ -186,6 +192,7 @@ namespace doris {
 #define DORIS_BUILD_VERSION_MAJOR       ${build_version_major};
 #define DORIS_BUILD_VERSION_MINOR       ${build_version_minor};
 #define DORIS_BUILD_VERSION_PATCH       ${build_version_patch};
+#define DORIS_BUILD_VERSION_HOTFIX      ${build_version_hotfix};
 #define DORIS_BUILD_VERSION_RC_VERSION  "${build_version_rc_version}";
 
 #define DORIS_BUILD_VERSION             "${build_version}"
@@ -203,11 +210,12 @@ EOF
 #                      doris cloud version info
 ################################################################################
 
-build_version_prefix="doris_cloud"
-build_version_major=0
-build_version_minor=0
-build_version_patch=0
-build_version_rc_version=""
+# build_version_prefix="doris_cloud"
+# build_version_major=0
+# build_version_minor=0
+# build_version_patch=0
+# build_version_hotfix=0
+# build_version_rc_version=""
 
 if [[ -f /etc/os-release ]]; then
     build_os_version=$(head -n2 </etc/os-release | tr '\n' ' ')
@@ -217,6 +225,10 @@ fi
 
 build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}"
 
+if [[ ${build_version_hotfix} > 0 ]]; then
+    build_version+=".${build_version_hotfix}"
+fi
+
 if [[ "${build_version_rc_version}" != "" ]]; then
     build_version=${build_version}"-${build_version_rc_version}"
 fi
@@ -247,6 +259,7 @@ namespace doris::cloud {
 #define DORIS_CLOUD_BUILD_VERSION_MAJOR       ${build_version_major}
 #define DORIS_CLOUD_BUILD_VERSION_MINOR       ${build_version_minor}
 #define DORIS_CLOUD_BUILD_VERSION_PATCH       ${build_version_patch}
+#define DORIS_CLOUD_BUILD_VERSION_HOTFIX      ${build_version_hotfix}
 #define DORIS_CLOUD_BUILD_VERSION_RC_VERSION  R"(${build_version_rc_version})"
 
 #define DORIS_CLOUD_BUILD_VERSION             R"(${build_version})"

From a6da226cfa951c78ee765bab6fb55b21faadfe90 Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Wed, 18 Dec 2024 07:11:52 +0800
Subject: [PATCH 376/399] [enchement](iceberg)support read iceberg partition
 evolution table. (#45367)

### What problem does this PR solve?
Problem Summary:
Supports reading iceberg tables after `ADD DROP REPLACE` partition.
Before this, reading such tables will cause be core.

https://iceberg.apache.org/docs/1.7.0/spark-ddl/#alter-table-add-partition-field

### Release note
[fix](iceberg) Supports reading Iceberg tables after partition evolution.
---
 .../writer/iceberg/partition_transformers.cpp |   2 +-
 .../iceberg/run07.sql                         |  83 ++++++++
 .../iceberg/source/IcebergScanNode.java       |  82 +++++---
 .../test_iceberg_partition_evolution.out      | 190 ++++++++++++++++++
 .../write/test_iceberg_write_partitions.out   |  24 +--
 .../test_iceberg_partition_evolution.groovy   | 179 +++++++++++++++++
 6 files changed, 517 insertions(+), 43 deletions(-)
 create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run07.sql
 create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_partition_evolution.out
 create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_partition_evolution.groovy

diff --git a/be/src/vec/sink/writer/iceberg/partition_transformers.cpp b/be/src/vec/sink/writer/iceberg/partition_transformers.cpp
index ee8268d30f73b5..06b22a4bbbce9a 100644
--- a/be/src/vec/sink/writer/iceberg/partition_transformers.cpp
+++ b/be/src/vec/sink/writer/iceberg/partition_transformers.cpp
@@ -211,7 +211,7 @@ std::string PartitionColumnTransform::get_partition_value(const TypeDescriptor&
     if (value.has_value()) {
         switch (type.type) {
         case TYPE_BOOLEAN: {
-            return std::to_string(std::any_cast<bool>(value));
+            return std::any_cast<bool>(value) ? "true" : "false";
         }
         case TYPE_TINYINT: {
             return std::to_string(std::any_cast<Int8>(value));
diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run07.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run07.sql
new file mode 100644
index 00000000000000..3c4131ba619f84
--- /dev/null
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run07.sql
@@ -0,0 +1,83 @@
+
+use demo.test_db;
+CREATE TABLE iceberg_add_partition (
+    id INT,
+    name STRING,
+    age INT
+) USING iceberg;
+INSERT INTO iceberg_add_partition VALUES(1, 'Alice', 30),(2, 'Bob', 25);
+ALTER TABLE iceberg_add_partition ADD PARTITION FIELD age;
+ALTER TABLE iceberg_add_partition ADD COLUMNS address STRING;
+INSERT INTO iceberg_add_partition VALUES (4, 'Charlie', 45, '123 Street Name');
+ALTER TABLE iceberg_add_partition ADD PARTITION FIELD bucket(10, id);
+INSERT INTO iceberg_add_partition VALUES (5, 'Eve', 29, '789 Third St');
+ALTER TABLE iceberg_add_partition ADD PARTITION FIELD truncate(5, address);
+INSERT INTO iceberg_add_partition VALUES (6, 'Frank', 33,"xx"),(7, 'Grace', 28,"yyyyyyyyy");
+
+
+
+CREATE TABLE iceberg_drop_partition (
+    id INT,
+    name STRING,
+    amount DOUBLE,
+    created_date DATE
+) 
+USING iceberg
+PARTITIONED BY (year(created_date),bucket(10,created_date));
+INSERT INTO iceberg_drop_partition VALUES
+    (1, 'Alice', 100.0, DATE '2023-12-01'),
+    (2, 'Bob', 200.0, DATE '2023-12-02'),
+    (3, 'Charlie', 300.0, DATE '2024-12-03');
+ALTER TABLE iceberg_drop_partition  DROP PARTITION FIELD year(created_date);
+INSERT INTO iceberg_drop_partition VALUES
+    (4, 'David', 400.0, DATE '2023-12-02'),
+    (5, 'Eve', 500.0, DATE '2024-12-03');
+ALTER TABLE iceberg_drop_partition  DROP PARTITION FIELD bucket(10,created_date);
+INSERT INTO iceberg_drop_partition VALUES
+    (6, 'David', 400.0, DATE '2025-12-12'),
+    (7, 'Eve', 500.0, DATE '2025-12-23');
+
+
+CREATE TABLE iceberg_replace_partition (
+    id INT,
+    name STRING,
+    amount DOUBLE,
+    created_date DATE
+)
+USING iceberg
+PARTITIONED BY (year(created_date),bucket(10,created_date));
+INSERT INTO iceberg_replace_partition VALUES
+    (1, 'Alice', 100.0, DATE '2023-01-01'),
+    (2, 'Bob', 200.0, DATE '2023-12-02'),
+    (3, 'Charlie', 300.0, DATE '2024-12-03');
+ALTER TABLE iceberg_replace_partition  REPLACE PARTITION FIELD year(created_date) WITH month(created_date);
+INSERT INTO iceberg_replace_partition VALUES
+    (4, 'David', 400.0, DATE '2023-12-02'),
+    (5, 'Eve', 500.0, DATE '2024-07-03');
+ALTER TABLE iceberg_replace_partition  REPLACE PARTITION FIELD bucket(10,created_date) WITH bucket(10,id);
+INSERT INTO iceberg_replace_partition VALUES
+    (6, 'David', 400.0, DATE '2025-10-12'),
+    (7, 'Eve', 500.0, DATE '2025-09-23');
+
+
+
+
+CREATE TABLE iceberg_evolution_partition (
+    id INT,
+    name STRING,
+    age INT
+) USING iceberg;
+INSERT INTO iceberg_evolution_partition VALUES(1, 'Alice', 30),(2, 'Bob', 25);
+ALTER TABLE iceberg_evolution_partition ADD PARTITION FIELD age;
+ALTER TABLE iceberg_evolution_partition ADD COLUMNS address STRING;
+INSERT INTO iceberg_evolution_partition VALUES (4, 'Charlie', 45, '123 Street Name');
+ALTER TABLE iceberg_evolution_partition ADD PARTITION FIELD bucket(10, id);
+INSERT INTO iceberg_evolution_partition VALUES (5, 'Eve', 29, '789 Third St');
+ALTER TABLE iceberg_evolution_partition REPLACE PARTITION FIELD bucket(10, id) WITH truncate(5, address);
+INSERT INTO iceberg_evolution_partition VALUES (6, 'Frank', 33,"xx"),(7, 'Grace', 28,"yyyyyyyyy");
+ALTER TABLE iceberg_evolution_partition DROP PARTITION FIELD truncate(5, address);
+INSERT INTO iceberg_evolution_partition VALUES (8, 'Hank', 40, "zz"), (9, 'Ivy', 22, "aaaaaa");
+ALTER TABLE iceberg_evolution_partition DROP COLUMNS address;
+-- INSERT INTO iceberg_evolution_partition VALUES (10, 'Jack', 35), (11, 'Kara', 30);
+-- spark error.
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index c78140b9d3cd99..e43e8893332a67 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -35,6 +35,7 @@
 import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
 import org.apache.doris.datasource.iceberg.IcebergExternalTable;
 import org.apache.doris.datasource.iceberg.IcebergUtils;
+import org.apache.doris.nereids.exceptions.NotSupportedException;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.spi.Split;
@@ -56,7 +57,6 @@
 import org.apache.iceberg.FileContent;
 import org.apache.iceberg.FileScanTask;
 import org.apache.iceberg.MetadataColumns;
-import org.apache.iceberg.PartitionField;
 import org.apache.iceberg.Snapshot;
 import org.apache.iceberg.StructLike;
 import org.apache.iceberg.Table;
@@ -64,11 +64,10 @@
 import org.apache.iceberg.expressions.Expression;
 import org.apache.iceberg.io.CloseableIterable;
 import org.apache.iceberg.types.Conversions;
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-import org.apache.iceberg.util.DateTimeUtil;
 import org.apache.iceberg.util.SnapshotUtil;
 import org.apache.iceberg.util.TableScanUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -78,11 +77,11 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalLong;
-import java.util.stream.Collectors;
 
 public class IcebergScanNode extends FileQueryScanNode {
 
     public static final int MIN_DELETE_FILE_SUPPORT_VERSION = 2;
+    private static final Logger LOG = LogManager.getLogger(IcebergScanNode.class);
 
     private IcebergSource source;
     private Table icebergTable;
@@ -217,32 +216,44 @@ private List<Split> doGetSplits(int numBackends) throws UserException {
         boolean isPartitionedTable = icebergTable.spec().isPartitioned();
 
         long realFileSplitSize = getRealFileSplitSize(0);
-        CloseableIterable<FileScanTask> fileScanTasks = TableScanUtil.splitFiles(scan.planFiles(), realFileSplitSize);
+        CloseableIterable<FileScanTask> fileScanTasks = null;
+        try {
+            fileScanTasks = TableScanUtil.splitFiles(scan.planFiles(), realFileSplitSize);
+        } catch (NullPointerException e) {
+            /*
+        Caused by: java.lang.NullPointerException: Type cannot be null
+            at org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull
+                (Preconditions.java:921) ~[iceberg-bundled-guava-1.4.3.jar:?]
+            at org.apache.iceberg.types.Types$NestedField.<init>(Types.java:447) ~[iceberg-api-1.4.3.jar:?]
+            at org.apache.iceberg.types.Types$NestedField.optional(Types.java:416) ~[iceberg-api-1.4.3.jar:?]
+            at org.apache.iceberg.PartitionSpec.partitionType(PartitionSpec.java:132) ~[iceberg-api-1.4.3.jar:?]
+            at org.apache.iceberg.DeleteFileIndex.lambda$new$0(DeleteFileIndex.java:97) ~[iceberg-core-1.4.3.jar:?]
+            at org.apache.iceberg.relocated.com.google.common.collect.RegularImmutableMap.forEach
+                (RegularImmutableMap.java:297) ~[iceberg-bundled-guava-1.4.3.jar:?]
+            at org.apache.iceberg.DeleteFileIndex.<init>(DeleteFileIndex.java:97) ~[iceberg-core-1.4.3.jar:?]
+            at org.apache.iceberg.DeleteFileIndex.<init>(DeleteFileIndex.java:71) ~[iceberg-core-1.4.3.jar:?]
+            at org.apache.iceberg.DeleteFileIndex$Builder.build(DeleteFileIndex.java:578) ~[iceberg-core-1.4.3.jar:?]
+            at org.apache.iceberg.ManifestGroup.plan(ManifestGroup.java:183) ~[iceberg-core-1.4.3.jar:?]
+            at org.apache.iceberg.ManifestGroup.planFiles(ManifestGroup.java:170) ~[iceberg-core-1.4.3.jar:?]
+            at org.apache.iceberg.DataTableScan.doPlanFiles(DataTableScan.java:89) ~[iceberg-core-1.4.3.jar:?]
+            at org.apache.iceberg.SnapshotScan.planFiles(SnapshotScan.java:139) ~[iceberg-core-1.4.3.jar:?]
+            at org.apache.doris.datasource.iceberg.source.IcebergScanNode.doGetSplits
+                (IcebergScanNode.java:209) ~[doris-fe.jar:1.2-SNAPSHOT]
+        EXAMPLE:
+             CREATE TABLE iceberg_tb(col1 INT,col2 STRING) USING ICEBERG PARTITIONED BY (bucket(10,col2));
+             INSERT INTO iceberg_tb VALUES( ... );
+             ALTER  TABLE iceberg_tb DROP PARTITION FIELD bucket(10,col2);
+             ALTER TABLE iceberg_tb DROP COLUMNS col2 STRING;
+        Link: https://github.com/apache/iceberg/pull/10755
+        */
+            LOG.warn("Iceberg TableScanUtil.splitFiles throw NullPointerException. Cause : ", e);
+            throw new NotSupportedException("Unable to read Iceberg table with dropped old partition column.");
+        }
         try (CloseableIterable<CombinedScanTask> combinedScanTasks =
                 TableScanUtil.planTasks(fileScanTasks, realFileSplitSize, 1, 0)) {
             combinedScanTasks.forEach(taskGrp -> taskGrp.files().forEach(splitTask -> {
-                List<String> partitionValues = new ArrayList<>();
                 if (isPartitionedTable) {
                     StructLike structLike = splitTask.file().partition();
-                    List<PartitionField> fields = splitTask.spec().fields();
-                    Types.StructType structType = icebergTable.schema().asStruct();
-
-                    // set partitionValue for this IcebergSplit
-                    for (int i = 0; i < structLike.size(); i++) {
-                        Object obj = structLike.get(i, Object.class);
-                        String value = String.valueOf(obj);
-                        PartitionField partitionField = fields.get(i);
-                        if (partitionField.transform().isIdentity()) {
-                            Type type = structType.fieldType(partitionField.name());
-                            if (type != null && type.typeId().equals(Type.TypeID.DATE)) {
-                                // iceberg use integer to store date,
-                                // we need transform it to string
-                                value = DateTimeUtil.daysToIsoDate((Integer) obj);
-                            }
-                        }
-                        partitionValues.add(value);
-                    }
-
                     // Counts the number of partitions read
                     partitionPathSet.add(structLike.toString());
                 }
@@ -256,7 +267,7 @@ private List<Split> doGetSplits(int numBackends) throws UserException {
                         new String[0],
                         formatVersion,
                         source.getCatalog().getProperties(),
-                        partitionValues,
+                        new ArrayList<>(),
                         originalPath);
                 split.setTargetSplitSize(realFileSplitSize);
                 if (formatVersion >= MIN_DELETE_FILE_SUPPORT_VERSION) {
@@ -289,7 +300,6 @@ private List<Split> doGetSplits(int numBackends) throws UserException {
                 return pushDownCountSplits;
             }
         }
-
         selectedPartitionNum = partitionPathSet.size();
         return splits;
     }
@@ -351,8 +361,20 @@ public TFileFormatType getFileFormatType() throws UserException {
 
     @Override
     public List<String> getPathPartitionKeys() throws UserException {
-        return icebergTable.spec().fields().stream().map(PartitionField::name).map(String::toLowerCase)
-                .collect(Collectors.toList());
+        // return icebergTable.spec().fields().stream().map(PartitionField::name).map(String::toLowerCase)
+        //         .collect(Collectors.toList());
+        /**First, iceberg partition columns are based on existing fields, which will be stored in the actual data file.
+         * Second, iceberg partition columns support Partition transforms. In this case, the path partition key is not
+         * equal to the column name of the partition column, so remove this code and get all the columns you want to
+         * read from the file.
+         * Related code:
+         *  be/src/vec/exec/scan/vfile_scanner.cpp:
+         *      VFileScanner::_init_expr_ctxes()
+         *          if (slot_info.is_file_slot) {
+         *              xxxx
+         *          }
+         */
+        return new ArrayList<>();
     }
 
     @Override
diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_partition_evolution.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_partition_evolution.out
new file mode 100644
index 00000000000000..c9eb9a8db62d39
--- /dev/null
+++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_partition_evolution.out
@@ -0,0 +1,190 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !add_partition_1 --
+id	int	Yes	true	\N	
+name	text	Yes	true	\N	
+age	int	Yes	true	\N	
+address	text	Yes	true	\N	
+
+-- !add_partition_2 --
+1	Alice	30	\N
+2	Bob	25	\N
+4	Charlie	45	123 Street Name
+5	Eve	29	789 Third St
+6	Frank	33	xx
+7	Grace	28	yyyyyyyyy
+
+-- !add_partition_3 --
+1
+2
+4
+5
+6
+7
+
+-- !add_partition_4 --
+1	Alice
+2	Bob
+4	Charlie
+5	Eve
+6	Frank
+7	Grace
+
+-- !add_partition_5 --
+30
+25
+45
+29
+33
+28
+
+-- !add_partition_6 --
+123 Street Name
+789 Third St
+xx
+yyyyyyyyy
+
+-- !add_partition_7 --
+4	Charlie	123 Street Name
+5	Eve	789 Third St
+7	Grace	yyyyyyyyy
+
+-- !add_partition_8 --
+6	Frank	xx
+
+-- !add_partition_9 --
+4	Charlie	45	123 Street Name
+6	Frank	33	xx
+
+-- !add_partition_10 --
+1	Alice	30	\N
+2	Bob	25	\N
+4	Charlie	45	123 Street Name
+
+-- !drop_partition_1 --
+id	int	Yes	true	\N	
+name	text	Yes	true	\N	
+amount	double	Yes	true	\N	
+created_date	date	Yes	true	\N	
+
+-- !drop_partition_2 --
+1	Alice	100.0	2023-12-01
+2	Bob	200.0	2023-12-02
+3	Charlie	300.0	2024-12-03
+4	David	400.0	2023-12-02
+5	Eve	500.0	2024-12-03
+6	David	400.0	2025-12-12
+7	Eve	500.0	2025-12-23
+
+-- !drop_partition_3 --
+1	Alice	2023-12-01
+2	Bob	2023-12-02
+3	Charlie	2024-12-03
+4	David	2023-12-02
+5	Eve	2024-12-03
+6	David	2025-12-12
+7	Eve	2025-12-23
+
+-- !drop_partition_4 --
+6	David	400.0	2025-12-12
+7	Eve	500.0	2025-12-23
+
+-- !drop_partition_5 --
+1	Alice	100.0	2023-12-01
+2	Bob	200.0	2023-12-02
+
+-- !drop_partition_6 --
+3	Charlie	300.0	2024-12-03
+5	Eve	500.0	2024-12-03
+6	David	400.0	2025-12-12
+7	Eve	500.0	2025-12-23
+
+-- !drop_partition_7 --
+3	Charlie	300.0	2024-12-03
+5	Eve	500.0	2024-12-03
+6	David	400.0	2025-12-12
+7	Eve	500.0	2025-12-23
+
+-- !drop_partition_8 --
+1	Alice	100.0	2023-12-01
+2	Bob	200.0	2023-12-02
+4	David	400.0	2023-12-02
+
+-- !drop_partition_9 --
+1	Alice	100.0	2023-12-01
+2	Bob	200.0	2023-12-02
+3	Charlie	300.0	2024-12-03
+4	David	400.0	2023-12-02
+5	Eve	500.0	2024-12-03
+
+-- !drop_partition_10 --
+3	Charlie	300.0	2024-12-03
+5	Eve	500.0	2024-12-03
+6	David	400.0	2025-12-12
+7	Eve	500.0	2025-12-23
+
+-- !replace_partition_1 --
+id	int	Yes	true	\N	
+name	text	Yes	true	\N	
+amount	double	Yes	true	\N	
+created_date	date	Yes	true	\N	
+
+-- !replace_partition_2 --
+1	Alice	100.0	2023-01-01
+2	Bob	200.0	2023-12-02
+3	Charlie	300.0	2024-12-03
+4	David	400.0	2023-12-02
+5	Eve	500.0	2024-07-03
+6	David	400.0	2025-10-12
+7	Eve	500.0	2025-09-23
+
+-- !replace_partition_3 --
+1	Alice	2023-01-01
+2	Bob	2023-12-02
+3	Charlie	2024-12-03
+4	David	2023-12-02
+5	Eve	2024-07-03
+6	David	2025-10-12
+7	Eve	2025-09-23
+
+-- !replace_partition_4 --
+6	David	400.0	2025-10-12
+7	Eve	500.0	2025-09-23
+
+-- !replace_partition_5 --
+1	Alice	100.0	2023-01-01
+2	Bob	200.0	2023-12-02
+
+-- !replace_partition_6 --
+3	Charlie	300.0	2024-12-03
+6	David	400.0	2025-10-12
+7	Eve	500.0	2025-09-23
+
+-- !replace_partition_7 --
+3	Charlie	300.0	2024-12-03
+5	Eve	500.0	2024-07-03
+6	David	400.0	2025-10-12
+7	Eve	500.0	2025-09-23
+
+-- !replace_partition_8 --
+1	Alice	100.0	2023-01-01
+2	Bob	200.0	2023-12-02
+4	David	400.0	2023-12-02
+5	Eve	500.0	2024-07-03
+
+-- !replace_partition_9 --
+1	Alice	100.0	2023-01-01
+2	Bob	200.0	2023-12-02
+3	Charlie	300.0	2024-12-03
+4	David	400.0	2023-12-02
+5	Eve	500.0	2024-07-03
+
+-- !replace_partition_10 --
+3	Charlie	300.0	2024-12-03
+5	Eve	500.0	2024-07-03
+6	David	400.0	2025-10-12
+7	Eve	500.0	2025-09-23
+
+-- !replace_partition_11 --
+6	David	400.0	2025-10-12
+7	Eve	500.0	2025-09-23
+
diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partitions.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partitions.out
index 1ec82236951bb9..c101079be333ed 100644
--- a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partitions.out
+++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partitions.out
@@ -9,13 +9,13 @@ false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
 -- !q03 --
 false	1	-2147483648	-9223372036854775808	-123.45	-123456.789
 false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
-false	3	2147483647	9223372036854775807	123.45	123456.789
+true	3	2147483647	9223372036854775807	123.45	123456.789
 
 -- !q04 --
 false	1	-2147483648	-9223372036854775808	-123.45	-123456.789
 false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
-false	3	2147483647	\N	123.45	\N
-false	3	2147483647	9223372036854775807	123.45	123456.789
+true	3	2147483647	\N	123.45	\N
+true	3	2147483647	9223372036854775807	123.45	123456.789
 
 -- !q01 --
 -123456.789012	1	string_value	2024-03-22
@@ -51,13 +51,13 @@ false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
 -- !q03 --
 false	1	-2147483648	-9223372036854775808	-123.45	-123456.789
 false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
-false	3	2147483647	9223372036854775807	123.45	123456.789
+true	3	2147483647	9223372036854775807	123.45	123456.789
 
 -- !q04 --
 false	1	-2147483648	-9223372036854775808	-123.45	-123456.789
 false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
-false	3	2147483647	\N	123.45	\N
-false	3	2147483647	9223372036854775807	123.45	123456.789
+true	3	2147483647	\N	123.45	\N
+true	3	2147483647	9223372036854775807	123.45	123456.789
 
 -- !q01 --
 -123456.789012	1	string_value	2024-03-22
@@ -93,13 +93,13 @@ false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
 -- !q03 --
 false	1	-2147483648	-9223372036854775808	-123.45	-123456.789
 false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
-false	3	2147483647	9223372036854775807	123.45	123456.789
+true	3	2147483647	9223372036854775807	123.45	123456.789
 
 -- !q04 --
 false	1	-2147483648	-9223372036854775808	-123.45	-123456.789
 false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
-false	3	2147483647	\N	123.45	\N
-false	3	2147483647	9223372036854775807	123.45	123456.789
+true	3	2147483647	\N	123.45	\N
+true	3	2147483647	9223372036854775807	123.45	123456.789
 
 -- !q01 --
 -123456.789012	1	string_value	2024-03-22
@@ -135,13 +135,13 @@ false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
 -- !q03 --
 false	1	-2147483648	-9223372036854775808	-123.45	-123456.789
 false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
-false	3	2147483647	9223372036854775807	123.45	123456.789
+true	3	2147483647	9223372036854775807	123.45	123456.789
 
 -- !q04 --
 false	1	-2147483648	-9223372036854775808	-123.45	-123456.789
 false	2	-2147483648	-9223372036854775808	-123.45	-123456.789
-false	3	2147483647	\N	123.45	\N
-false	3	2147483647	9223372036854775807	123.45	123456.789
+true	3	2147483647	\N	123.45	\N
+true	3	2147483647	9223372036854775807	123.45	123456.789
 
 -- !q01 --
 -123456.789012	1	string_value	2024-03-22
diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_partition_evolution.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_partition_evolution.groovy
new file mode 100644
index 00000000000000..985f9e035cb192
--- /dev/null
+++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_partition_evolution.groovy
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_partition_evolution", "p0,external,doris,external_docker,external_docker_doris") {
+    String enabled = context.config.otherConfigs.get("enableIcebergTest")
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+        logger.info("disable iceberg test.")
+        return
+    }
+
+    String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+    String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+    String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+    String catalog_name = "test_iceberg_partition_evolution"
+
+    sql """drop catalog if exists ${catalog_name}"""
+    sql """
+    CREATE CATALOG ${catalog_name} PROPERTIES (
+        'type'='iceberg',
+        'iceberg.catalog.type'='rest',
+        'uri' = 'http://${externalEnvIp}:${rest_port}',
+        "s3.access_key" = "admin",
+        "s3.secret_key" = "password",
+        "s3.endpoint" = "http://${externalEnvIp}:${minio_port}",
+        "s3.region" = "us-east-1"
+    );"""
+
+    logger.info("catalog " + catalog_name + " created")
+    sql """switch ${catalog_name};"""
+    logger.info("switched to catalog " + catalog_name)
+    sql """ use test_db;""" 
+
+    def tbs = sql """ show tables  """ 
+    logger.info("tables = " +tbs )
+
+
+    qt_add_partition_1  """desc iceberg_add_partition;"""
+    qt_add_partition_2  """select * from iceberg_add_partition order by id;"""
+    qt_add_partition_3  """select id from iceberg_add_partition order by id;"""
+    qt_add_partition_4  """select id,name from iceberg_add_partition order by id;"""
+    qt_add_partition_5  """select age from iceberg_add_partition order by id;"""
+    qt_add_partition_6  """select address from iceberg_add_partition where address is not null order by id;"""
+    qt_add_partition_7  """select id,name,address from iceberg_add_partition where length(address) > 5 order by id;"""
+    qt_add_partition_8  """select id,name,address from iceberg_add_partition where length(address) < 5 order by id;"""
+    qt_add_partition_9  """select * from iceberg_add_partition  where age > 30 order by id;"""
+    qt_add_partition_10  """select * from iceberg_add_partition  where id < 5 order by id;"""
+
+    qt_drop_partition_1  """desc iceberg_drop_partition;"""
+    qt_drop_partition_2  """select * from iceberg_drop_partition order by id;"""
+    qt_drop_partition_3  """select id,name,created_date from iceberg_drop_partition order by id;"""
+    qt_drop_partition_4  """select * from iceberg_drop_partition where id > 5 order by id;"""
+    qt_drop_partition_5  """select * from iceberg_drop_partition where id < 3 order by id;"""
+    qt_drop_partition_6  """select * from iceberg_drop_partition where created_date > "2024-12-01" order by id;"""
+    qt_drop_partition_7  """select * from iceberg_drop_partition where created_date > "2023-12-02" order by id;"""
+    qt_drop_partition_8  """select * from iceberg_drop_partition where created_date < "2024-12-02" order by id;"""
+    qt_drop_partition_9  """select * from iceberg_drop_partition where created_date <= "2024-12-03" order by id;"""
+    qt_drop_partition_10  """select * from iceberg_drop_partition where created_date >= "2024-07-03" order by id;"""
+
+    qt_replace_partition_1  """desc iceberg_replace_partition;"""
+    qt_replace_partition_2  """select * from iceberg_replace_partition order by id;"""
+    qt_replace_partition_3  """select id,name,created_date from iceberg_replace_partition order by id;"""
+    qt_replace_partition_4  """select * from iceberg_replace_partition where id > 5 order by id;"""
+    qt_replace_partition_5  """select * from iceberg_replace_partition where id < 3 order by id;"""
+    qt_replace_partition_6  """select * from iceberg_replace_partition where created_date > "2024-12-01" order by id;"""
+    qt_replace_partition_7  """select * from iceberg_replace_partition where created_date > "2023-12-02" order by id;"""
+    qt_replace_partition_8  """select * from iceberg_replace_partition where created_date < "2024-12-02" order by id;"""
+    qt_replace_partition_9  """select * from iceberg_replace_partition where created_date <= "2024-12-03" order by id;"""
+    qt_replace_partition_10  """select * from iceberg_replace_partition where created_date >= "2024-07-03" order by id;"""
+    qt_replace_partition_11  """select * from iceberg_replace_partition where created_date >= "2025-09-23" order by id;"""
+
+
+
+
+    try {
+        sql """ select * from iceberg_evolution_partition """ 
+    }catch (Exception e) {
+        assertTrue(e.getMessage().contains("Unable to read Iceberg table with dropped old partition column."), e.getMessage())
+    }
+
+}
+/*
+
+
+CREATE TABLE iceberg_add_partition (
+    id INT,
+    name STRING,
+    age INT
+) USING iceberg;
+INSERT INTO iceberg_add_partition VALUES(1, 'Alice', 30),(2, 'Bob', 25);
+ALTER TABLE iceberg_add_partition ADD PARTITION FIELD age;
+INSERT INTO iceberg_add_partition VALUES (3, 'Diana', 30, '456');
+ALTER TABLE iceberg_add_partition ADD COLUMNS address STRING;
+INSERT INTO iceberg_add_partition VALUES (4, 'Charlie', 45, '123 Street Name');
+ALTER TABLE iceberg_add_partition ADD PARTITION FIELD bucket(10, id);
+INSERT INTO iceberg_add_partition VALUES (5, 'Eve', 29, '789 Third St');
+ALTER TABLE iceberg_add_partition ADD PARTITION FIELD truncate(5, address);
+INSERT INTO iceberg_add_partition VALUES (6, 'Frank', 33,"xx"),(7, 'Grace', 28,"yyyyyyyyy");
+
+
+CREATE TABLE iceberg_drop_partition (
+    id INT,
+    name STRING,
+    amount DOUBLE,
+    created_date DATE
+) 
+USING iceberg
+PARTITIONED BY (year(created_date),bucket(10,created_date));
+INSERT INTO iceberg_drop_partition VALUES
+    (1, 'Alice', 100.0, DATE '2023-12-01'),
+    (2, 'Bob', 200.0, DATE '2023-12-02'),
+    (3, 'Charlie', 300.0, DATE '2024-12-03');
+ALTER TABLE iceberg_drop_partition  DROP PARTITION FIELD year(created_date);
+INSERT INTO iceberg_drop_partition VALUES
+    (4, 'David', 400.0, DATE '2023-12-02'),
+    (5, 'Eve', 500.0, DATE '2024-12-03');
+ALTER TABLE iceberg_drop_partition  DROP PARTITION FIELD bucket(10,created_date);
+INSERT INTO iceberg_drop_partition VALUES
+    (6, 'David', 400.0, DATE '2025-12-12'),
+    (7, 'Eve', 500.0, DATE '2025-12-23');
+
+
+CREATE TABLE iceberg_replace_partition (
+    id INT,
+    name STRING,
+    amount DOUBLE,
+    created_date DATE
+)
+USING iceberg
+PARTITIONED BY (year(created_date),bucket(10,created_date));
+INSERT INTO iceberg_replace_partition VALUES
+    (1, 'Alice', 100.0, DATE '2023-01-01'),
+    (2, 'Bob', 200.0, DATE '2023-12-02'),
+    (3, 'Charlie', 300.0, DATE '2024-12-03');
+ALTER TABLE iceberg_replace_partition  REPLACE PARTITION FIELD year(created_date) WITH month(created_date);
+INSERT INTO iceberg_replace_partition VALUES
+    (4, 'David', 400.0, DATE '2023-12-02'),
+    (5, 'Eve', 500.0, DATE '2024-07-03');
+ALTER TABLE iceberg_replace_partition  REPLACE PARTITION FIELD bucket(10,created_date) WITH bucket(10,id);
+INSERT INTO iceberg_replace_partition VALUES
+    (6, 'David', 400.0, DATE '2025-10-12'),
+    (7, 'Eve', 500.0, DATE '2025-09-23');
+
+CREATE TABLE iceberg_evolution_partition (
+    id INT,
+    name STRING,
+    age INT
+) USING iceberg;
+INSERT INTO iceberg_evolution_partition VALUES(1, 'Alice', 30),(2, 'Bob', 25);
+ALTER TABLE iceberg_evolution_partition ADD PARTITION FIELD age;
+INSERT INTO iceberg_evolution_partition VALUES (3, 'Diana', 30, '456');
+ALTER TABLE iceberg_evolution_partition ADD COLUMNS address STRING;
+INSERT INTO iceberg_evolution_partition VALUES (4, 'Charlie', 45, '123 Street Name');
+ALTER TABLE iceberg_evolution_partition ADD PARTITION FIELD bucket(10, id);
+INSERT INTO iceberg_evolution_partition VALUES (5, 'Eve', 29, '789 Third St');
+ALTER TABLE iceberg_evolution_partition REPLACE PARTITION FIELD bucket(10, id) WITH truncate(5, address);
+INSERT INTO iceberg_evolution_partition VALUES (6, 'Frank', 33,"xx"),(7, 'Grace', 28,"yyyyyyyyy");
+ALTER TABLE iceberg_evolution_partition DROP PARTITION FIELD truncate(5, address);
+INSERT INTO iceberg_evolution_partition VALUES (8, 'Hank', 40, "zz"), (9, 'Ivy', 22, "aaaaaa");
+ALTER TABLE iceberg_evolution_partition DROP COLUMNS address;
+-- INSERT INTO iceberg_evolution_partition VALUES (10, 'Jack', 35), (11, 'Kara', 30);
+
+
+*/
+
+

From 36230be7a1e7ca17810fef6570346de8a6cbfcb5 Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchi@selectdb.com>
Date: Wed, 18 Dec 2024 07:57:44 +0800
Subject: [PATCH 377/399] [fix](external)fix split and get the schema (#45408)

### What problem does this PR solve?

Related PR: #39116

Problem Summary:

Split and get the schema according to `://` and `:/`.
Like `file://ab/c`, `file:/ab/c`
---
 .../org/apache/doris/common/util/LocationPath.java   |  2 +-
 .../apache/doris/common/util/LocationPathTest.java   | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index 4604e4deabb2b7..2318532cba697f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -90,7 +90,7 @@ public LocationPath(String location, Map<String, String> props) {
     private LocationPath(String originLocation, Map<String, String> props, boolean convertPath) {
         isBindBroker = props.containsKey(HMSExternalCatalog.BIND_BROKER_NAME);
         String tmpLocation = originLocation;
-        if (!originLocation.contains(SCHEME_DELIM)) {
+        if (!(originLocation.contains(SCHEME_DELIM) || originLocation.contains(NONSTANDARD_SCHEME_DELIM))) {
             // Sometimes the file path does not contain scheme, need to add default fs
             // eg, /path/to/file.parquet -> hdfs://nn/path/to/file.parquet
             // the default fs is from the catalog properties
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
index 9d1edadd919b59..4457b7dd1efd60 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
@@ -205,4 +205,16 @@ public void testNoSchemeLocation() {
         String beLocation = locationPath.toStorageLocation().toString();
         Assertions.assertTrue(beLocation.equalsIgnoreCase("/path/to/local"));
     }
+
+    @Test
+    public void testLocalFileSystem() {
+        HashMap<String, String> props = new HashMap<>();
+        props.put("fs.defaultFS", "hdfs:///xyz");
+        LocationPath p1 = new LocationPath("file:///abc/def", props);
+        Assertions.assertEquals(Scheme.LOCAL, p1.getScheme());
+        LocationPath p2 = new LocationPath("file:/abc/def", props);
+        Assertions.assertEquals(Scheme.LOCAL, p2.getScheme());
+        LocationPath p3 = new LocationPath("file://authority/abc/def", props);
+        Assertions.assertEquals(Scheme.LOCAL, p3.getScheme());
+    }
 }

From b87d0bfa7a1461e1365eea1aa2631b7efe09e026 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Wed, 18 Dec 2024 09:47:40 +0800
Subject: [PATCH 378/399] [fix](suites) Enable common expr pushdown in the
 restore inverted idx case (#45547)

`set enable_common_expr_pushdown = false` will affect the execution of
match_any
---
 .../backup_restore/test_backup_restore_inverted_idx.groovy  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/regression-test/suites/backup_restore/test_backup_restore_inverted_idx.groovy b/regression-test/suites/backup_restore/test_backup_restore_inverted_idx.groovy
index 0dadc99dd21b4f..9c5718e3bf725a 100644
--- a/regression-test/suites/backup_restore/test_backup_restore_inverted_idx.groovy
+++ b/regression-test/suites/backup_restore/test_backup_restore_inverted_idx.groovy
@@ -117,13 +117,13 @@ suite("test_backup_restore_inverted_idx", "backup_restore") {
 
         // 1. query with inverted index
         sql """ set enable_match_without_inverted_index = false """
-        def res = sql """ SELECT /*+ SET_VAR(inverted_index_skip_threshold = 0) */ * FROM ${dbName}.${tableName} WHERE value MATCH_ANY "10" """
+        def res = sql """ SELECT /*+ SET_VAR(inverted_index_skip_threshold = 0, enable_common_expr_pushdown = true) */ * FROM ${dbName}.${tableName} WHERE value MATCH_ANY "10" """
         assertTrue(res.size() > 0)
 
         // 2. add partition and query
         sql """ ALTER TABLE ${dbName}.${tableName} ADD PARTITION p8 VALUES LESS THAN ("80") """
         sql """ INSERT INTO ${dbName}.${tableName} VALUES (75, "75 750", "76 77") """
-        res = sql """ SELECT /*+ SET_VAR(inverted_index_skip_threshold = 0) */ * FROM ${dbName}.${tableName} WHERE value MATCH_ANY "75" """
+        res = sql """ SELECT /*+ SET_VAR(inverted_index_skip_threshold = 0, enable_common_expr_pushdown = true) */ * FROM ${dbName}.${tableName} WHERE value MATCH_ANY "75" """
         assertTrue(res.size() > 0)
 
         // 3. add new index
@@ -159,7 +159,7 @@ suite("test_backup_restore_inverted_idx", "backup_restore") {
             logger.info("the build index status: ${build_status}")
             assertTrue(false)
         }
-        res = sql """ SELECT /*+ SET_VAR(inverted_index_skip_threshold = 0) */ * FROM ${dbName}.${tableName} WHERE value1 MATCH_ANY "12321" """
+        res = sql """ SELECT /*+ SET_VAR(inverted_index_skip_threshold = 0, enable_common_expr_pushdown = true) */ * FROM ${dbName}.${tableName} WHERE value1 MATCH_ANY "12321" """
         assertTrue(res.size() > 0)
 
     } finally {

From fe42ec9c104aee83292714d2a4a55eaed5841bc0 Mon Sep 17 00:00:00 2001
From: Tiewei Fang <fangtiewei@selectdb.com>
Date: Wed, 18 Dec 2024 09:55:42 +0800
Subject: [PATCH 379/399] [fix](tvf) Tvf supports to parse the enclose
 character in csv files (#45407)

### What problem does this PR solve?

Problem Summary:

Tvf supports to parse the enclose character in csv files
---
 be/src/vec/exec/format/csv/csv_reader.cpp     |  6 +-
 .../common/util/FileFormatConstants.java      |  1 +
 .../ExternalFileTableValuedFunction.java      | 15 ++++
 .../external_table_p0/tvf/enclose_csv.csv     |  6 ++
 .../tvf/test_local_tvf_enclose.out            | 15 ++++
 .../tvf/test_local_tvf_enclose.groovy         | 72 +++++++++++++++++++
 6 files changed, 114 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/data/external_table_p0/tvf/enclose_csv.csv
 create mode 100644 regression-test/data/external_table_p0/tvf/test_local_tvf_enclose.out
 create mode 100644 regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy

diff --git a/be/src/vec/exec/format/csv/csv_reader.cpp b/be/src/vec/exec/format/csv/csv_reader.cpp
index bf0e543d650142..b27bb050dc6e0c 100644
--- a/be/src/vec/exec/format/csv/csv_reader.cpp
+++ b/be/src/vec/exec/format/csv/csv_reader.cpp
@@ -928,9 +928,13 @@ Status CsvReader::_prepare_parse(size_t* read_line, bool* is_parse_name) {
                 _trim_tailing_spaces, _trim_double_quotes, _value_separator,
                 _value_separator_length);
     } else {
+        // If we pass `_file_slot_descs.size() - 1` to EncloseCsvTextFieldSplitter, it will cause BE core dump
+        // because currently _file_slot_descs is an empty vector.
+        // The _file_slot_descs.size() is only used to reserve space,
+        // so it's ok to pass 0 to EncloseCsvLineReaderContext
         text_line_reader_ctx = std::make_shared<EncloseCsvLineReaderContext>(
                 _line_delimiter, _line_delimiter_length, _value_separator, _value_separator_length,
-                _file_slot_descs.size() - 1, _enclose, _escape, _keep_cr);
+                0, _enclose, _escape, _keep_cr);
         _fields_splitter = std::make_unique<EncloseCsvTextFieldSplitter>(
                 _trim_tailing_spaces, false,
                 std::static_pointer_cast<EncloseCsvLineReaderContext>(text_line_reader_ctx),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java
index bdb2e97b9f2518..7050bec9d77c52 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java
@@ -50,6 +50,7 @@ public class FileFormatConstants {
     public static final String PROP_COMPRESS = "compress";
     public static final String PROP_COMPRESS_TYPE = "compress_type";
     public static final String PROP_PATH_PARTITION_KEYS = "path_partition_keys";
+    public static final String PROP_ENCLOSE = "enclose";
 
     // decimal(p,s)
     public static final Pattern DECIMAL_TYPE_PATTERN = Pattern.compile("decimal\\((\\d+),(\\d+)\\)");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index 6f45a1cc0eb03d..55d046c2ed94f5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -120,6 +120,7 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
     private TTextSerdeType textSerdeType = TTextSerdeType.JSON_TEXT_SERDE;
     private String columnSeparator = FileFormatConstants.DEFAULT_COLUMN_SEPARATOR;
     private String lineDelimiter = FileFormatConstants.DEFAULT_LINE_DELIMITER;
+    private byte enclose = 0;
     private String jsonRoot = "";
     private String jsonPaths = "";
     private boolean stripOuterArray;
@@ -231,6 +232,17 @@ protected Map<String, String> parseCommonProperties(Map<String, String> properti
         }
         lineDelimiter = Separator.convertSeparator(lineDelimiter);
 
+        String enclosedString = getOrDefaultAndRemove(copiedProps, FileFormatConstants.PROP_ENCLOSE, "");
+        if (!Strings.isNullOrEmpty(enclosedString)) {
+            if (enclosedString.length() > 1) {
+                throw new AnalysisException("enclose should not be longer than one byte.");
+            }
+            enclose = (byte) enclosedString.charAt(0);
+            if (enclose == 0) {
+                throw new AnalysisException("enclose should not be byte [0].");
+            }
+        }
+
         jsonRoot = getOrDefaultAndRemove(copiedProps, FileFormatConstants.PROP_JSON_ROOT, "");
         jsonPaths = getOrDefaultAndRemove(copiedProps, FileFormatConstants.PROP_JSON_PATHS, "");
         readJsonByLine = Boolean.valueOf(
@@ -285,6 +297,9 @@ public TFileAttributes getFileAttributes() {
         TFileTextScanRangeParams fileTextScanRangeParams = new TFileTextScanRangeParams();
         fileTextScanRangeParams.setColumnSeparator(this.columnSeparator);
         fileTextScanRangeParams.setLineDelimiter(this.lineDelimiter);
+        if (enclose != 0) {
+            fileTextScanRangeParams.setEnclose(enclose);
+        }
         fileAttributes.setTextParams(fileTextScanRangeParams);
         if (this.fileFormatType == TFileFormatType.FORMAT_CSV_PLAIN) {
             fileAttributes.setHeaderType(this.headerType);
diff --git a/regression-test/data/external_table_p0/tvf/enclose_csv.csv b/regression-test/data/external_table_p0/tvf/enclose_csv.csv
new file mode 100644
index 00000000000000..ca787200ff18ca
--- /dev/null
+++ b/regression-test/data/external_table_p0/tvf/enclose_csv.csv
@@ -0,0 +1,6 @@
+id, field1, field2
+"1", "hello", "same, field"
+"2", "doris", "same, field2"
+"3", "nereids", "same, field3"
+"4", "pipeline", "same, field4"
+"5", "storage", "same, field5"
\ No newline at end of file
diff --git a/regression-test/data/external_table_p0/tvf/test_local_tvf_enclose.out b/regression-test/data/external_table_p0/tvf/test_local_tvf_enclose.out
new file mode 100644
index 00000000000000..6e5d10e4858a9c
--- /dev/null
+++ b/regression-test/data/external_table_p0/tvf/test_local_tvf_enclose.out
@@ -0,0 +1,15 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !enclose_1 --
+"1"	"hello"	"same, field"
+"2"	"doris"	"same, field2"
+"3"	"nereids"	"same, field3"
+"4"	"pipeline"	"same, field4"
+"5"	"storage"	"same, field5"
+
+-- !enclose_2 --
+1	hello	same, field
+2	doris	same, field2
+3	nereids	same, field3
+4	pipeline	same, field4
+5	storage	same, field5
+
diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy
new file mode 100644
index 00000000000000..e7437cd20ec13e
--- /dev/null
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy
@@ -0,0 +1,72 @@
+import org.junit.Assert
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_local_tvf_enclose", "p0,tvf") {
+    List<List<Object>> backends =  sql """ show backends """
+    assertTrue(backends.size() > 0)
+    def be_id = backends[0][0]
+
+    String filename = "enclose_csv.csv"
+
+    def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/${filename}"
+
+    def outFilePath="/"
+
+    for (List<Object> backend : backends) {
+         def be_host = backend[1]
+         scpFiles ("root", be_host, dataFilePath, outFilePath, false);
+    }
+
+
+    sql """set enable_nereids_planner=true"""
+    sql """set enable_fallback_to_original_planner=false"""
+
+    qt_enclose_1 """
+                    select * from local(
+                        "file_path" = "${filename}",
+                        "backend_id" = "${be_id}",
+                        "format" = "csv_with_names",
+                        "column_separator" = ", ",
+                        "enclose" = "\\\"") order by id;            
+                """
+
+    qt_enclose_2 """
+                    select * from local(
+                        "file_path" = "${filename}",
+                        "backend_id" = "${be_id}",
+                        "format" = "csv_with_names",
+                        "column_separator" = ", ",
+                        "enclose" = "\\\"",
+                        "trim_double_quotes" = "true") order by id;            
+                """
+
+    // test error case
+    test {
+        sql """
+                select * from local(
+                        "file_path" = "${filename}",
+                        "backend_id" = "${be_id}",
+                        "format" = "csv_with_names",
+                        "column_separator" = ", ",
+                        "enclose" = "\\\"\\\"") order by id;            
+                """
+        // check exception message contains
+        exception "enclose should not be longer than one byte."
+    }
+}

From b65de37d7497ea6bd6fa62ef57e8c6e150b03f0c Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Wed, 18 Dec 2024 10:07:26 +0800
Subject: [PATCH 380/399] [env](compile) open compile_check in some file
 (#45357)

---
 be/src/util/simd/vstring_function.h           |  7 +++-
 be/src/vec/functions/function_rpc.cpp         |  2 +-
 .../functions/function_split_by_regexp.cpp    |  9 +++--
 be/src/vec/functions/function_string.cpp      | 38 +++++++++++++------
 be/src/vec/functions/functions_comparison.h   |  6 ++-
 .../functions_multi_string_position.cpp       | 11 ++++--
 6 files changed, 48 insertions(+), 25 deletions(-)

diff --git a/be/src/util/simd/vstring_function.h b/be/src/util/simd/vstring_function.h
index 99313132382e5c..bfa75b728d5620 100644
--- a/be/src/util/simd/vstring_function.h
+++ b/be/src/util/simd/vstring_function.h
@@ -309,8 +309,11 @@ class VStringFunctions {
     // is to say, counting bytes which do not match 10xx_xxxx pattern.
     // All 0xxx_xxxx, 110x_xxxx, 1110_xxxx and 1111_0xxx are greater than 1011_1111 when use int8_t arithmetic,
     // so just count bytes greater than 1011_1111 in a byte string as the result of utf8_length.
-    static inline size_t get_char_len(const char* src, size_t len) {
-        size_t char_len = 0;
+    // get_char_len is used to return the UTF-8 length of a string.
+    // The return value will never exceed len.
+    template <typename T>
+    static inline T get_char_len(const char* src, T len) {
+        T char_len = 0;
         const char* p = src;
         const char* end = p + len;
 #if defined(__SSE2__) || defined(__aarch64__)
diff --git a/be/src/vec/functions/function_rpc.cpp b/be/src/vec/functions/function_rpc.cpp
index 6d3a64f78409c6..d88266cc2bf647 100644
--- a/be/src/vec/functions/function_rpc.cpp
+++ b/be/src/vec/functions/function_rpc.cpp
@@ -33,7 +33,7 @@
 #include "vec/data_types/serde/data_type_serde.h"
 
 namespace doris::vectorized {
-
+#include "common/compile_check_begin.h"
 RPCFnImpl::RPCFnImpl(const TFunction& fn) : _fn(fn) {
     _function_name = _fn.scalar_fn.symbol;
     _server_addr = _fn.hdfs_location;
diff --git a/be/src/vec/functions/function_split_by_regexp.cpp b/be/src/vec/functions/function_split_by_regexp.cpp
index a4c46b59ac1f75..89791d398ece50 100644
--- a/be/src/vec/functions/function_split_by_regexp.cpp
+++ b/be/src/vec/functions/function_split_by_regexp.cpp
@@ -30,6 +30,7 @@
 #include "vec/functions/simple_function_factory.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 struct Match {
     std::string::size_type offset;
@@ -194,7 +195,7 @@ struct ExecuteImpl {
                 unpack_if_const(block.get_by_position(arguments[1]).column);
         const auto& [three_column, three_is_const] =
                 unpack_if_const(block.get_by_position(arguments[2]).column);
-        auto limit_value = assert_cast<const ColumnInt32&>(*three_column).get_int(0);
+        auto limit_value = assert_cast<const ColumnInt32&>(*three_column).get_element(0);
         const auto& src_column = assert_cast<const ColumnString&>(*first_column);
         const auto& pattern_column = assert_cast<const ColumnString&>(*second_column);
 
@@ -238,7 +239,7 @@ struct ExecuteImpl {
                                             const StringRef& pattern_ref,
                                             ColumnString& dest_column_string,
                                             ColumnArray::Offsets64& dest_offsets,
-                                            NullMapType* dest_nested_null_map, Int64 limit_value,
+                                            NullMapType* dest_nested_null_map, Int32 limit_value,
                                             size_t input_rows_count, RE2::Options* opts) {
         const char* token_begin = nullptr;
         const char* token_end = nullptr;
@@ -270,7 +271,7 @@ struct ExecuteImpl {
                                              const ColumnString& pattern_column,
                                              ColumnString& dest_column_string,
                                              ColumnArray::Offsets64& dest_offsets,
-                                             NullMapType* dest_nested_null_map, Int64 limit_value,
+                                             NullMapType* dest_nested_null_map, Int32 limit_value,
                                              size_t input_rows_count, RE2::Options* opts) {
         const char* token_begin = nullptr;
         const char* token_end = nullptr;
@@ -307,7 +308,7 @@ struct ExecuteImpl {
                                        const ColumnString& pattern_column,
                                        ColumnString& dest_column_string,
                                        ColumnArray::Offsets64& dest_offsets,
-                                       NullMapType* dest_nested_null_map, Int64 limit_value,
+                                       NullMapType* dest_nested_null_map, Int32 limit_value,
                                        size_t input_rows_count, RE2::Options* opts) {
         const char* token_begin = nullptr;
         const char* token_end = nullptr;
diff --git a/be/src/vec/functions/function_string.cpp b/be/src/vec/functions/function_string.cpp
index c434d344daa40a..f34d9f91b4d91d 100644
--- a/be/src/vec/functions/function_string.cpp
+++ b/be/src/vec/functions/function_string.cpp
@@ -25,6 +25,7 @@
 #include <cstddef>
 #include <string_view>
 
+#include "common/cast_set.h"
 #include "common/status.h"
 #include "runtime/string_search.hpp"
 #include "util/url_coding.h"
@@ -38,6 +39,7 @@
 #include "vec/functions/simple_function_factory.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 struct NameStringASCII {
     static constexpr auto name = "ascii";
 };
@@ -68,7 +70,7 @@ struct NameQuoteImpl {
     static Status vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets,
                          ColumnString::Chars& res_data, ColumnString::Offsets& res_offsets) {
         size_t offset_size = offsets.size();
-        size_t pos = 0;
+        ColumnString::Offset pos = 0;
         res_offsets.resize(offset_size);
         res_data.resize(data.size() + offset_size * 2);
         for (int i = 0; i < offset_size; i++) {
@@ -285,9 +287,9 @@ struct StringInStrImpl {
 
             /// We check that the entry does not pass through the boundaries of strings.
             if (pos + rdata.size <= begin + loffsets[i]) {
-                int loc = pos - begin - loffsets[i - 1];
+                int loc = (int)(pos - begin) - loffsets[i - 1];
                 int l_str_size = loffsets[i] - loffsets[i - 1];
-                size_t len = std::min(l_str_size, loc);
+                auto len = std::min(l_str_size, loc);
                 loc = simd::VStringFunctions::get_char_len((char*)(begin + loffsets[i - 1]), len);
                 res[i] = loc + 1;
             }
@@ -332,7 +334,7 @@ struct StringInStrImpl {
         // Hive returns positions starting from 1.
         int loc = search.search(&strl);
         if (loc > 0) {
-            size_t len = std::min((size_t)loc, strl.size);
+            int len = std::min(loc, (int)strl.size);
             loc = simd::VStringFunctions::get_char_len(strl.data, len);
         }
 
@@ -489,7 +491,16 @@ struct InitcapImpl {
                 if (!::isalnum(res_data[i])) {
                     need_capitalize = true;
                 } else if (need_capitalize) {
-                    res_data[i] = ::toupper(res_data[i]);
+                    /*
+                    https://en.cppreference.com/w/cpp/string/byte/toupper
+                    Like all other functions from <cctype>, the behavior of std::toupper is undefined if the argument's value is neither representable as unsigned char nor equal to EOF. 
+                    To use these functions safely with plain chars (or signed chars), the argument should first be converted to unsigned char:
+                    char my_toupper(char ch)
+                    {
+                        return static_cast<char>(std::toupper(static_cast<unsigned char>(ch)));
+                    }
+                    */
+                    res_data[i] = static_cast<unsigned char>(::toupper(res_data[i]));
                     need_capitalize = false;
                 }
             }
@@ -540,7 +551,8 @@ struct TrimUtil {
             }
 
             res_data.insert_assume_reserved(str_begin, str_end);
-            res_offsets[i] = res_data.size();
+            // The length of the result of the trim function will never exceed the length of the input.
+            res_offsets[i] = (ColumnString::Offset)res_data.size();
         }
         return Status::OK();
     }
@@ -606,7 +618,8 @@ struct TrimInUtil {
             }
 
             res_data.insert_assume_reserved(left_trim_pos, right_trim_pos);
-            res_offsets[i] = res_data.size();
+            // The length of the result of the trim function will never exceed the length of the input.
+            res_offsets[i] = (ColumnString::Offset)res_data.size();
         }
 
         return Status::OK();
@@ -669,7 +682,8 @@ struct TrimInUtil {
             }
 
             res_data.insert_assume_reserved(left_trim_pos, right_trim_pos);
-            res_offsets[i] = res_data.size();
+            // The length of the result of the trim function will never exceed the length of the input.
+            res_offsets[i] = (ColumnString::Offset)res_data.size();
         }
         return Status::OK();
     }
@@ -820,7 +834,7 @@ struct UnHexImpl {
         return false;
     }
 
-    static int hex_decode(const char* src_str, size_t src_len, char* dst_str) {
+    static int hex_decode(const char* src_str, ColumnString::Offset src_len, char* dst_str) {
         // if str length is odd or 0, return empty string like mysql dose.
         if ((src_len & 1) != 0 or src_len == 0) {
             return 0;
@@ -848,7 +862,7 @@ struct UnHexImpl {
 
         for (int i = 0; i < rows_count; ++i) {
             const auto* source = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
-            size_t srclen = offsets[i] - offsets[i - 1];
+            ColumnString::Offset srclen = offsets[i] - offsets[i - 1];
 
             if (srclen == 0) {
                 StringOP::push_empty_string(i, dst_data, dst_offsets);
@@ -898,7 +912,7 @@ struct StringSpace {
         for (size_t i = 0; i < input_size; ++i) {
             if (data[i] > 0) [[likely]] {
                 res_data.resize_fill(res_data.size() + data[i], ' ');
-                res_offsets[i] = res_data.size();
+                cast_set(res_offsets[i], res_data.size());
             } else {
                 StringOP::push_empty_string(i, res_data, res_offsets);
             }
@@ -962,7 +976,7 @@ struct FromBase64Impl {
             }
 
             const auto* source = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
-            size_t srclen = offsets[i] - offsets[i - 1];
+            ColumnString::Offset srclen = offsets[i] - offsets[i - 1];
 
             if (srclen == 0) {
                 StringOP::push_empty_string(i, dst_data, dst_offsets);
diff --git a/be/src/vec/functions/functions_comparison.h b/be/src/vec/functions/functions_comparison.h
index cb56f176c71eb1..2da767fa7ef909 100644
--- a/be/src/vec/functions/functions_comparison.h
+++ b/be/src/vec/functions/functions_comparison.h
@@ -42,6 +42,7 @@
 //#include "olap/rowset/segment_v2/inverted_index_reader.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 
 /** Comparison functions: ==, !=, <, >, <=, >=.
   * The comparison functions always return 0 or 1 (UInt8).
@@ -441,7 +442,7 @@ class FunctionComparison : public IFunction {
 
             if (c0_const_string) {
                 c0_const_chars = &c0_const_string->get_chars();
-                c0_const_size = c0_const_string->get_data_at(0).size;
+                c0_const_size = c0_const_string->get_offsets()[0];
             } else {
                 return Status::NotSupported("Illegal columns {}, of argument of function {}",
                                             c0->get_name(), name);
@@ -454,7 +455,7 @@ class FunctionComparison : public IFunction {
 
             if (c1_const_string) {
                 c1_const_chars = &c1_const_string->get_chars();
-                c1_const_size = c1_const_string->get_data_at(0).size;
+                c1_const_size = c1_const_string->get_offsets()[0];
             } else {
                 return Status::NotSupported("Illegal columns {}, of argument of function {}",
                                             c1->get_name(), name);
@@ -714,4 +715,5 @@ class FunctionComparison : public IFunction {
     }
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/functions/functions_multi_string_position.cpp b/be/src/vec/functions/functions_multi_string_position.cpp
index 77dccff5eb763e..fdef27bc2be873 100644
--- a/be/src/vec/functions/functions_multi_string_position.cpp
+++ b/be/src/vec/functions/functions_multi_string_position.cpp
@@ -55,6 +55,7 @@
 #include "vec/functions/simple_function_factory.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class FunctionContext;
 } // namespace doris
 
@@ -106,10 +107,11 @@ class FunctionMultiStringPosition : public IFunction {
         const ColumnConst* col_needles_const =
                 check_and_get_column_const<ColumnArray>(needles_ptr.get());
 
-        if (!col_needles_const && !col_needles_vector)
+        if (!col_needles_const && !col_needles_vector) {
             return Status::InvalidArgument(
                     "function '{}' encountered unsupported needles column, found {}", name,
                     needles_column->get_name());
+        }
 
         if (col_haystack_const && col_needles_vector) {
             return Status::InvalidArgument(
@@ -219,9 +221,9 @@ struct FunctionMultiSearchAllPositionsImpl {
                 const auto* haystack_end =
                         haystack - prev_haystack_offset + haystack_offsets[haystack_index];
 
-                auto ans_now = searcher.search(haystack, haystack_end);
+                const auto* ans_now = searcher.search(haystack, haystack_end);
                 vec_res[res_index] =
-                        ans_now >= haystack_end ? 0 : std::distance(haystack, ans_now) + 1;
+                        ans_now >= haystack_end ? 0 : (Int32)std::distance(haystack, ans_now) + 1;
                 prev_haystack_offset = haystack_offsets[haystack_index];
             }
         }
@@ -296,7 +298,7 @@ struct FunctionMultiSearchAllPositionsImpl {
 
                 auto ans_now = searcher.search(haystack, haystack_end);
                 vec_res[ans_row_begin + ans_slot_in_row] =
-                        ans_now >= haystack_end ? 0 : std::distance(haystack, ans_now) + 1;
+                        ans_now >= haystack_end ? 0 : (Int32)std::distance(haystack, ans_now) + 1;
             }
 
             prev_haystack_offset = haystack_offsets[haystack_index];
@@ -315,4 +317,5 @@ void register_function_multi_string_position(SimpleFunctionFactory& factory) {
     factory.register_function<FunctionMultiSearchAllPositions>();
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::vectorized

From ff0c9a47d38c82f8ea7ece6560e42b303e492a07 Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Wed, 18 Dec 2024 10:11:09 +0800
Subject: [PATCH 381/399] [env](compile) open compile_check in pipeline
 operator (#45386)

---
 .../pipeline/exec/assert_num_rows_operator.h  |  2 ++
 be/src/pipeline/exec/cache_sink_operator.h    |  2 ++
 be/src/pipeline/exec/cache_source_operator.h  |  2 ++
 be/src/pipeline/exec/data_queue.h             |  2 ++
 be/src/pipeline/exec/datagen_operator.h       |  2 ++
 be/src/pipeline/exec/empty_set_operator.cpp   |  2 ++
 be/src/pipeline/exec/empty_set_operator.h     |  2 ++
 be/src/pipeline/exec/es_scan_operator.h       |  2 ++
 be/src/pipeline/exec/exchange_sink_buffer.cpp |  2 ++
 be/src/pipeline/exec/exchange_sink_buffer.h   |  2 ++
 be/src/pipeline/exec/exchange_sink_operator.h |  2 ++
 .../pipeline/exec/exchange_source_operator.h  |  2 ++
 be/src/pipeline/exec/file_scan_operator.h     |  2 ++
 .../exec/group_commit_block_sink_operator.h   |  4 ++-
 .../exec/group_commit_scan_operator.h         |  2 ++
 .../pipeline/exec/hive_table_sink_operator.h  |  2 ++
 .../exec/iceberg_table_sink_operator.h        |  2 ++
 be/src/pipeline/exec/jdbc_scan_operator.h     |  2 ++
 .../pipeline/exec/jdbc_table_sink_operator.h  |  2 ++
 .../pipeline/exec/join_build_sink_operator.h  |  2 ++
 be/src/pipeline/exec/join_probe_operator.h    |  2 ++
 .../exec/memory_scratch_sink_operator.h       |  2 ++
 be/src/pipeline/exec/meta_scan_operator.h     |  2 ++
 .../exec/multi_cast_data_stream_sink.h        |  2 ++
 .../pipeline/exec/multi_cast_data_streamer.h  |  2 ++
 .../exec/nested_loop_join_build_operator.h    |  2 ++
 be/src/pipeline/exec/olap_scan_operator.cpp   | 10 +++---
 be/src/pipeline/exec/olap_scan_operator.h     |  2 ++
 .../pipeline/exec/olap_table_sink_operator.h  |  2 ++
 .../exec/olap_table_sink_v2_operator.h        |  2 ++
 be/src/pipeline/exec/operator.cpp             |  2 ++
 be/src/pipeline/exec/operator.h               |  2 ++
 .../exec/partition_sort_sink_operator.cpp     |  6 ++--
 .../exec/partition_sort_sink_operator.h       |  2 ++
 .../exec/partition_sort_source_operator.cpp   |  2 ++
 .../exec/partition_sort_source_operator.h     |  2 ++
 .../partitioned_aggregation_sink_operator.cpp |  2 ++
 .../partitioned_aggregation_sink_operator.h   |  2 ++
 ...artitioned_aggregation_source_operator.cpp |  2 ++
 .../partitioned_aggregation_source_operator.h |  2 ++
 .../partitioned_hash_join_probe_operator.cpp  |  2 ++
 .../partitioned_hash_join_probe_operator.h    |  2 ++
 .../partitioned_hash_join_sink_operator.cpp   |  6 ++--
 .../partitioned_hash_join_sink_operator.h     |  2 ++
 be/src/pipeline/exec/repeat_operator.cpp      |  5 +--
 be/src/pipeline/exec/repeat_operator.h        |  2 ++
 .../exec/result_file_sink_operator.cpp        |  2 ++
 .../pipeline/exec/result_file_sink_operator.h |  2 ++
 be/src/pipeline/exec/result_sink_operator.cpp |  2 ++
 be/src/pipeline/exec/result_sink_operator.h   |  2 ++
 be/src/pipeline/exec/scan_operator.h          |  4 ++-
 be/src/pipeline/exec/schema_scan_operator.cpp |  4 ++-
 be/src/pipeline/exec/schema_scan_operator.h   |  2 ++
 be/src/pipeline/exec/select_operator.h        |  2 ++
 .../pipeline/exec/set_probe_sink_operator.cpp |  8 +++--
 .../pipeline/exec/set_probe_sink_operator.h   |  2 ++
 be/src/pipeline/exec/sort_sink_operator.cpp   |  2 ++
 be/src/pipeline/exec/sort_sink_operator.h     |  2 ++
 be/src/pipeline/exec/sort_source_operator.cpp |  2 ++
 be/src/pipeline/exec/sort_source_operator.h   |  2 ++
 be/src/pipeline/exec/spill_utils.h            |  2 ++
 .../exec/streaming_aggregation_operator.cpp   | 32 ++++++++++---------
 .../exec/streaming_aggregation_operator.h     |  2 ++
 .../pipeline/exec/table_function_operator.cpp | 11 +++++--
 .../pipeline/exec/table_function_operator.h   |  2 ++
 be/src/pipeline/exec/union_sink_operator.cpp  |  6 +++-
 be/src/pipeline/exec/union_sink_operator.h    |  2 ++
 .../pipeline/exec/union_source_operator.cpp   |  4 ++-
 be/src/pipeline/exec/union_source_operator.h  |  2 ++
 69 files changed, 178 insertions(+), 36 deletions(-)

diff --git a/be/src/pipeline/exec/assert_num_rows_operator.h b/be/src/pipeline/exec/assert_num_rows_operator.h
index dcc64f57878d38..a7408d695928c5 100644
--- a/be/src/pipeline/exec/assert_num_rows_operator.h
+++ b/be/src/pipeline/exec/assert_num_rows_operator.h
@@ -20,6 +20,7 @@
 #include "operator.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class AssertNumRowsLocalState final : public PipelineXLocalState<FakeSharedState> {
 public:
@@ -55,4 +56,5 @@ class AssertNumRowsOperatorX final : public StreamingOperatorX<AssertNumRowsLoca
     bool _should_convert_output_to_nullable;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/cache_sink_operator.h b/be/src/pipeline/exec/cache_sink_operator.h
index 9c4beb48df2e4c..6966b22d8a47a3 100644
--- a/be/src/pipeline/exec/cache_sink_operator.h
+++ b/be/src/pipeline/exec/cache_sink_operator.h
@@ -26,6 +26,7 @@
 #include "vec/core/block.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -70,4 +71,5 @@ class CacheSinkOperatorX final : public DataSinkOperatorX<CacheSinkLocalState> {
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/pipeline/exec/cache_source_operator.h b/be/src/pipeline/exec/cache_source_operator.h
index e764323846b153..146c984d04aa3f 100644
--- a/be/src/pipeline/exec/cache_source_operator.h
+++ b/be/src/pipeline/exec/cache_source_operator.h
@@ -25,6 +25,7 @@
 #include "pipeline/query_cache/query_cache.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace vectorized {
@@ -101,4 +102,5 @@ class CacheSourceOperatorX final : public OperatorX<CacheSourceLocalState> {
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/data_queue.h b/be/src/pipeline/exec/data_queue.h
index f5bd84cc278d0a..d97f58c0debdb6 100644
--- a/be/src/pipeline/exec/data_queue.h
+++ b/be/src/pipeline/exec/data_queue.h
@@ -29,6 +29,7 @@
 #include "vec/core/block.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class Dependency;
 
@@ -108,4 +109,5 @@ class DataQueue {
     SpinLock _source_lock;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/datagen_operator.h b/be/src/pipeline/exec/datagen_operator.h
index bada5ec4080d08..ffc2c6f946fb3a 100644
--- a/be/src/pipeline/exec/datagen_operator.h
+++ b/be/src/pipeline/exec/datagen_operator.h
@@ -24,6 +24,7 @@
 #include "pipeline/exec/operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 } // namespace doris
 
@@ -70,4 +71,5 @@ class DataGenSourceOperatorX final : public OperatorX<DataGenLocalState> {
     std::vector<TRuntimeFilterDesc> _runtime_filter_descs;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
\ No newline at end of file
diff --git a/be/src/pipeline/exec/empty_set_operator.cpp b/be/src/pipeline/exec/empty_set_operator.cpp
index 7233e46dfd1e52..2dfe9701558da0 100644
--- a/be/src/pipeline/exec/empty_set_operator.cpp
+++ b/be/src/pipeline/exec/empty_set_operator.cpp
@@ -22,6 +22,7 @@
 #include "pipeline/exec/operator.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 Status EmptySetSourceOperatorX::get_block(RuntimeState* state, vectorized::Block* block,
                                           bool* eos) {
@@ -29,4 +30,5 @@ Status EmptySetSourceOperatorX::get_block(RuntimeState* state, vectorized::Block
     return Status::OK();
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/empty_set_operator.h b/be/src/pipeline/exec/empty_set_operator.h
index 6b200bfdbde249..d8e920b256494d 100644
--- a/be/src/pipeline/exec/empty_set_operator.h
+++ b/be/src/pipeline/exec/empty_set_operator.h
@@ -22,6 +22,7 @@
 #include "operator.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class EmptySetLocalState final : public PipelineXLocalState<FakeSharedState> {
 public:
@@ -43,4 +44,5 @@ class EmptySetSourceOperatorX final : public OperatorX<EmptySetLocalState> {
     [[nodiscard]] bool is_source() const override { return true; }
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/es_scan_operator.h b/be/src/pipeline/exec/es_scan_operator.h
index 2ae562e4fc7f32..6e64110997e3af 100644
--- a/be/src/pipeline/exec/es_scan_operator.h
+++ b/be/src/pipeline/exec/es_scan_operator.h
@@ -26,6 +26,7 @@
 #include "pipeline/exec/scan_operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 namespace vectorized {
 class NewEsScanner;
@@ -86,4 +87,5 @@ class EsScanOperatorX final : public ScanOperatorX<EsScanLocalState> {
     std::vector<std::string> _column_names;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/exchange_sink_buffer.cpp b/be/src/pipeline/exec/exchange_sink_buffer.cpp
index 65e7698737076e..e3f895444d4168 100644
--- a/be/src/pipeline/exec/exchange_sink_buffer.cpp
+++ b/be/src/pipeline/exec/exchange_sink_buffer.cpp
@@ -47,6 +47,7 @@
 #include "vec/sink/vdata_stream_sender.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 namespace vectorized {
 BroadcastPBlockHolder::~BroadcastPBlockHolder() {
@@ -576,4 +577,5 @@ void ExchangeSinkBuffer::update_profile(RuntimeProfile* profile) {
 }
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/exchange_sink_buffer.h b/be/src/pipeline/exec/exchange_sink_buffer.h
index b2eb32414feca2..458c7c3f66e3ee 100644
--- a/be/src/pipeline/exec/exchange_sink_buffer.h
+++ b/be/src/pipeline/exec/exchange_sink_buffer.h
@@ -40,6 +40,7 @@
 #include "util/ref_count_closure.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class PTransmitDataParams;
 class TUniqueId;
 
@@ -318,4 +319,5 @@ class ExchangeSinkBuffer : public HasTaskExecutionCtx {
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/exchange_sink_operator.h b/be/src/pipeline/exec/exchange_sink_operator.h
index e88389b1d7bb5a..85575beb9f7e47 100644
--- a/be/src/pipeline/exec/exchange_sink_operator.h
+++ b/be/src/pipeline/exec/exchange_sink_operator.h
@@ -31,6 +31,7 @@
 #include "vec/sink/vdata_stream_sender.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 class TDataSink;
 
@@ -263,4 +264,5 @@ class ExchangeSinkOperatorX final : public DataSinkOperatorX<ExchangeSinkLocalSt
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/exchange_source_operator.h b/be/src/pipeline/exec/exchange_source_operator.h
index f938f5007d1643..ff9c5840033777 100644
--- a/be/src/pipeline/exec/exchange_source_operator.h
+++ b/be/src/pipeline/exec/exchange_source_operator.h
@@ -22,6 +22,7 @@
 #include "operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class ExecNode;
 } // namespace doris
 
@@ -109,4 +110,5 @@ class ExchangeSourceOperatorX final : public OperatorX<ExchangeLocalState> {
     std::vector<bool> _nulls_first;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/file_scan_operator.h b/be/src/pipeline/exec/file_scan_operator.h
index 2777a013d62f61..87c5bcd2e54de5 100644
--- a/be/src/pipeline/exec/file_scan_operator.h
+++ b/be/src/pipeline/exec/file_scan_operator.h
@@ -29,6 +29,7 @@
 #include "vec/exec/scan/split_source_connector.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 namespace vectorized {
 class VFileScanner;
 } // namespace vectorized
@@ -86,4 +87,5 @@ class FileScanOperatorX final : public ScanOperatorX<FileScanLocalState> {
     const std::string _table_name;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/group_commit_block_sink_operator.h b/be/src/pipeline/exec/group_commit_block_sink_operator.h
index e469aee8df595c..5eabb280c4315d 100644
--- a/be/src/pipeline/exec/group_commit_block_sink_operator.h
+++ b/be/src/pipeline/exec/group_commit_block_sink_operator.h
@@ -22,8 +22,9 @@
 #include "runtime/group_commit_mgr.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 class OlapTableBlockConvertor;
-}
+} // namespace doris::vectorized
 
 namespace doris::pipeline {
 
@@ -125,4 +126,5 @@ class GroupCommitBlockSinkOperatorX final
     TGroupCommitMode::type _group_commit_mode;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/group_commit_scan_operator.h b/be/src/pipeline/exec/group_commit_scan_operator.h
index 46f50f3772440a..d1428899ede6b9 100644
--- a/be/src/pipeline/exec/group_commit_scan_operator.h
+++ b/be/src/pipeline/exec/group_commit_scan_operator.h
@@ -27,6 +27,7 @@
 #include "runtime/group_commit_mgr.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class GroupCommitOperatorX;
 class GroupCommitLocalState final : public ScanLocalState<GroupCommitLocalState> {
@@ -60,4 +61,5 @@ class GroupCommitOperatorX final : public ScanOperatorX<GroupCommitLocalState> {
     const int64_t _table_id;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/hive_table_sink_operator.h b/be/src/pipeline/exec/hive_table_sink_operator.h
index 58e705fd8e46c7..8af3e5bd5e9764 100644
--- a/be/src/pipeline/exec/hive_table_sink_operator.h
+++ b/be/src/pipeline/exec/hive_table_sink_operator.h
@@ -21,6 +21,7 @@
 #include "vec/sink/writer/vhive_table_writer.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class HiveTableSinkOperatorX;
 
@@ -83,4 +84,5 @@ class HiveTableSinkOperatorX final : public DataSinkOperatorX<HiveTableSinkLocal
     ObjectPool* _pool = nullptr;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/iceberg_table_sink_operator.h b/be/src/pipeline/exec/iceberg_table_sink_operator.h
index 24fb3d178a8953..4240c98d0434d9 100644
--- a/be/src/pipeline/exec/iceberg_table_sink_operator.h
+++ b/be/src/pipeline/exec/iceberg_table_sink_operator.h
@@ -21,6 +21,7 @@
 #include "vec/sink/writer/iceberg/viceberg_table_writer.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class IcebergTableSinkOperatorX;
 
@@ -82,4 +83,5 @@ class IcebergTableSinkOperatorX final : public DataSinkOperatorX<IcebergTableSin
     ObjectPool* _pool = nullptr;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/jdbc_scan_operator.h b/be/src/pipeline/exec/jdbc_scan_operator.h
index 9ceff69ef4cd4b..6564b8a1277cd4 100644
--- a/be/src/pipeline/exec/jdbc_scan_operator.h
+++ b/be/src/pipeline/exec/jdbc_scan_operator.h
@@ -26,6 +26,7 @@
 #include "pipeline/exec/scan_operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 namespace vectorized {
 class NewJdbcScanner;
@@ -62,4 +63,5 @@ class JDBCScanOperatorX final : public ScanOperatorX<JDBCScanLocalState> {
     TOdbcTableType::type _table_type;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/jdbc_table_sink_operator.h b/be/src/pipeline/exec/jdbc_table_sink_operator.h
index 3ea702fd0baf0a..a0dae301a5fcad 100644
--- a/be/src/pipeline/exec/jdbc_table_sink_operator.h
+++ b/be/src/pipeline/exec/jdbc_table_sink_operator.h
@@ -23,6 +23,7 @@
 #include "vec/sink/writer/vjdbc_table_writer.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class JdbcTableSinkOperatorX;
 class JdbcTableSinkLocalState final
@@ -59,4 +60,5 @@ class JdbcTableSinkOperatorX final : public DataSinkOperatorX<JdbcTableSinkLocal
     vectorized::VExprContextSPtrs _output_vexpr_ctxs;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/join_build_sink_operator.h b/be/src/pipeline/exec/join_build_sink_operator.h
index 9d79a97397ff77..2a24f6a0492f3b 100644
--- a/be/src/pipeline/exec/join_build_sink_operator.h
+++ b/be/src/pipeline/exec/join_build_sink_operator.h
@@ -20,6 +20,7 @@
 #include "operator.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 template <typename LocalStateType>
 class JoinBuildSinkOperatorX;
 
@@ -78,4 +79,5 @@ class JoinBuildSinkOperatorX : public DataSinkOperatorX<LocalStateType> {
     const std::vector<TRuntimeFilterDesc> _runtime_filter_descs;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/join_probe_operator.h b/be/src/pipeline/exec/join_probe_operator.h
index 078806cea4fc5a..161fd18fa1dab8 100644
--- a/be/src/pipeline/exec/join_probe_operator.h
+++ b/be/src/pipeline/exec/join_probe_operator.h
@@ -20,6 +20,7 @@
 #include "operator.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 template <typename LocalStateType>
 class JoinProbeOperatorX;
 template <typename SharedStateArg, typename Derived>
@@ -123,4 +124,5 @@ class JoinProbeOperatorX : public StatefulOperatorX<LocalStateType> {
     const bool _use_specific_projections;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/memory_scratch_sink_operator.h b/be/src/pipeline/exec/memory_scratch_sink_operator.h
index c74659d15b96f2..352826955fca99 100644
--- a/be/src/pipeline/exec/memory_scratch_sink_operator.h
+++ b/be/src/pipeline/exec/memory_scratch_sink_operator.h
@@ -23,6 +23,7 @@
 #include "runtime/result_queue_mgr.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class MemoryScratchSinkOperatorX;
 class MemoryScratchSinkLocalState final : public PipelineXSinkLocalState<FakeSharedState> {
@@ -67,4 +68,5 @@ class MemoryScratchSinkOperatorX final : public DataSinkOperatorX<MemoryScratchS
     vectorized::VExprContextSPtrs _output_vexpr_ctxs;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/meta_scan_operator.h b/be/src/pipeline/exec/meta_scan_operator.h
index d717a5ab09ba7b..133fd4daabe5f5 100644
--- a/be/src/pipeline/exec/meta_scan_operator.h
+++ b/be/src/pipeline/exec/meta_scan_operator.h
@@ -26,6 +26,7 @@
 #include "pipeline/exec/scan_operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 namespace vectorized {
 class NewOlapScanner;
@@ -65,4 +66,5 @@ class MetaScanOperatorX final : public ScanOperatorX<MetaScanLocalState> {
     TUserIdentity _user_identity;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/multi_cast_data_stream_sink.h b/be/src/pipeline/exec/multi_cast_data_stream_sink.h
index 57b5974064b6a2..9d69b3fb5bdc9e 100644
--- a/be/src/pipeline/exec/multi_cast_data_stream_sink.h
+++ b/be/src/pipeline/exec/multi_cast_data_stream_sink.h
@@ -20,6 +20,7 @@
 #include "operator.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class MultiCastDataStreamSinkOperatorX;
 class MultiCastDataStreamSinkLocalState final
@@ -75,4 +76,5 @@ class MultiCastDataStreamSinkOperatorX final
     std::atomic<size_t> _num_dests;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/multi_cast_data_streamer.h b/be/src/pipeline/exec/multi_cast_data_streamer.h
index 51a73cf0c2b053..380538d0ac0805 100644
--- a/be/src/pipeline/exec/multi_cast_data_streamer.h
+++ b/be/src/pipeline/exec/multi_cast_data_streamer.h
@@ -20,6 +20,7 @@
 #include "vec/sink/vdata_stream_sender.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class Dependency;
 struct MultiCastBlock {
@@ -84,4 +85,5 @@ class MultiCastDataStreamer {
 
     std::vector<Dependency*> _dependencies;
 };
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
\ No newline at end of file
diff --git a/be/src/pipeline/exec/nested_loop_join_build_operator.h b/be/src/pipeline/exec/nested_loop_join_build_operator.h
index 5c41088a7059d4..11bcba2bd8fc3a 100644
--- a/be/src/pipeline/exec/nested_loop_join_build_operator.h
+++ b/be/src/pipeline/exec/nested_loop_join_build_operator.h
@@ -23,6 +23,7 @@
 #include "pipeline/exec/join_build_sink_operator.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class NestedLoopJoinBuildSinkOperatorX;
 
@@ -89,4 +90,5 @@ class NestedLoopJoinBuildSinkOperatorX final
     RowDescriptor _row_descriptor;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp b/be/src/pipeline/exec/olap_scan_operator.cpp
index 7b06e216b81bf7..34fa741ff1ec00 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -40,6 +40,7 @@
 #include "vec/functions/in.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 Status OlapScanLocalState::_init_profile() {
     RETURN_IF_ERROR(ScanLocalState<OlapScanLocalState>::_init_profile());
@@ -347,13 +348,13 @@ Status OlapScanLocalState::_init_scanners(std::list<vectorized::VScannerSPtr>* s
         int ranges_per_scanner =
                 std::max(1, (int)ranges->size() /
                                     std::min(scanners_per_tablet, size_based_scanners_per_tablet));
-        int num_ranges = ranges->size();
-        for (int i = 0; i < num_ranges;) {
+        int64_t num_ranges = ranges->size();
+        for (int64_t i = 0; i < num_ranges;) {
             std::vector<doris::OlapScanRange*> scanner_ranges;
             scanner_ranges.push_back((*ranges)[i].get());
             ++i;
-            for (int j = 1; i < num_ranges && j < ranges_per_scanner &&
-                            (*ranges)[i]->end_include == (*ranges)[i - 1]->end_include;
+            for (int64_t j = 1; i < num_ranges && j < ranges_per_scanner &&
+                                (*ranges)[i]->end_include == (*ranges)[i - 1]->end_include;
                  ++j, ++i) {
                 scanner_ranges.push_back((*ranges)[i].get());
             }
@@ -587,4 +588,5 @@ OlapScanOperatorX::OlapScanOperatorX(ObjectPool* pool, const TPlanNode& tnode, i
     }
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/olap_scan_operator.h b/be/src/pipeline/exec/olap_scan_operator.h
index 9e8624b3a0b255..91980d6a3f172b 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -26,6 +26,7 @@
 #include "pipeline/exec/scan_operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 
 namespace vectorized {
 class NewOlapScanner;
@@ -198,4 +199,5 @@ class OlapScanOperatorX final : public ScanOperatorX<OlapScanLocalState> {
     TQueryCacheParam _cache_param;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/olap_table_sink_operator.h b/be/src/pipeline/exec/olap_table_sink_operator.h
index 8a9ffaaf769c31..3453a57a67b9bc 100644
--- a/be/src/pipeline/exec/olap_table_sink_operator.h
+++ b/be/src/pipeline/exec/olap_table_sink_operator.h
@@ -21,6 +21,7 @@
 #include "vec/sink/writer/vtablet_writer.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class OlapTableSinkOperatorX;
 
@@ -75,4 +76,5 @@ class OlapTableSinkOperatorX final : public DataSinkOperatorX<OlapTableSinkLocal
     ObjectPool* _pool = nullptr;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/olap_table_sink_v2_operator.h b/be/src/pipeline/exec/olap_table_sink_v2_operator.h
index 6b5dadae399b77..4b55ec09efa40a 100644
--- a/be/src/pipeline/exec/olap_table_sink_v2_operator.h
+++ b/be/src/pipeline/exec/olap_table_sink_v2_operator.h
@@ -21,6 +21,7 @@
 #include "vec/sink/writer/vtablet_writer_v2.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class OlapTableSinkV2OperatorX;
 
@@ -76,4 +77,5 @@ class OlapTableSinkV2OperatorX final : public DataSinkOperatorX<OlapTableSinkV2L
     ObjectPool* _pool = nullptr;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/operator.cpp b/be/src/pipeline/exec/operator.cpp
index 09d0eef0f04340..f6664e147a3dab 100644
--- a/be/src/pipeline/exec/operator.cpp
+++ b/be/src/pipeline/exec/operator.cpp
@@ -83,6 +83,7 @@
 #include "vec/utils/util.hpp"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RowDescriptor;
 class RuntimeState;
 } // namespace doris
@@ -780,4 +781,5 @@ template class AsyncWriterSink<doris::vectorized::VTabletWriterV2, OlapTableSink
 template class AsyncWriterSink<doris::vectorized::VHiveTableWriter, HiveTableSinkOperatorX>;
 template class AsyncWriterSink<doris::vectorized::VIcebergTableWriter, IcebergTableSinkOperatorX>;
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h
index c84c4e7b43f981..a2c8e110cedac3 100644
--- a/be/src/pipeline/exec/operator.h
+++ b/be/src/pipeline/exec/operator.h
@@ -39,6 +39,7 @@
 #include "vec/runtime/vdata_stream_recvr.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RowDescriptor;
 class RuntimeState;
 class TDataSink;
@@ -859,4 +860,5 @@ class AsyncWriterSink : public PipelineXSinkLocalState<BasicSharedState> {
     std::shared_ptr<Dependency> _finish_dependency;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/partition_sort_sink_operator.cpp b/be/src/pipeline/exec/partition_sort_sink_operator.cpp
index 48b8fe9cb765a1..d0c28afe9de5ba 100644
--- a/be/src/pipeline/exec/partition_sort_sink_operator.cpp
+++ b/be/src/pipeline/exec/partition_sort_sink_operator.cpp
@@ -24,6 +24,7 @@
 #include "vec/common/hash_table/hash.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 Status PartitionSortSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) {
     RETURN_IF_ERROR(PipelineXSinkLocalState<PartitionSortNodeSharedState>::init(state, info));
@@ -66,7 +67,7 @@ PartitionSortSinkOperatorX::PartitionSortSinkOperatorX(ObjectPool* pool, int ope
           _pool(pool),
           _row_descriptor(descs, tnode.row_tuples, tnode.nullable_tuples),
           _limit(tnode.limit),
-          _partition_exprs_num(tnode.partition_sort_node.partition_exprs.size()),
+          _partition_exprs_num(cast_set<int>(tnode.partition_sort_node.partition_exprs.size())),
           _topn_phase(tnode.partition_sort_node.ptopn_phase),
           _has_global_limit(tnode.partition_sort_node.has_global_limit),
           _top_n_algorithm(tnode.partition_sort_node.top_n_algorithm),
@@ -212,7 +213,7 @@ Status PartitionSortSinkOperatorX::_emplace_into_hash_table(
                         };
 
                         SCOPED_TIMER(local_state._emplace_key_timer);
-                        int row = num_rows;
+                        int64_t row = num_rows;
                         for (row = row - 1; row >= 0 && !local_state._is_need_passthrough; --row) {
                             auto& mapped = *agg_method.lazy_emplace(state, row, creator,
                                                                     creator_for_null_key);
@@ -274,4 +275,5 @@ bool PartitionSortSinkLocalState::check_whether_need_passthrough() {
 }
 // NOLINTEND(readability-simplify-boolean-expr)
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/partition_sort_sink_operator.h b/be/src/pipeline/exec/partition_sort_sink_operator.h
index 6926445f18f2f4..32bbf38202713f 100644
--- a/be/src/pipeline/exec/partition_sort_sink_operator.h
+++ b/be/src/pipeline/exec/partition_sort_sink_operator.h
@@ -24,6 +24,7 @@
 #include "vec/common/sort/partition_sorter.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class PartitionSortSinkOperatorX;
 class PartitionSortSinkLocalState : public PipelineXSinkLocalState<PartitionSortNodeSharedState> {
@@ -110,4 +111,5 @@ class PartitionSortSinkOperatorX final : public DataSinkOperatorX<PartitionSortS
                                     PartitionSortSinkLocalState& local_state, bool eos);
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/partition_sort_source_operator.cpp b/be/src/pipeline/exec/partition_sort_source_operator.cpp
index 6d355477ab871c..f2da39f81afba6 100644
--- a/be/src/pipeline/exec/partition_sort_source_operator.cpp
+++ b/be/src/pipeline/exec/partition_sort_source_operator.cpp
@@ -20,6 +20,7 @@
 #include "pipeline/exec/operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -105,4 +106,5 @@ Status PartitionSortSourceOperatorX::get_sorted_block(RuntimeState* state,
 }
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/pipeline/exec/partition_sort_source_operator.h b/be/src/pipeline/exec/partition_sort_source_operator.h
index 1f75e1f49d4cf7..5ff654568e9ead 100644
--- a/be/src/pipeline/exec/partition_sort_source_operator.h
+++ b/be/src/pipeline/exec/partition_sort_source_operator.h
@@ -23,6 +23,7 @@
 #include "operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -63,4 +64,5 @@ class PartitionSortSourceOperatorX final : public OperatorX<PartitionSortSourceL
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp
index 5273960a5c1c29..3d763ce9f6309a 100644
--- a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp
@@ -27,6 +27,7 @@
 #include "vec/spill/spill_stream_manager.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 PartitionedAggSinkLocalState::PartitionedAggSinkLocalState(DataSinkOperatorXBase* parent,
                                                            RuntimeState* state)
         : Base(parent, state) {
@@ -319,4 +320,5 @@ Status PartitionedAggSinkLocalState::revoke_memory(RuntimeState* state) {
             std::move(spill_runnable));
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.h b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.h
index 15f6b22387a8e2..63cd95534dc9f5 100644
--- a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.h
+++ b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.h
@@ -23,6 +23,7 @@
 #include "vec/spill/spill_stream_manager.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 class PartitionedAggSinkOperatorX;
 class PartitionedAggSinkLocalState
         : public PipelineXSpillSinkLocalState<PartitionedAggSharedState> {
@@ -324,4 +325,5 @@ class PartitionedAggSinkOperatorX : public DataSinkOperatorX<PartitionedAggSinkL
 
     size_t _spill_partition_count_bits = 4;
 };
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
\ No newline at end of file
diff --git a/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp b/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp
index cdc6ef881d436d..55a06f98add1db 100644
--- a/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp
@@ -29,6 +29,7 @@
 #include "vec/spill/spill_stream_manager.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 PartitionedAggLocalState::PartitionedAggLocalState(RuntimeState* state, OperatorXBase* parent)
         : Base(state, parent) {}
@@ -295,4 +296,5 @@ Status PartitionedAggLocalState::initiate_merge_spill_partition_agg_data(Runtime
             std::make_shared<SpillRunnable>(state, _shared_state->shared_from_this(),
                                             exception_catch_func));
 }
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/partitioned_aggregation_source_operator.h b/be/src/pipeline/exec/partitioned_aggregation_source_operator.h
index 7e73241745e029..6fb0ecaba01e20 100644
--- a/be/src/pipeline/exec/partitioned_aggregation_source_operator.h
+++ b/be/src/pipeline/exec/partitioned_aggregation_source_operator.h
@@ -22,6 +22,7 @@
 #include "operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -99,4 +100,5 @@ class PartitionedAggSourceOperatorX : public OperatorX<PartitionedAggLocalState>
     std::unique_ptr<AggSourceOperatorX> _agg_source_operator;
 };
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp
index 20b25d54ff9f16..3e7f95374f53d2 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp
@@ -23,6 +23,7 @@
 #include "vec/spill/spill_stream_manager.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 PartitionedHashJoinProbeLocalState::PartitionedHashJoinProbeLocalState(RuntimeState* state,
                                                                        OperatorXBase* parent)
@@ -866,4 +867,5 @@ Status PartitionedHashJoinProbeOperatorX::get_block(RuntimeState* state, vectori
     return Status::OK();
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h
index f8fc0780b6fc3f..a19e88d7203e62 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h
+++ b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h
@@ -27,6 +27,7 @@
 #include "pipeline/exec/spill_utils.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -213,4 +214,5 @@ class PartitionedHashJoinProbeOperatorX final
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp
index 878c3870946f1c..852dccae71ca3b 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp
@@ -23,6 +23,7 @@
 #include "vec/spill/spill_stream_manager.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 Status PartitionedHashJoinSinkLocalState::init(doris::RuntimeState* state,
                                                doris::pipeline::LocalSinkStateInfo& info) {
@@ -246,11 +247,11 @@ Status PartitionedHashJoinSinkLocalState::revoke_memory(RuntimeState* state) {
         return _revoke_unpartitioned_block(state);
     }
 
-    _spilling_streams_count = _shared_state->partitioned_build_blocks.size();
+    _spilling_streams_count = cast_set<int>(_shared_state->partitioned_build_blocks.size());
 
     auto query_id = state->query_id();
 
-    for (size_t i = 0; i != _shared_state->partitioned_build_blocks.size(); ++i) {
+    for (int i = 0; i != _shared_state->partitioned_build_blocks.size(); ++i) {
         vectorized::SpillStreamSPtr& spilling_stream = _shared_state->spilled_streams[i];
         auto& mutable_block = _shared_state->partitioned_build_blocks[i];
 
@@ -555,4 +556,5 @@ Status PartitionedHashJoinSinkOperatorX::revoke_memory(RuntimeState* state) {
     return local_state.revoke_memory(state);
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h
index d1fe30e06f2dd2..e16e52dcaf9453 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h
+++ b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h
@@ -28,6 +28,7 @@
 #include "vec/runtime/partitioner.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -148,4 +149,5 @@ class PartitionedHashJoinSinkOperatorX
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/repeat_operator.cpp b/be/src/pipeline/exec/repeat_operator.cpp
index 5c94d43f0d1e05..48131e0d96e4c6 100644
--- a/be/src/pipeline/exec/repeat_operator.cpp
+++ b/be/src/pipeline/exec/repeat_operator.cpp
@@ -24,6 +24,7 @@
 #include "vec/core/block.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 } // namespace doris
 
@@ -221,8 +222,7 @@ Status RepeatOperatorX::pull(doris::RuntimeState* state, vectorized::Block* outp
 
             _repeat_id_idx++;
 
-            int size = _repeat_id_list.size();
-            if (_repeat_id_idx >= size) {
+            if (_repeat_id_idx >= _repeat_id_list.size()) {
                 _intermediate_block->clear();
                 _child_block.clear_column_data(_child->row_desc().num_materialized_slots());
                 _repeat_id_idx = 0;
@@ -251,4 +251,5 @@ Status RepeatOperatorX::pull(doris::RuntimeState* state, vectorized::Block* outp
     return Status::OK();
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/repeat_operator.h b/be/src/pipeline/exec/repeat_operator.h
index 31f88f37231aaa..2c2af32de0b0fb 100644
--- a/be/src/pipeline/exec/repeat_operator.h
+++ b/be/src/pipeline/exec/repeat_operator.h
@@ -23,6 +23,7 @@
 #include "pipeline/exec/operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -92,4 +93,5 @@ class RepeatOperatorX final : public StatefulOperatorX<RepeatLocalState> {
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/result_file_sink_operator.cpp b/be/src/pipeline/exec/result_file_sink_operator.cpp
index c65b9dda89d0ec..f806d9533d9e4c 100644
--- a/be/src/pipeline/exec/result_file_sink_operator.cpp
+++ b/be/src/pipeline/exec/result_file_sink_operator.cpp
@@ -28,6 +28,7 @@
 #include "vec/sink/vdata_stream_sender.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 ResultFileSinkLocalState::ResultFileSinkLocalState(DataSinkOperatorXBase* parent,
                                                    RuntimeState* state)
@@ -143,4 +144,5 @@ Status ResultFileSinkOperatorX::sink(RuntimeState* state, vectorized::Block* in_
     return local_state.sink(state, in_block, eos);
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/result_file_sink_operator.h b/be/src/pipeline/exec/result_file_sink_operator.h
index e9f2b8eeb9c670..c3c5e345f77e1a 100644
--- a/be/src/pipeline/exec/result_file_sink_operator.h
+++ b/be/src/pipeline/exec/result_file_sink_operator.h
@@ -21,6 +21,7 @@
 #include "vec/sink/writer/vfile_result_writer.h"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 class BroadcastPBlockHolder;
 } // namespace doris::vectorized
 
@@ -88,4 +89,5 @@ class ResultFileSinkOperatorX final : public DataSinkOperatorX<ResultFileSinkLoc
     std::shared_ptr<BufferControlBlock> _sender = nullptr;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/result_sink_operator.cpp b/be/src/pipeline/exec/result_sink_operator.cpp
index f8196910021b2c..8aeecbbddc12dc 100644
--- a/be/src/pipeline/exec/result_sink_operator.cpp
+++ b/be/src/pipeline/exec/result_sink_operator.cpp
@@ -35,6 +35,7 @@
 #include "vec/sink/vmysql_result_writer.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 Status ResultSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) {
     RETURN_IF_ERROR(Base::init(state, info));
@@ -208,4 +209,5 @@ Status ResultSinkLocalState::close(RuntimeState* state, Status exec_status) {
     return final_status;
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/result_sink_operator.h b/be/src/pipeline/exec/result_sink_operator.h
index 339c167825643b..479343ed6d5ea5 100644
--- a/be/src/pipeline/exec/result_sink_operator.h
+++ b/be/src/pipeline/exec/result_sink_operator.h
@@ -25,6 +25,7 @@
 #include "runtime/result_writer.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class BufferControlBlock;
 
 namespace pipeline {
@@ -172,4 +173,5 @@ class ResultSinkOperatorX final : public DataSinkOperatorX<ResultSinkLocalState>
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h
index 4519a3ca283f6f..c6c9cdf405d5a4 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -35,8 +35,9 @@
 #include "vec/utils/util.hpp"
 
 namespace doris::vectorized {
+#include "common/compile_check_begin.h"
 class ScannerDelegate;
-}
+} // namespace doris::vectorized
 
 namespace doris::pipeline {
 
@@ -436,4 +437,5 @@ class ScanOperatorX : public OperatorX<LocalStateType> {
     std::vector<int> topn_filter_source_node_ids;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/schema_scan_operator.cpp b/be/src/pipeline/exec/schema_scan_operator.cpp
index ddc2821cac14a1..2e2f80f5e24838 100644
--- a/be/src/pipeline/exec/schema_scan_operator.cpp
+++ b/be/src/pipeline/exec/schema_scan_operator.cpp
@@ -26,6 +26,7 @@
 #include "vec/data_types/data_type_factory.hpp"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 } // namespace doris
 
@@ -144,7 +145,7 @@ Status SchemaScanOperatorX::open(RuntimeState* state) {
         return Status::InternalError("Failed to get tuple descriptor.");
     }
 
-    _slot_num = _dest_tuple_desc->slots().size();
+    _slot_num = cast_set<int>(_dest_tuple_desc->slots().size());
     // get src tuple desc
     const auto* schema_table =
             static_cast<const SchemaTableDescriptor*>(_dest_tuple_desc->table_desc());
@@ -269,4 +270,5 @@ Status SchemaScanOperatorX::get_block(RuntimeState* state, vectorized::Block* bl
     return Status::OK();
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/schema_scan_operator.h b/be/src/pipeline/exec/schema_scan_operator.h
index c8ddf885e98a0f..2d861002748163 100644
--- a/be/src/pipeline/exec/schema_scan_operator.h
+++ b/be/src/pipeline/exec/schema_scan_operator.h
@@ -24,6 +24,7 @@
 #include "operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 } // namespace doris
 
@@ -88,4 +89,5 @@ class SchemaScanOperatorX final : public OperatorX<SchemaScanLocalState> {
     std::unique_ptr<SchemaScanner> _schema_scanner;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
\ No newline at end of file
diff --git a/be/src/pipeline/exec/select_operator.h b/be/src/pipeline/exec/select_operator.h
index 5370cd9e293c34..584a6f74308903 100644
--- a/be/src/pipeline/exec/select_operator.h
+++ b/be/src/pipeline/exec/select_operator.h
@@ -22,6 +22,7 @@
 #include "operator.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class SelectOperatorX;
 class SelectLocalState final : public PipelineXLocalState<FakeSharedState> {
@@ -55,4 +56,5 @@ class SelectOperatorX final : public StreamingOperatorX<SelectLocalState> {
     [[nodiscard]] bool is_source() const override { return false; }
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/set_probe_sink_operator.cpp b/be/src/pipeline/exec/set_probe_sink_operator.cpp
index 9ca87db925a44a..db487b0f9e7252 100644
--- a/be/src/pipeline/exec/set_probe_sink_operator.cpp
+++ b/be/src/pipeline/exec/set_probe_sink_operator.cpp
@@ -25,6 +25,7 @@
 #include "vec/common/hash_table/hash_table_set_probe.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace vectorized {
@@ -69,7 +70,7 @@ Status SetProbeSinkOperatorX<is_intersect>::sink(RuntimeState* state, vectorized
     SCOPED_TIMER(local_state.exec_time_counter());
     COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)in_block->rows());
 
-    auto probe_rows = in_block->rows();
+    uint32_t probe_rows = cast_set<uint32_t>(in_block->rows());
     if (probe_rows > 0) {
         {
             SCOPED_TIMER(local_state._extract_probe_data_timer);
@@ -220,8 +221,8 @@ void SetProbeSinkOperatorX<is_intersect>::_refresh_hash_table(
                                     ? (valid_element_in_hash_tbl <
                                        arg.hash_table
                                                ->size()) // When intersect, shrink as long as the element decreases
-                                    : (valid_element_in_hash_tbl <
-                                       arg.hash_table->size() *
+                                    : ((double)valid_element_in_hash_tbl <
+                                       (double)arg.hash_table->size() *
                                                need_shrink_ratio); // When except, element decreases need to within the 'need_shrink_ratio' before shrinking
 
                     if (is_need_shrink) {
@@ -269,4 +270,5 @@ template class SetProbeSinkLocalState<false>;
 template class SetProbeSinkOperatorX<true>;
 template class SetProbeSinkOperatorX<false>;
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/set_probe_sink_operator.h b/be/src/pipeline/exec/set_probe_sink_operator.h
index 368ea812cdfe01..6b764c1e509951 100644
--- a/be/src/pipeline/exec/set_probe_sink_operator.h
+++ b/be/src/pipeline/exec/set_probe_sink_operator.h
@@ -23,6 +23,7 @@
 #include "operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace vectorized {
@@ -116,4 +117,5 @@ class SetProbeSinkOperatorX final : public DataSinkOperatorX<SetProbeSinkLocalSt
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/sort_sink_operator.cpp b/be/src/pipeline/exec/sort_sink_operator.cpp
index 072f28723a36ea..6bec42ac62d192 100644
--- a/be/src/pipeline/exec/sort_sink_operator.cpp
+++ b/be/src/pipeline/exec/sort_sink_operator.cpp
@@ -25,6 +25,7 @@
 #include "vec/common/sort/topn_sorter.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 Status SortSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) {
     RETURN_IF_ERROR(Base::init(state, info));
@@ -176,4 +177,5 @@ void SortSinkOperatorX::reset(RuntimeState* state) {
     auto& local_state = get_local_state(state);
     local_state._shared_state->sorter->reset();
 }
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/sort_sink_operator.h b/be/src/pipeline/exec/sort_sink_operator.h
index 6bf87164e71026..766c6c0ffc9a59 100644
--- a/be/src/pipeline/exec/sort_sink_operator.h
+++ b/be/src/pipeline/exec/sort_sink_operator.h
@@ -23,6 +23,7 @@
 #include "vec/core/field.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 class SortSinkOperatorX;
 
@@ -109,4 +110,5 @@ class SortSinkOperatorX final : public DataSinkOperatorX<SortSinkLocalState> {
     const bool _reuse_mem;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/sort_source_operator.cpp b/be/src/pipeline/exec/sort_source_operator.cpp
index 7f801b79c0b12b..2fb09d7278fda8 100644
--- a/be/src/pipeline/exec/sort_source_operator.cpp
+++ b/be/src/pipeline/exec/sort_source_operator.cpp
@@ -22,6 +22,7 @@
 #include "pipeline/exec/operator.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 SortLocalState::SortLocalState(RuntimeState* state, OperatorXBase* parent)
         : PipelineXLocalState<SortSharedState>(state, parent) {}
@@ -79,4 +80,5 @@ Status SortSourceOperatorX::build_merger(RuntimeState* state,
     return Status::OK();
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/sort_source_operator.h b/be/src/pipeline/exec/sort_source_operator.h
index 20714eb44e5e60..a638b04b368eaa 100644
--- a/be/src/pipeline/exec/sort_source_operator.h
+++ b/be/src/pipeline/exec/sort_source_operator.h
@@ -23,6 +23,7 @@
 #include "operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -69,4 +70,5 @@ class SortSourceOperatorX final : public OperatorX<SortLocalState> {
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/spill_utils.h b/be/src/pipeline/exec/spill_utils.h
index 925e7df44e607e..2ba6f22a60b10c 100644
--- a/be/src/pipeline/exec/spill_utils.h
+++ b/be/src/pipeline/exec/spill_utils.h
@@ -26,6 +26,7 @@
 #include "vec/runtime/partitioner.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 using SpillPartitionerType = vectorized::Crc32HashPartitioner<vectorized::SpillPartitionChannelIds>;
 
 class SpillRunnable : public Runnable {
@@ -70,4 +71,5 @@ class SpillRunnable : public Runnable {
     std::function<void()> _func;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
\ No newline at end of file
diff --git a/be/src/pipeline/exec/streaming_aggregation_operator.cpp b/be/src/pipeline/exec/streaming_aggregation_operator.cpp
index 1c8d2c47bc698a..b6e5788a07c626 100644
--- a/be/src/pipeline/exec/streaming_aggregation_operator.cpp
+++ b/be/src/pipeline/exec/streaming_aggregation_operator.cpp
@@ -29,6 +29,7 @@
 #include "vec/exprs/vslot_ref.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 } // namespace doris
 
@@ -228,7 +229,7 @@ Status StreamingAggLocalState::_merge_with_serialized_key_helper(vectorized::Blo
         }
     }
 
-    int rows = block->rows();
+    size_t rows = block->rows();
     if (_places.size() < rows) {
         _places.resize(rows);
     }
@@ -270,7 +271,7 @@ Status StreamingAggLocalState::_merge_with_serialized_key_helper(vectorized::Blo
 
         for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
             if (_aggregate_evaluators[i]->is_merge() || for_spill) {
-                int col_id = 0;
+                size_t col_id = 0;
                 if constexpr (for_spill) {
                     col_id = _probe_expr_ctxs.size() + i;
                 } else {
@@ -403,7 +404,7 @@ Status StreamingAggLocalState::_execute_with_serialized_key_helper(vectorized::B
         }
     }
 
-    int rows = block->rows();
+    size_t rows = block->rows();
     if (_places.size() < rows) {
         _places.resize(rows);
     }
@@ -542,8 +543,8 @@ bool StreamingAggLocalState::_should_expand_preagg_hash_tables() {
                         const int64_t aggregated_input_rows = input_rows - _cur_num_rows_returned;
                         // TODO chenhao
                         //  const int64_t expected_input_rows = estimated_input_cardinality_ - num_rows_returned_;
-                        double current_reduction =
-                                static_cast<double>(aggregated_input_rows) / ht_rows;
+                        double current_reduction = static_cast<double>(aggregated_input_rows) /
+                                                   static_cast<double>(ht_rows);
 
                         // TODO: workaround for IMPALA-2490: subplan node rows_returned counter may be
                         // inaccurate, which could lead to a divide by zero below.
@@ -615,7 +616,7 @@ Status StreamingAggLocalState::_pre_agg_with_serialized_key(doris::vectorized::B
         }
     }
 
-    int rows = in_block->rows();
+    size_t rows = in_block->rows();
     _places.resize(rows);
 
     // Stop expanding hash tables if we're not reducing the input sufficiently. As our
@@ -739,7 +740,7 @@ Status StreamingAggLocalState::_get_with_serialized_key_result(RuntimeState* sta
 
     auto columns_with_schema =
             vectorized::VectorizedUtils::create_columns_with_type_and_name(p._row_descriptor);
-    int key_size = _probe_expr_ctxs.size();
+    size_t key_size = _probe_expr_ctxs.size();
 
     vectorized::MutableColumns key_columns;
     for (int i = 0; i < key_size; ++i) {
@@ -750,7 +751,7 @@ Status StreamingAggLocalState::_get_with_serialized_key_result(RuntimeState* sta
         }
     }
     vectorized::MutableColumns value_columns;
-    for (int i = key_size; i < columns_with_schema.size(); ++i) {
+    for (size_t i = key_size; i < columns_with_schema.size(); ++i) {
         if (!mem_reuse) {
             value_columns.emplace_back(columns_with_schema[i].type->create_column());
         } else {
@@ -852,7 +853,7 @@ Status StreamingAggLocalState::_get_results_without_key(RuntimeState* state,
     block->clear();
 
     DCHECK(_agg_data->without_key != nullptr);
-    int agg_size = _aggregate_evaluators.size();
+    const auto agg_size = _aggregate_evaluators.size();
 
     vectorized::MutableColumns value_columns(agg_size);
     std::vector<vectorized::DataTypePtr> data_types(agg_size);
@@ -888,8 +889,8 @@ Status StreamingAggLocalState::_get_results_with_serialized_key(RuntimeState* st
                                                                 bool* eos) {
     SCOPED_TIMER(_get_results_timer);
     auto& p = _parent->cast<StreamingAggOperatorX>();
-    int key_size = _probe_expr_ctxs.size();
-    int agg_size = _aggregate_evaluators.size();
+    const auto key_size = _probe_expr_ctxs.size();
+    const auto agg_size = _aggregate_evaluators.size();
     vectorized::MutableColumns value_columns(agg_size);
     vectorized::DataTypes value_data_types(agg_size);
 
@@ -1013,7 +1014,7 @@ Status StreamingAggLocalState::_get_without_key_result(RuntimeState* state,
 
     auto& p = _parent->cast<StreamingAggOperatorX>();
     *block = vectorized::VectorizedUtils::create_empty_columnswithtypename(p._row_descriptor);
-    int agg_size = _aggregate_evaluators.size();
+    const auto agg_size = _aggregate_evaluators.size();
 
     vectorized::MutableColumns columns(agg_size);
     std::vector<vectorized::DataTypePtr> data_types(agg_size);
@@ -1170,8 +1171,8 @@ Status StreamingAggOperatorX::open(RuntimeState* state) {
     DCHECK_EQ(_intermediate_tuple_desc->slots().size(), _output_tuple_desc->slots().size());
     RETURN_IF_ERROR(vectorized::VExpr::prepare(_probe_expr_ctxs, state, _child->row_desc()));
 
-    int j = _probe_expr_ctxs.size();
-    for (int i = 0; i < j; ++i) {
+    size_t j = _probe_expr_ctxs.size();
+    for (size_t i = 0; i < j; ++i) {
         auto nullable_output = _output_tuple_desc->slots()[i]->is_nullable();
         auto nullable_input = _probe_expr_ctxs[i]->root()->is_nullable();
         if (nullable_output != nullable_input) {
@@ -1179,7 +1180,7 @@ Status StreamingAggOperatorX::open(RuntimeState* state) {
             _make_nullable_keys.emplace_back(i);
         }
     }
-    for (int i = 0; i < _aggregate_evaluators.size(); ++i, ++j) {
+    for (size_t i = 0; i < _aggregate_evaluators.size(); ++i, ++j) {
         SlotDescriptor* intermediate_slot_desc = _intermediate_tuple_desc->slots()[j];
         SlotDescriptor* output_slot_desc = _output_tuple_desc->slots()[j];
         RETURN_IF_ERROR(_aggregate_evaluators[i]->prepare(
@@ -1290,4 +1291,5 @@ bool StreamingAggOperatorX::need_more_input_data(RuntimeState* state) const {
     return local_state._pre_aggregated_block->empty() && !local_state._child_eos;
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/streaming_aggregation_operator.h b/be/src/pipeline/exec/streaming_aggregation_operator.h
index b695880ac2857b..bd35cd940f2974 100644
--- a/be/src/pipeline/exec/streaming_aggregation_operator.h
+++ b/be/src/pipeline/exec/streaming_aggregation_operator.h
@@ -27,6 +27,7 @@
 #include "vec/core/block.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -237,4 +238,5 @@ class StreamingAggOperatorX final : public StatefulOperatorX<StreamingAggLocalSt
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/table_function_operator.cpp b/be/src/pipeline/exec/table_function_operator.cpp
index c1621470f435b4..acd867fcc6d158 100644
--- a/be/src/pipeline/exec/table_function_operator.cpp
+++ b/be/src/pipeline/exec/table_function_operator.cpp
@@ -24,6 +24,7 @@
 #include "vec/exprs/table_function/table_function_factory.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 } // namespace doris
 
@@ -195,7 +196,10 @@ Status TableFunctionLocalState::get_expanded_block(RuntimeState* state,
             DCHECK_LE(1, p._fn_num);
             auto repeat_times = _fns[p._fn_num - 1]->get_value(
                     columns[p._child_slots.size() + p._fn_num - 1],
-                    state->batch_size() - columns[p._child_slots.size()]->size());
+                    //// It has already been checked that
+                    // columns[p._child_slots.size()]->size() < state->batch_size(),
+                    // so columns[p._child_slots.size()]->size() will not exceed the range of int.
+                    state->batch_size() - (int)columns[p._child_slots.size()]->size());
             _current_row_insert_times += repeat_times;
             for (int i = 0; i < p._fn_num - 1; i++) {
                 _fns[i]->get_same_many_values(columns[i + p._child_slots.size()], repeat_times);
@@ -276,7 +280,7 @@ Status TableFunctionOperatorX::init(const TPlanNode& tnode, RuntimeState* state)
         fn->set_expr_context(ctx);
         _fns.push_back(fn);
     }
-    _fn_num = _fns.size();
+    _fn_num = cast_set<int>(_fns.size());
 
     // Prepare output slot ids
     RETURN_IF_ERROR(_prepare_output_slot_ids(tnode));
@@ -304,7 +308,7 @@ Status TableFunctionOperatorX::open(doris::RuntimeState* state) {
         }
     }
 
-    for (size_t i = 0; i < _child_slots.size(); i++) {
+    for (int i = 0; i < _child_slots.size(); i++) {
         if (_slot_need_copy(i)) {
             _output_slot_indexs.push_back(i);
         } else {
@@ -315,4 +319,5 @@ Status TableFunctionOperatorX::open(doris::RuntimeState* state) {
     return vectorized::VExpr::open(_vfn_ctxs, state);
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/table_function_operator.h b/be/src/pipeline/exec/table_function_operator.h
index 81160acb7f7611..9aa26e9ae22b10 100644
--- a/be/src/pipeline/exec/table_function_operator.h
+++ b/be/src/pipeline/exec/table_function_operator.h
@@ -24,6 +24,7 @@
 #include "vec/exprs/table_function/table_function.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 } // namespace doris
 
@@ -154,4 +155,5 @@ class TableFunctionOperatorX final : public StatefulOperatorX<TableFunctionLocal
     std::vector<int> _child_slot_sizes;
 };
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/union_sink_operator.cpp b/be/src/pipeline/exec/union_sink_operator.cpp
index 8467eeb1d5467a..56491b5258bc55 100644
--- a/be/src/pipeline/exec/union_sink_operator.cpp
+++ b/be/src/pipeline/exec/union_sink_operator.cpp
@@ -19,6 +19,7 @@
 
 #include <utility>
 
+#include "common/cast_set.h"
 #include "common/compiler_util.h" // IWYU pragma: keep
 #include "common/status.h"
 #include "pipeline/exec/data_queue.h"
@@ -27,6 +28,7 @@
 #include "util/runtime_profile.h"
 
 namespace doris::pipeline {
+#include "common/compile_check_begin.h"
 
 Status UnionSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) {
     RETURN_IF_ERROR(Base::init(state, info));
@@ -54,7 +56,8 @@ Status UnionSinkLocalState::open(RuntimeState* state) {
 UnionSinkOperatorX::UnionSinkOperatorX(int child_id, int sink_id, ObjectPool* pool,
                                        const TPlanNode& tnode, const DescriptorTbl& descs)
         : Base(sink_id, tnode.node_id, tnode.node_id),
-          _first_materialized_child_idx(tnode.union_node.first_materialized_child_idx),
+          _first_materialized_child_idx(
+                  cast_set<int>(tnode.union_node.first_materialized_child_idx)),
           _row_descriptor(descs, tnode.row_tuples, tnode.nullable_tuples),
           _cur_child_id(child_id),
           _child_size(tnode.num_children) {}
@@ -130,4 +133,5 @@ Status UnionSinkOperatorX::sink(RuntimeState* state, vectorized::Block* in_block
     return Status::OK();
 }
 
+#include "common/compile_check_end.h"
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/union_sink_operator.h b/be/src/pipeline/exec/union_sink_operator.h
index aa94ed9a73038f..3a8880622cb108 100644
--- a/be/src/pipeline/exec/union_sink_operator.h
+++ b/be/src/pipeline/exec/union_sink_operator.h
@@ -26,6 +26,7 @@
 #include "vec/core/block.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -152,4 +153,5 @@ class UnionSinkOperatorX final : public DataSinkOperatorX<UnionSinkLocalState> {
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/pipeline/exec/union_source_operator.cpp b/be/src/pipeline/exec/union_source_operator.cpp
index ecaaf22922b657..d13658488e2c9b 100644
--- a/be/src/pipeline/exec/union_source_operator.cpp
+++ b/be/src/pipeline/exec/union_source_operator.cpp
@@ -30,6 +30,7 @@
 #include "vec/core/block.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace pipeline {
@@ -148,7 +149,7 @@ Status UnionSourceOperatorX::get_next_const(RuntimeState* state, vectorized::Blo
         vectorized::Block tmp_block;
         tmp_block.insert({vectorized::ColumnUInt8::create(1),
                           std::make_shared<vectorized::DataTypeUInt8>(), ""});
-        int const_expr_lists_size = _const_expr_lists[_const_expr_list_idx].size();
+        int const_expr_lists_size = cast_set<int>(_const_expr_lists[_const_expr_list_idx].size());
         if (_const_expr_list_idx && const_expr_lists_size != _const_expr_lists[0].size()) {
             return Status::InternalError(
                     "[UnionNode]const expr at {}'s count({}) not matched({} expected)",
@@ -183,4 +184,5 @@ Status UnionSourceOperatorX::get_next_const(RuntimeState* state, vectorized::Blo
 }
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
diff --git a/be/src/pipeline/exec/union_source_operator.h b/be/src/pipeline/exec/union_source_operator.h
index 200e7de8597b91..0ee66c3da7447b 100644
--- a/be/src/pipeline/exec/union_source_operator.h
+++ b/be/src/pipeline/exec/union_source_operator.h
@@ -24,6 +24,7 @@
 #include "operator.h"
 
 namespace doris {
+#include "common/compile_check_begin.h"
 class RuntimeState;
 
 namespace vectorized {
@@ -123,4 +124,5 @@ class UnionSourceOperatorX final : public OperatorX<UnionSourceLocalState> {
 };
 
 } // namespace pipeline
+#include "common/compile_check_end.h"
 } // namespace doris
\ No newline at end of file

From eefa66d0f0a835e2fb96af4acc499a1da61a3602 Mon Sep 17 00:00:00 2001
From: Yongqiang YANG <yangyongqiang@selectdb.com>
Date: Wed, 18 Dec 2024 10:27:48 +0800
Subject: [PATCH 382/399] [fix](vault) avoid encrypt twice when altering vault
 (#45156)

---
 .../meta-service/meta_service_resource.cpp    |  44 +++--
 cloud/test/meta_service_test.cpp              |   5 +-
 docker/runtime/doris-compose/command.py       |   4 +-
 .../apache/doris/catalog/StorageVaultMgr.java |   3 +
 .../vault_p0/alter/test_alter_s3_vault.groovy | 157 +++++++++++++++++-
 .../alter/test_alter_use_path_style.groovy    |  24 +++
 6 files changed, 212 insertions(+), 25 deletions(-)

diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp
index f26c2d26200dd3..4fa8cc5a132296 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -648,10 +648,19 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Tran
         obj_info.has_provider()) {
         code = MetaServiceCode::INVALID_ARGUMENT;
         std::stringstream ss;
-        ss << "Only ak, sk can be altered";
+        ss << "Bucket, endpoint, prefix and provider can not be altered";
         msg = ss.str();
         return -1;
     }
+
+    if (obj_info.has_ak() ^ obj_info.has_sk()) {
+        code = MetaServiceCode::INVALID_ARGUMENT;
+        std::stringstream ss;
+        ss << "Accesskey and secretkey must be alter together";
+        msg = ss.str();
+        return -1;
+    }
+
     const auto& name = vault.name();
     // Here we try to get mutable iter since we might need to alter the vault name
     auto name_itr = std::find_if(instance.mutable_storage_vault_names()->begin(),
@@ -703,22 +712,25 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Tran
         *name_itr = vault.alter_name();
     }
     auto origin_vault_info = new_vault.DebugString();
-    AkSkPair pre {new_vault.obj_info().ak(), new_vault.obj_info().sk()};
-    const auto& plain_ak = obj_info.has_ak() ? obj_info.ak() : new_vault.obj_info().ak();
-    const auto& plain_sk = obj_info.has_ak() ? obj_info.sk() : new_vault.obj_info().sk();
-    AkSkPair plain_ak_sk_pair {plain_ak, plain_sk};
-    AkSkPair cipher_ak_sk_pair;
-    EncryptionInfoPB encryption_info;
-    auto ret = encrypt_ak_sk_helper(plain_ak, plain_sk, &encryption_info, &cipher_ak_sk_pair, code,
-                                    msg);
-    if (ret != 0) {
-        msg = "failed to encrypt";
-        code = MetaServiceCode::ERR_ENCRYPT;
-        LOG(WARNING) << msg;
-        return -1;
+
+    // For ak or sk is not altered.
+    EncryptionInfoPB encryption_info = new_vault.obj_info().encryption_info();
+    AkSkPair new_ak_sk_pair {new_vault.obj_info().ak(), new_vault.obj_info().sk()};
+
+    if (obj_info.has_ak()) {
+        // ak and sk must be altered together, there is check before.
+        auto ret = encrypt_ak_sk_helper(obj_info.ak(), obj_info.sk(), &encryption_info,
+                                        &new_ak_sk_pair, code, msg);
+        if (ret != 0) {
+            msg = "failed to encrypt";
+            code = MetaServiceCode::ERR_ENCRYPT;
+            LOG(WARNING) << msg;
+            return -1;
+        }
     }
-    new_vault.mutable_obj_info()->set_ak(cipher_ak_sk_pair.first);
-    new_vault.mutable_obj_info()->set_sk(cipher_ak_sk_pair.second);
+
+    new_vault.mutable_obj_info()->set_ak(new_ak_sk_pair.first);
+    new_vault.mutable_obj_info()->set_sk(new_ak_sk_pair.second);
     new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info);
     if (obj_info.has_use_path_style()) {
         new_vault.mutable_obj_info()->set_use_path_style(obj_info.use_path_style());
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index ce5eda9ed7a46c..3bf20facca1217 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -564,7 +564,7 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) {
         AlterObjStoreInfoResponse res;
         meta_service->alter_storage_vault(
                 reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
-        ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+        ASSERT_NE(res.status().code(), MetaServiceCode::OK) << res.status().msg();
         InstanceInfoPB instance;
         get_test_instance(instance);
 
@@ -575,7 +575,7 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) {
                   TxnErrorCode::TXN_OK);
         StorageVaultPB get_obj;
         get_obj.ParseFromString(val);
-        ASSERT_EQ(get_obj.obj_info().ak(), "new_ak") << get_obj.obj_info().ak();
+        ASSERT_EQ(get_obj.obj_info().ak(), "ak") << get_obj.obj_info().ak();
     }
 
     {
@@ -627,6 +627,7 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) {
         vault.set_alter_name(new_vault_name);
         ObjectStoreInfoPB obj;
         obj_info.set_ak("new_ak");
+        obj_info.set_sk("new_sk");
         vault.mutable_obj_info()->MergeFrom(obj);
         vault.set_name(vault_name);
         req.mutable_vault()->CopyFrom(vault);
diff --git a/docker/runtime/doris-compose/command.py b/docker/runtime/doris-compose/command.py
index 638c1c465d75b1..df3d47cabd9cf3 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -942,8 +942,8 @@ def run(self, args):
 metaServiceHttpAddress = "{ms_endpoint}"
 metaServiceToken = "greedisgood9999"
 recycleServiceHttpAddress = "{recycle_endpoint}"
-instanceId = "default_instance_id"
-multiClusterInstance = "default_instance_id"
+instanceId = "12345678"
+multiClusterInstance = "12345678"
 multiClusterBes = "{multi_cluster_bes}"
 cloudUniqueId= "{fe_cloud_unique_id}"
 '''
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
index 762acc7bed28d0..4014219c5a49b6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
@@ -168,6 +168,9 @@ public void alterStorageVault(StorageVaultType type, Map<String, String> propert
             }
             LOG.info("Succeed to alter storage vault {}, id {}, origin default vault replaced {}",
                     name, response.getStorageVaultId(), response.getDefaultStorageVaultReplaced());
+
+            // Make BE eagerly fetch the storage vault info from Meta Service
+            ALTER_BE_SYNC_THREAD_POOL.execute(() -> alterSyncVaultTask());
         } catch (RpcException e) {
             LOG.warn("failed to alter storage vault due to RpcException: {}", e);
             throw new DdlException(e.getMessage());
diff --git a/regression-test/suites/vault_p0/alter/test_alter_s3_vault.groovy b/regression-test/suites/vault_p0/alter/test_alter_s3_vault.groovy
index 723422c6e0b84d..ffe67c77bc1fce 100644
--- a/regression-test/suites/vault_p0/alter/test_alter_s3_vault.groovy
+++ b/regression-test/suites/vault_p0/alter/test_alter_s3_vault.groovy
@@ -42,6 +42,43 @@ suite("test_alter_s3_vault", "nonConcurrent") {
         );
     """
 
+    def dupVaultName = "${suiteName}" + "_dup"
+    sql """
+        CREATE STORAGE VAULT IF NOT EXISTS ${dupVaultName}
+        PROPERTIES (
+            "type"="S3",
+            "s3.endpoint"="${getS3Endpoint()}",
+            "s3.region" = "${getS3Region()}",
+            "s3.access_key" = "${getS3AK()}",
+            "s3.secret_key" = "${getS3SK()}",
+            "s3.root.path" = "${suiteName}",
+            "s3.bucket" = "${getS3BucketName()}",
+            "s3.external_endpoint" = "",
+            "provider" = "${getS3Provider()}"
+        );
+    """
+
+    sql """
+        DROP TABLE IF EXISTS alter_s3_vault_tbl
+        """
+
+    sql """
+        CREATE TABLE IF NOT EXISTS alter_s3_vault_tbl
+        (
+        `k1` INT NULL,
+        `v1` INT NULL
+        )
+        UNIQUE KEY (k1)
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+        PROPERTIES (
+        "replication_num" = "1",
+        "disable_auto_compaction" = "true",
+        "storage_vault_name" = "${suiteName}"
+        );
+    """
+
+    sql """insert into alter_s3_vault_tbl values(2, 2); """
+
     expectExceptionLike({
         sql """
             ALTER STORAGE VAULT ${suiteName}
@@ -62,33 +99,125 @@ suite("test_alter_s3_vault", "nonConcurrent") {
         """
     }, "Alter property")
 
+    expectExceptionLike({
+        sql """
+            ALTER STORAGE VAULT ${suiteName}
+            PROPERTIES (
+            "type"="S3",
+            "s3.access_key" = "new_ak"
+            );
+        """
+    }, "Accesskey and secretkey must be alter together")
 
     def vaultName = suiteName
-    String properties;
+    def String properties;
 
-    def vaultInfos = try_sql """show storage vault"""
+    def vaultInfos = try_sql """show storage vaults"""
 
     for (int i = 0; i < vaultInfos.size(); i++) {
         def name = vaultInfos[i][0]
+        logger.info("name is ${name}, info ${vaultInfos[i]}")
         if (name.equals(vaultName)) {
             properties = vaultInfos[i][2]
         }
     }
 
-    def newVaultName = suiteName + "_new";
+    // alter ak sk
+    sql """
+        ALTER STORAGE VAULT ${vaultName}
+        PROPERTIES (
+            "type"="S3",
+            "s3.access_key" = "${getS3AK()}",
+            "s3.secret_key" = "${getS3SK()}"
+        );
+    """
+
+    vaultInfos = sql """SHOW STORAGE VAULT;"""
+
+    for (int i = 0; i < vaultInfos.size(); i++) {
+        def name = vaultInfos[i][0]
+        logger.info("name is ${name}, info ${vaultInfos[i]}")
+        if (name.equals(vaultName)) {
+            def newProperties = vaultInfos[i][2]
+            assert properties == newProperties, "Properties are not the same"
+        }
+    }
+
+    sql """insert into alter_s3_vault_tbl values("2", "2"); """
+
+
+    // rename
+    newVaultName = vaultName + "_new";
+
+    sql """
+        ALTER STORAGE VAULT ${vaultName}
+        PROPERTIES (
+            "type"="S3",
+            "VAULT_NAME" = "${newVaultName}"
+        );
+    """
+
+    vaultInfos = sql """SHOW STORAGE VAULT;"""
+    for (int i = 0; i < vaultInfos.size(); i++) {
+        def name = vaultInfos[i][0]
+        logger.info("name is ${name}, info ${vaultInfos[i]}")
+        if (name.equals(newVaultName)) {
+            def newProperties = vaultInfos[i][2]
+            assert properties == newProperties, "Properties are not the same"
+        }
+        if (name.equals(vaultName)) {
+            assertTrue(false);
+        }
+    }
+
+    sql """insert into alter_s3_vault_tbl values("2", "2"); """
+
+    // rename + aksk
+    vaultName = newVaultName
+    newVaultName = vaultName + "_new";
 
     sql """
         ALTER STORAGE VAULT ${vaultName}
         PROPERTIES (
             "type"="S3",
             "VAULT_NAME" = "${newVaultName}",
-            "s3.access_key" = "new_ak"
+            "s3.access_key" = "${getS3AK()}",
+            "s3.secret_key" = "${getS3SK()}"
         );
     """
 
+    vaultInfos = sql """SHOW STORAGE VAULT;"""
+    for (int i = 0; i < vaultInfos.size(); i++) {
+        def name = vaultInfos[i][0]
+        logger.info("name is ${name}, info ${vaultInfos[i]}")
+        if (name.equals(newVaultName)) {
+            def newProperties = vaultInfos[i][2]
+            assert properties == newProperties, "Properties are not the same"
+        }
+        if (name.equals(vaultName)) {
+            assertTrue(false);
+        }
+    }
+    sql """insert into alter_s3_vault_tbl values("2", "2"); """
+
+
+    vaultName = newVaultName;
+
+    newVaultName = vaultName + "_new";
+
     vaultInfos = sql """SHOW STORAGE VAULT;"""
     boolean exist = false
 
+    sql """
+        ALTER STORAGE VAULT ${vaultName}
+        PROPERTIES (
+            "type"="S3",
+            "VAULT_NAME" = "${newVaultName}",
+            "s3.access_key" = "new_ak_ak",
+            "s3.secret_key" = "sk"
+        );
+    """
+
     for (int i = 0; i < vaultInfos.size(); i++) {
         def name = vaultInfos[i][0]
         logger.info("name is ${name}, info ${vaultInfos[i]}")
@@ -96,11 +225,29 @@ suite("test_alter_s3_vault", "nonConcurrent") {
             assertTrue(false);
         }
         if (name.equals(newVaultName)) {
-            assertTrue(vaultInfos[i][2].contains("new_ak"))
+            assertTrue(vaultInfos[i][2].contains("new_ak_ak"))
             exist = true
         }
     }
     assertTrue(exist)
+
+    vaultName = newVaultName;
+
+    expectExceptionLike({
+        sql """
+            ALTER STORAGE VAULT ${vaultName}
+            PROPERTIES (
+                "type"="S3",
+                "VAULT_NAME" = "${dupVaultName}",
+                "s3.access_key" = "new_ak_ak",
+                "s3.secret_key" = "sk"
+            );
+        """
+    }, "already exists")
+
+    def count = sql """ select count() from alter_s3_vault_tbl; """
+    assertTrue(res[0][0] == 4)
+
     // failed to insert due to the wrong ak
     expectExceptionLike({ sql """insert into alter_s3_vault_tbl values("2", "2");""" }, "")
 }
diff --git a/regression-test/suites/vault_p0/alter/test_alter_use_path_style.groovy b/regression-test/suites/vault_p0/alter/test_alter_use_path_style.groovy
index cc9289f49e0c9a..4aaeb7ec472034 100644
--- a/regression-test/suites/vault_p0/alter/test_alter_use_path_style.groovy
+++ b/regression-test/suites/vault_p0/alter/test_alter_use_path_style.groovy
@@ -43,6 +43,23 @@ suite("test_alter_use_path_style", "nonConcurrent") {
         );
     """
 
+    sql """
+        CREATE TABLE IF NOT EXISTS alter_use_path_style_tbl
+        (
+        `k1` INT NULL,
+        `v1` INT NULL
+        )
+        UNIQUE KEY (k1)
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+        PROPERTIES (
+        "replication_num" = "1",
+        "disable_auto_compaction" = "true",
+        "storage_vault_name" = "${suiteName}"
+        );
+    """
+
+    sql """ insert into alter_use_path_style_tbl values(2, 2); """
+
     sql """
         ALTER STORAGE VAULT ${suiteName}
         PROPERTIES (
@@ -51,6 +68,8 @@ suite("test_alter_use_path_style", "nonConcurrent") {
         );
     """
 
+    sql """ insert into alter_use_path_style_tbl values(2, 2); """
+
     def vaultInfos = sql """ SHOW STORAGE VAULT; """
     boolean exist = false
 
@@ -73,6 +92,8 @@ suite("test_alter_use_path_style", "nonConcurrent") {
         );
     """
 
+    sql """ insert into alter_use_path_style_tbl values(2, 2); """
+
     vaultInfos = sql """ SHOW STORAGE VAULT; """
     exist = false
 
@@ -105,4 +126,7 @@ suite("test_alter_use_path_style", "nonConcurrent") {
             );
         """
     }, "Invalid use_path_style value")
+
+    def count = sql """ select count() from alter_use_path_style_tbl; """
+    assertTrue(res[0][0] == 3)
 }
\ No newline at end of file

From 1253ef3dc674f4c2b7ac9824dedb708d7c1bb6d0 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Wed, 18 Dec 2024 10:32:32 +0800
Subject: [PATCH 383/399] [refactor](exchange) remove duplicate code (#45466)

---
 be/src/pipeline/exec/exchange_sink_operator.cpp | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/be/src/pipeline/exec/exchange_sink_operator.cpp b/be/src/pipeline/exec/exchange_sink_operator.cpp
index aa893fc0a26f2e..e7fed76be8fa16 100644
--- a/be/src/pipeline/exec/exchange_sink_operator.cpp
+++ b/be/src/pipeline/exec/exchange_sink_operator.cpp
@@ -112,6 +112,7 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf
         }
     }
     only_local_exchange = local_size == channels.size();
+    _rpc_channels_num = channels.size() - local_size;
 
     if (!only_local_exchange) {
         _sink_buffer = p.get_sink_buffer(state->fragment_instance_id().lo);
@@ -206,17 +207,12 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) {
         std::mt19937 g(rd());
         shuffle(channels.begin(), channels.end(), g);
     }
-    size_t local_size = 0;
     for (int i = 0; i < channels.size(); ++i) {
         RETURN_IF_ERROR(channels[i]->open(state));
         if (channels[i]->is_local()) {
-            local_size++;
             _last_local_channel_idx = i;
         }
     }
-    only_local_exchange = local_size == channels.size();
-
-    _rpc_channels_num = channels.size() - local_size;
 
     PUniqueId id;
     id.set_hi(_state->query_id().hi);
@@ -228,7 +224,7 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) {
                 _parent->operator_id(), _parent->node_id(), "BroadcastDependency", true);
         _broadcast_pb_mem_limiter =
                 vectorized::BroadcastPBlockHolderMemLimiter::create_shared(_broadcast_dependency);
-    } else if (local_size > 0) {
+    } else if (!only_local_exchange) {
         size_t dep_id = 0;
         for (auto& channel : channels) {
             if (channel->is_local()) {

From 3f94c8a5fd7810959cec5b9296366adcc6da7eb4 Mon Sep 17 00:00:00 2001
From: Mryange <yanxuecheng@selectdb.com>
Date: Wed, 18 Dec 2024 11:16:47 +0800
Subject: [PATCH 384/399] [fix](expr)Remove the _can_fast_execute flag from
 VExpr. (#45542)

In the past, a _can_fast_execute flag was maintained in VExpr.
However, since exec executes concurrently, errors would occur when using
the _can_fast_execute judgment.

In fact, we don't need the _can_fast_execute variable
because fast_execute will perform the necessary checks itself.
---
 be/src/vec/exprs/vcompound_pred.h       |  4 +---
 be/src/vec/exprs/vectorized_fn_call.cpp |  2 +-
 be/src/vec/exprs/vexpr.cpp              |  2 --
 be/src/vec/exprs/vexpr.h                |  1 -
 be/src/vec/exprs/vin_predicate.cpp      | 28 +++++++++----------------
 be/src/vec/exprs/vin_predicate.h        |  2 --
 be/src/vec/exprs/vmatch_predicate.cpp   |  2 +-
 7 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/be/src/vec/exprs/vcompound_pred.h b/be/src/vec/exprs/vcompound_pred.h
index 8c6832a7a35a37..ff7649600b4c7f 100644
--- a/be/src/vec/exprs/vcompound_pred.h
+++ b/be/src/vec/exprs/vcompound_pred.h
@@ -147,15 +147,13 @@ class VCompoundPred : public VectorizedFnCall {
         }
 
         if (all_pass && !res.is_empty()) {
-            // set fast_execute when expr evaluated by inverted index correctly
-            _can_fast_execute = true;
             context->get_inverted_index_context()->set_inverted_index_result_for_expr(this, res);
         }
         return Status::OK();
     }
 
     Status execute(VExprContext* context, Block* block, int* result_column_id) override {
-        if (_can_fast_execute && fast_execute(context, block, result_column_id)) {
+        if (fast_execute(context, block, result_column_id)) {
             return Status::OK();
         }
         if (get_num_children() == 1 || _has_const_child()) {
diff --git a/be/src/vec/exprs/vectorized_fn_call.cpp b/be/src/vec/exprs/vectorized_fn_call.cpp
index fc69b4047ceb5f..43aa24090393a1 100644
--- a/be/src/vec/exprs/vectorized_fn_call.cpp
+++ b/be/src/vec/exprs/vectorized_fn_call.cpp
@@ -149,7 +149,7 @@ Status VectorizedFnCall::_do_execute(doris::vectorized::VExprContext* context,
     if (is_const_and_have_executed()) { // const have executed in open function
         return get_result_from_const(block, _expr_name, result_column_id);
     }
-    if (_can_fast_execute && fast_execute(context, block, result_column_id)) {
+    if (fast_execute(context, block, result_column_id)) {
         return Status::OK();
     }
     DBUG_EXECUTE_IF("VectorizedFnCall.must_in_slow_path", {
diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp
index c40bcbcab57f23..7cfb96e77118fb 100644
--- a/be/src/vec/exprs/vexpr.cpp
+++ b/be/src/vec/exprs/vexpr.cpp
@@ -735,8 +735,6 @@ Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBase
         for (int column_id : column_ids) {
             index_context->set_true_for_inverted_index_status(this, column_id);
         }
-        // set fast_execute when expr evaluated by inverted index correctly
-        _can_fast_execute = true;
     }
     return Status::OK();
 }
diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h
index 2580cf7ddc88f9..953fbaa9c38c8d 100644
--- a/be/src/vec/exprs/vexpr.h
+++ b/be/src/vec/exprs/vexpr.h
@@ -328,7 +328,6 @@ class VExpr {
 
     // ensuring uniqueness during index traversal
     uint32_t _index_unique_id = 0;
-    bool _can_fast_execute = false;
     bool _enable_inverted_index_query = true;
 };
 
diff --git a/be/src/vec/exprs/vin_predicate.cpp b/be/src/vec/exprs/vin_predicate.cpp
index ce6270c2e62dea..4b1622589be608 100644
--- a/be/src/vec/exprs/vin_predicate.cpp
+++ b/be/src/vec/exprs/vin_predicate.cpp
@@ -84,10 +84,6 @@ Status VInPredicate::prepare(RuntimeState* state, const RowDescriptor& desc,
     if (state->query_options().__isset.in_list_value_count_threshold) {
         _in_list_value_count_threshold = state->query_options().in_list_value_count_threshold;
     }
-
-    const auto in_list_value_count = _children.size() - 1;
-    // When the number of values in the IN condition exceeds this threshold, fast_execute will not be used
-    _can_fast_execute = in_list_value_count <= _in_list_value_count_threshold;
     return Status::OK();
 }
 
@@ -108,17 +104,6 @@ Status VInPredicate::open(RuntimeState* state, VExprContext* context,
     return Status::OK();
 }
 
-size_t VInPredicate::skip_constant_args_size() const {
-    if (_is_args_all_constant && !_can_fast_execute) {
-        // This is an optimization. For expressions like colA IN (1, 2, 3, 4),
-        // where all values inside the IN clause are constants,
-        // a hash set is created during open, and it will not be accessed again during execute
-        //  Here, _children[0] is colA
-        return 1;
-    }
-    return _children.size();
-}
-
 void VInPredicate::close(VExprContext* context, FunctionContext::FunctionStateScope scope) {
     VExpr::close_function_context(context, scope, _function);
     VExpr::close(context, scope);
@@ -133,12 +118,19 @@ Status VInPredicate::execute(VExprContext* context, Block* block, int* result_co
     if (is_const_and_have_executed()) { // const have execute in open function
         return get_result_from_const(block, _expr_name, result_column_id);
     }
-    if (_can_fast_execute && fast_execute(context, block, result_column_id)) {
+    if (fast_execute(context, block, result_column_id)) {
         return Status::OK();
     }
     DCHECK(_open_finished || _getting_const_col);
-    doris::vectorized::ColumnNumbers arguments(skip_constant_args_size());
-    for (int i = 0; i < skip_constant_args_size(); ++i) {
+
+    // This is an optimization. For expressions like colA IN (1, 2, 3, 4),
+    // where all values inside the IN clause are constants,
+    // a hash set is created during open, and it will not be accessed again during execute
+    //  Here, _children[0] is colA
+    const size_t args_size = _is_args_all_constant ? 1 : _children.size();
+
+    doris::vectorized::ColumnNumbers arguments(args_size);
+    for (int i = 0; i < args_size; ++i) {
         int column_id = -1;
         RETURN_IF_ERROR(_children[i]->execute(context, block, &column_id));
         arguments[i] = column_id;
diff --git a/be/src/vec/exprs/vin_predicate.h b/be/src/vec/exprs/vin_predicate.h
index 024ad68f2ba576..d9bc6e3cab9978 100644
--- a/be/src/vec/exprs/vin_predicate.h
+++ b/be/src/vec/exprs/vin_predicate.h
@@ -51,8 +51,6 @@ class VInPredicate final : public VExpr {
 
     std::string debug_string() const override;
 
-    size_t skip_constant_args_size() const;
-
     const FunctionBasePtr function() { return _function; }
 
     bool is_not_in() const { return _is_not_in; };
diff --git a/be/src/vec/exprs/vmatch_predicate.cpp b/be/src/vec/exprs/vmatch_predicate.cpp
index 333d0ef389f724..61541909883c56 100644
--- a/be/src/vec/exprs/vmatch_predicate.cpp
+++ b/be/src/vec/exprs/vmatch_predicate.cpp
@@ -136,7 +136,7 @@ Status VMatchPredicate::evaluate_inverted_index(VExprContext* context, uint32_t
 
 Status VMatchPredicate::execute(VExprContext* context, Block* block, int* result_column_id) {
     DCHECK(_open_finished || _getting_const_col);
-    if (_can_fast_execute && fast_execute(context, block, result_column_id)) {
+    if (fast_execute(context, block, result_column_id)) {
         return Status::OK();
     }
     DBUG_EXECUTE_IF("VMatchPredicate.execute", {

From 9a4253f0f668b5376af154345304bc7e3c0f33af Mon Sep 17 00:00:00 2001
From: Uniqueyou <wangyixuan@selectdb.com>
Date: Wed, 18 Dec 2024 11:27:20 +0800
Subject: [PATCH 385/399] [fix](restore) Filter table property storage_policy
 (#44756)

---
 .../src/main/java/org/apache/doris/catalog/TableProperty.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 1ac556c6846c05..9c1e09d7d1624f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -467,6 +467,7 @@ public boolean isBeingSynced() {
 
     public void removeInvalidProperties() {
         properties.remove(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY);
+        storagePolicy = "";
         properties.remove(PropertyAnalyzer.PROPERTIES_COLOCATE_WITH);
     }
 

From fa10bdde5d9e4d105cf4d43c6298c9f712a46b53 Mon Sep 17 00:00:00 2001
From: Petrichor <xiaowenli@selectdb.com>
Date: Wed, 18 Dec 2024 11:29:49 +0800
Subject: [PATCH 386/399] [feat](nereids)support switch command in nereids
 (#45561)

Issue Number: close https://github.com/apache/doris/issues/42525
---
 .../org/apache/doris/nereids/DorisParser.g4   |  6 +-
 .../nereids/parser/LogicalPlanBuilder.java    | 11 +++
 .../doris/nereids/trees/plans/PlanType.java   |  3 +-
 .../plans/commands/use/SwitchCommand.java     | 94 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 +
 .../ddl/use/switch_command_nereids.groovy     | 22 +++++
 6 files changed, 139 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/use/SwitchCommand.java
 create mode 100644 regression-test/suites/nereids_p0/ddl/use/switch_command_nereids.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 11796c5df3b5b3..64ff9209bfe6f2 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -62,6 +62,7 @@ statementBase
     | supportedCancelStatement          #supportedCancelStatementAlias
     | supportedRecoverStatement         #supportedRecoverStatementAlias
     | supportedAdminStatement           #supportedAdminStatementAlias
+    | supportedUseStatement             #supportedUseStatementAlias
     | unsupportedStatement              #unsupported
     ;
 
@@ -867,10 +868,13 @@ supportedUnsetStatement
     | UNSET DEFAULT STORAGE VAULT
     ;
 
+supportedUseStatement
+     : SWITCH catalog=identifier                                                      #switchCatalog
+     ;
+
 unsupportedUseStatement
     : USE (catalog=identifier DOT)? database=identifier                              #useDatabase
     | USE ((catalog=identifier DOT)? database=identifier)? ATSIGN cluster=identifier #useCloudCluster
-    | SWITCH catalog=identifier                                                      #switchCatalog
     ;
 
 unsupportedDmlStatement
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 085ad4458d0b3d..951c8b342d5d9e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -302,6 +302,7 @@
 import org.apache.doris.nereids.DorisParser.SubqueryContext;
 import org.apache.doris.nereids.DorisParser.SubqueryExpressionContext;
 import org.apache.doris.nereids.DorisParser.SupportedUnsetStatementContext;
+import org.apache.doris.nereids.DorisParser.SwitchCatalogContext;
 import org.apache.doris.nereids.DorisParser.SyncContext;
 import org.apache.doris.nereids.DorisParser.SystemVariableContext;
 import org.apache.doris.nereids.DorisParser.TableAliasContext;
@@ -669,6 +670,7 @@
 import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
+import org.apache.doris.nereids.trees.plans.commands.use.SwitchCommand;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
 import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
@@ -5105,5 +5107,14 @@ public LogicalPlan visitShowQueryProfile(ShowQueryProfileContext ctx) {
         String queryIdPath = stripQuotes(ctx.queryIdPath.getText());
         return new ShowQueryProfileCommand(queryIdPath);
     }
+
+    @Override
+    public Object visitSwitchCatalog(SwitchCatalogContext ctx) {
+        String catalogName = ctx.catalog.getText();
+        if (catalogName != null) {
+            return new SwitchCommand(catalogName);
+        }
+        throw new AnalysisException("catalog name can not be null");
+    }
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index e9488239b8ac2f..9ed408dfe05b45 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -259,5 +259,6 @@ public enum PlanType {
     CREATE_FILE_COMMAND,
     CREATE_ROUTINE_LOAD_COMMAND,
     SHOW_TABLE_CREATION_COMMAND,
-    SHOW_QUERY_PROFILE_COMMAND
+    SHOW_QUERY_PROFILE_COMMAND,
+    SWITCH_COMMAND
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/use/SwitchCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/use/SwitchCommand.java
new file mode 100644
index 00000000000000..61e822e8c58060
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/use/SwitchCommand.java
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.use;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.NoForward;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * switch command.
+ */
+public class SwitchCommand extends Command implements NoForward {
+    private static final Logger LOG = LogManager.getLogger(SwitchCommand.class);
+    private final String catalogName;
+
+    public SwitchCommand(String catalogName) {
+        super(PlanType.SWITCH_COMMAND);
+        this.catalogName = catalogName;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // validate catalog access
+        validate(ctx);
+        handleSwitchStmt(ctx);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitSwitchCommand(this, context);
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.SWITCH;
+    }
+
+    public String toSql() {
+        return "SWITCH `" + catalogName + "`";
+    }
+
+    private void validate(ConnectContext context) throws AnalysisException {
+        Util.checkCatalogAllRules(catalogName);
+
+        if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv(
+                ConnectContext.get(), catalogName, PrivPredicate.SHOW)) {
+            ErrorReport.reportAnalysisException(
+                    ErrorCode.ERR_CATALOG_ACCESS_DENIED, context.getQualifiedUser(), catalogName);
+        }
+    }
+
+    /**
+     * Process switch catalog.
+     */
+    private void handleSwitchStmt(ConnectContext context) {
+        try {
+            context.getEnv().changeCatalog(context, catalogName);
+        } catch (DdlException e) {
+            LOG.warn("handle switch command failed! ", e);
+            context.getState().setError(e.getMysqlErrorCode(), e.getMessage());
+            return;
+        }
+        context.getState().setOk();
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index ff71331b7afc5e..eb764be57c0097 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -144,6 +144,7 @@
 import org.apache.doris.nereids.trees.plans.commands.load.CreateRoutineLoadCommand;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
+import org.apache.doris.nereids.trees.plans.commands.use.SwitchCommand;
 
 /** CommandVisitor. */
 public interface CommandVisitor<R, C> {
@@ -662,4 +663,8 @@ default R visitShowQueryProfileCommand(ShowQueryProfileCommand showQueryProfileC
                                            C context) {
         return visitCommand(showQueryProfileCommand, context);
     }
+
+    default R visitSwitchCommand(SwitchCommand switchCommand, C context) {
+        return visitCommand(switchCommand, context);
+    }
 }
diff --git a/regression-test/suites/nereids_p0/ddl/use/switch_command_nereids.groovy b/regression-test/suites/nereids_p0/ddl/use/switch_command_nereids.groovy
new file mode 100644
index 00000000000000..c7658983c61b06
--- /dev/null
+++ b/regression-test/suites/nereids_p0/ddl/use/switch_command_nereids.groovy
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("switch_command_nereids") {
+    // switch internal;
+    checkNereidsExecute("""switch internal;""")
+    checkNereidsExecute("""switch internal;""")
+}
\ No newline at end of file

From a8864637701093e5aefe8996bc68ad88a944e143 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Wed, 18 Dec 2024 14:33:10 +0800
Subject: [PATCH 387/399] [fix](Nereids) set correct sort key for aggregate 
 (#45369)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What problem does this PR solve?
in previous #44042, we supported more patterns for PushTopnToAgg rule.
the new pattern：
topn
  +-->agg(global)
      +-->shuffle
          +-->agg(local)

In order to support this new pattern, the group by keys and orderkeys
are identical, but group keys can be in different order.
that is
topn(orderkey=[B,A])->agg(groupkey=[A,B,C])
=>
topn(orderkey=[B, A, C]) ->agg(groupKey=[A, B, C])
---
 .../doris/nereids/jobs/executor/Analyzer.java |   2 +-
 .../doris/nereids/jobs/executor/Rewriter.java |   2 +
 .../nereids/processor/post/PushTopnToAgg.java | 161 ++++---------
 .../apache/doris/nereids/rules/RuleType.java  |   1 +
 .../rules/analysis/CompressedMaterialize.java |  12 +-
 .../rules/rewrite/LimitAggToTopNAgg.java      | 221 ++++++++++++------
 .../rules/rewrite/SimplifyEncodeDecode.java   |  68 ++++++
 .../nereids/trees/plans/algebra/Project.java  |  13 ++
 .../trees/plans/logical/LogicalAggregate.java |   5 +
 .../org/apache/doris/qe/SessionVariable.java  |   2 -
 .../query_p0/limit/test_group_by_limit.out    |  26 +--
 ...distinct_through_join_one_side_cust.groovy |  48 ++--
 .../tpch/push_topn_to_agg.groovy              | 111 ++-------
 .../query_p0/limit/test_group_by_limit.groovy | 101 ++++++--
 14 files changed, 428 insertions(+), 345 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyEncodeDecode.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
index 8985dadc0bd2e9..03dbb6c7110a7c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
@@ -167,7 +167,7 @@ private static List<RewriteJob> buildAnalyzerJobs(Optional<CustomTableResolver>
             topDown(new EliminateGroupByConstant()),
 
             topDown(new SimplifyAggGroupBy()),
-            topDown(new CompressedMaterialize()),
+            bottomUp(new CompressedMaterialize()),
             topDown(new NormalizeAggregate()),
             topDown(new HavingToFilter()),
             topDown(new QualifyToFilter()),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index dbaaf2a6b32244..5b276258263f37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -132,6 +132,7 @@
 import org.apache.doris.nereids.rules.rewrite.ReduceAggregateChildOutputRows;
 import org.apache.doris.nereids.rules.rewrite.ReorderJoin;
 import org.apache.doris.nereids.rules.rewrite.RewriteCteChildren;
+import org.apache.doris.nereids.rules.rewrite.SimplifyEncodeDecode;
 import org.apache.doris.nereids.rules.rewrite.SimplifyWindowExpression;
 import org.apache.doris.nereids.rules.rewrite.SplitLimit;
 import org.apache.doris.nereids.rules.rewrite.SumLiteralRewrite;
@@ -371,6 +372,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
                         //       generate one PhysicalLimit if current distribution is gather or two
                         //       PhysicalLimits with gather exchange
                         topDown(new LimitSortToTopN()),
+                        topDown(new SimplifyEncodeDecode()),
                         topDown(new LimitAggToTopNAgg()),
                         topDown(new MergeTopNs()),
                         topDown(new SplitLimit()),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushTopnToAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushTopnToAgg.java
index aca3f21a7d175f..d3161a0dba36b6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushTopnToAgg.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushTopnToAgg.java
@@ -21,157 +21,84 @@
 package org.apache.doris.nereids.processor.post;
 
 import org.apache.doris.nereids.CascadesContext;
-import org.apache.doris.nereids.properties.DistributionSpecGather;
-import org.apache.doris.nereids.properties.OrderKey;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.plans.AggMode;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate.TopnPushInfo;
-import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
 import org.apache.doris.qe.ConnectContext;
 
-import org.apache.hadoop.util.Lists;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
 /**
- * Add SortInfo to Agg. This SortInfo is used as boundary, not used to sort elements.
+ * Add TopNInfo to Agg. This TopNInfo is used as boundary, not used to sort elements.
  * example
  * sql: select count(*) from orders group by o_clerk order by o_clerk limit 1;
  * plan: topn(1) -> aggGlobal -> shuffle -> aggLocal -> scan
  * optimization: aggLocal and aggGlobal only need to generate the smallest row with respect to o_clerk.
  *
- * TODO: the following case is not covered:
- * sql: select sum(o_shippriority) from orders group by o_clerk limit 1;
- * plan: limit -> aggGlobal -> shuffle -> aggLocal -> scan
- * aggGlobal may receive partial aggregate results, and hence is not supported now
- * instance1: input (key=2, v=1) => localAgg => (2, 1) => aggGlobal inst1 => (2, 1)
- * instance2: input (key=1, v=1), (key=2, v=2) => localAgg inst2 => (1, 1)
- * (2,1),(1,1) => limit => may output (2, 1), which is not complete, missing (2, 2) in instance2
- *
- *TOPN:
- *  Precondition: topn orderkeys are the prefix of group keys
- *  TODO: topnKeys could be subset of groupKeys. This will be implemented in future
- *  Pattern 2-phase agg:
- *     topn -> aggGlobal -> distribute -> aggLocal
- *     =>
- *     topn(n) -> aggGlobal(topn=n) -> distribute -> aggLocal(topn=n)
- *  Pattern 1-phase agg:
- *     topn->agg->Any(not agg) -> topn -> agg(topn=n) -> any
- *
- *  LIMIT:
- *  Pattern 1: limit->agg(1phase)->any
- *  Pattern 2: limit->agg(global)->gather->agg(local)
+ * This rule only applies to the patterns
+ * 1. topn->project->agg, or
+ * 2. topn->agg
+ * that
+ * 1. orderKeys and groupkeys are one-one mapping
+ * 2. aggregate is not scalar agg
+ * Refer to LimitAggToTopNAgg rule.
  */
 public class PushTopnToAgg extends PlanPostProcessor {
     @Override
     public Plan visitPhysicalTopN(PhysicalTopN<? extends Plan> topN, CascadesContext ctx) {
         topN.child().accept(this, ctx);
-        if (ConnectContext.get().getSessionVariable().topnOptLimitThreshold <= topN.getLimit() + topN.getOffset()) {
+        if (ConnectContext.get().getSessionVariable().topnOptLimitThreshold <= topN.getLimit() + topN.getOffset()
+                && !ConnectContext.get().getSessionVariable().pushTopnToAgg) {
             return topN;
         }
-        Plan topnChild = topN.child();
-        if (topnChild instanceof PhysicalProject) {
-            topnChild = topnChild.child(0);
+        Plan topNChild = topN.child();
+        if (topNChild instanceof PhysicalProject) {
+            topNChild = topNChild.child(0);
         }
-        if (topnChild instanceof PhysicalHashAggregate) {
-            PhysicalHashAggregate<? extends Plan> upperAgg = (PhysicalHashAggregate<? extends Plan>) topnChild;
-            List<OrderKey> orderKeys = tryGenerateOrderKeyByGroupKeyAndTopnKey(topN, upperAgg);
-            if (!orderKeys.isEmpty()) {
-
-                if (upperAgg.getAggPhase().isGlobal() && upperAgg.getAggMode() == AggMode.BUFFER_TO_RESULT) {
-                    upperAgg.setTopnPushInfo(new TopnPushInfo(
-                            orderKeys,
-                            topN.getLimit() + topN.getOffset()));
-                    if (upperAgg.child() instanceof PhysicalDistribute
-                            && upperAgg.child().child(0) instanceof PhysicalHashAggregate) {
-                        PhysicalHashAggregate<? extends Plan> bottomAgg =
-                                (PhysicalHashAggregate<? extends Plan>) upperAgg.child().child(0);
+        if (topNChild instanceof PhysicalHashAggregate) {
+            PhysicalHashAggregate<? extends Plan> upperAgg = (PhysicalHashAggregate<? extends Plan>) topNChild;
+            if (isGroupKeyIdenticalToOrderKey(topN, upperAgg)) {
+                upperAgg.setTopnPushInfo(new TopnPushInfo(
+                        topN.getOrderKeys(),
+                        topN.getLimit() + topN.getOffset()));
+                if (upperAgg.child() instanceof PhysicalDistribute
+                        && upperAgg.child().child(0) instanceof PhysicalHashAggregate) {
+                    PhysicalHashAggregate<? extends Plan> bottomAgg =
+                            (PhysicalHashAggregate<? extends Plan>) upperAgg.child().child(0);
+                    if (isGroupKeyIdenticalToOrderKey(topN, bottomAgg)) {
+                        bottomAgg.setTopnPushInfo(new TopnPushInfo(
+                                topN.getOrderKeys(),
+                                topN.getLimit() + topN.getOffset()));
+                    }
+                } else if (upperAgg.child() instanceof PhysicalHashAggregate) {
+                    // multi-distinct plan
+                    PhysicalHashAggregate<? extends Plan> bottomAgg =
+                            (PhysicalHashAggregate<? extends Plan>) upperAgg.child();
+                    if (isGroupKeyIdenticalToOrderKey(topN, bottomAgg)) {
                         bottomAgg.setTopnPushInfo(new TopnPushInfo(
-                                orderKeys,
+                                topN.getOrderKeys(),
                                 topN.getLimit() + topN.getOffset()));
                     }
-                } else if (upperAgg.getAggPhase().isLocal() && upperAgg.getAggMode() == AggMode.INPUT_TO_RESULT) {
-                    // one phase agg
-                    upperAgg.setTopnPushInfo(new TopnPushInfo(
-                            orderKeys,
-                            topN.getLimit() + topN.getOffset()));
                 }
             }
         }
         return topN;
     }
 
-    /**
-     return true, if topn order-key is prefix of agg group-key, ignore asc/desc and null_first
-     TODO order-key can be subset of group-key. BE does not support now.
-     */
-    private List<OrderKey> tryGenerateOrderKeyByGroupKeyAndTopnKey(PhysicalTopN<? extends Plan> topN,
-            PhysicalHashAggregate<? extends Plan> agg) {
-        List<OrderKey> orderKeys = Lists.newArrayListWithCapacity(agg.getGroupByExpressions().size());
-        if (topN.getOrderKeys().size() > agg.getGroupByExpressions().size()) {
-            return orderKeys;
-        }
-        List<Expression> topnKeys = topN.getOrderKeys().stream()
-                .map(OrderKey::getExpr).collect(Collectors.toList());
-        for (int i = 0; i < topN.getOrderKeys().size(); i++) {
-            // prefix check
-            if (!topnKeys.get(i).equals(agg.getGroupByExpressions().get(i))) {
-                return Lists.newArrayList();
-            }
-            orderKeys.add(topN.getOrderKeys().get(i));
-        }
-        for (int i = topN.getOrderKeys().size(); i < agg.getGroupByExpressions().size(); i++) {
-            orderKeys.add(new OrderKey(agg.getGroupByExpressions().get(i), true, false));
-        }
-        return orderKeys;
-    }
-
-    @Override
-    public Plan visitPhysicalLimit(PhysicalLimit<? extends Plan> limit, CascadesContext ctx) {
-        limit.child().accept(this, ctx);
-        if (ConnectContext.get().getSessionVariable().topnOptLimitThreshold <= limit.getLimit() + limit.getOffset()) {
-            return limit;
+    private boolean isGroupKeyIdenticalToOrderKey(PhysicalTopN<? extends Plan> topN,
+                                                                PhysicalHashAggregate<? extends Plan> agg) {
+        if (topN.getOrderKeys().size() != agg.getGroupByExpressions().size()) {
+            return false;
         }
-        Plan limitChild = limit.child();
-        if (limitChild instanceof PhysicalProject) {
-            limitChild = limitChild.child(0);
-        }
-        if (limitChild instanceof PhysicalHashAggregate) {
-            PhysicalHashAggregate<? extends Plan> upperAgg = (PhysicalHashAggregate<? extends Plan>) limitChild;
-            if (upperAgg.getAggPhase().isGlobal() && upperAgg.getAggMode() == AggMode.BUFFER_TO_RESULT) {
-                Plan child = upperAgg.child();
-                Plan grandChild = child.child(0);
-                if (child instanceof PhysicalDistribute
-                        && ((PhysicalDistribute<?>) child).getDistributionSpec() instanceof DistributionSpecGather
-                        && grandChild instanceof PhysicalHashAggregate) {
-                    upperAgg.setTopnPushInfo(new TopnPushInfo(
-                            generateOrderKeyByGroupKey(upperAgg),
-                            limit.getLimit() + limit.getOffset()));
-                    PhysicalHashAggregate<? extends Plan> bottomAgg =
-                            (PhysicalHashAggregate<? extends Plan>) grandChild;
-                    bottomAgg.setTopnPushInfo(new TopnPushInfo(
-                            generateOrderKeyByGroupKey(bottomAgg),
-                            limit.getLimit() + limit.getOffset()));
-                }
-            } else if (upperAgg.getAggMode() == AggMode.INPUT_TO_RESULT) {
-                // 1-phase agg
-                upperAgg.setTopnPushInfo(new TopnPushInfo(
-                        generateOrderKeyByGroupKey(upperAgg),
-                        limit.getLimit() + limit.getOffset()));
+        for (int i = 0; i < agg.getGroupByExpressions().size(); i++) {
+            Expression groupByKey = agg.getGroupByExpressions().get(i);
+            Expression orderKey = topN.getOrderKeys().get(i).getExpr();
+            if (!groupByKey.equals(orderKey)) {
+                return false;
             }
         }
-        return limit;
-    }
-
-    private List<OrderKey> generateOrderKeyByGroupKey(PhysicalHashAggregate<? extends Plan> agg) {
-        return agg.getGroupByExpressions().stream()
-                .map(key -> new OrderKey(key, true, false))
-                .collect(Collectors.toList());
+        return true;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index e25050b012dc5f..d348889818a5dd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -108,6 +108,7 @@ public enum RuleType {
     // rewrite rules
     COMPRESSED_MATERIALIZE_AGG(RuleTypeClass.REWRITE),
     COMPRESSED_MATERIALIZE_SORT(RuleTypeClass.REWRITE),
+    SIMPLIFY_ENCODE_DECODE(RuleTypeClass.REWRITE),
     NORMALIZE_AGGREGATE(RuleTypeClass.REWRITE),
     NORMALIZE_SORT(RuleTypeClass.REWRITE),
     NORMALIZE_REPEAT(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CompressedMaterialize.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CompressedMaterialize.java
index 7d8a7664f821f6..814269a63984a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CompressedMaterialize.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CompressedMaterialize.java
@@ -78,7 +78,7 @@ public List<Rule> buildRules() {
 
     private LogicalSort<Plan> compressMaterializeSort(LogicalSort<Plan> sort) {
         List<OrderKey> newOrderKeys = Lists.newArrayList();
-        boolean changed = false;
+        List<Expression> orderKeysToEncode = Lists.newArrayList();
         for (OrderKey orderKey : sort.getOrderKeys()) {
             Expression expr = orderKey.getExpr();
             Optional<Expression> encode = getEncodeExpression(expr);
@@ -86,12 +86,18 @@ private LogicalSort<Plan> compressMaterializeSort(LogicalSort<Plan> sort) {
                 newOrderKeys.add(new OrderKey(encode.get(),
                         orderKey.isAsc(),
                         orderKey.isNullFirst()));
-                changed = true;
+                orderKeysToEncode.add(expr);
             } else {
                 newOrderKeys.add(orderKey);
             }
         }
-        return changed ? sort.withOrderKeys(newOrderKeys) : sort;
+        if (orderKeysToEncode.isEmpty()) {
+            return sort;
+        } else {
+            sort = sort.withOrderKeys(newOrderKeys);
+            return sort;
+        }
+
     }
 
     private Optional<Expression> getEncodeExpression(Expression expression) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java
index dfa1230a8f8f0e..f0f9c4347252d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LimitAggToTopNAgg.java
@@ -17,24 +17,29 @@
 
 package org.apache.doris.nereids.rules.rewrite;
 
+import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.properties.OrderKey;
 import org.apache.doris.nereids.rules.Rule;
 import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
-import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
 import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
+import java.util.HashMap;
 import java.util.List;
-import java.util.Optional;
+import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
@@ -54,109 +59,171 @@ public List<Rule> buildRules() {
                                 && ConnectContext.get().getSessionVariable().pushTopnToAgg
                                 && ConnectContext.get().getSessionVariable().topnOptLimitThreshold
                                 >= limit.getLimit() + limit.getOffset())
+                        .when(limit -> {
+                            LogicalAggregate<? extends Plan> agg = limit.child();
+                            return !agg.getGroupByExpressions().isEmpty() && !agg.getSourceRepeat().isPresent();
+                        })
                         .then(limit -> {
                             LogicalAggregate<? extends Plan> agg = limit.child();
-                            Optional<OrderKey> orderKeysOpt = tryGenerateOrderKeyByTheFirstGroupKey(agg);
-                            if (!orderKeysOpt.isPresent()) {
-                                return null;
-                            }
-                            List<OrderKey> orderKeys = Lists.newArrayList(orderKeysOpt.get());
+                            List<OrderKey> orderKeys = generateOrderKeyByGroupKey(agg);
                             return new LogicalTopN<>(orderKeys, limit.getLimit(), limit.getOffset(), agg);
                         }).toRule(RuleType.LIMIT_AGG_TO_TOPN_AGG),
-                //limit->project->agg to topn->project->agg
+                //limit->project->agg to project->topn->agg
                 logicalLimit(logicalProject(logicalAggregate()))
                         .when(limit -> ConnectContext.get() != null
                                 && ConnectContext.get().getSessionVariable().pushTopnToAgg
                                 && ConnectContext.get().getSessionVariable().topnOptLimitThreshold
                                 >= limit.getLimit() + limit.getOffset())
+                        .when(limit -> {
+                            LogicalAggregate<? extends Plan> agg = limit.child().child();
+                            return !agg.getGroupByExpressions().isEmpty() && !agg.getSourceRepeat().isPresent();
+                        })
                         .then(limit -> {
                             LogicalProject<? extends Plan> project = limit.child();
-                            LogicalAggregate<? extends Plan> agg
-                                    = (LogicalAggregate<? extends Plan>) project.child();
-                            Optional<OrderKey> orderKeysOpt = tryGenerateOrderKeyByTheFirstGroupKey(agg);
-                            if (!orderKeysOpt.isPresent()) {
-                                return null;
-                            }
-                            List<OrderKey> orderKeys = Lists.newArrayList(orderKeysOpt.get());
-                            Plan result;
-
-                            if (outputAllGroupKeys(limit, agg)) {
-                                result = new LogicalTopN<>(orderKeys, limit.getLimit(),
-                                        limit.getOffset(), project);
-                            } else {
-                                // add the first group by key to topn, and prune this key by upper project
-                                // topn order keys are prefix of group by keys
-                                // refer to PushTopnToAgg.tryGenerateOrderKeyByGroupKeyAndTopnKey()
-                                Expression firstGroupByKey = agg.getGroupByExpressions().get(0);
-                                if (!(firstGroupByKey instanceof SlotReference)) {
-                                    return null;
-                                }
-                                boolean shouldPruneFirstGroupByKey = true;
-                                if (project.getOutputs().contains(firstGroupByKey)) {
-                                    shouldPruneFirstGroupByKey = false;
-                                } else {
-                                    List<NamedExpression> bottomProjections = Lists.newArrayList(project.getProjects());
-                                    bottomProjections.add((SlotReference) firstGroupByKey);
-                                    project = project.withProjects(bottomProjections);
-                                }
-                                LogicalTopN topn = new LogicalTopN<>(orderKeys, limit.getLimit(),
-                                        limit.getOffset(), project);
-                                if (shouldPruneFirstGroupByKey) {
-                                    List<NamedExpression> limitOutput = limit.getOutput().stream()
-                                            .map(e -> (NamedExpression) e).collect(Collectors.toList());
-                                    result = new LogicalProject<>(limitOutput, topn);
-                                } else {
-                                    result = topn;
-                                }
-                            }
-                            return result;
+                            LogicalAggregate<? extends Plan> agg = (LogicalAggregate<? extends Plan>) project.child();
+                            List<OrderKey> orderKeys = generateOrderKeyByGroupKey(agg);
+                            LogicalTopN topn = new LogicalTopN<>(orderKeys, limit.getLimit(),
+                                    limit.getOffset(), agg);
+                            project = (LogicalProject<? extends Plan>) project.withChildren(topn);
+                            return project;
                         }).toRule(RuleType.LIMIT_AGG_TO_TOPN_AGG),
-                // topn -> agg: add all group key to sort key, if sort key is prefix of group key
+                // topn -> agg: append group key(if it is not sort key) to sort key
                 logicalTopN(logicalAggregate())
                         .when(topn -> ConnectContext.get() != null
                                 && ConnectContext.get().getSessionVariable().pushTopnToAgg
                                 && ConnectContext.get().getSessionVariable().topnOptLimitThreshold
                                 >= topn.getLimit() + topn.getOffset())
+                        .when(topn -> {
+                            LogicalAggregate<? extends Plan> agg = topn.child();
+                            return !agg.getGroupByExpressions().isEmpty() && !agg.getSourceRepeat().isPresent();
+                        })
                         .then(topn -> {
-                            LogicalAggregate<? extends Plan> agg = (LogicalAggregate<? extends Plan>) topn.child();
-                            List<OrderKey> newOrders = tryGenerateOrderKeyByGroupKeyAndTopnKey(topn, agg);
-                            if (newOrders.isEmpty()) {
-                                return topn;
+                            LogicalAggregate<? extends Plan> agg = topn.child();
+                            Pair<List<OrderKey>, List<Expression>> pair =
+                                    supplementOrderKeyByGroupKeyIfCompatible(topn, agg);
+                            if (pair != null) {
+                                agg = agg.withGroupBy(pair.second);
+                                topn = (LogicalTopN) topn.withChildren(agg);
+                                topn = (LogicalTopN) topn.withOrderKeys(pair.first);
+                            }
+                            return topn;
+                        }).toRule(RuleType.LIMIT_AGG_TO_TOPN_AGG),
+                //topn -> project ->agg: add all group key to sort key, and prune column
+                logicalTopN(logicalProject(logicalAggregate()))
+                        .when(topn -> ConnectContext.get() != null
+                                && ConnectContext.get().getSessionVariable().pushTopnToAgg
+                                && ConnectContext.get().getSessionVariable().topnOptLimitThreshold
+                                >= topn.getLimit() + topn.getOffset())
+                        .when(topn -> {
+                            LogicalAggregate<? extends Plan> agg = topn.child().child();
+                            return !agg.getGroupByExpressions().isEmpty() && !agg.getSourceRepeat().isPresent();
+                        })
+                        .then(topn -> {
+                            LogicalTopN originTopn = topn;
+                            LogicalProject<? extends Plan> project = topn.child();
+                            LogicalAggregate<? extends Plan> agg = (LogicalAggregate) project.child();
+                            if (!project.isAllSlots()) {
+                                /*
+                                    topn(orderKey=[a])
+                                        +-->project(b as a)
+                                           +--> agg(groupKey[b]
+                                     =>
+                                     topn(orderKey=[b])
+                                        +-->project(b as a)
+                                           +-->agg(groupKey[b])
+                                     and then exchange topn and project
+                                 */
+                                Map<SlotReference, SlotReference> keyAsKey = new HashMap<>();
+                                for (NamedExpression e : project.getProjects()) {
+                                    if (e instanceof Alias && e.child(0) instanceof SlotReference) {
+                                        keyAsKey.put((SlotReference) e.toSlot(), (SlotReference) e.child(0));
+                                    }
+                                }
+                                List<OrderKey> projectOrderKeys = Lists.newArrayList();
+                                boolean hasNew = false;
+                                for (OrderKey orderKey : topn.getOrderKeys()) {
+                                    if (keyAsKey.containsKey(orderKey.getExpr())) {
+                                        projectOrderKeys.add(orderKey.withExpression(keyAsKey.get(orderKey.getExpr())));
+                                        hasNew = true;
+                                    } else {
+                                        projectOrderKeys.add(orderKey);
+                                    }
+                                }
+                                if (hasNew) {
+                                    topn = (LogicalTopN) topn.withOrderKeys(projectOrderKeys);
+                                }
+                            }
+                            Pair<List<OrderKey>, List<Expression>> pair =
+                                    supplementOrderKeyByGroupKeyIfCompatible(topn, agg);
+                            Plan result;
+                            if (pair == null) {
+                                result = originTopn;
                             } else {
-                                return topn.withOrderKeys(newOrders);
+                                agg = agg.withGroupBy(pair.second);
+                                topn = (LogicalTopN) topn.withOrderKeys(pair.first);
+                                if (isOrderKeysInProject(topn, project)) {
+                                    project = (LogicalProject<? extends Plan>) project.withChildren(agg);
+                                    topn = (LogicalTopN<LogicalProject<LogicalAggregate<Plan>>>)
+                                            topn.withChildren(project);
+                                    result = topn;
+                                } else {
+                                    topn = (LogicalTopN) topn.withChildren(agg);
+                                    project = (LogicalProject<? extends Plan>) project.withChildren(topn);
+                                    result = project;
+                                }
                             }
-                        }).toRule(RuleType.LIMIT_AGG_TO_TOPN_AGG));
+                            return result;
+                        }).toRule(RuleType.LIMIT_AGG_TO_TOPN_AGG)
+        );
     }
 
-    private List<OrderKey> tryGenerateOrderKeyByGroupKeyAndTopnKey(LogicalTopN<? extends Plan> topN,
-            LogicalAggregate<? extends Plan> agg) {
-        List<OrderKey> orderKeys = Lists.newArrayListWithCapacity(agg.getGroupByExpressions().size());
-        if (topN.getOrderKeys().size() > agg.getGroupByExpressions().size()) {
-            return orderKeys;
-        }
-        List<Expression> topnKeys = topN.getOrderKeys().stream()
-                .map(OrderKey::getExpr).collect(Collectors.toList());
-        for (int i = 0; i < topN.getOrderKeys().size(); i++) {
-            // prefix check
-            if (!topnKeys.get(i).equals(agg.getGroupByExpressions().get(i))) {
-                return Lists.newArrayList();
+    private boolean isOrderKeysInProject(LogicalTopN<? extends Plan> topn, LogicalProject project) {
+        Set<Slot> projectSlots = project.getOutputSet();
+        for (OrderKey orderKey : topn.getOrderKeys()) {
+            if (!projectSlots.contains(orderKey.getExpr())) {
+                return false;
             }
-            orderKeys.add(topN.getOrderKeys().get(i));
-        }
-        for (int i = topN.getOrderKeys().size(); i < agg.getGroupByExpressions().size(); i++) {
-            orderKeys.add(new OrderKey(agg.getGroupByExpressions().get(i), true, false));
         }
-        return orderKeys;
+        return true;
     }
 
-    private boolean outputAllGroupKeys(LogicalLimit limit, LogicalAggregate agg) {
-        return limit.getOutputSet().containsAll(agg.getGroupByExpressions());
+    private List<OrderKey> generateOrderKeyByGroupKey(LogicalAggregate<? extends Plan> agg) {
+        return agg.getGroupByExpressions().stream()
+                .map(key -> new OrderKey(key, true, false))
+                .collect(Collectors.toList());
     }
 
-    private Optional<OrderKey> tryGenerateOrderKeyByTheFirstGroupKey(LogicalAggregate<? extends Plan> agg) {
-        if (agg.getGroupByExpressions().isEmpty()) {
-            return Optional.empty();
+    /**
+     * compatible: if order key is subset of group by keys
+     * example:
+     * 1. orderKey[a, b], groupKeys[b, a, c]
+     *    compatible, return Pair(orderKey[a, b, c], groupKey[a, b, c])
+     * 2. orderKey[a, b+1], groupKeys[a, b]
+     *    not compatible, return null
+     */
+    private Pair<List<OrderKey>, List<Expression>> supplementOrderKeyByGroupKeyIfCompatible(
+            LogicalTopN<? extends Plan> topn, LogicalAggregate<? extends Plan> agg) {
+        Set<Expression> groupKeySet = Sets.newHashSet(agg.getGroupByExpressions());
+        List<Expression> orderKeyList = topn.getOrderKeys().stream()
+                .map(OrderKey::getExpr).collect(Collectors.toList());
+        Set<Expression> orderKeySet = Sets.newHashSet(orderKeyList);
+        boolean compatible = groupKeySet.containsAll(orderKeyList);
+        if (compatible) {
+            List<OrderKey> newOrderKeys = Lists.newArrayList(topn.getOrderKeys());
+            List<Expression> newGroupExpressions = Lists.newArrayListWithCapacity(agg.getGroupByExpressions().size());
+            for (OrderKey orderKey : newOrderKeys) {
+                newGroupExpressions.add(orderKey.getExpr());
+            }
+
+            for (Expression groupKey : agg.getGroupByExpressions()) {
+                if (!orderKeySet.contains(groupKey)) {
+                    newOrderKeys.add(new OrderKey(groupKey, true, false));
+                    newGroupExpressions.add(groupKey);
+                }
+            }
+            return Pair.of(newOrderKeys, newGroupExpressions);
+        } else {
+            return null;
         }
-        return Optional.of(new OrderKey(agg.getGroupByExpressions().get(0), true, false));
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyEncodeDecode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyEncodeDecode.java
new file mode 100644
index 00000000000000..b6513fc7580707
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyEncodeDecode.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DecodeAsVarchar;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.EncodeStrToInteger;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * project (..., encode(decode(A)) as B, ...)
+ * =>
+ * project (..., A as B,...)
+ */
+public class SimplifyEncodeDecode implements RewriteRuleFactory {
+
+    @Override
+    public List<Rule> buildRules() {
+        return ImmutableList.of(
+                RuleType.SIMPLIFY_ENCODE_DECODE.build(
+                        logicalProject()
+                                .then(project -> {
+                                    List<NamedExpression> newProjections =
+                                            Lists.newArrayListWithCapacity(project.getProjects().size());
+                                    boolean changed = false;
+                                    for (NamedExpression namedExpression : project.getProjects()) {
+                                        if (namedExpression instanceof Alias
+                                                && namedExpression.child(0) instanceof EncodeStrToInteger
+                                                && namedExpression.child(0).child(0)
+                                                instanceof DecodeAsVarchar) {
+                                            Alias alias = (Alias) namedExpression;
+                                            Expression body = namedExpression.child(0)
+                                                    .child(0).child(0);
+                                            newProjections.add((Alias) alias.withChildren(body));
+                                            changed = true;
+                                        } else {
+                                            newProjections.add(namedExpression);
+                                        }
+                                    }
+                                    return changed ? project.withProjects(newProjections) : project;
+                                })
+                )
+        );
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java
index a5d15f1d515608..606253148358a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.algebra;
 
 import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.Alias;
 import org.apache.doris.nereids.trees.expressions.ExprId;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -108,6 +109,18 @@ default boolean isAllSlots() {
         return true;
     }
 
+    /**
+     * project(A as B) is eventually slot project, where A is a slot
+     */
+    default boolean isEventuallyAllSlots() {
+        for (NamedExpression project : getProjects()) {
+            if (!project.isSlot() && !(project instanceof Alias && project.child(0) instanceof Slot)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     /** containsNoneMovableFunction */
     default boolean containsNoneMovableFunction() {
         for (NamedExpression expression : getProjects()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
index d80b4f3166df86..d7d7767e602c74 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
@@ -273,6 +273,11 @@ public LogicalAggregate<Plan> withGroupByAndOutput(List<Expression> groupByExprL
                 hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), child());
     }
 
+    public LogicalAggregate<Plan> withGroupBy(List<Expression> groupByExprList) {
+        return new LogicalAggregate<>(groupByExprList, outputExpressions, normalized, ordinalIsResolved, generated,
+                hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), child());
+    }
+
     public LogicalAggregate<Plan> withChildGroupByAndOutput(List<Expression> groupByExprList,
             List<NamedExpression> outputExpressionList, Plan newChild) {
         return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 380c758e5751c8..126ed1135e9dc0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -2391,13 +2391,11 @@ public void initFuzzyModeVariables() {
             this.rewriteOrToInPredicateThreshold = 100000;
             this.enableFunctionPushdown = false;
             this.enableDeleteSubPredicateV2 = false;
-            this.topnOptLimitThreshold = 0;
             this.enableSyncRuntimeFilterSize = true;
         } else {
             this.rewriteOrToInPredicateThreshold = 2;
             this.enableFunctionPushdown = true;
             this.enableDeleteSubPredicateV2 = true;
-            this.topnOptLimitThreshold = 1024;
             this.enableSyncRuntimeFilterSize = false;
         }
 
diff --git a/regression-test/data/query_p0/limit/test_group_by_limit.out b/regression-test/data/query_p0/limit/test_group_by_limit.out
index d9ac2a2481a993..4a396cdaca812a 100644
--- a/regression-test/data/query_p0/limit/test_group_by_limit.out
+++ b/regression-test/data/query_p0/limit/test_group_by_limit.out
@@ -1,65 +1,65 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
--- !select --
+-- !select1 --
 253967024	8491	AIR
 259556658	8641	FOB
 260402265	8669	MAIL
 
--- !select --
+-- !select2 --
 449872500	15000	1
 386605746	12900	2
 320758616	10717	3
 
--- !select --
+-- !select3 --
 198674527	6588	0.0
 198679731	6563	0.01
 198501055	6622	0.02
 
--- !select --
+-- !select4 --
 27137	1	1992-02-02
 45697	1	1992-02-04
 114452	5	1992-02-05
 
--- !select --
+-- !select5 --
 27137	1	1992-02-02T00:00
 45697	1	1992-02-04T00:00
 114452	5	1992-02-05T00:00
 
--- !select --
+-- !select6 --
 139015016	4632	1
 130287219	4313	2
 162309750	5334	3
 
--- !select --
+-- !select7 --
 64774969	2166	AIR	1
 54166166	1804	AIR	2
 45538267	1532	AIR	3
 
--- !select --
+-- !select8 --
 6882631	228	AIR	1	0.0
 6756423	228	AIR	1	0.01
 7920028	254	AIR	1	0.02
 
--- !select --
+-- !select9 --
 7618	1	AIR	1	0.0	1992-02-06
 2210	1	AIR	1	0.0	1992-03-24
 16807	1	AIR	1	0.0	1992-03-29
 
--- !select --
+-- !select10 --
 6882631	228	AIR	1	0.0
 6756423	228	AIR	1	0.01
 7920028	254	AIR	1	0.02
 
--- !select --
+-- !select11 --
 6882631	228	AIR	1	0.0
 6756423	228	AIR	1	0.01
 7920028	254	AIR	1	0.02
 
--- !select --
+-- !select12 --
 7707018	238	TRUCK	1	0.0
 7467045	233	TRUCK	1	0.01
 6927206	245	TRUCK	1	0.02
 
--- !select --
+-- !select13 --
 7661562	249	TRUCK	1	0.08
 6673139	228	TRUCK	1	0.07
 8333862	265	TRUCK	1	0.06
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy
index 9134d66b76d751..cc75e289a605fc 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy
@@ -87,18 +87,22 @@ suite("push_down_aggr_distinct_through_join_one_side_cust") {
     );
     """
 
+    sql """
+      set topn_opt_limit_threshold=1024;
+      """
+
     explain {
-        sql("physical PLAN SELECT /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/" +
-                "COUNT(DISTINCT dwd_tracking_sensor_init_tmp_ymds.gz_user_id) AS a2c1a830_1," +
-                "dwd_com_abtest_result_inc_ymds.group_name AS ab1011d6," +
-                "dwd_tracking_sensor_init_tmp_ymds.dt AS ad466123 " +
-                "FROM dwd_tracking_sensor_init_tmp_ymds " +
-                "LEFT JOIN dwd_com_abtest_result_inc_ymds " +
-                "ON dwd_tracking_sensor_init_tmp_ymds.gz_user_id = dwd_com_abtest_result_inc_ymds.user_key " +
-                "AND dwd_tracking_sensor_init_tmp_ymds.dt = dwd_com_abtest_result_inc_ymds.dt " +
-                "WHERE dwd_tracking_sensor_init_tmp_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' " +
-                "AND dwd_com_abtest_result_inc_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' " +
-                "GROUP BY 2, 3 ORDER BY 3 asc limit 10000;");
+        sql("""physical PLAN SELECT /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/
+                COUNT(DISTINCT dwd_tracking_sensor_init_tmp_ymds.gz_user_id) AS a2c1a830_1,
+                dwd_com_abtest_result_inc_ymds.group_name AS ab1011d6,
+                dwd_tracking_sensor_init_tmp_ymds.dt AS ad466123 
+                FROM dwd_tracking_sensor_init_tmp_ymds 
+                LEFT JOIN dwd_com_abtest_result_inc_ymds 
+                ON dwd_tracking_sensor_init_tmp_ymds.gz_user_id = dwd_com_abtest_result_inc_ymds.user_key 
+                AND dwd_tracking_sensor_init_tmp_ymds.dt = dwd_com_abtest_result_inc_ymds.dt 
+                WHERE dwd_tracking_sensor_init_tmp_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' 
+                AND dwd_com_abtest_result_inc_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' 
+                GROUP BY 2, 3 ORDER BY 3 asc limit 10000;""");
         contains"groupByExpr=[gz_user_id#1, dt#2]"
         contains"groupByExpr=[gz_user_id#1, dt#2, group_name#5], outputExpr=[gz_user_id#1, dt#2, group_name#5]"
         contains"[group_name#5, dt#2]"
@@ -106,17 +110,17 @@ suite("push_down_aggr_distinct_through_join_one_side_cust") {
     }
 
     explain {
-        sql("physical PLAN SELECT /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/" +
-                "COUNT(DISTINCT dwd_tracking_sensor_init_tmp_ymds.ip) AS a2c1a830_1," +
-                "dwd_com_abtest_result_inc_ymds.group_name AS ab1011d6," +
-                "dwd_tracking_sensor_init_tmp_ymds.dt AS ad466123 " +
-                "FROM dwd_tracking_sensor_init_tmp_ymds " +
-                "LEFT JOIN dwd_com_abtest_result_inc_ymds " +
-                "ON dwd_tracking_sensor_init_tmp_ymds.gz_user_id = dwd_com_abtest_result_inc_ymds.user_key " +
-                "AND dwd_tracking_sensor_init_tmp_ymds.dt = dwd_com_abtest_result_inc_ymds.dt " +
-                "WHERE dwd_tracking_sensor_init_tmp_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' " +
-                "AND dwd_com_abtest_result_inc_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' " +
-                "GROUP BY 2, 3 ORDER BY 3 asc limit 10000;");
+        sql("""physical PLAN SELECT /*+use_cbo_rule(PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)*/
+                COUNT(DISTINCT dwd_tracking_sensor_init_tmp_ymds.ip) AS a2c1a830_1,
+                dwd_com_abtest_result_inc_ymds.group_name AS ab1011d6,
+                dwd_tracking_sensor_init_tmp_ymds.dt AS ad466123 
+                FROM dwd_tracking_sensor_init_tmp_ymds 
+                LEFT JOIN dwd_com_abtest_result_inc_ymds 
+                ON dwd_tracking_sensor_init_tmp_ymds.gz_user_id = dwd_com_abtest_result_inc_ymds.user_key 
+                AND dwd_tracking_sensor_init_tmp_ymds.dt = dwd_com_abtest_result_inc_ymds.dt 
+                WHERE dwd_tracking_sensor_init_tmp_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' 
+                AND dwd_com_abtest_result_inc_ymds.dt BETWEEN '2024-08-15' AND '2024-08-15' 
+                GROUP BY 2, 3 ORDER BY 3 asc limit 10000;""");
         contains"groupByExpr=[ip#0, gz_user_id#1, dt#2], outputExpr=[ip#0, gz_user_id#1, dt#2]"
         contains"groupByExpr=[ip#0, dt#2, group_name#5], outputExpr=[ip#0, dt#2, group_name#5]"
         contains"groupByExpr=[group_name#5, dt#2], outputExpr=[group_name#5, dt#2, partial_count(ip#0) AS `partial_count(ip)`#12]"
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy b/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy
index 631656a6b1921b..5ae587910b6ce5 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/push_topn_to_agg.groovy
@@ -50,22 +50,31 @@ suite("push_topn_to_agg") {
         notContains("STREAMING")
     }
 
-    // order key should be prefix of group key
+    // order keys are part of group keys, 
+    // 1. adjust group keys (o_custkey, o_clerk) -> o_clerk, o_custkey
+    // 2. append o_custkey to order key 
     explain{
-        sql "select o_custkey, sum(o_shippriority), o_clerk  from orders group by o_custkey, o_clerk order by o_clerk, o_custkey limit 11;"
-        multiContains("sortByGroupKey:false", 2)
+        sql "select sum(o_shippriority)  from orders group by o_custkey, o_clerk order by o_clerk limit 11;"
+        contains("sortByGroupKey:true")
+        contains("group by: o_clerk[#10], o_custkey[#9]")
+        contains("order by: o_clerk[#18] ASC, o_custkey[#19] ASC")
     }
 
-    // order key should be prefix of group key
-    explain{
-        sql "select o_custkey, o_clerk, sum(o_shippriority) as x from orders group by o_custkey, o_clerk order by o_custkey, x limit 12;"
-        multiContains("sortByGroupKey:false", 2)
+
+    // one distinct 
+    explain {
+        sql "select sum(distinct o_shippriority) from orders group by o_orderkey limit 13; "
+        contains("VTOP-N")
+        contains("order by: o_orderkey")
+        multiContains("sortByGroupKey:true", 1)
     }
 
-    // one phase agg is optimized 
+    // multi distinct 
     explain {
-        sql "select sum(o_shippriority) from orders group by o_orderkey limit 13; "
-        contains("sortByGroupKey:true")
+        sql "select count(distinct o_clerk), sum(distinct o_shippriority) from orders group by o_orderkey limit 14; "
+        contains("VTOP-N")
+        contains("order by: o_orderkey")
+        multiContains("sortByGroupKey:true", 2)
     }
 
     // use group key as sort key to enable topn-push opt
@@ -74,22 +83,17 @@ suite("push_topn_to_agg") {
         contains("sortByGroupKey:true")
     }
 
-    // group key is part of output of limit, apply opt
+    // group key is expression
     explain {
-        sql "select sum(o_shippriority), o_clerk from orders group by o_clerk limit 15; "
+        sql "select sum(o_shippriority), o_clerk+1 from orders group by o_clerk+1 limit 15; "
         contains("sortByGroupKey:true")
     }
 
-    // order key is not prefix of group key
+    // order key is not part of group key
     explain {
         sql "select o_custkey, sum(o_shippriority) from orders group by o_custkey order by o_custkey+1 limit 16; "
         contains("sortByGroupKey:false")
-    }
-
-    // order key is not prefix of group key
-    explain {
-        sql "select o_custkey, sum(o_shippriority) from orders group by o_custkey order by o_custkey+1 limit 17; "
-        contains("sortByGroupKey:false")
+        notContains("sortByGroupKey:true")
     }
 
     // topn + one phase agg
@@ -97,73 +101,4 @@ suite("push_topn_to_agg") {
         sql "select sum(ps_availqty), ps_partkey, ps_suppkey from partsupp group by ps_partkey, ps_suppkey order by ps_partkey, ps_suppkey limit 18;"
         contains("sortByGroupKey:true")
     }
-
-    // sort key is prefix of group key, make all group key to sort key(ps_suppkey) and then apply push-topn-agg rule
-    explain {
-        sql "select sum(ps_availqty), ps_partkey, ps_suppkey from partsupp group by ps_partkey, ps_suppkey order by ps_partkey limit 19;"
-        contains("sortByGroupKey:true")
-    }
-
-    explain {
-        sql "select sum(ps_availqty), ps_suppkey, ps_availqty from partsupp group by ps_suppkey, ps_availqty order by ps_suppkey limit 19;"
-        contains("sortByGroupKey:true")
-    }
-
-    // sort key is not prefix of group key, deny
-    explain {
-        sql "select sum(ps_availqty), ps_partkey, ps_suppkey from partsupp group by ps_partkey, ps_suppkey order by ps_suppkey limit 20;"
-        contains("sortByGroupKey:false")
-    }
-
-    multi_sql """
-    drop table if exists t1;
-    CREATE TABLE IF NOT EXISTS t1
-        (
-        k1 TINYINT
-        )
-        ENGINE=olap
-        AGGREGATE KEY(k1)
-        DISTRIBUTED BY HASH(k1) BUCKETS 1
-        PROPERTIES (
-            "replication_num" = "1"
-        );
-
-    insert into t1 values (0),(1);
-
-    drop table if exists t2;
-    CREATE TABLE IF NOT EXISTS t2
-        (
-        k1 TINYINT
-        )
-        ENGINE=olap
-        AGGREGATE KEY(k1)
-        DISTRIBUTED BY HASH(k1) BUCKETS 1
-        PROPERTIES (
-            "replication_num" = "1"
-        );
-    insert into t2 values(5),(6);
-    """
-
-    // the result of following sql may be unstable, run 3 times
-    qt_stable_1 """
-    select * from (
-        select k1 from t1
-        UNION
-        select k1 from t2
-    ) as b order by k1  limit 2;
-    """
-    qt_stable_2 """
-    select * from (
-        select k1 from t1
-        UNION
-        select k1 from t2
-    ) as b order by k1  limit 2;
-    """
-    qt_stable_3 """
-    select * from (
-        select k1 from t1
-        UNION
-        select k1 from t2
-    ) as b order by k1  limit 2;
-    """
 }
\ No newline at end of file
diff --git a/regression-test/suites/query_p0/limit/test_group_by_limit.groovy b/regression-test/suites/query_p0/limit/test_group_by_limit.groovy
index 271619c4a93cba..d6f49f9d7a397f 100644
--- a/regression-test/suites/query_p0/limit/test_group_by_limit.groovy
+++ b/regression-test/suites/query_p0/limit/test_group_by_limit.groovy
@@ -23,42 +23,99 @@ sql 'set enable_force_spill=false'
 
 sql 'set topn_opt_limit_threshold=10'
 
+sql "set experimental_ENABLE_COMPRESS_MATERIALIZE=true;"
 
 // different types
-qt_select """ select  sum(orderkey), count(partkey), shipmode from tpch_tiny_lineitem group by shipmode limit 3; """
-
-qt_select """ select  sum(orderkey), count(partkey),  linenumber from tpch_tiny_lineitem group by linenumber limit 3; """
+qt_select1 """ select  sum(orderkey), count(partkey), shipmode from tpch_tiny_lineitem group by shipmode limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey), shipmode from tpch_tiny_lineitem group by shipmode limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 
-qt_select """ select  sum(orderkey), count(partkey),  tax from tpch_tiny_lineitem group by tax limit 3; """
+qt_select2 """ select  sum(orderkey), count(partkey),  linenumber from tpch_tiny_lineitem group by linenumber limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey),  linenumber from tpch_tiny_lineitem group by linenumber limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 
-qt_select """ select  sum(orderkey), count(partkey),  commitdate from tpch_tiny_lineitem group by commitdate limit 3; """
+qt_select3 """ select  sum(orderkey), count(partkey),  tax from tpch_tiny_lineitem group by tax limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey),  tax from tpch_tiny_lineitem group by tax limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 
+qt_select4 """ select  sum(orderkey), count(partkey),  commitdate from tpch_tiny_lineitem group by commitdate limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey),  commitdate from tpch_tiny_lineitem group by commitdate limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 
 // group by functions
-qt_select """ select  sum(orderkey), count(partkey),  cast(commitdate as datetime) from tpch_tiny_lineitem group by cast(commitdate as datetime) limit 3; """
-
-qt_select """ select  sum(orderkey), count(partkey),  month(commitdate) from tpch_tiny_lineitem group by month(commitdate) limit 3; """
+qt_select5 """ select  sum(orderkey), count(partkey),  cast(commitdate as datetime) from tpch_tiny_lineitem group by cast(commitdate as datetime) limit 3; """
+explain {
+    sql " select  sum(orderkey), count(partkey),  cast(commitdate as datetime) from tpch_tiny_lineitem group by cast(commitdate as datetime) limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 
+qt_select6 """ select  sum(orderkey), count(partkey),  month(commitdate) from tpch_tiny_lineitem group by month(commitdate) limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey),  month(commitdate) from tpch_tiny_lineitem group by month(commitdate) limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 
 // mutli column
-qt_select """ select  sum(orderkey), count(partkey), shipmode, linenumber from tpch_tiny_lineitem group by shipmode, linenumber limit 3; """
-
-qt_select """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax limit 3; """
-
-qt_select """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax , commitdate from tpch_tiny_lineitem group by shipmode, linenumber, tax, commitdate  limit 3; """
-
+qt_select7 """ select  sum(orderkey), count(partkey), shipmode, linenumber from tpch_tiny_lineitem group by shipmode, linenumber limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey), shipmode, linenumber from tpch_tiny_lineitem group by shipmode, linenumber limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
+qt_select8 """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
+qt_select9 """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax , commitdate from tpch_tiny_lineitem group by shipmode, linenumber, tax, commitdate  limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey), shipmode, linenumber , tax , commitdate from tpch_tiny_lineitem group by shipmode, linenumber, tax, commitdate  limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 
 // group by + order by 
 
 // group by columns eq order by columns
-qt_select """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode, linenumber, tax limit 3; """
-
+qt_select10 """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode, linenumber, tax limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode, linenumber, tax limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 // group by columns contains order by columns
-qt_select """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode limit 3; """
-
+qt_select11 """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 // desc order by column
-qt_select """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax limit 3; """
-
-qt_select """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax desc limit 3; """
-
+qt_select12 """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
+qt_select13 """ select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax desc limit 3; """
+explain{
+    sql " select  sum(orderkey), count(partkey), shipmode, linenumber , tax from tpch_tiny_lineitem group by shipmode, linenumber, tax order by shipmode desc, linenumber, tax desc limit 3; "
+    contains("VTOP-N")
+    contains("sortByGroupKey:true")
+}
 }

From c58a1b7491ee9118027b9a316289c1a28a9801c5 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Wed, 18 Dec 2024 14:34:32 +0800
Subject: [PATCH 388/399] [reafactor](runtime filter) Delete unused targets
 parameter (#45573)

`TRuntimeFilterTargetParams` will never be used since version 3.0.
---
 be/src/runtime/runtime_filter_mgr.cpp         | 68 +++++--------------
 be/src/runtime/runtime_filter_mgr.h           |  7 +-
 .../java/org/apache/doris/qe/Coordinator.java | 10 ---
 .../runtime/RuntimeFiltersThriftBuilder.java  | 10 ---
 gensrc/thrift/PaloInternalService.thrift      |  4 +-
 5 files changed, 19 insertions(+), 80 deletions(-)

diff --git a/be/src/runtime/runtime_filter_mgr.cpp b/be/src/runtime/runtime_filter_mgr.cpp
index c16db7c67d3420..b4a38173d72222 100644
--- a/be/src/runtime/runtime_filter_mgr.cpp
+++ b/be/src/runtime/runtime_filter_mgr.cpp
@@ -219,35 +219,14 @@ Status RuntimeFilterMgr::get_merge_addr(TNetworkAddress* addr) {
 
 Status RuntimeFilterMergeControllerEntity::_init_with_desc(
         const TRuntimeFilterDesc* runtime_filter_desc, const TQueryOptions* query_options,
-        const std::vector<doris::TRuntimeFilterTargetParams>* target_info,
-        const int producer_size) {
-    std::unique_lock<std::shared_mutex> guard(_filter_map_mutex);
-    std::shared_ptr<RuntimeFilterCntlVal> cnt_val = std::make_shared<RuntimeFilterCntlVal>();
-    // runtime_filter_desc and target will be released,
-    // so we need to copy to cnt_val
-    cnt_val->producer_size = producer_size;
-    cnt_val->runtime_filter_desc = *runtime_filter_desc;
-    cnt_val->pool.reset(new ObjectPool());
-    cnt_val->filter = cnt_val->pool->add(new IRuntimeFilter(_state, runtime_filter_desc));
-
-    auto filter_id = runtime_filter_desc->filter_id;
-    RETURN_IF_ERROR(
-            cnt_val->filter->init_with_desc(&cnt_val->runtime_filter_desc, query_options, -1));
-    cnt_val->filter->set_ignored();
-    _filter_map.emplace(filter_id, cnt_val);
-    return Status::OK();
-}
-
-Status RuntimeFilterMergeControllerEntity::_init_with_desc(
-        const TRuntimeFilterDesc* runtime_filter_desc, const TQueryOptions* query_options,
-        const std::vector<doris::TRuntimeFilterTargetParamsV2>* targetv2_info,
+        const std::vector<doris::TRuntimeFilterTargetParamsV2>&& targetv2_info,
         const int producer_size) {
     std::shared_ptr<RuntimeFilterCntlVal> cnt_val = std::make_shared<RuntimeFilterCntlVal>();
     // runtime_filter_desc and target will be released,
     // so we need to copy to cnt_val
     cnt_val->producer_size = producer_size;
     cnt_val->runtime_filter_desc = *runtime_filter_desc;
-    cnt_val->targetv2_info = *targetv2_info;
+    cnt_val->targetv2_info = targetv2_info;
     cnt_val->pool.reset(new ObjectPool());
     cnt_val->filter = cnt_val->pool->add(new IRuntimeFilter(_state, runtime_filter_desc));
     auto filter_id = runtime_filter_desc->filter_id;
@@ -268,36 +247,21 @@ Status RuntimeFilterMergeControllerEntity::init(UniqueId query_id,
     if (runtime_filter_params.__isset.rid_to_runtime_filter) {
         for (const auto& filterid_to_desc : runtime_filter_params.rid_to_runtime_filter) {
             int filter_id = filterid_to_desc.first;
-            const auto& target_iter = runtime_filter_params.rid_to_target_param.find(filter_id);
-            if (target_iter == runtime_filter_params.rid_to_target_param.end() &&
-                !runtime_filter_params.__isset.rid_to_target_paramv2) {
-                // This runtime filter has to target info
-                return Status::InternalError("runtime filter params meet error");
-            } else if (target_iter == runtime_filter_params.rid_to_target_param.end()) {
-                const auto& targetv2_iter =
-                        runtime_filter_params.rid_to_target_paramv2.find(filter_id);
-                if (targetv2_iter == runtime_filter_params.rid_to_target_paramv2.end()) {
-                    // This runtime filter has to target info
-                    return Status::InternalError("runtime filter params meet error");
-                }
-                const auto& build_iter =
-                        runtime_filter_params.runtime_filter_builder_num.find(filter_id);
-                if (build_iter == runtime_filter_params.runtime_filter_builder_num.end()) {
-                    // This runtime filter has to builder info
-                    return Status::InternalError("runtime filter params meet error");
-                }
-
-                RETURN_IF_ERROR(_init_with_desc(&filterid_to_desc.second, &query_options,
-                                                &targetv2_iter->second, build_iter->second));
-            } else {
-                const auto& build_iter =
-                        runtime_filter_params.runtime_filter_builder_num.find(filter_id);
-                if (build_iter == runtime_filter_params.runtime_filter_builder_num.end()) {
-                    return Status::InternalError("runtime filter params meet error");
-                }
-                RETURN_IF_ERROR(_init_with_desc(&filterid_to_desc.second, &query_options,
-                                                &target_iter->second, build_iter->second));
+            const auto& targetv2_iter = runtime_filter_params.rid_to_target_paramv2.find(filter_id);
+            const auto& build_iter =
+                    runtime_filter_params.runtime_filter_builder_num.find(filter_id);
+            if (build_iter == runtime_filter_params.runtime_filter_builder_num.end()) {
+                // This runtime filter has no builder info
+                return Status::InternalError(
+                        "Runtime filter has a wrong parameter. Maybe FE version is mismatched.");
             }
+
+            RETURN_IF_ERROR(_init_with_desc(
+                    &filterid_to_desc.second, &query_options,
+                    targetv2_iter == runtime_filter_params.rid_to_target_paramv2.end()
+                            ? std::vector<doris::TRuntimeFilterTargetParamsV2> {}
+                            : std::move(targetv2_iter->second),
+                    build_iter->second));
         }
     }
     return Status::OK();
diff --git a/be/src/runtime/runtime_filter_mgr.h b/be/src/runtime/runtime_filter_mgr.h
index 9f4cf5f4e22a07..c54be905f28f08 100644
--- a/be/src/runtime/runtime_filter_mgr.h
+++ b/be/src/runtime/runtime_filter_mgr.h
@@ -192,12 +192,7 @@ class RuntimeFilterMergeControllerEntity {
 private:
     Status _init_with_desc(const TRuntimeFilterDesc* runtime_filter_desc,
                            const TQueryOptions* query_options,
-                           const std::vector<doris::TRuntimeFilterTargetParams>* target_info,
-                           const int producer_size);
-
-    Status _init_with_desc(const TRuntimeFilterDesc* runtime_filter_desc,
-                           const TQueryOptions* query_options,
-                           const std::vector<doris::TRuntimeFilterTargetParamsV2>* target_info,
+                           const std::vector<doris::TRuntimeFilterTargetParamsV2>&& target_info,
                            const int producer_size);
 
     UniqueId _query_id;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 1e327c469e7140..472f2462e4d39d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -111,7 +111,6 @@
 import org.apache.doris.thrift.TReportExecStatusParams;
 import org.apache.doris.thrift.TResourceLimit;
 import org.apache.doris.thrift.TRuntimeFilterParams;
-import org.apache.doris.thrift.TRuntimeFilterTargetParams;
 import org.apache.doris.thrift.TRuntimeFilterTargetParamsV2;
 import org.apache.doris.thrift.TScanRange;
 import org.apache.doris.thrift.TScanRangeLocation;
@@ -3262,15 +3261,6 @@ Map<TNetworkAddress, TPipelineFragmentParams> toThrift(int backendNum) {
 
                             localParams.runtime_filter_params.putToRidToTargetParamv2(rf.getFilterId().asInt(),
                                     new ArrayList<TRuntimeFilterTargetParamsV2>(targetParamsV2.values()));
-                        } else {
-                            List<TRuntimeFilterTargetParams> targetParams = Lists.newArrayList();
-                            for (FRuntimeFilterTargetParam targetParam : fParams) {
-                                // Instance id make no sense if this runtime filter doesn't have remote targets.
-                                targetParams.add(new TRuntimeFilterTargetParams(new TUniqueId(),
-                                        targetParam.targetFragmentInstanceAddr));
-                            }
-                            localParams.runtime_filter_params.putToRidToTargetParam(rf.getFilterId().asInt(),
-                                    targetParams);
                         }
                     }
                     for (Map.Entry<RuntimeFilterId, Integer> entry : ridToBuilderNum.entrySet()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/RuntimeFiltersThriftBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/RuntimeFiltersThriftBuilder.java
index 47c01ef8eb34a6..f9ab8e83f07e99 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/RuntimeFiltersThriftBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/RuntimeFiltersThriftBuilder.java
@@ -27,11 +27,8 @@
 import org.apache.doris.system.Backend;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TRuntimeFilterParams;
-import org.apache.doris.thrift.TRuntimeFilterTargetParams;
 import org.apache.doris.thrift.TRuntimeFilterTargetParamsV2;
-import org.apache.doris.thrift.TUniqueId;
 
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 import java.util.ArrayList;
@@ -94,13 +91,6 @@ public void setRuntimeFilterThriftParams(TRuntimeFilterParams runtimeFilterParam
                 runtimeFilterParams.putToRidToTargetParamv2(
                         rf.getFilterId().asInt(), new ArrayList<>(targetToParams.values())
                 );
-            } else {
-                List<TRuntimeFilterTargetParams> targetParams = Lists.newArrayList();
-                for (RuntimeFilterTarget target : targets) {
-                    // Instance id make no sense if this runtime filter doesn't have remote targets.
-                    targetParams.add(new TRuntimeFilterTargetParams(new TUniqueId(), target.address));
-                }
-                runtimeFilterParams.putToRidToTargetParam(rf.getFilterId().asInt(), targetParams);
             }
         }
         for (Map.Entry<RuntimeFilterId, Integer> entry : ridToBuilderNum.entrySet()) {
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index 85abbf9b66ddba..0a1ea4a98fca94 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -373,6 +373,7 @@ struct TScanRangeParams {
   2: optional i32 volume_id = -1
 }
 
+// deprecated
 struct TRuntimeFilterTargetParams {
   1: required Types.TUniqueId target_fragment_instance_id
   // The address of the instance where the fragment is expected to run
@@ -390,8 +391,7 @@ struct TRuntimeFilterParams {
   // Runtime filter merge instance address
   1: optional Types.TNetworkAddress runtime_filter_merge_addr
 
-  // Runtime filter ID to the instance address of the fragment,
-  // that is expected to use this runtime filter
+  // deprecated
   2: optional map<i32, list<TRuntimeFilterTargetParams>> rid_to_target_param
 
   // Runtime filter ID to the runtime filter desc

From ad814d236e9102cef606e17ddf0c05c4f06a2c15 Mon Sep 17 00:00:00 2001
From: yujun <yujun@selectdb.com>
Date: Wed, 18 Dec 2024 14:39:46 +0800
Subject: [PATCH 389/399] [fix](nereids) fix compare with date like literal
 (#45382)

when compare with date like literal,  the literal may be wrong cut.
for example:

if a is datetimev1,   then:
`a = '2020-01-01 00:00:00.12'` should opt as `FALSE`, but it opt as `a =
'2020-01-01 00:00:00`;
`a >= '2020-01-01 00:00:00.12'` should opt as `a >= '2020-01-01
00:00:01`, but it opt as `a >= '2020-01-01 00:00:00`;

if a is date / datev2, then:
`a = '2020-01-01 00:00:12'` should opt as `FALSE`, but it don't opt it;
---
 .../rules/SimplifyComparisonPredicate.java    |  92 ++++++++--------
 .../SimplifyComparisonPredicateTest.java      | 101 +++++++++++++++++-
 .../pull_up_predicate_literal.out             |   2 +-
 .../pull_up_predicate_literal.groovy          |   2 +-
 4 files changed, 145 insertions(+), 52 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java
index 5f14d9625e0849..fbe0d44417363a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java
@@ -46,6 +46,7 @@
 import org.apache.doris.nereids.trees.expressions.literal.NumericLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
+import org.apache.doris.nereids.types.DataType;
 import org.apache.doris.nereids.types.DateTimeType;
 import org.apache.doris.nereids.types.DateTimeV2Type;
 import org.apache.doris.nereids.types.DateType;
@@ -71,12 +72,6 @@
 public class SimplifyComparisonPredicate extends AbstractExpressionRewriteRule implements ExpressionPatternRuleFactory {
     public static SimplifyComparisonPredicate INSTANCE = new SimplifyComparisonPredicate();
 
-    enum AdjustType {
-        LOWER,
-        UPPER,
-        NONE
-    }
-
     @Override
     public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
         return ImmutableList.of(
@@ -119,18 +114,23 @@ public static Expression simplify(ComparisonPredicate cp) {
         return result;
     }
 
-    private static Expression processComparisonPredicateDateTimeV2Literal(
+    private static Expression processDateTimeLikeComparisonPredicateDateTimeV2Literal(
             ComparisonPredicate comparisonPredicate, Expression left, DateTimeV2Literal right) {
-        DateTimeV2Type leftType = (DateTimeV2Type) left.getDataType();
+        DataType leftType = left.getDataType();
+        int toScale = 0;
+        if (leftType instanceof DateTimeType) {
+            toScale = 0;
+        } else if (leftType instanceof DateTimeV2Type) {
+            toScale = ((DateTimeV2Type) leftType).getScale();
+        } else {
+            return comparisonPredicate;
+        }
         DateTimeV2Type rightType = right.getDataType();
-        if (leftType.getScale() < rightType.getScale()) {
-            int toScale = leftType.getScale();
+        if (toScale < rightType.getScale()) {
             if (comparisonPredicate instanceof EqualTo) {
                 long originValue = right.getMicroSecond();
                 right = right.roundCeiling(toScale);
-                if (right.getMicroSecond() == originValue) {
-                    return comparisonPredicate.withChildren(left, right);
-                } else {
+                if (right.getMicroSecond() != originValue) {
                     // TODO: the ideal way is to return an If expr like:
                     // return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
                     // BooleanLiteral.of(false));
@@ -142,50 +142,55 @@ private static Expression processComparisonPredicateDateTimeV2Literal(
             } else if (comparisonPredicate instanceof NullSafeEqual) {
                 long originValue = right.getMicroSecond();
                 right = right.roundCeiling(toScale);
-                if (right.getMicroSecond() == originValue) {
-                    return comparisonPredicate.withChildren(left, right);
-                } else {
+                if (right.getMicroSecond() != originValue) {
                     return BooleanLiteral.of(false);
                 }
             } else if (comparisonPredicate instanceof GreaterThan
                     || comparisonPredicate instanceof LessThanEqual) {
-                return comparisonPredicate.withChildren(left, right.roundFloor(toScale));
+                right = right.roundFloor(toScale);
             } else if (comparisonPredicate instanceof LessThan
                     || comparisonPredicate instanceof GreaterThanEqual) {
-                return comparisonPredicate.withChildren(left, right.roundCeiling(toScale));
+                right = right.roundCeiling(toScale);
+            } else {
+                return comparisonPredicate;
+            }
+            Expression newRight = leftType instanceof DateTimeType ? migrateToDateTime(right) : right;
+            return comparisonPredicate.withChildren(left, newRight);
+        } else {
+            if (leftType instanceof DateTimeType) {
+                return comparisonPredicate.withChildren(left, migrateToDateTime(right));
+            } else {
+                return comparisonPredicate;
             }
         }
-        return comparisonPredicate;
     }
 
     private static Expression processDateLikeTypeCoercion(ComparisonPredicate cp, Expression left, Expression right) {
         if (left instanceof Cast && right instanceof DateLiteral) {
             Cast cast = (Cast) left;
-            if (cast.child().getDataType() instanceof DateTimeType) {
+            if (cast.child().getDataType() instanceof DateTimeType
+                    || cast.child().getDataType() instanceof DateTimeV2Type) {
                 if (right instanceof DateTimeV2Literal) {
-                    left = cast.child();
-                    right = migrateToDateTime((DateTimeV2Literal) right);
-                }
-            }
-            if (cast.child().getDataType() instanceof DateTimeV2Type) {
-                if (right instanceof DateTimeV2Literal) {
-                    left = cast.child();
-                    return processComparisonPredicateDateTimeV2Literal(cp, left, (DateTimeV2Literal) right);
+                    return processDateTimeLikeComparisonPredicateDateTimeV2Literal(
+                            cp, cast.child(), (DateTimeV2Literal) right);
                 }
             }
+
             // datetime to datev2
             if (cast.child().getDataType() instanceof DateType || cast.child().getDataType() instanceof DateV2Type) {
                 if (right instanceof DateTimeLiteral) {
-                    if (cannotAdjust((DateTimeLiteral) right, cp)) {
-                        return cp;
-                    }
-                    AdjustType type = AdjustType.NONE;
-                    if (cp instanceof GreaterThanEqual || cp instanceof LessThan) {
-                        type = AdjustType.UPPER;
-                    } else if (cp instanceof GreaterThan || cp instanceof LessThanEqual) {
-                        type = AdjustType.LOWER;
+                    DateTimeLiteral dateTimeLiteral = (DateTimeLiteral) right;
+                    right = migrateToDateV2(dateTimeLiteral);
+                    if (dateTimeLiteral.getHour() != 0 || dateTimeLiteral.getMinute() != 0
+                            || dateTimeLiteral.getSecond() != 0) {
+                        if (cp instanceof EqualTo) {
+                            return ExpressionUtils.falseOrNull(cast.child());
+                        } else if (cp instanceof NullSafeEqual) {
+                            return BooleanLiteral.FALSE;
+                        } else if (cp instanceof GreaterThanEqual || cp instanceof LessThan) {
+                            right = ((DateV2Literal) right).plusDays(1);
+                        }
                     }
-                    right = migrateToDateV2((DateTimeLiteral) right, type);
                     if (cast.child().getDataType() instanceof DateV2Type) {
                         left = cast.child();
                     }
@@ -416,17 +421,8 @@ private static Expression migrateToDateTime(DateTimeV2Literal l) {
         return new DateTimeLiteral(l.getYear(), l.getMonth(), l.getDay(), l.getHour(), l.getMinute(), l.getSecond());
     }
 
-    private static boolean cannotAdjust(DateTimeLiteral l, ComparisonPredicate cp) {
-        return cp instanceof EqualTo && (l.getHour() != 0 || l.getMinute() != 0 || l.getSecond() != 0);
-    }
-
-    private static Expression migrateToDateV2(DateTimeLiteral l, AdjustType type) {
-        DateV2Literal d = new DateV2Literal(l.getYear(), l.getMonth(), l.getDay());
-        if (type == AdjustType.UPPER && (l.getHour() != 0 || l.getMinute() != 0 || l.getSecond() != 0)) {
-            return d.plusDays(1);
-        } else {
-            return d;
-        }
+    private static Expression migrateToDateV2(DateTimeLiteral l) {
+        return new DateV2Literal(l.getYear(), l.getMonth(), l.getDay());
     }
 
     private static Expression migrateToDate(DateV2Literal l) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java
index 9202c1e202469d..028f1c4864f099 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java
@@ -48,7 +48,10 @@
 import org.apache.doris.nereids.types.BigIntType;
 import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.DateTimeType;
 import org.apache.doris.nereids.types.DateTimeV2Type;
+import org.apache.doris.nereids.types.DateType;
+import org.apache.doris.nereids.types.DateV2Type;
 import org.apache.doris.nereids.types.DecimalV3Type;
 import org.apache.doris.nereids.types.DoubleType;
 import org.apache.doris.nereids.types.IntegerType;
@@ -95,11 +98,11 @@ void testSimplifyComparisonPredicateRule() {
                 new LessThan(dv2, dv2PlusOne));
         assertRewrite(
                 new EqualTo(new Cast(dv2, DateTimeV2Type.SYSTEM_DEFAULT), dtv2),
-                new EqualTo(new Cast(dv2, DateTimeV2Type.SYSTEM_DEFAULT), dtv2));
+                BooleanLiteral.FALSE);
 
         assertRewrite(
                 new EqualTo(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT), dtv2),
-                new EqualTo(new Cast(d, DateTimeV2Type.SYSTEM_DEFAULT), dtv2));
+                BooleanLiteral.FALSE);
 
         // test hour, minute and second all zero
         Expression dtv2AtZeroClock = new DateTimeV2Literal(1, 1, 1, 0, 0, 0, 0);
@@ -140,6 +143,100 @@ void testDateTimeV2CmpDateTimeV2() {
         expression = new GreaterThan(left, right);
         rewrittenExpression = executor.rewrite(typeCoercion(expression), context);
         Assertions.assertEquals(dt.getDataType(), rewrittenExpression.child(0).getDataType());
+
+        Expression date = new SlotReference("a", DateV2Type.INSTANCE);
+        Expression datev1 = new SlotReference("a", DateType.INSTANCE);
+        Expression datetime0 = new SlotReference("a", DateTimeV2Type.of(0));
+        Expression datetime2 = new SlotReference("a", DateTimeV2Type.of(2));
+        Expression datetimev1 = new SlotReference("a", DateTimeType.INSTANCE);
+
+        // date
+        // cast (date as datetimev1) cmp datetimev1
+        assertRewrite(new EqualTo(new Cast(date, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:00")),
+                new EqualTo(date, new DateV2Literal("2020-01-01")));
+        assertRewrite(new EqualTo(new Cast(date, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                ExpressionUtils.falseOrNull(date));
+        assertRewrite(new NullSafeEqual(new Cast(date, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                BooleanLiteral.FALSE);
+        assertRewrite(new GreaterThan(new Cast(date, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                new GreaterThan(date, new DateV2Literal("2020-01-01")));
+        assertRewrite(new GreaterThanEqual(new Cast(date, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                new GreaterThanEqual(date, new DateV2Literal("2020-01-02")));
+        assertRewrite(new LessThan(new Cast(date, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                new LessThan(date, new DateV2Literal("2020-01-02")));
+        assertRewrite(new LessThanEqual(new Cast(date, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                new LessThanEqual(date, new DateV2Literal("2020-01-01")));
+        // cast (date as datev1) = datev1-literal
+        // assertRewrite(new EqualTo(new Cast(date, DateType.INSTANCE), new DateLiteral("2020-01-01")),
+        //        new EqualTo(date, new DateV2Literal("2020-01-01")));
+        // assertRewrite(new GreaterThan(new Cast(date, DateType.INSTANCE), new DateLiteral("2020-01-01")),
+        //        new GreaterThan(date, new DateV2Literal("2020-01-01")));
+
+        // cast (datev1 as datetimev1) cmp datetimev1
+        assertRewrite(new EqualTo(new Cast(datev1, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:00")),
+                new EqualTo(datev1, new DateLiteral("2020-01-01")));
+        assertRewrite(new EqualTo(new Cast(datev1, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                ExpressionUtils.falseOrNull(datev1));
+        assertRewrite(new NullSafeEqual(new Cast(datev1, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                BooleanLiteral.FALSE);
+        assertRewrite(new GreaterThan(new Cast(datev1, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                new GreaterThan(datev1, new DateLiteral("2020-01-01")));
+        assertRewrite(new GreaterThanEqual(new Cast(datev1, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                new GreaterThanEqual(datev1, new DateLiteral("2020-01-02")));
+        assertRewrite(new LessThan(new Cast(datev1, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                new LessThan(datev1, new DateLiteral("2020-01-02")));
+        assertRewrite(new LessThanEqual(new Cast(datev1, DateTimeType.INSTANCE), new DateTimeLiteral("2020-01-01 00:00:01")),
+                new LessThanEqual(datev1, new DateLiteral("2020-01-01")));
+        assertRewrite(new EqualTo(new Cast(datev1, DateV2Type.INSTANCE), new DateV2Literal("2020-01-01")),
+                new EqualTo(datev1, new DateLiteral("2020-01-01")));
+        assertRewrite(new GreaterThan(new Cast(datev1, DateV2Type.INSTANCE), new DateV2Literal("2020-01-01")),
+                new GreaterThan(datev1, new DateLiteral("2020-01-01")));
+
+        // cast (datetimev1 as datetime) cmp datetime
+        assertRewrite(new EqualTo(new Cast(datetimev1, DateTimeV2Type.of(0)), new DateTimeV2Literal("2020-01-01 00:00:00")),
+                new EqualTo(datetimev1, new DateTimeLiteral("2020-01-01 00:00:00")));
+        assertRewrite(new GreaterThan(new Cast(datetimev1, DateTimeV2Type.of(0)), new DateTimeV2Literal("2020-01-01 00:00:00")),
+                new GreaterThan(datetimev1, new DateTimeLiteral("2020-01-01 00:00:00")));
+        assertRewrite(new EqualTo(new Cast(datetimev1, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                ExpressionUtils.falseOrNull(datetimev1));
+        assertRewrite(new NullSafeEqual(new Cast(datetimev1, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                BooleanLiteral.FALSE);
+        assertRewrite(new GreaterThan(new Cast(datetimev1, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                new GreaterThan(datetimev1, new DateTimeLiteral("2020-01-01 00:00:00")));
+        assertRewrite(new GreaterThanEqual(new Cast(datetimev1, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                new GreaterThanEqual(datetimev1, new DateTimeLiteral("2020-01-01 00:00:01")));
+        assertRewrite(new LessThan(new Cast(datetimev1, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                new LessThan(datetimev1, new DateTimeLiteral("2020-01-01 00:00:01")));
+        assertRewrite(new LessThanEqual(new Cast(datetimev1, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                new LessThanEqual(datetimev1, new DateTimeLiteral("2020-01-01 00:00:00")));
+
+        // cast (datetime0 as datetime) cmp datetime
+        assertRewrite(new EqualTo(new Cast(datetime0, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                ExpressionUtils.falseOrNull(datetime0));
+        assertRewrite(new NullSafeEqual(new Cast(datetime0, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                BooleanLiteral.FALSE);
+        assertRewrite(new GreaterThan(new Cast(datetime0, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                new GreaterThan(datetime0, new DateTimeV2Literal("2020-01-01 00:00:00")));
+        assertRewrite(new GreaterThanEqual(new Cast(datetime0, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                new GreaterThanEqual(datetime0, new DateTimeV2Literal("2020-01-01 00:00:01")));
+        assertRewrite(new LessThan(new Cast(datetime0, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                new LessThan(datetime0, new DateTimeV2Literal("2020-01-01 00:00:01")));
+        assertRewrite(new LessThanEqual(new Cast(datetime0, DateTimeV2Type.of(2)), new DateTimeV2Literal("2020-01-01 00:00:00.12")),
+                new LessThanEqual(datetime0, new DateTimeV2Literal("2020-01-01 00:00:00")));
+
+        // cast (datetime2 as datetime) cmp datetime
+        assertRewrite(new EqualTo(new Cast(datetime2, DateTimeV2Type.of(3)), new DateTimeV2Literal("2020-01-01 00:00:00.123")),
+                ExpressionUtils.falseOrNull(datetime2));
+        assertRewrite(new NullSafeEqual(new Cast(datetime2, DateTimeV2Type.of(3)), new DateTimeV2Literal("2020-01-01 00:00:00.123")),
+                BooleanLiteral.FALSE);
+        assertRewrite(new GreaterThan(new Cast(datetime2, DateTimeV2Type.of(3)), new DateTimeV2Literal("2020-01-01 00:00:00.123")),
+                new GreaterThan(datetime2, new DateTimeV2Literal("2020-01-01 00:00:00.12")));
+        assertRewrite(new GreaterThanEqual(new Cast(datetime2, DateTimeV2Type.of(3)), new DateTimeV2Literal("2020-01-01 00:00:00.123")),
+                new GreaterThanEqual(datetime2, new DateTimeV2Literal("2020-01-01 00:00:00.13")));
+        assertRewrite(new LessThan(new Cast(datetime2, DateTimeV2Type.of(3)), new DateTimeV2Literal("2020-01-01 00:00:00.123")),
+                new LessThan(datetime2, new DateTimeV2Literal("2020-01-01 00:00:00.13")));
+        assertRewrite(new LessThanEqual(new Cast(datetime2, DateTimeV2Type.of(3)), new DateTimeV2Literal("2020-01-01 00:00:00.123")),
+                new LessThanEqual(datetime2, new DateTimeV2Literal("2020-01-01 00:00:00.12")));
     }
 
     @Test
diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
index 920289be66c360..68562a821b81c4 100644
--- a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
+++ b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out
@@ -1528,7 +1528,7 @@ PhysicalResultSink
 ----------PhysicalProject
 ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal]
 ------PhysicalProject
---------filter((cast(d_date as DATETIMEV2(0)) = '2024-08-02 10:10:00'))
+--------filter((t2.d_date = '2024-08-02'))
 ----------PhysicalOlapScan[test_types]
 
 -- !const_value_and_join_column_type170 --
diff --git a/regression-test/suites/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.groovy b/regression-test/suites/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.groovy
index 06998e0e438be1..bf0d6e744202e7 100644
--- a/regression-test/suites/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.groovy
+++ b/regression-test/suites/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.groovy
@@ -997,7 +997,7 @@ select c1 from (select
     qt_const_value_and_join_column_type169 """
 explain shape plan
 select c1 from (select 
-'2024-08-02 10:10:00.123332' as c1 from test_pull_up_predicate_literal limit 10) t inner join test_types t2 on d_date=t.c1"""
+'2024-08-02 00:00:00.000000' as c1 from test_pull_up_predicate_literal limit 10) t inner join test_types t2 on d_date=t.c1"""
 
     qt_const_value_and_join_column_type170 """
 explain shape plan

From 3d1f0c67376c231e764ab214215106784fc0f2cb Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Wed, 18 Dec 2024 15:13:00 +0800
Subject: [PATCH 390/399] [fix](binlog) Avoid clear binlog dir (#45581)

Related PR: #45445

The binlog dir is shared with all rowsets, so the existing binlog dir
should not be cleared directly.

Since the publish_txn might fail even if the add_to_binlog success, so
we need to check whether a file already exists before linking.
---
 be/src/olap/rowset/beta_rowset.cpp | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp
index 2546c87492de84..a328b1b9e8b90e 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -18,6 +18,7 @@
 #include "olap/rowset/beta_rowset.h"
 
 #include <ctype.h>
+#include <errno.h>
 #include <fmt/format.h>
 
 #include <algorithm>
@@ -578,6 +579,10 @@ Status BetaRowset::add_to_binlog() {
         }
     }};
 
+    // The publish_txn might fail even if the add_to_binlog success, so we need to check
+    // whether a file already exists before linking.
+    auto errno_is_file_exists = []() { return Errno::no() == EEXIST; };
+
     // all segments are in the same directory, so cache binlog_dir without multi times check
     std::string binlog_dir;
     for (int i = 0; i < segments_num; ++i) {
@@ -586,20 +591,18 @@ Status BetaRowset::add_to_binlog() {
         if (binlog_dir.empty()) {
             binlog_dir = std::filesystem::path(seg_file).parent_path().append("_binlog").string();
 
-            // Delete all existing files in binlog dir, to keep binlog dir clean.
             bool exists = true;
             RETURN_IF_ERROR(fs->exists(binlog_dir, &exists));
-            if (exists) {
-                RETURN_IF_ERROR(fs->delete_directory(binlog_dir));
+            if (!exists) {
+                RETURN_IF_ERROR(fs->create_directory(binlog_dir));
             }
-            RETURN_IF_ERROR(fs->create_directory(binlog_dir));
         }
 
         auto binlog_file =
                 (std::filesystem::path(binlog_dir) / std::filesystem::path(seg_file).filename())
                         .string();
         VLOG_DEBUG << "link " << seg_file << " to " << binlog_file;
-        if (!fs->link_file(seg_file, binlog_file).ok()) {
+        if (!fs->link_file(seg_file, binlog_file).ok() && !errno_is_file_exists()) {
             status = Status::Error<OS_ERROR>("fail to create hard link. from={}, to={}, errno={}",
                                              seg_file, binlog_file, Errno::no());
             return status;
@@ -616,7 +619,7 @@ Status BetaRowset::add_to_binlog() {
                                           std::filesystem::path(index_file).filename())
                                                  .string();
                 VLOG_DEBUG << "link " << index_file << " to " << binlog_index_file;
-                if (!fs->link_file(index_file, binlog_index_file).ok()) {
+                if (!fs->link_file(index_file, binlog_index_file).ok() && !errno_is_file_exists()) {
                     status = Status::Error<OS_ERROR>(
                             "fail to create hard link. from={}, to={}, errno={}", index_file,
                             binlog_index_file, Errno::no());
@@ -632,7 +635,7 @@ Status BetaRowset::add_to_binlog() {
                                           std::filesystem::path(index_file).filename())
                                                  .string();
                 VLOG_DEBUG << "link " << index_file << " to " << binlog_index_file;
-                if (!fs->link_file(index_file, binlog_index_file).ok()) {
+                if (!fs->link_file(index_file, binlog_index_file).ok() && !errno_is_file_exists()) {
                     status = Status::Error<OS_ERROR>(
                             "fail to create hard link. from={}, to={}, errno={}", index_file,
                             binlog_index_file, Errno::no());

From 58f1df285408163787bb9d52772064c18e5d9560 Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Wed, 18 Dec 2024 15:32:17 +0800
Subject: [PATCH 391/399] [fix](orc) ignore null values when the literals of
 in_predicate contains (#45104)

### What problem does this PR solve?

Related PR: #43255

Problem Summary:
Should ignore null values when the literals of in_predicate contains
null value, like `in (1, null)`
For example, init table in hive:
```sql
CREATE TABLE sample_orc_table (
    id INT,
    name STRING,
    age INT
)
STORED AS ORC;
INSERT INTO TABLE sample_orc_table VALUES
    (1, 'Alice', 25),
    (2, NULL, NULL);
```
select result in Doris should be:
```sql
mysql> select * from sample_orc_table where age in (null,25);
+------+-------+------+
| id   | name  | age  |
+------+-------+------+
|    1 | Alice |   25 |
+------+-------+------+
1 row in set (0.30 sec)

mysql> select * from sample_orc_table where age in (25);
+------+-------+------+
| id   | name  | age  |
+------+-------+------+
|    1 | Alice |   25 |
+------+-------+------+
1 row in set (0.27 sec)

mysql> select * from sample_orc_table where age in (null);
Empty set (0.01 sec)

mysql> select * from sample_orc_table where age is null;
+------+------+------+
| id   | name | age  |
+------+------+------+
|    2 | NULL | NULL |
+------+------+------+
1 row in set (0.11 sec)
```
---
 be/src/apache-orc                             |   2 +-
 be/src/vec/exec/format/orc/vorc_reader.cpp    |  24 +++--
 be/test/vec/exec/orc_reader_test.cpp          |   3 +
 .../external_table_p0/hive/test_hive_orc.out  | 102 ++++++++++++++++++
 .../hive/test_hive_orc.groovy                 |  22 ++++
 5 files changed, 145 insertions(+), 8 deletions(-)

diff --git a/be/src/apache-orc b/be/src/apache-orc
index db01184f765c03..2f937bdc76406f 160000
--- a/be/src/apache-orc
+++ b/be/src/apache-orc
@@ -1 +1 @@
-Subproject commit db01184f765c03496e4107bd3ac37c077ac4bc5f
+Subproject commit 2f937bdc76406f150b484b6e57629aa8a03d48b6
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index df8ec52efc2896..a1ecb1ae0dcf8b 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -586,6 +586,10 @@ std::tuple<bool, orc::Literal, orc::PredicateDataType> OrcReader::_make_orc_lite
         // only get the predicate_type
         return std::make_tuple(true, orc::Literal(true), predicate_type);
     }
+    // this only happens when the literals of in_predicate contains null value, like in (1, null)
+    if (literal->get_column_ptr()->is_null_at(0)) {
+        return std::make_tuple(false, orc::Literal(false), predicate_type);
+    }
     auto literal_data = literal->get_column_ptr()->get_data_at(0);
     auto* slot = _tuple_descriptor->slots()[slot_ref->column_id()];
     auto slot_type = slot->type();
@@ -666,12 +670,13 @@ bool OrcReader::_check_rest_children_can_push_down(const VExprSPtr& expr) {
         return false;
     }
 
+    bool at_least_one_child_can_push_down = false;
     for (size_t i = 1; i < expr->children().size(); ++i) {
-        if (!_check_literal_can_push_down(expr, i)) {
-            return false;
+        if (_check_literal_can_push_down(expr, i)) {
+            at_least_one_child_can_push_down = true;
         }
     }
-    return true;
+    return at_least_one_child_can_push_down;
 }
 
 // check if the expr can be pushed down to orc reader
@@ -774,12 +779,17 @@ void OrcReader::_build_filter_in(const VExprSPtr& expr,
     for (size_t i = 1; i < expr->children().size(); ++i) {
         DCHECK(expr->children()[i]->is_literal());
         const auto* literal = static_cast<const VLiteral*>(expr->children()[i].get());
-        DCHECK(_vliteral_to_orc_literal.contains(literal));
-        auto orc_literal = _vliteral_to_orc_literal.find(literal)->second;
-        literals.emplace_back(orc_literal);
+        if (_vliteral_to_orc_literal.contains(literal)) {
+            auto orc_literal = _vliteral_to_orc_literal.find(literal)->second;
+            literals.emplace_back(orc_literal);
+        }
     }
     DCHECK(!literals.empty());
-    builder->in(slot_ref->expr_name(), predicate_type, literals);
+    if (literals.size() == 1) {
+        builder->equals(slot_ref->expr_name(), predicate_type, literals[0]);
+    } else {
+        builder->in(slot_ref->expr_name(), predicate_type, literals);
+    }
 }
 
 void OrcReader::_build_is_null(const VExprSPtr& expr,
diff --git a/be/test/vec/exec/orc_reader_test.cpp b/be/test/vec/exec/orc_reader_test.cpp
index f2bba434368ae1..ff7452ae625428 100644
--- a/be/test/vec/exec/orc_reader_test.cpp
+++ b/be/test/vec/exec/orc_reader_test.cpp
@@ -111,6 +111,8 @@ TEST_F(OrcReaderTest, test_build_search_argument) {
                             R"|({"1":{"lst":["rec",11,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":2},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":30},"3":{"i32":12},"4":{"i32":2}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":30},"3":{"i32":12},"4":{"i32":2}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(decimalv3(12,2), decimalv3(12,2))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":30},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":30},"3":{"i32":12},"4":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":3},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_totalprice"}},{"1":{"i32":10},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":30},"3":{"i32":12},"4":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"19":{"rec":{"1":{"str":"173665.47"}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":14},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"ge"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"ge(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":1},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_custkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":36901}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
                             // select count(o_orderkey) from tpch1_orc.orders where o_orderkey < 1 + 1;
                             R"|({"1":{"lst":["rec",3,{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":11},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"lt"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"lt(int, int)"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":5},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":2}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
+                            // select count(o_orderkey) from tpch1_orc.orders where o_orderkey in (null, 25);
+                            R"|({"1":{"lst":["rec",4,{"1":{"i32":11},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":5},"4":{"i32":3},"11":{"rec":{"1":{"tf":0}}},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":0},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderkey"}},{"1":{"i32":15},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":9},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":5}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"10":{"rec":{"1":{"i64":25}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
                             // SELECT count(o_orderkey) FROM tpch1_orc.orders WHERE o_comment LIKE '%delayed%' OR o_orderpriority = '1-URGENT';
                             R"|({"1":{"lst":["rec",7,{"1":{"i32":6},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":3},"4":{"i32":2},"20":{"i32":-1},"29":{"tf":1}},{"1":{"i32":20},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"like"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":9}}}}]},"3":{"i64":9}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"like(text, varchar(9))"},"9":{"rec":{"1":{"str":""}}},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":8},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_comment"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"%delayed%"}}},"20":{"i32":-1},"29":{"tf":0}},{"1":{"i32":2},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"3":{"i32":9},"4":{"i32":2},"20":{"i32":-1},"26":{"rec":{"1":{"rec":{"2":{"str":"eq"}}},"2":{"i32":0},"3":{"lst":["rec",2,{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}},{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}]},"4":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":2}}}}]},"3":{"i64":-1}}},"5":{"tf":0},"7":{"str":"eq(text, varchar(65533))"},"11":{"i64":0},"13":{"tf":1},"14":{"tf":0},"15":{"tf":0},"16":{"i64":360}}},"28":{"i32":23},"29":{"tf":1}},{"1":{"i32":16},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":23},"2":{"i32":2147483643}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"15":{"rec":{"1":{"i32":5},"2":{"i32":0},"3":{"i32":-1}}},"20":{"i32":-1},"29":{"tf":1},"36":{"str":"o_orderpriority"}},{"1":{"i32":17},"2":{"rec":{"1":{"lst":["rec",1,{"1":{"i32":0},"2":{"rec":{"1":{"i32":15},"2":{"i32":65533}}}}]},"3":{"i64":-1}}},"4":{"i32":0},"16":{"rec":{"1":{"str":"1-URGENT"}}},"20":{"i32":-1},"29":{"tf":0}}]}})|",
                             // select count(o_orderkey) from tpch1_orc.orders where o_orderkey between 1 and 100 or random() > 0.5;
@@ -143,6 +145,7 @@ TEST_F(OrcReaderTest, test_build_search_argument) {
             "leaf-0 = (o_orderkey < 2), leaf-1 = (o_custkey < 36901), expr = (or leaf-0 (not "
             "leaf-1))",
             "leaf-0 = (o_orderkey < 2), expr = leaf-0",
+            "leaf-0 = (o_orderkey = 25), expr = leaf-0",
             CANNOT_PUSH_DOWN_ERROR,
             CANNOT_PUSH_DOWN_ERROR,
             CANNOT_PUSH_DOWN_ERROR,
diff --git a/regression-test/data/external_table_p0/hive/test_hive_orc.out b/regression-test/data/external_table_p0/hive/test_hive_orc.out
index b34f276020c7cc..03942dbe9fb979 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_orc.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_orc.out
@@ -134,6 +134,57 @@ tablets	tinyint_col	179	182	182	187	183	181	177	183	177	187	183	202	202	186	528
 -- !predicate_pushdown8 --
 1533
 
+-- !predicate_pushdown_in1 --
+0
+
+-- !predicate_pushdown_in2 --
+1634
+
+-- !predicate_pushdown_in3 --
+1597
+
+-- !orc_all_types_tinyint_col_is_null --
+366
+
+-- !orc_all_types_smallint_col_is_null --
+359
+
+-- !orc_all_types_int_col_is_null --
+361
+
+-- !orc_all_types_bigint_col_is_null --
+377
+
+-- !orc_all_types_boolean_col_is_null --
+369
+
+-- !orc_all_types_float_col_is_null --
+340
+
+-- !orc_all_types_double_col_is_null --
+395
+
+-- !orc_all_types_string_col_is_null --
+347
+
+-- !orc_all_types_binary_col_is_null --
+362
+
+-- !orc_all_types_timestamp_col_is_null --
+339
+
+-- !orc_all_types_decimal_col_is_null --
+367
+
+-- !orc_all_types_char_col_is_null --
+0
+
+-- !orc_all_types_varchar_col_is_null --
+0
+
+-- !orc_all_types_date_col_is_null --
+378
+
 -- !select_top50 --
 4	55	999742610	400899305488827731	false	6.5976813E8	7.8723304616937395E17	\N	base tennis pit vertical friday	2022-08-19T07:29:58	\N	tablets	smallint_col	2019-02-07	[7.53124931825377e+17]	["NbSSBtwzpxNSkkwga"]	tablets	smallint_col
 2	49	999613702	105493714032727452	\N	6.3322381E8	9.8642324410240179E17	Unveil bright recruit participate. Suspect impression camera mathematical revelation. Fault live2 elbow debt west hydrogen current.	how literary	2022-09-03T17:20:21	481707.1065	tablets	boolean_col	2020-01-12	[]	["HoMrAnn", "wteEFvIwoZsVpVQdscMb", null, "zcGFmv", "kGEBBckbMtX", "hrEtCGFdPWZK"]	tablets	boolean_col
@@ -321,6 +372,57 @@ tablets	tinyint_col	179	182	182	187	183	181	177	183	177	187	183	202	202	186	528
 -- !predicate_pushdown8 --
 1533
 
+-- !predicate_pushdown_in1 --
+0
+
+-- !predicate_pushdown_in2 --
+1634
+
+-- !predicate_pushdown_in3 --
+1597
+
+-- !orc_all_types_tinyint_col_is_null --
+366
+
+-- !orc_all_types_smallint_col_is_null --
+359
+
+-- !orc_all_types_int_col_is_null --
+361
+
+-- !orc_all_types_bigint_col_is_null --
+377
+
+-- !orc_all_types_boolean_col_is_null --
+369
+
+-- !orc_all_types_float_col_is_null --
+340
+
+-- !orc_all_types_double_col_is_null --
+395
+
+-- !orc_all_types_string_col_is_null --
+347
+
+-- !orc_all_types_binary_col_is_null --
+362
+
+-- !orc_all_types_timestamp_col_is_null --
+339
+
+-- !orc_all_types_decimal_col_is_null --
+367
+
+-- !orc_all_types_char_col_is_null --
+0
+
+-- !orc_all_types_varchar_col_is_null --
+0
+
+-- !orc_all_types_date_col_is_null --
+378
+
 -- !select_top50 --
 4	55	999742610	400899305488827731	false	6.5976813E8	7.8723304616937395E17	\N	base tennis pit vertical friday	2022-08-19T07:29:58	\N	tablets	smallint_col	2019-02-07	[7.53124931825377e+17]	["NbSSBtwzpxNSkkwga"]	tablets	smallint_col
 2	49	999613702	105493714032727452	\N	6.3322381E8	9.8642324410240179E17	Unveil bright recruit participate. Suspect impression camera mathematical revelation. Fault live2 elbow debt west hydrogen current.	how literary	2022-09-03T17:20:21	481707.1065	tablets	boolean_col	2020-01-12	[]	["HoMrAnn", "wteEFvIwoZsVpVQdscMb", null, "zcGFmv", "kGEBBckbMtX", "hrEtCGFdPWZK"]	tablets	boolean_col
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_orc.groovy b/regression-test/suites/external_table_p0/hive/test_hive_orc.groovy
index 6457d2b3edd5d8..8d85feaa77ae9a 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_orc.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_orc.groovy
@@ -90,6 +90,28 @@ suite("test_hive_orc", "all_types,p0,external,hive,external_docker,external_dock
         qt_predicate_pushdown6 """ SELECT count(o_orderkey) FROM tpch1_orc.orders WHERE o_orderstatus <> 'F' AND o_custkey < 54321; """
         qt_predicate_pushdown7 """ SELECT count(o_orderkey) FROM tpch1_orc.orders WHERE o_comment LIKE '%delayed%' OR o_orderpriority = '1-URGENT'; """
         qt_predicate_pushdown8 """ SELECT count(o_orderkey) FROM tpch1_orc.orders WHERE o_orderkey IN (1000000, 2000000, 3000000) OR o_clerk = 'Clerk#000000470'; """
+
+        qt_predicate_pushdown_in1 """ select count(*)  from orc_all_types where boolean_col in (null); """
+        qt_predicate_pushdown_in2 """ select count(*)  from orc_all_types where boolean_col in (null, 0); """
+        qt_predicate_pushdown_in3 """ select count(*)  from orc_all_types where boolean_col in (null, 1); """
+
+        def test_col_is_null = { String col ->
+            "qt_orc_all_types_${col}_is_null" """ select count(*)  from orc_all_types where ${col} is null; """
+        }
+        test_col_is_null("tinyint_col")
+        test_col_is_null("smallint_col")
+        test_col_is_null("int_col")
+        test_col_is_null("bigint_col")
+        test_col_is_null("boolean_col")
+        test_col_is_null("float_col")
+        test_col_is_null("double_col")
+        test_col_is_null("string_col")
+        test_col_is_null("binary_col")
+        test_col_is_null("timestamp_col")
+        test_col_is_null("decimal_col")
+        test_col_is_null("char_col")
+        test_col_is_null("varchar_col")
+        test_col_is_null("date_col")
     }
 
     String enabled = context.config.otherConfigs.get("enableHiveTest")

From f296a02f6ae7e90ad8776357aa55748f650a2cc7 Mon Sep 17 00:00:00 2001
From: smiletan <smiletan@selectdb.com>
Date: Wed, 18 Dec 2024 19:21:04 +0800
Subject: [PATCH 392/399] [improve](docker)add iperf for base image when debug
 network (#45503)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [x] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [x] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [x] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 docker/runtime/base-image/Dockerfile_base | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/runtime/base-image/Dockerfile_base b/docker/runtime/base-image/Dockerfile_base
index d589a28946d446..2c035239a56112 100644
--- a/docker/runtime/base-image/Dockerfile_base
+++ b/docker/runtime/base-image/Dockerfile_base
@@ -28,7 +28,7 @@ ARG TARGETARCH
 
 RUN sed -i -e 's/^APT/# APT/' -e 's/^DPkg/# DPkg/' /etc/apt/apt.conf.d/docker-clean && \
     apt-get update -y &&  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    patchelf gdb binutils binutils-common mysql-client \
+    patchelf gdb binutils binutils-common mysql-client iperf3\
     curl wget less vim htop iproute2 numactl jq iotop sysstat \
     tcpdump iputils-ping dnsutils strace lsof blktrace tzdata \
     bpfcc-tools linux-headers-realtime linux-tools-realtime silversearcher-ag \
@@ -43,4 +43,4 @@ RUN sed -i -e 's/^APT/# APT/' -e 's/^DPkg/# DPkg/' /etc/apt/apt.conf.d/docker-cl
 
 COPY --from=selectdb/doris-debug-ubuntu:latest /doris-debug /opt/apache-doris/
 
-WORKDIR /opt/apache-doris
\ No newline at end of file
+WORKDIR /opt/apache-doris

From f1677000f5d67db1ac4c8c4c36b674afe186ba49 Mon Sep 17 00:00:00 2001
From: Sun Chenyang <sunchenyang@selectdb.com>
Date: Wed, 18 Dec 2024 19:35:09 +0800
Subject: [PATCH 393/399] Revert "[fix] (compaction) fix compaction score in
 time series policy(#40242)" (#44525)

revert #40242
---
 ...mulative_compaction_time_series_policy.cpp | 68 ++++++++++++++++++-
 ...ive_compaction_time_series_policy_test.cpp |  2 +-
 2 files changed, 68 insertions(+), 2 deletions(-)

diff --git a/be/src/olap/cumulative_compaction_time_series_policy.cpp b/be/src/olap/cumulative_compaction_time_series_policy.cpp
index 6fa4b8d014313f..64e51c77641311 100644
--- a/be/src/olap/cumulative_compaction_time_series_policy.cpp
+++ b/be/src/olap/cumulative_compaction_time_series_policy.cpp
@@ -27,11 +27,14 @@ namespace doris {
 
 uint32_t TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score(Tablet* tablet) {
     uint32_t score = 0;
+    uint32_t level0_score = 0;
     bool base_rowset_exist = false;
     const int64_t point = tablet->cumulative_layer_point();
 
+    int64_t level0_total_size = 0;
     RowsetMetaSharedPtr first_meta;
     int64_t first_version = INT64_MAX;
+    std::list<RowsetMetaSharedPtr> checked_rs_metas;
     // NOTE: tablet._meta_lock is hold
     auto& rs_metas = tablet->tablet_meta()->all_rs_metas();
     // check the base rowset and collect the rowsets of cumulative part
@@ -50,6 +53,12 @@ uint32_t TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score(
         } else {
             // collect the rowsets of cumulative part
             score += rs_meta->get_compaction_score();
+            if (rs_meta->compaction_level() == 0) {
+                level0_total_size += rs_meta->total_disk_size();
+                level0_score += rs_meta->get_compaction_score();
+            } else {
+                checked_rs_metas.push_back(rs_meta);
+            }
         }
     }
 
@@ -64,7 +73,64 @@ uint32_t TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score(
         return 0;
     }
 
-    return score;
+    // Condition 1: the size of input files for compaction meets the requirement of parameter compaction_goal_size
+    int64_t compaction_goal_size_mbytes =
+            tablet->tablet_meta()->time_series_compaction_goal_size_mbytes();
+    if (level0_total_size >= compaction_goal_size_mbytes * 1024 * 1024) {
+        return score;
+    }
+
+    // Condition 2: the number of input files reaches the threshold specified by parameter compaction_file_count_threshold
+    if (level0_score >= tablet->tablet_meta()->time_series_compaction_file_count_threshold()) {
+        return score;
+    }
+
+    // Condition 3: level1 achieve compaction_goal_size
+    if (tablet->tablet_meta()->time_series_compaction_level_threshold() >= 2) {
+        checked_rs_metas.sort([](const RowsetMetaSharedPtr& a, const RowsetMetaSharedPtr& b) {
+            return a->version().first < b->version().first;
+        });
+        int32_t rs_meta_count = 0;
+        int64_t continuous_size = 0;
+        for (const auto& rs_meta : checked_rs_metas) {
+            rs_meta_count++;
+            continuous_size += rs_meta->total_disk_size();
+            if (rs_meta_count >= 2) {
+                if (continuous_size >= compaction_goal_size_mbytes * 1024 * 1024) {
+                    return score;
+                }
+            }
+        }
+    }
+
+    int64_t now = UnixMillis();
+    int64_t last_cumu = tablet->last_cumu_compaction_success_time();
+    if (last_cumu != 0) {
+        int64_t cumu_interval = now - last_cumu;
+
+        // Condition 4: the time interval between compactions exceeds the value specified by parameter _compaction_time_threshold_second
+        if (cumu_interval >
+            (tablet->tablet_meta()->time_series_compaction_time_threshold_seconds() * 1000)) {
+            return score;
+        }
+    } else if (score > 0) {
+        // If the compaction process has not been successfully executed,
+        // the condition for triggering compaction based on the last successful compaction time (condition 3) will never be met
+        tablet->set_last_cumu_compaction_success_time(now);
+    }
+
+    // Condition 5: If there is a continuous set of empty rowsets, prioritize merging.
+    std::vector<RowsetSharedPtr> input_rowsets;
+    std::vector<RowsetSharedPtr> candidate_rowsets =
+            tablet->pick_candidate_rowsets_to_cumulative_compaction();
+    tablet->calc_consecutive_empty_rowsets(
+            &input_rowsets, candidate_rowsets,
+            tablet->tablet_meta()->time_series_compaction_empty_rowsets_threshold());
+    if (!input_rowsets.empty()) {
+        return score;
+    }
+
+    return 0;
 }
 
 void TimeSeriesCumulativeCompactionPolicy::calculate_cumulative_point(
diff --git a/be/test/olap/cumulative_compaction_time_series_policy_test.cpp b/be/test/olap/cumulative_compaction_time_series_policy_test.cpp
index 79e540fca77d05..386a7a2d211c3a 100644
--- a/be/test/olap/cumulative_compaction_time_series_policy_test.cpp
+++ b/be/test/olap/cumulative_compaction_time_series_policy_test.cpp
@@ -404,7 +404,7 @@ TEST_F(TestTimeSeriesCumulativeCompactionPolicy, calc_cumulative_compaction_scor
     bool ret = _tablet->suitable_for_compaction(CompactionType::CUMULATIVE_COMPACTION,
                                                 cumulative_compaction_policy);
 
-    EXPECT_EQ(true, ret);
+    EXPECT_EQ(false, ret);
 }
 
 TEST_F(TestTimeSeriesCumulativeCompactionPolicy, calc_cumulative_compaction_score_big_rowset) {

From 011d3934e100d4668403649d892f1a2c2cdb7dae Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Wed, 18 Dec 2024 20:34:46 +0800
Subject: [PATCH 394/399] [fix](shell) clean ShellCheck (#45608)

### What problem does this PR solve?
Related PR: #45490

Problem Summary:
```shell
In gensrc/script/gen_build_version.sh line 38:
if [[ ${build_version_hotfix} > 0 ]]; then
                              ^-- SC2071 (error): > is for string comparisons. Use -gt instead.


In gensrc/script/gen_build_version.sh line 228:
if [[ ${build_version_hotfix} > 0 ]]; then
                              ^-- SC2071 (error): > is for string comparisons. Use -gt instead.
```
---
 gensrc/script/gen_build_version.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh
index a85effebd5e3bb..6b6852199682b0 100755
--- a/gensrc/script/gen_build_version.sh
+++ b/gensrc/script/gen_build_version.sh
@@ -35,7 +35,7 @@ build_version_hotfix=0
 build_version_rc_version=""
 
 build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}"
-if [[ ${build_version_hotfix} > 0 ]]; then
+if [[ ${build_version_hotfix} -gt 0 ]]; then
     build_version+=".${build_version_hotfix}"
 fi
 build_version+="-${build_version_rc_version}"
@@ -225,7 +225,7 @@ fi
 
 build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}"
 
-if [[ ${build_version_hotfix} > 0 ]]; then
+if [[ ${build_version_hotfix} -gt 0 ]]; then
     build_version+=".${build_version_hotfix}"
 fi
 

From 14a01734e08cc81c47e323df8ea8fc5048386d73 Mon Sep 17 00:00:00 2001
From: Petrichor <vinleexiao@163.com>
Date: Thu, 19 Dec 2024 11:36:12 +0800
Subject: [PATCH 395/399] [feat](nereids)support refresh table command (#45546)

Issue Number: close https://github.com/apache/doris/issues/42578
---
 .../org/apache/doris/nereids/DorisParser.g4   |  4 +-
 .../nereids/parser/LogicalPlanBuilder.java    | 14 ++++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/refresh/RefreshTableCommand.java | 84 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../test_nereids_refresh_catalog.out          | 21 +++++
 .../test_nereids_refresh_catalog.groovy       | 18 ++++
 7 files changed, 145 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshTableCommand.java

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 64ff9209bfe6f2..ad884eefee18de 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -457,6 +457,7 @@ channelDescription
 supportedRefreshStatement
     : REFRESH CATALOG name=identifier propertyClause?                               #refreshCatalog
     | REFRESH DATABASE name=multipartIdentifier propertyClause?                     #refreshDatabase
+    | REFRESH TABLE name=multipartIdentifier                                        #refreshTable
     ;
 
 supportedCleanStatement
@@ -464,8 +465,7 @@ supportedCleanStatement
     ;
 
 unsupportedRefreshStatement
-    : REFRESH TABLE name=multipartIdentifier                                        #refreshTable
-    | REFRESH LDAP (ALL | (FOR user=identifierOrText))                              #refreshLdap
+    : REFRESH LDAP (ALL | (FOR user=identifierOrText))                              #refreshLdap
     ;
 
 unsupportedCleanStatement
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 951c8b342d5d9e..4c5d43cb4dbefe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -208,6 +208,7 @@
 import org.apache.doris.nereids.DorisParser.RefreshMTMVContext;
 import org.apache.doris.nereids.DorisParser.RefreshMethodContext;
 import org.apache.doris.nereids.DorisParser.RefreshScheduleContext;
+import org.apache.doris.nereids.DorisParser.RefreshTableContext;
 import org.apache.doris.nereids.DorisParser.RefreshTriggerContext;
 import org.apache.doris.nereids.DorisParser.RegularQuerySpecificationContext;
 import org.apache.doris.nereids.DorisParser.RelationContext;
@@ -670,6 +671,7 @@
 import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
+import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.use.SwitchCommand;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
@@ -4464,6 +4466,18 @@ public RefreshDatabaseCommand visitRefreshDatabase(RefreshDatabaseContext ctx) {
         throw new ParseException("Only one dot can be in the name: " + String.join(".", parts));
     }
 
+    @Override
+    public Object visitRefreshTable(RefreshTableContext ctx) {
+        List<String> parts = visitMultipartIdentifier(ctx.name);
+        int size = parts.size();
+        if (size == 0) {
+            throw new ParseException("table name can't be empty");
+        } else if (size <= 3) {
+            return new RefreshTableCommand(new TableNameInfo(parts));
+        }
+        throw new ParseException("Only one or two dot can be in the name: " + String.join(".", parts));
+    }
+
     @Override
     public LogicalPlan visitShowCreateRepository(ShowCreateRepositoryContext ctx) {
         return new ShowCreateRepositoryCommand(ctx.identifier().getText());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 9ed408dfe05b45..0cec8c48caf767 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -194,6 +194,7 @@ public enum PlanType {
     SET_DEFAULT_STORAGE_VAULT_COMMAND,
     REFRESH_CATALOG_COMMAND,
     REFRESH_DATABASE_COMMAND,
+    REFRESH_TABLE_COMMAND,
     PREPARED_COMMAND,
     EXECUTE_COMMAND,
     DROP_SQL_BLOCK_RULE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshTableCommand.java
new file mode 100644
index 00000000000000..71d77ab22b36fd
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshTableCommand.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.refresh;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
+import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Refresh table command.
+ */
+public class RefreshTableCommand extends Command implements ForwardWithSync {
+    private static final Logger LOG = LogManager.getLogger(RefreshTableCommand.class);
+    private TableNameInfo tableNameInfo;
+
+    public RefreshTableCommand(TableNameInfo tableName) {
+        super(PlanType.REFRESH_TABLE_COMMAND);
+        this.tableNameInfo = tableName;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        tableNameInfo.analyze(ctx);
+        checkRefreshTableAccess();
+        // refresh table execute logic
+        Env.getCurrentEnv().getRefreshManager()
+                .refreshTable(tableNameInfo.getCtl(), tableNameInfo.getDb(), tableNameInfo.getTbl(), false);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitRefreshTableCommand(this, context);
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.REFRESH;
+    }
+
+    /**
+     * check access.
+     */
+    private void checkRefreshTableAccess() throws AnalysisException {
+        if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(),
+                tableNameInfo.getCtl(), tableNameInfo.getDb(),
+                tableNameInfo.getTbl(), PrivPredicate.SHOW)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLE_ACCESS_DENIED_ERROR,
+                    PrivPredicate.SHOW.getPrivs().toString(), tableNameInfo.getTbl());
+        }
+    }
+
+    public String toSql() {
+        return "REFRESH TABLE " + tableNameInfo.toSql();
+    }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index eb764be57c0097..f988055f0f613d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -144,6 +144,7 @@
 import org.apache.doris.nereids.trees.plans.commands.load.CreateRoutineLoadCommand;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
+import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.use.SwitchCommand;
 
 /** CommandVisitor. */
@@ -415,6 +416,10 @@ default R visitRefreshDatabaseCommand(RefreshDatabaseCommand refreshDatabaseComm
         return visitCommand(refreshDatabaseCommand, context);
     }
 
+    default R visitRefreshTableCommand(RefreshTableCommand refreshTableCommand, C context) {
+        return visitCommand(refreshTableCommand, context);
+    }
+
     default R visitShowBackendsCommand(ShowBackendsCommand showBackendsCommand, C context) {
         return visitCommand(showBackendsCommand, context);
     }
diff --git a/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
index c3f57d7ba810b8..66ca9363d711dc 100644
--- a/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
+++ b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
@@ -54,6 +54,27 @@ new_mysql_table1
 new_mysql_table1
 new_mysql_table2
 
+-- !show_create_table --
+new_mysql_table2	CREATE TABLE `new_mysql_table2` (\n  `id` int NULL,\n  `name` varchar(20) NULL\n) ENGINE=JDBC_EXTERNAL_TABLE;
+
+-- !preceding_refresh_table --
+new_mysql_table2	CREATE TABLE `new_mysql_table2` (\n  `id` int NULL,\n  `name` varchar(20) NULL\n) ENGINE=JDBC_EXTERNAL_TABLE;
+
+-- !subsequent_refresh_table --
+new_mysql_table2	CREATE TABLE `new_mysql_table2` (\n  `id` int NULL,\n  `name` varchar(20) NULL,\n  `new_column` int NULL\n) ENGINE=JDBC_EXTERNAL_TABLE;
+
+-- !preceding_refresh_table --
+new_mysql_table2	CREATE TABLE `new_mysql_table2` (\n  `id` int NULL,\n  `name` varchar(20) NULL,\n  `new_column` int NULL\n) ENGINE=JDBC_EXTERNAL_TABLE;
+
+-- !subsequent_refresh_table --
+new_mysql_table2	CREATE TABLE `new_mysql_table2` (\n  `id` int NULL,\n  `name` varchar(20) NULL,\n  `new_column` int NULL,\n  `new_column_1` int NULL\n) ENGINE=JDBC_EXTERNAL_TABLE;
+
+-- !preceding_refresh_table --
+new_mysql_table2	CREATE TABLE `new_mysql_table2` (\n  `id` int NULL,\n  `name` varchar(20) NULL,\n  `new_column` int NULL,\n  `new_column_1` int NULL\n) ENGINE=JDBC_EXTERNAL_TABLE;
+
+-- !subsequent_refresh_table --
+new_mysql_table2	CREATE TABLE `new_mysql_table2` (\n  `id` int NULL,\n  `name` varchar(20) NULL,\n  `new_column` int NULL,\n  `new_column_1` int NULL,\n  `new_column_2` int NULL\n) ENGINE=JDBC_EXTERNAL_TABLE;
+
 -- !preceding_drop_external_database --
 DORIS
 Doris
diff --git a/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
index 87dacd4142501e..8fbaa486610109 100644
--- a/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
+++ b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
@@ -75,6 +75,24 @@ suite("test_nereids_refresh_catalog", "p0,external,mysql,external_docker,externa
         checkNereidsExecute("refresh database ${catalog_name}.${new_mysql_db} ;")
         qt_subsequent_refresh_database """show tables;"""
 
+        qt_show_create_table """show create table ${new_mysql_db}.${new_mysql_table2};"""
+        // add a column in external table
+        sql """CALL EXECUTE_STMT("${catalog_name}", "alter table ${new_mysql_db}.${new_mysql_table2} add new_column int;");"""
+        qt_preceding_refresh_table """show create table ${new_mysql_db}.${new_mysql_table2};"""
+        //refresh table ctl.db.tb1
+        checkNereidsExecute("refresh table ${catalog_name}.${new_mysql_db}.${new_mysql_table2} ;")
+        qt_subsequent_refresh_table """show create table ${new_mysql_db}.${new_mysql_table2};"""
+        sql """CALL EXECUTE_STMT("${catalog_name}", "alter table ${new_mysql_db}.${new_mysql_table2} add new_column_1 int;");"""
+        qt_preceding_refresh_table """show create table ${new_mysql_db}.${new_mysql_table2};"""
+        // refresh table db.tbl
+        checkNereidsExecute("refresh table ${new_mysql_db}.${new_mysql_table2} ;")
+        qt_subsequent_refresh_table """show create table ${new_mysql_db}.${new_mysql_table2};"""
+        sql """CALL EXECUTE_STMT("${catalog_name}", "alter table ${new_mysql_db}.${new_mysql_table2} add new_column_2 int;");"""
+        qt_preceding_refresh_table """show create table ${new_mysql_db}.${new_mysql_table2};"""
+        //refresh table tbl1
+        checkNereidsExecute("refresh table ${new_mysql_table2};")
+        qt_subsequent_refresh_table """show create table ${new_mysql_db}.${new_mysql_table2};"""
+
         sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists ${new_mysql_db} ;");"""
         qt_preceding_drop_external_database """show databases;"""
 

From a05582f990117ec6b19925ce72dd81c8ff2164f4 Mon Sep 17 00:00:00 2001
From: Sridhar R Manikarnike <sridhar.nike@gmail.com>
Date: Thu, 19 Dec 2024 09:06:49 +0530
Subject: [PATCH 396/399] [Enhancement] (nereids)implement showSyncJobCommand
 in nereids (#44817)

issue Number: close #42775
---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  12 ++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../plans/commands/ShowSyncJobCommand.java    | 115 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../show/test_show_sync_job_command.groovy    |  56 +++++++++
 6 files changed, 190 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSyncJobCommand.java
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_sync_job_command.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index ad884eefee18de..6c37a2b276b0dd 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -239,6 +239,7 @@ supportedShowStatement
     | SHOW DELETE ((FROM | IN) database=multipartIdentifier)?                       #showDelete
     | SHOW ALL? GRANTS                                                              #showGrants
     | SHOW GRANTS FOR userIdentify                                                  #showGrantsForUser
+    | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)?                     #showSyncJob    
     | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL                                   #showLoadProfile
     | SHOW CREATE REPOSITORY FOR identifier                                         #showCreateRepository
     | SHOW VIEW
@@ -366,7 +367,6 @@ unsupportedShowStatement
         ((FROM | IN) database=multipartIdentifier)?                                 #showIndex
     | SHOW TRANSACTION ((FROM | IN) database=multipartIdentifier)? wildWhere?       #showTransaction
     | SHOW CACHE HOTSPOT tablePath=STRING_LITERAL                                   #showCacheHotSpot
-    | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)?                     #showSyncJob
     | SHOW CATALOG RECYCLE BIN wildWhere?                                           #showCatalogRecycleBin
     | SHOW QUERY STATS ((FOR database=identifier)
             | (FROM tableName=multipartIdentifier (ALL VERBOSE?)?))?                #showQueryStats
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 4c5d43cb4dbefe..a9ce9215d4d9be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -279,6 +279,7 @@
 import org.apache.doris.nereids.DorisParser.ShowSmallFilesContext;
 import org.apache.doris.nereids.DorisParser.ShowSqlBlockRuleContext;
 import org.apache.doris.nereids.DorisParser.ShowStorageEnginesContext;
+import org.apache.doris.nereids.DorisParser.ShowSyncJobContext;
 import org.apache.doris.nereids.DorisParser.ShowTableCreationContext;
 import org.apache.doris.nereids.DorisParser.ShowTableIdContext;
 import org.apache.doris.nereids.DorisParser.ShowTabletStorageFormatContext;
@@ -593,6 +594,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowSyncJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTabletStorageFormatCommand;
@@ -4979,6 +4981,16 @@ public LogicalPlan visitCreateWorkloadGroup(CreateWorkloadGroupContext ctx) {
         return new CreateWorkloadGroupCommand(workloadGroupName, ifNotExists, properties);
     }
 
+    @Override
+    public LogicalPlan visitShowSyncJob(ShowSyncJobContext ctx) {
+        String databaseName = null;
+        if (ctx.multipartIdentifier() != null) {
+            List<String> databaseParts = visitMultipartIdentifier(ctx.multipartIdentifier());
+            databaseName = databaseParts.get(0);
+        }
+        return new ShowSyncJobCommand(databaseName);
+    }
+
     @Override
     public LogicalPlan visitDropFile(DropFileContext ctx) {
         String dbName = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 0cec8c48caf767..bfa0163e7d267e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -237,6 +237,7 @@ public enum PlanType {
     SHOW_ROLE_COMMAND,
     SHOW_SMALL_FILES_COMMAND,
     SHOW_STORAGE_ENGINES_COMMAND,
+    SHOW_SYNC_JOB_COMMAND,
     SHOW_TABLE_ID_COMMAND,
     SHOW_TRASH_COMMAND,
     SHOW_TABLET_STORAGE_FORMAT_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSyncJobCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSyncJobCommand.java
new file mode 100644
index 00000000000000..a673164707f5fa
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowSyncJobCommand.java
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.RedirectStatus;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.ListComparator;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents the command for SHOW SYNC JOB.
+ */
+public class ShowSyncJobCommand extends ShowCommand {
+    public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
+            .add("JobId").add("JobName").add("Type").add("State").add("Channel").add("Status")
+            .add("JobConfig").add("CreateTime").add("LastStartTime").add("LastStopTime").add("FinishTime").add("Msg")
+            .build();
+
+    private String databaseName;
+
+    public ShowSyncJobCommand(String databaseName) {
+        super(PlanType.SHOW_SYNC_JOB_COMMAND);
+        this.databaseName = databaseName;
+    }
+
+    private void validate(ConnectContext ctx) throws AnalysisException {
+        if (Strings.isNullOrEmpty(databaseName)) {
+            databaseName = ctx.getDatabase();
+            if (Strings.isNullOrEmpty(databaseName)) {
+                ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
+            }
+        }
+        if (!Env.getCurrentEnv().getAccessManager()
+                .checkDbPriv(ConnectContext.get(), InternalCatalog.INTERNAL_CATALOG_NAME,
+                    databaseName, PrivPredicate.SHOW)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED_ERROR,
+                    PrivPredicate.SHOW.getPrivs().toString(), databaseName);
+        }
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        Env env = Env.getCurrentEnv();
+        DatabaseIf db = Env.getCurrentInternalCatalog().getDbOrAnalysisException(databaseName);
+
+        List<List<Comparable>> syncInfos = env.getSyncJobManager().getSyncJobsInfoByDbId(db.getId());
+        Collections.sort(syncInfos, new ListComparator<List<Comparable>>(0));
+
+        List<List<String>> rows = Lists.newArrayList();
+        for (List<Comparable> syncInfo : syncInfos) {
+            List<String> row = new ArrayList<String>(syncInfo.size());
+
+            for (Comparable element : syncInfo) {
+                row.add(element.toString());
+            }
+            rows.add(row);
+        }
+        return new ShowResultSet(this.getMetaData(), rows);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowSyncJobCommand(this, context);
+    }
+
+    public ShowResultSetMetaData getMetaData() {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        for (String title : TITLE_NAMES) {
+            builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
+        }
+        return builder.build();
+    }
+
+    @Override
+    public RedirectStatus toRedirectStatus() {
+        return RedirectStatus.FORWARD_NO_SYNC;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index f988055f0f613d..6efa58c1b8eb91 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -123,6 +123,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowSyncJobCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowTabletStorageFormatCommand;
@@ -299,6 +300,10 @@ default R visitCallCommand(CallCommand callCommand, C context) {
         return visitCommand(callCommand, context);
     }
 
+    default R visitShowSyncJobCommand(ShowSyncJobCommand showSyncJobCommand, C context) {
+        return visitCommand(showSyncJobCommand, context);
+    }
+
     default R visitCreateProcedureCommand(CreateProcedureCommand createProcedureCommand, C context) {
         return visitCommand(createProcedureCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/show/test_show_sync_job_command.groovy b/regression-test/suites/nereids_p0/show/test_show_sync_job_command.groovy
new file mode 100644
index 00000000000000..32b460be490575
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_sync_job_command.groovy
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_sync_job_command", "query,sync_job") {
+    try {
+        sql """CREATE DATABASE IF NOT EXISTS test_db_sync_job;"""
+        
+        sql """CREATE TABLE IF NOT EXISTS test_db_sync_job.test_tbl1_sync_job (
+                  id INT,
+                  name STRING
+              ) UNIQUE KEY(id)
+              DISTRIBUTED BY HASH(id) BUCKETS 1
+              PROPERTIES("replication_num" = "1");"""
+
+        // Create the sync job
+        sql """CREATE SYNC test_db_sync_job.job1
+        (
+            FROM mysql_db1.tbl1 INTO test_tbl1_sync_job
+        )
+        FROM BINLOG
+        (
+            "type" = "canal",
+            "canal.server.ip" = "127.0.0.1",
+            "canal.server.port" = "11111",
+            "canal.destination" = "example",
+            "canal.username" = "",
+            "canal.password" = ""
+        );"""
+
+        checkNereidsExecute("SHOW SYNC JOB FROM test_db_sync_job")
+
+    } catch (Exception e) {
+        // Log any exceptions that occur during testing
+        log.error("Failed to execute CREATE SYNC JOB command", e)
+        throw e
+    } finally {
+        // Cleanup
+        try_sql("STOP SYNC JOB IF EXISTS job1;")
+        try_sql("DROP TABLE IF EXISTS test_db_sync_job.test_tbl1_sync_job;")
+        try_sql("DROP DATABASE IF EXISTS test_db_sync_job;")
+    }
+}

From 9d23c1390fc9701565b464a2d6989dcfea443e3a Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Thu, 19 Dec 2024 11:43:15 +0530
Subject: [PATCH 397/399] [Enhancement] (nereids)implement
 adminSetTableStatusCommand in nereids (#45272)

Issue Number: close #42851
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 .../nereids/parser/LogicalPlanBuilder.java    | 10 ++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/AdminSetTableStatusCommand.java  | 98 +++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 +
 .../test_nereids_admin_set_tbl_status.groovy  | 49 ++++++++++
 6 files changed, 164 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminSetTableStatusCommand.java
 create mode 100644 regression-test/suites/nereids_p0/ddl/admin/test_nereids_admin_set_tbl_status.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 6c37a2b276b0dd..0cd32f3820fda4 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -508,6 +508,7 @@ supportedAdminStatement
     | ADMIN CLEAN TRASH
         (ON LEFT_PAREN backends+=STRING_LITERAL
               (COMMA backends+=STRING_LITERAL)* RIGHT_PAREN)?                       #adminCleanTrash
+    | ADMIN SET TABLE name=multipartIdentifier STATUS properties=propertyClause?    #adminSetTableStatus    
     ;
 
 supportedRecoverStatement
@@ -528,7 +529,6 @@ unsupportedAdminStatement
     | ADMIN SET TABLE name=multipartIdentifier
         PARTITION VERSION properties=propertyClause?                                #adminSetPartitionVersion
     | ADMIN COPY TABLET tabletId=INTEGER_VALUE properties=propertyClause?           #adminCopyTablet
-    | ADMIN SET TABLE name=multipartIdentifier STATUS properties=propertyClause?    #adminSetTableStatus
     ;
 
 baseTableRef
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index a9ce9215d4d9be..68226f156c7f25 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -55,6 +55,7 @@
 import org.apache.doris.nereids.DorisParser.AdminCompactTableContext;
 import org.apache.doris.nereids.DorisParser.AdminDiagnoseTabletContext;
 import org.apache.doris.nereids.DorisParser.AdminRebalanceDiskContext;
+import org.apache.doris.nereids.DorisParser.AdminSetTableStatusContext;
 import org.apache.doris.nereids.DorisParser.AdminShowReplicaDistributionContext;
 import org.apache.doris.nereids.DorisParser.AdminShowReplicaStatusContext;
 import org.apache.doris.nereids.DorisParser.AdminShowTabletStorageFormatContext;
@@ -491,6 +492,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminRebalanceDiskCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminSetTableStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterCatalogCommentCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
@@ -4844,6 +4846,14 @@ public LogicalPlan visitShowCharset(ShowCharsetContext ctx) {
         return new ShowCharsetCommand();
     }
 
+    @Override
+    public LogicalPlan visitAdminSetTableStatus(AdminSetTableStatusContext ctx) {
+        List<String> dbTblNameParts = visitMultipartIdentifier(ctx.name);
+        Map<String, String> properties = ctx.propertyClause() != null
+                        ? Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+        return new AdminSetTableStatusCommand(new TableNameInfo(dbTblNameParts), properties);
+    }
+
     @Override
     public LogicalPlan visitShowFrontends(ShowFrontendsContext ctx) {
         String detail = (ctx.name != null) ? ctx.name.getText() : null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index bfa0163e7d267e..6395f429db29de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -201,6 +201,7 @@ public enum PlanType {
     DROP_USER_COMMAND,
     DROP_WORKLOAD_GROUP_NAME,
     DROP_WORKLOAD_POLICY_COMMAND,
+    ADMIN_SET_TABLE_STATUS_COMMAND,
     ALTER_CATALOG_COMMENT_COMMAND,
     ALTER_SQL_BLOCK_RULE_COMMAND,
     SHOW_BACKENDS_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminSetTableStatusCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminSetTableStatusCommand.java
new file mode 100644
index 00000000000000..36c46c333468c7
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminSetTableStatusCommand.java
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable.OlapTableState;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import java.util.Map;
+
+/**
+ * AdminSetTableStatusCommand
+ */
+public class AdminSetTableStatusCommand extends Command implements ForwardWithSync {
+    public static final String TABLE_STATE = "state";
+    private final TableNameInfo tableNameInfo;
+    private final Map<String, String> properties;
+    private OlapTableState tableState;
+
+    /**
+    * constructor
+    */
+    public AdminSetTableStatusCommand(TableNameInfo tableNameInfo, Map<String, String> properties) {
+        super(PlanType.ADMIN_SET_TABLE_STATUS_COMMAND);
+        this.tableNameInfo = tableNameInfo;
+        this.properties = properties;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        validate(ctx);
+        Env.getCurrentEnv().setTableStatusInternal(tableNameInfo.getDb(), tableNameInfo.getTbl(), tableState, false);
+    }
+
+    private void validate(ConnectContext ctx) throws UserException {
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+        }
+
+        tableNameInfo.analyze(ctx);
+        Util.prohibitExternalCatalog(tableNameInfo.getCtl(), this.getClass().getSimpleName());
+
+        checkProperties();
+    }
+
+    private void checkProperties() throws AnalysisException {
+        for (Map.Entry<String, String> entry : properties.entrySet()) {
+            String key = entry.getKey();
+            String val = entry.getValue();
+
+            if (key.equalsIgnoreCase(TABLE_STATE)) {
+                try {
+                    tableState = OlapTableState.valueOf(val.toUpperCase());
+                } catch (IllegalArgumentException e) {
+                    throw new AnalysisException("Invalid table state: " + val);
+                }
+            } else {
+                throw new AnalysisException("Unsupported property: " + key);
+            }
+        }
+
+        if (tableState == null) {
+            throw new AnalysisException("Should add properties: STATE.");
+        }
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitAdminSetTableStatusCommand(this, context);
+    }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 6efa58c1b8eb91..a9340894c33590 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminRebalanceDiskCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminSetTableStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterCatalogCommentCommand;
 import org.apache.doris.nereids.trees.plans.commands.AlterJobStatusCommand;
@@ -236,6 +237,10 @@ default R visitAdminCleanTrashCommand(AdminCleanTrashCommand adminCleanTrashComm
         return visitCommand(adminCleanTrashCommand, context);
     }
 
+    default R visitAdminSetTableStatusCommand(AdminSetTableStatusCommand cmd, C context) {
+        return visitCommand(cmd, context);
+    }
+
     default R visitDropConstraintCommand(DropConstraintCommand dropConstraintCommand, C context) {
         return visitCommand(dropConstraintCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/ddl/admin/test_nereids_admin_set_tbl_status.groovy b/regression-test/suites/nereids_p0/ddl/admin/test_nereids_admin_set_tbl_status.groovy
new file mode 100644
index 00000000000000..8e2a8b67aaed0f
--- /dev/null
+++ b/regression-test/suites/nereids_p0/ddl/admin/test_nereids_admin_set_tbl_status.groovy
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_nereids_admin_set_tbl_status") {
+    def tbName1 = "test_nereids_admin_set_tbl_status"
+
+    try {
+        sql "DROP TABLE IF EXISTS ${tbName1}"
+        sql """
+                CREATE TABLE IF NOT EXISTS ${tbName1} (
+                    k1 INT,
+                    v1 INT,
+                    v2 INT
+                )
+                DUPLICATE KEY (k1)
+                DISTRIBUTED BY HASH(k1) BUCKETS 1 properties("replication_num" = "1", "light_schema_change" = "false", "disable_auto_compaction" = "true");
+            """
+
+        // set table state to ROLLUP
+        checkNereidsExecute("ADMIN SET TABLE ${tbName1} STATUS PROPERTIES ('state' = 'rollup');")
+        // try alter table comment
+        test {
+            sql """ ALTER TABLE ${tbName1} MODIFY COMMENT 'test'; """
+            exception "Table[test_nereids_admin_set_tbl_status]'s state(ROLLUP) is not NORMAL. Do not allow doing ALTER ops"
+        }
+
+        // set table state to NORMAL
+        checkNereidsExecute("ADMIN SET TABLE ${tbName1} STATUS PROPERTIES ('state' = 'normal');")
+        // try alter table comment
+        sql """ ALTER TABLE ${tbName1} MODIFY COMMENT 'test'; """
+    } finally {
+        // drop table
+        sql """ DROP TABLE  ${tbName1} force"""
+    }
+}

From b124c560f78320ba8982698c06a07e5af6c50cd4 Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Thu, 19 Dec 2024 15:11:02 +0800
Subject: [PATCH 398/399] [fix](nereids) fix
 ExtractAndNormalizeWindowExpression bug (#45553)

Problem Summary:
In ExtractAndNormalizeWindowExpression, some expressions are pushed down
and output by the bottom LogicalProject. If a window expression depends
on these pushed-down expressions, those parts should be replaced
accordingly. However, when a Literal is pushed down, it should not be
used as a replacement.

Example:
For the window expression:
last_value(c1, false) over()

If the expression false is pushed down as alias1, the window expression
would incorrectly be replaced as:
last_value(c1, alias1) over()

This PR fixes the issue by ensuring Literals are not replaced when
pushed down.
---
 .../ExtractAndNormalizeWindowExpression.java  |  3 +-
 .../normalize_window_nullable_agg_test.out    |  4 ++
 .../normalize_window_nullable_agg_test.groovy | 57 +++++++++++++++++++
 3 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java
index 2cfe4523003879..a74ebe4b76b9cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java
@@ -27,6 +27,7 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.WindowExpression;
 import org.apache.doris.nereids.trees.expressions.functions.agg.NullableAggregateFunction;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
@@ -117,7 +118,7 @@ private Plan normalize(LogicalProject<Plan> project) {
         // we need replace alias's child expr with corresponding alias's slot in output
         // so create a customNormalizeMap alias's child -> alias.toSlot to do it
         Map<Expression, Slot> customNormalizeMap = toBePushedDown.stream()
-                .filter(expr -> expr instanceof Alias)
+                .filter(expr -> expr instanceof Alias && !(expr.child(0) instanceof Literal))
                 .collect(Collectors.toMap(expr -> ((Alias) expr).child(), expr -> ((Alias) expr).toSlot(),
                         (oldExpr, newExpr) -> oldExpr));
 
diff --git a/regression-test/data/nereids_rules_p0/normalize_window/normalize_window_nullable_agg_test.out b/regression-test/data/nereids_rules_p0/normalize_window/normalize_window_nullable_agg_test.out
index 2df25bb0d3bed2..f910333a57e176 100644
--- a/regression-test/data/nereids_rules_p0/normalize_window/normalize_window_nullable_agg_test.out
+++ b/regression-test/data/nereids_rules_p0/normalize_window/normalize_window_nullable_agg_test.out
@@ -291,3 +291,7 @@ false
 false
 false
 
+-- !fold_window --
+0	false	0	be
+9999-12-31 23:59:59	false	9999-12-31 23:59:59	b
+
diff --git a/regression-test/suites/nereids_rules_p0/normalize_window/normalize_window_nullable_agg_test.groovy b/regression-test/suites/nereids_rules_p0/normalize_window/normalize_window_nullable_agg_test.groovy
index 9617e8bb19cd95..df3fd63d750742 100644
--- a/regression-test/suites/nereids_rules_p0/normalize_window/normalize_window_nullable_agg_test.groovy
+++ b/regression-test/suites/nereids_rules_p0/normalize_window/normalize_window_nullable_agg_test.groovy
@@ -72,4 +72,61 @@ suite("normalize_window_nullable_agg") {
         sql "select group_concat(xwho order by xwhat) over(partition by xwhen) from windowfunnel_test_normalize_window;"
         exception "order by is not supported"
     }
+
+    sql "set enable_fold_constant_by_be = 1;"
+    sql "drop table if exists fold_window1"
+    sql """create table fold_window1 (
+            pk int,
+                    col_char_255__undef_signed char(255)  null  ,
+            col_char_100__undef_signed char(100)  null  ,
+            col_varchar_255__undef_signed varchar(255)  null  ,
+            col_char_255__undef_signed_not_null char(255)  not null  ,
+                    col_char_100__undef_signed_not_null char(100)  not null  ,
+            col_varchar_255__undef_signed_not_null varchar(255)  not null  ,
+            col_varchar_1000__undef_signed varchar(1000)  null  ,
+            col_varchar_1000__undef_signed_not_null varchar(1000)  not null  ,
+                    col_varchar_1001__undef_signed varchar(1001)  null  ,
+            col_varchar_1001__undef_signed_not_null varchar(1001)  not null  ,
+                    col_string_undef_signed string  null  ,
+                    col_string_undef_signed_not_null string  not null
+    ) engine=olap
+    DUPLICATE KEY(pk, col_char_255__undef_signed, col_char_100__undef_signed, col_varchar_255__undef_signed)
+    distributed by hash(pk) buckets 10
+    properties("bloom_filter_columns" = "col_char_255__undef_signed, col_char_100__undef_signed, col_varchar_255__undef_signed ", "replication_num" = "1");"""
+    sql """insert into fold_window1(pk,col_char_255__undef_signed,col_char_255__undef_signed_not_null,col_char_100__undef_signed
+    ,col_char_100__undef_signed_not_null,col_varchar_255__undef_signed,col_varchar_255__undef_signed_not_null,col_varchar_1000__undef_signed,col_varchar_1000__undef_signed_not_null
+    ,col_varchar_1001__undef_signed,col_varchar_1001__undef_signed_not_null,col_string_undef_signed,col_string_undef_signed_not_null)
+     values (0,'like','9999-12-31 23:59:59','9999-12-31 23:59:59','c','20240803','2024-08-03 13:08:30','300.343','2024-07-01','that''s','9999-12-31 23:59:59','s','b'),
+     (1,'be','g','f','not','20240803','20240803','2024-08-03 13:08:30','g','20240803','0','2024-07-01','be')"""
+
+    sql "drop table if exists fold_window2"
+    sql """create table fold_window2 (
+            pk int,
+                    col_char_255__undef_signed char(255)  null  ,
+            col_char_255__undef_signed_not_null char(255)  not null  ,
+                    col_char_100__undef_signed char(100)  null  ,
+            col_char_100__undef_signed_not_null char(100)  not null  ,
+                    col_varchar_255__undef_signed varchar(255)  null  ,
+            col_varchar_255__undef_signed_not_null varchar(255)  not null  ,
+                    col_varchar_1000__undef_signed varchar(1000)  null  ,
+            col_varchar_1000__undef_signed_not_null varchar(1000)  not null  ,
+                    col_varchar_1001__undef_signed varchar(1001)  null  ,
+            col_varchar_1001__undef_signed_not_null varchar(1001)  not null  ,
+                    col_string_undef_signed string  null  ,
+                    col_string_undef_signed_not_null string  not null
+    ) engine=olap
+    DUPLICATE KEY(pk)
+    distributed by hash(pk) buckets 10
+    properties ("bloom_filter_columns" = "col_char_255__undef_signed, col_char_100__undef_signed, col_varchar_255__undef_signed ", "replication_num" = "1");"""
+    sql """insert into fold_window2(pk,col_char_255__undef_signed,col_char_255__undef_signed_not_null,col_char_100__undef_signed
+    ,col_char_100__undef_signed_not_null,col_varchar_255__undef_signed,col_varchar_255__undef_signed_not_null,col_varchar_1000__undef_signed
+    ,col_varchar_1000__undef_signed_not_null,col_varchar_1001__undef_signed,col_varchar_1001__undef_signed_not_null,col_string_undef_signed,col_string_undef_signed_not_null)
+    values (0,'some','2024-07-01','9999-12-31 23:59:59','9999-12-31 23:59:59','9999-12-31 23:59:59','300.343','2024-07-01','1','1','2024-07-01','2024-08-03 13:08:30','2024-08-03 13:08:30');"""
+
+    qt_fold_window """
+    select initcap(col_varchar_1001__undef_signed_not_null)  col_alias97650 , starts_with('ourBZbRijD', "e")  AS col_alias97651 ,
+    col_varchar_1001__undef_signed_not_null  AS col_alias97652 , LAST_VALUE(col_string_undef_signed_not_null , false) over ( order by pk ) 
+     AS col_alias97653  from fold_window1 where 'DCOFMrybqf' <> (select min (  col_char_255__undef_signed ) 
+     from fold_window2)  ORDER BY col_alias97650,col_alias97651,col_alias97652,col_alias97653 ;
+    """
 }
\ No newline at end of file

From 2a7c2e3f410a715ff9c22890de06d809fa411c52 Mon Sep 17 00:00:00 2001
From: yujun <yujun@selectdb.com>
Date: Thu, 19 Dec 2024 15:47:50 +0800
Subject: [PATCH 399/399] [fix](create table) fix create table fail msg
 (#45623)

---
 .../org/apache/doris/clone/TabletScheduler.java |  2 +-
 .../java/org/apache/doris/system/Backend.java   |  3 +++
 .../apache/doris/system/SystemInfoService.java  |  2 +-
 .../apache/doris/catalog/CreateTableTest.java   | 17 +++++++++++++++++
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
index 504e1d36a65af3..1545236aa59cd0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -1545,7 +1545,7 @@ private RootPathLoadStatistic doChooseAvailableDestPath(TabletSchedCtx tabletCtx
                 !allFitPathsSameMedium.isEmpty() ? allFitPathsSameMedium : allFitPathsDiffMedium;
         if (allFitPaths.isEmpty()) {
             List<String> backendsInfo = Env.getCurrentSystemInfo().getAllClusterBackendsNoException().values().stream()
-                    .filter(be -> be.getLocationTag() == tag)
+                    .filter(be -> be.getLocationTag().equals(tag))
                     .map(Backend::getDetailsForCreateReplica)
                     .collect(Collectors.toList());
             throw new SchedException(Status.UNRECOVERABLE, String.format("unable to find dest path for new replica"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
index 974c0e0cae13a7..c864e1ba2ae0ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
@@ -28,6 +28,7 @@
 import org.apache.doris.common.util.PrintableMap;
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.qe.SimpleScheduler;
 import org.apache.doris.resource.Tag;
 import org.apache.doris.system.HeartbeatResponse.HbStatus;
 import org.apache.doris.thrift.TDisk;
@@ -340,6 +341,8 @@ public String getDetailsForCreateReplica() {
             sb.append(", isDecommissioned=true, exclude it");
         } else if (isComputeNode()) {
             sb.append(", isComputeNode=true, exclude it");
+        } else if (!Config.disable_backend_black_list && !SimpleScheduler.isAvailable(this)) {
+            sb.append(", is in black list, exclude it");
         } else {
             sb.append(", hdd disks count={");
             if (hddOk > 0) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
index 447dc3457aefe1..f7359293621e3d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
@@ -567,7 +567,7 @@ public String getDetailsForCreateReplica(ReplicaAllocation replicaAlloc) {
         StringBuilder sb = new StringBuilder(" Backends details: ");
         for (Tag tag : replicaAlloc.getAllocMap().keySet()) {
             sb.append("backends with tag ").append(tag).append(" is ");
-            sb.append(idToBackendRef.values().stream().filter(be -> be.getLocationTag() == tag)
+            sb.append(idToBackendRef.values().stream().filter(be -> be.getLocationTag().equals(tag))
                     .map(Backend::getDetailsForCreateReplica)
                     .collect(Collectors.toList()));
             sb.append(", ");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 50e9ac40bc7d93..76acb4ad76e196 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -25,12 +25,15 @@
 import org.apache.doris.common.ExceptionChecker;
 import org.apache.doris.common.FeConstants;
 import org.apache.doris.common.UserException;
+import org.apache.doris.resource.Tag;
 import org.apache.doris.utframe.TestWithFeService;
 
+import com.google.common.collect.Maps;
 import org.junit.Assert;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 
@@ -789,6 +792,20 @@ public void testCreateTableWithForceReplica() throws DdlException  {
         }
     }
 
+    @Test
+    public void testCreateTableDetailMsg() throws Exception {
+        Map<Tag, Short> allocMap = Maps.newHashMap();
+        allocMap.put(Tag.create(Tag.TYPE_LOCATION, "group_a"),  (short) 6);
+        Assert.assertEquals(" Backends details: backends with tag {\"location\" : \"group_a\"} is [], ",
+                Env.getCurrentSystemInfo().getDetailsForCreateReplica(new ReplicaAllocation(allocMap)));
+
+        allocMap.clear();
+        allocMap.put(Tag.create(Tag.TYPE_LOCATION, new String(Tag.VALUE_DEFAULT_TAG)),  (short) 6);
+        String msg = Env.getCurrentSystemInfo().getDetailsForCreateReplica(new ReplicaAllocation(allocMap));
+        Assert.assertTrue("msg: " + msg, msg.contains("Backends details: backends with tag {\"location\" : \"default\"} is [[backendId=")
+                && msg.contains("hdd disks count={ok=1,}, ssd disk count={}], [backendId="));
+    }
+
     @Test
     public void testCreateTableWithMinLoadReplicaNum() throws Exception {
         ExceptionChecker.expectThrowsNoException(